#pragma once
/*
 * (C) Copyright 2020-2021 UCAR
 *
 * This software is licensed under the terms of the Apache Licence Version 2.0
 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
 */
/*! \defgroup ioda_cxx_variable Variables, Data Access, and Selections
 * \brief The main data storage methods and objects in IODA.
 * \ingroup ioda_cxx_api
 *
 * @{
 * \file Variable.h
 * \brief @link ioda_cxx_variable Interfaces @endlink for ioda::Variable and related classes.
 */
#include <cstring>
#include <gsl/gsl-lite.hpp>
#include <list>
#include <map>
#include <memory>
#include <string>
#include <tuple>
#include <utility>
#include <vector>

#include "ioda/Attributes/Has_Attributes.h"
#include "ioda/Exception.h"
#include "ioda/Misc/Eigen_Compat.h"
#include "ioda/Python/Var_ext.h"
#include "ioda/Types/Marshalling.h"
#include "ioda/Types/Type.h"
#include "ioda/Types/Type_Provider.h"
#include "ioda/Variables/Fill.h"
#include "ioda/Variables/Selection.h"
#include "ioda/defs.h"

namespace ioda {
class Attribute;
class Variable;
struct VariableCreationParameters;

struct Named_Variable;

namespace detail {
class Attribute_Backend;
class Variable_Backend;

/// \brief Exists to prevent constructor conflicts when passing a backend into
///   a frontend object.
/// \ingroup ioda_cxx_variable
/// \bug Need to add a virtual resize function.
template <class Variable_Implementation = Variable>
class Variable_Base {
protected:
  /// Using an opaque object to implement the backend.
  std::shared_ptr<Variable_Backend> backend_;

  /// @name General Functions
  /// @{

  Variable_Base(std::shared_ptr<Variable_Backend>);

public:
  virtual ~Variable_Base();

  /// @}
  /// @name Metadata manipulation
  /// @{

  /// Attributes
  Has_Attributes atts;

  /// @}

  /// @name General Functions
  /// @{

  /// Gets a handle to the underlying object that implements the backend functionality.
  std::shared_ptr<Variable_Backend> get() const;

  /// @}
  /// @name Type-querying Functions
  /// @{

  /// Get type
  virtual Type getType() const;
  /// Get type
  inline Type type() const { return getType(); }

  /// Query the backend and get the type provider.
  virtual detail::Type_Provider* getTypeProvider() const;

  /// \brief Convenience function to check a Variable's storage type.
  /// \param DataType is the type of the data. I.e. float, int, int32_t, uint16_t, std::string, etc.
  /// \returns True if the type matches
  /// \returns False (0) if the type does not match
  /// \throws ioda::Exception if an error occurred.
  template <class DataType>
  bool isA() const {
    Type templateType = Types::GetType_Wrapper<DataType>::GetType(getTypeProvider());

    return isA(templateType);
  }
  /// Hand-off to the backend to check equivalence
  virtual bool isA(Type lhs) const;

  /// Python compatability function
  inline bool isA(BasicTypes dataType) const { return isA(Type(dataType, getTypeProvider())); }
  /// \internal pybind11
  inline bool _py_isA2(BasicTypes dataType) { return isA(dataType); }
  /// Convenience function to query type
  BasicTypes getBasicType() const;

  /// @}
  /// @name Querying Functions for Fill Values, Chunking and Compression
  /// @{

  /// @brief Convenience function to get fill value, attributes,
  ///   chunk sizes, and compression in a collective call.
  /// @details This function has better performance on some engines
  ///   for bulk operations than calling separately.
  /// @param doAtts includes attributes in the creation parameters.
  /// @param doDims includes dimension scales in the creation parameters.
  /// Although dimensions are attributes on some backends, we treat them
  /// separately in this function.
  /// @returns the filled-in VariableCreationParameters object
  virtual VariableCreationParameters getCreationParameters(bool doAtts = true,
                                                           bool doDims = true) const;

  /// \brief Check if a variable has a fill value set
  /// \returns true if a fill value is set, false otherwise.
  virtual bool hasFillValue() const;

  /// Remap fill value storage type into this class.
  typedef detail::FillValueData_t FillValueData_t;

  /// \brief Retrieve the fill value
  /// \returns an object of type FillValueData_t that stores
  ///   the fill value. If there is no fill value, then
  ///   FillValueData_t::set_ will equal false. The fill
  ///   value data will be stored in
  ///   FillValueData_t::fillValue_ (for simple types), or in
  ///   FillValueData_t::stringFillValue_ (for strings only).
  /// \note Recommend querying isA to make sure that you
  ///   are reading the fill value as the correct type.
  virtual FillValueData_t getFillValue() const;

  /// \brief Retrieve the chunking options for the Variable.
  /// \note Not all backends support chunking, but they should
  ///   all store the desired chunk size information in case the
  ///   Variable is copied to a new backend.
  /// \returns a vector containing the chunk sizes.
  /// \returns an empty vector if chunking is not used.
  virtual std::vector<Dimensions_t> getChunkSizes() const;

  /// \brief Retrieve the GZIP compression options for the Variable.
  /// \note Not all backends support compression (and those that
  ///   do also require chunking support). They should store this
  ///   information anyways, in case the Variable is copied to
  ///   a new backend.
  /// \returns a pair indicating 1) whether GZIP is requested and
  ///   2) the compression level that is desired.
  virtual std::pair<bool, int> getGZIPCompression() const;

  /// \brief Retrieve the SZIP compression options for the Variable.
  /// \note Not all backends support compression (and those that
  ///   do also require chunking support). They should store this
  ///   information anyways, in case the Variable is copied to
  ///   a new backend.
  /// \returns a tuple indicating 1) whether SZIP is requested,
  ///   2) PixelsPerBlock, and 3) general SZIP filter option flags.
  virtual std::tuple<bool, unsigned, unsigned> getSZIPCompression() const;

  /// @}
  /// @name Data Space-Querying Functions
  /// @{

  // Get dataspace
  // JEDI_NODISCARD DataSpace getSpace() const;

  /// Get current and maximum dimensions, and number of total points.
  /// \note In Python, see the dims property.
  virtual Dimensions getDimensions() const;

  /// \brief Resize the variable.
  /// \note Not all variables are resizable. This depends
  ///   on backend support. For HDF5, the variable must be chunked
  ///   and must not exceed max_dims.
  /// \note Bad things may happen if a variable's dimension scales
  ///   have different lengths than its dimensions. Resize them
  ///   together, preferably using the ObsSpace resize function.
  /// \see ObsSpace::resize
  /// \param newDims are the new dimensions.
  virtual Variable resize(const std::vector<Dimensions_t>& newDims);

  /// Attach a dimension scale to this Variable.
  virtual Variable attachDimensionScale(unsigned int DimensionNumber, const Variable& scale);
  /// Detach a dimension scale
  virtual Variable detachDimensionScale(unsigned int DimensionNumber, const Variable& scale);
  /// Set dimensions (convenience function to several invocations of attachDimensionScale).
  Variable setDimScale(const std::vector<Variable>& dims);
  /// Set dimensions (convenience function to several invocations of attachDimensionScale).
  Variable setDimScale(const std::vector<Named_Variable>& dims);
  /// Set dimensions (convenience function to several invocations of attachDimensionScale).
  Variable setDimScale(const Variable& dims);
  /// Set dimensions (convenience function to several invocations of attachDimensionScale).
  Variable setDimScale(const Variable& dim1, const Variable& dim2);
  /// Set dimensions (convenience function to several invocations of attachDimensionScale).
  Variable setDimScale(const Variable& dim1, const Variable& dim2, const Variable& dim3);

  /// Is this Variable used as a dimension scale?
  virtual bool isDimensionScale() const;

  /// Designate this table as a dimension scale
  virtual Variable setIsDimensionScale(const std::string& dimensionScaleName);
  /// Get the name of this Variable's defined dimension scale
  inline std::string getDimensionScaleName() const {
    std::string r;
    getDimensionScaleName(r);
    return r;
  }
  virtual Variable getDimensionScaleName(std::string& res) const;

  /// Is a dimension scale attached to this Variable in a certain position?
  virtual bool isDimensionScaleAttached(unsigned int DimensionNumber, const Variable& scale) const;

  /// \brief Which dimensions are attached at which positions? This function may offer improved
  /// performance on some backends compared to serial isDimensionScaleAttached calls.
  /// \param scalesToQueryAgainst is a vector containing the scales. You can pass in
  ///   "tagged" strings that map the Variable to a name.
  ///   If you do not pass in a scale to check against, then this scale will not be checked and
  ///   will not be present in the function output.
  /// \param firstOnly is specified when only one dimension can be attached to each axis (the
  /// default).
  /// \returns a vector with the same length as the variable's dimensionality.
  ///   Each variable dimension in the vector can have one or more attached scales. These scales are
  ///   returned as their own, inner, vector of pair<string, Variable>.
  virtual std::vector<std::vector<Named_Variable>> getDimensionScaleMappings(
    const std::list<Named_Variable>& scalesToQueryAgainst,
    bool firstOnly = true) const;

  /// @}
  /// @name Writing Data
  /// @{

  /// \brief The fundamental write function. Backends overload this function to implement
  ///   all write operations.
  ///
  /// \details This function writes a span of bytes (characters) to the backend attribute
  ///   storage. No type conversions take place here (see the templated conversion function,
  ///   below).
  ///
  /// \param data is a span of data.
  /// \param in_memory_datatype is an opaque (backend-level) object that describes the
  ///   placement of the data in memory. Usually ignorable - needed for complex data structures.
  /// \param mem_selection is the user's memory layout representing the location where
  ///   the data is read from.
  /// \param file_selection is the backend's memory layout representing the
  ///   location where the data are written to.
  /// \throws ioda::xError if data has the wrong size.
  /// \returns The variable (for chaining).
  virtual Variable write(gsl::span<const char> data, const Type& in_memory_dataType,
                         const Selection& mem_selection  = Selection::all,
                         const Selection& file_selection = Selection::all);

  /// \brief Write the Variable
  /// \note Ensure that the correct dimension ordering is preserved.
  /// \note With default parameters, the entire Variable is written.
  /// \tparam DataType is the type of the data to be written.
  /// \tparam Marshaller is a class that serializes / deserializes data.
  /// \tparam TypeWrapper translates DataType into a form that the backend understands.
  /// \param data is a span of data.
  /// \param mem_selection is the user's memory layout representing the location where
  ///   the data is read from.
  /// \param file_selection is the backend's memory layout representing the
  ///   location where the data are written to.
  /// \throws ioda::xError if data has the wrong size.
  /// \returns The variable (for chaining).
  template <class DataType, class Marshaller = Object_Accessor<DataType>,
            class TypeWrapper = Types::GetType_Wrapper<DataType>>
  Variable_Implementation write(const gsl::span<DataType> data,
                                const Selection& mem_selection  = Selection::all,
                                const Selection& file_selection = Selection::all) {
    try {
      Marshaller m;
      auto d = m.serialize(data);
      return write(gsl::make_span<const char>(
                      reinterpret_cast<const char*>(d->DataPointers.data()),
                     d->DataPointers.size() * Marshaller::bytesPerElement_),
                   TypeWrapper::GetType(getTypeProvider()), mem_selection, file_selection);
    } catch (...) {
      std::throw_with_nested(Exception(ioda_Here()));
    }
  }

  /// \brief Write the Variable
  /// \note Ensure that the correct dimension ordering is preserved.
  /// \note With default parameters, the entire Variable is written.
  /// \tparam DataType is the type of the data to be written.
  /// \tparam Marshaller is a class that serializes / deserializes data.
  /// \tparam TypeWrapper translates DataType into a form that the backend understands.
  /// \param data is a span of data.
  /// \param mem_selection is the user's memory layout representing the location where
  ///   the data is read from.
  /// \param file_selection is the backend's memory layout representing the
  ///   location where the data are written to.
  /// \throws ioda::xError if data has the wrong size.
  /// \returns The variable (for chaining).
  template <class DataType, class Marshaller = Object_Accessor<DataType>,
            class TypeWrapper = Types::GetType_Wrapper<DataType>>
  Variable_Implementation write(const gsl::span<const DataType> data,
                                const Selection& mem_selection  = Selection::all,
                                const Selection& file_selection = Selection::all) {
    try {
      Marshaller m;
      auto d = m.serialize(data);
      return write(gsl::make_span<const char>(
                      reinterpret_cast<const char*>(d->DataPointers.data()),
                     d->DataPointers.size() * Marshaller::bytesPerElement_),
                   TypeWrapper::GetType(getTypeProvider()), mem_selection, file_selection);
    } catch (...) {
      std::throw_with_nested(Exception(ioda_Here()));
    }
  }

  /// \brief Write the variable
  /// \tparam DataType is the type of the data to be written.
  /// \tparam Marshaller is a class that serializes / deserializes data.
  /// \tparam TypeWrapper translates DataType into a form that the backend understands.
  /// \param data is a span of data.
  /// \param mem_selection is the user's memory layout representing the location where
  ///   the data is read from.
  /// \param file_selection is the backend's memory layout representing the
  ///   location where the data are written to.
  /// \throws ioda::xError if data has the wrong size.
  /// \returns The variable (for chaining).

  template <class DataType, class Marshaller = Object_Accessor<DataType>,
            class TypeWrapper = Types::GetType_Wrapper<DataType>>
  Variable_Implementation write(const std::vector<DataType>& data,
                                const Selection& mem_selection  = Selection::all,
                                const Selection& file_selection = Selection::all) {
    try {
      return this->write<DataType, Marshaller, TypeWrapper>(gsl::make_span(data), mem_selection,
                                                            file_selection);
    } catch (...) {
      std::throw_with_nested(Exception(ioda_Here()));
    }
  }

  /// \brief Write an Eigen object (a Matrix, an Array, a Block, a Map).
  /// \tparam EigenClass is the type of the Eigen object being written.
  /// \param d is the data to be written.
  /// \param mem_selection is the user's memory layout representing the location where
  ///   the data is read from.
  /// \param file_selection is the backend's memory layout representing the
  ///   location where the data are written to.
  /// \throws ioda::xError on a dimension mismatch.
  /// \returns the variable
  template <class EigenClass>
  Variable_Implementation writeWithEigenRegular(const EigenClass& d,
                                                const Selection& mem_selection  = Selection::all,
                                                const Selection& file_selection = Selection::all) {
#if 1  //__has_include("Eigen/Dense")
    try {
      typedef typename EigenClass::Scalar ScalarType;
      // If d is already in Row Major form, then this is optimized out.
      Eigen::Array<ScalarType, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor> dout;
      dout.resize(d.rows(), d.cols());
      dout               = d;
      const auto& dconst = dout;  // To make some compilers happy.
      auto sp            = gsl::make_span(dconst.data(), static_cast<int>(d.rows() * d.cols()));

      return write<ScalarType>(sp, mem_selection, file_selection);
    } catch (...) {
      std::throw_with_nested(Exception(ioda_Here()));
    }
#else
    static_assert(false, "The Eigen headers cannot be found, so this function cannot be used.");
#endif
  }

  /// \brief Write an Eigen Tensor-like object
  /// \tparam EigenClass is the type of the Eigen object being written.
  /// \param d is the data to be written.
  /// \param mem_selection is the user's memory layout representing the location where
  ///   the data is read from.
  /// \param file_selection is the backend's memory layout representing the
  ///   location where the data are written to.
  /// \throws ioda::xError on a dimension mismatch.
  /// \returns the variable
  template <class EigenClass>
  Variable_Implementation writeWithEigenTensor(const EigenClass& d,
                                               const Selection& mem_selection  = Selection::all,
                                               const Selection& file_selection = Selection::all) {
#if 1  //__has_include("unsupported/Eigen/CXX11/Tensor")
    try {
      ioda::Dimensions dims = detail::EigenCompat::getTensorDimensions(d);

      auto sp  = (gsl::make_span(d.data(), dims.numElements));
      auto res = write(sp, mem_selection, file_selection);
      return res;
    } catch (...) {
      std::throw_with_nested(Exception(ioda_Here()));
    }
#else
    static_assert(
      false, "The Eigen unsupported/ headers cannot be found, so this function cannot be used.");
#endif
  }

  /// @}
  /// @name Reading Data
  /// @{

  /// \brief Read the Variable - as char array. Ordering is row-major.
  /// \details This is the fundamental read function that has to be implemented.
  /// \param data is a byte-array that will hold the read data.
  /// \param in_memory_dataType describes how ioda should arrange the read data in memory.
  ///   As floats? As doubles? Strings?
  /// \param mem_selection is the user's memory layout representing the location where
  ///   the data is read from.
  /// \param file_selection is the backend's memory layout representing the
  ///   location where the data are written to.
  /// \note Ensure that the correct dimension ordering is preserved
  /// \note With default parameters, the entire Variable is read
  virtual Variable read(gsl::span<char> data, const Type& in_memory_dataType,
                        const Selection& mem_selection  = Selection::all,
                        const Selection& file_selection = Selection::all) const;

  /// \brief Read the variable into a span (range) or memory. Ordering is row-major.
  /// \tparam DataType is the type of the data to be written.
  /// \tparam Marshaller is a class that serializes / deserializes data.
  /// \tparam TypeWrapper translates DataType into a form that the backend understands.
  /// \param data is a byte-array that will hold the read data.
  /// \param mem_selection is the user's memory layout representing the location where
  ///   the data is read from.
  /// \param file_selection is the backend's memory layout representing the
  ///   location where the data are written to.
  /// \todo Add in the dataspaces!
  template <class DataType, class Marshaller = ioda::Object_Accessor<DataType>,
            class TypeWrapper = Types::GetType_Wrapper<DataType>>
  Variable_Implementation read(gsl::span<DataType> data,
                               const Selection& mem_selection  = Selection::all,
                               const Selection& file_selection = Selection::all) const {
    try {
      const size_t numObjects = data.size();

      detail::PointerOwner pointerOwner = getTypeProvider()->getReturnedPointerOwner();
      Marshaller m(pointerOwner);
      auto p = m.prep_deserialize(numObjects);
      read(gsl::make_span<char>(
             reinterpret_cast<char*>(p->DataPointers.data()),
             // Logic note: sizeof mutable data type. If we are
             // reading in a string, then mutable data type is char*,
             // which works because address pointers have the same size.
             p->DataPointers.size() * Marshaller::bytesPerElement_),
           TypeWrapper::GetType(getTypeProvider()), mem_selection, file_selection);
      m.deserialize(p, data);

      return Variable_Implementation{backend_};
    } catch (...) {
      std::throw_with_nested(Exception(ioda_Here()));
    }
  }

  /// \brief Read the variable into a vector. Resize if needed. For a non-resizing
  ///   version, use a gsl::span.
  /// \details Ordering is row-major.
  /// \tparam DataType is the type of the data to be written.
  /// \tparam Marshaller is a class that serializes / deserializes data.
  /// \tparam TypeWrapper translates DataType into a form that the backend understands.
  /// \param data is a byte-array that will hold the read data.
  /// \param mem_selection is the user's memory layout representing the location where
  ///   the data is read from.
  /// \param file_selection is the backend's memory layout representing the
  ///   location where the data are written to.
  /// \bug Resize only if needed, and resize to the proper extent depending on
  ///   mem_selection and file_selection.
  template <class DataType, class Marshaller = ioda::Object_Accessor<DataType>,
            class TypeWrapper = Types::GetType_Wrapper<DataType>>
  Variable_Implementation read(std::vector<DataType>& data,
                               const Selection& mem_selection  = Selection::all,
                               const Selection& file_selection = Selection::all) const {
    data.resize(getDimensions().numElements); // TODO(Ryan): remove
    return read<DataType, Marshaller, TypeWrapper>(gsl::make_span(data.data(), data.size()),
                                                   mem_selection, file_selection);
  }

  /// \brief Read the variable into a new vector. Python convenience function.
  /// \bug Get correct size based on selection operands.
  /// \tparam DataType is the type of the data to be written.
  /// \tparam Marshaller is a class that serializes / deserializes data.
  /// \tparam TypeWrapper translates DataType into a form that the backend understands.
  /// \param mem_selection is the user's memory layout representing the location where
  ///   the data is read from.
  /// \param file_selection is the backend's memory layout representing the
  ///   location where the data are written to.
  template <class DataType, class Marshaller = ioda::Object_Accessor<DataType>,
            class TypeWrapper = Types::GetType_Wrapper<DataType>>
  std::vector<DataType> readAsVector(const Selection& mem_selection  = Selection::all,
                                     const Selection& file_selection = Selection::all) const {
    std::vector<DataType> data(getDimensions().numElements);
    read<DataType, Marshaller, TypeWrapper>(gsl::make_span(data.data(), data.size()), mem_selection,
                                            file_selection);
    return data;
  }

  /// \brief Valarray read convenience function. Resize if needed.
  ///   For a non-resizing version, use a gsl::span.
  /// \tparam DataType is the type of the data to be written.
  /// \tparam Marshaller is a class that serializes / deserializes data.
  /// \tparam TypeWrapper translates DataType into a form that the backend understands.
  /// \param data is a valarray acting as a data buffer that is filled with the
  ///   metadata's contents. It gets resized as needed.
  /// \param mem_selection is the user's memory layout representing the location where
  ///   the data is read from.
  /// \param file_selection is the backend's memory layout representing the
  ///   location where the data are written to.
  /// \returns Another instance of this Attribute. Used for operation chaining.
  /// \note data will be stored in row-major order.
  template <class DataType, class Marshaller = ioda::Object_Accessor<DataType>,
            class TypeWrapper = Types::GetType_Wrapper<DataType>>
  Variable_Implementation read(std::valarray<DataType>& data,
                               const Selection& mem_selection  = Selection::all,
                               const Selection& file_selection = Selection::all) const {
    /// \bug Resize only if needed, and resize to the proper extent depending on
    /// mem_selection and file_selection.
    data.resize(getDimensions().numElements); // TODO(Ryan): remove
    return read<DataType, Marshaller, TypeWrapper>(gsl::make_span(std::begin(data), std::end(data)),
                                                   mem_selection, file_selection);
  }

  /// \brief Read data into an Eigen::Array, Eigen::Matrix, Eigen::Map, etc.
  /// \tparam EigenClass is a template pointing to the Eigen object.
  ///   This template must provide the EigenClass::Scalar typedef.
  /// \tparam Resize indicates whether the Eigen object should be resized
  ///   if there is a dimension mismatch. Not all Eigen objects can be resized.
  /// \param res is the Eigen object.
  /// \param mem_selection is the user's memory layout representing the location where
  ///   the data is read from.
  /// \param file_selection is the backend's memory layout representing the
  ///   location where the data are written to.
  /// \returns Another instance of this Variable. Used for operation chaining.
  /// \throws ioda::xError if the variable's dimensionality is
  ///   too high.
  /// \throws ioda::xError if resize = false and there is a dimension mismatch.
  /// \note When reading in a 1-D object, the data are read as a column vector.
  template <class EigenClass, bool Resize = detail::EigenCompat::CanResize<EigenClass>::value>
  Variable_Implementation readWithEigenRegular(EigenClass& res,
                                               const Selection& mem_selection = Selection::all,
                                               const Selection& file_selection
                                               = Selection::all) const {
    /// \bug Resize only if needed, and resize to the proper extent depending on
    /// mem_selection and file_selection.
#if 1  //__has_include("Eigen/Dense")
    try {
      typedef typename EigenClass::Scalar ScalarType;

      static_assert(
        !(Resize && !detail::EigenCompat::CanResize<EigenClass>::value),
        "This object cannot be resized, but you have specified that a resize is required.");

      // Check that the dimensionality is 1 or 2.
      const auto dims = getDimensions();
      if (dims.dimensionality > 2)
        throw Exception("Dimensionality too high for a regular Eigen read. Use "
          "Eigen::Tensor reads instead.", ioda_Here());

      int nDims[2] = {1, 1};
      if (dims.dimsCur.size() >= 1) nDims[0] = gsl::narrow<int>(dims.dimsCur[0]);
      if (dims.dimsCur.size() >= 2) nDims[1] = gsl::narrow<int>(dims.dimsCur[1]);

      // Resize if needed.
      if (Resize)
        detail::EigenCompat::DoEigenResize(res, nDims[0],
                                           nDims[1]);  // nullop if the size is already correct.
      else if (dims.numElements != (size_t)(res.rows() * res.cols()))
        throw Exception("Size mismatch", ioda_Here());

      // Array copy to preserve row vs column major format.
      // Should be optimized away by the compiler if unneeded.
      // Note to the reader: We are reading in the data to a temporary object.
      // We can size _this_ temporary object however we want.
      // The temporary is used to swap row / column indices if needed.
      // It should be optimized away if not needed... making sure this happens is a todo.
      Eigen::Array<ScalarType, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor> data_in(res.rows(),
                                                                                res.cols());

      auto ret = read<ScalarType>(gsl::span<ScalarType>(data_in.data(), dims.numElements),
                                  mem_selection, file_selection);
      res      = data_in;
      return ret;
    } catch (...) {
      std::throw_with_nested(Exception(ioda_Here()));
    }
#else
    static_assert(false, "The Eigen headers cannot be found, so this function cannot be used.");
#endif
  }

  /// \brief Read data into an Eigen::Array, Eigen::Matrix, Eigen::Map, etc.
  /// \tparam EigenClass is a template pointing to the Eigen object.
  ///   This template must provide the EigenClass::Scalar typedef.
  /// \param res is the Eigen object.
  /// \param mem_selection is the user's memory layout representing the location where
  ///   the data is read from.
  /// \param file_selection is the backend's memory layout representing the
  ///   location where the data are written to.
  /// \returns Another instance of this Variable. Used for operation chaining.
  /// \throws ioda::Exception if there is a size mismatch.
  /// \note When reading in a 1-D object, the data are read as a column vector.
  template <class EigenClass>
  Variable_Implementation readWithEigenTensor(EigenClass& res,
                                              const Selection& mem_selection = Selection::all,
                                              const Selection& file_selection
                                              = Selection::all) const {
#if 1  //__has_include("unsupported/Eigen/CXX11/Tensor")
    try {
      // Check dimensionality of source and destination
      const auto ioda_dims  = getDimensions();
      const auto eigen_dims = ioda::detail::EigenCompat::getTensorDimensions(res);
      if (ioda_dims.numElements != eigen_dims.numElements)
        throw Exception("Size mismatch for Eigen Tensor-like read.", ioda_Here());

      auto sp = (gsl::make_span(res.data(), eigen_dims.numElements));
      return read(sp, mem_selection, file_selection);
    } catch (...) {
      std::throw_with_nested(Exception(ioda_Here()));
    }
#else
    static_assert(
      false, "The Eigen unsupported/ headers cannot be found, so this function cannot be used.");
#endif
  }

  /// \internal Python binding function
  template <class EigenClass>
  EigenClass _readWithEigenRegular_python(const Selection& mem_selection  = Selection::all,
                                          const Selection& file_selection = Selection::all) const {
    EigenClass data;
    readWithEigenRegular(data, mem_selection, file_selection);
    return data;
  }

  /// @brief Convert a selection into its backend representation
  /// @param sel is the frontend selection.
  /// @return The cached backend selection.
  virtual Selections::SelectionBackend_t instantiateSelection(const Selection& sel) const;

  /// @}

private:
  /// \brief get the fill value from the netcdf specification (_FillValue attribute)
  FillValueData_t getNcFillValue() const;

  /// \brief check if fill data objects match, print warning if they don't match
  /// \param hdfFill fill value obtained from the hdf fill value property
  /// \param ncFill fill value obtained from the netcdf fill value attribute
  void checkWarnFillValue(FillValueData_t & hdfFill, FillValueData_t & ncFill) const;

  /// \brief run an action according to the current variable data type
  /// \details this function will call the function passed to it through the action
  /// parameter, giving this action a typed entity that can be identified by decltype.
  /// At this point, all of the types supported by Variable_Base::getBasicType() except
  /// for bool are handled in this function.
  /// \param action Function object callable with a single argument which is identifiable
  /// by decltype
  // TODO(srh) Additional development is required to properly handle the bool data type.
  template <typename Action>
  auto runForVarType(const Action & action) const {
    if (isA<int>()) {
      int typeMe;
      return action(typeMe);
    } else if (isA<unsigned int>()) {
      unsigned int typeMe;
      return action(typeMe);
    } else if (isA<float>()) {
      float typeMe;
      return action(typeMe);
    } else if (isA<double>()) {
      double typeMe;
      return action(typeMe);
    } else if (isA<std::string>()) {
      std::string typeMe;
      return action(typeMe);
    } else if (isA<long>()) {
      long typeMe;
      return action(typeMe);
    } else if (isA<unsigned long>()) {
      unsigned long typeMe;
      return action(typeMe);
    } else if (isA<short>()) {
      short typeMe;
      return action(typeMe);
    } else if (isA<unsigned short>()) {
      unsigned short typeMe;
      return action(typeMe);
    } else if (isA<long long>()) {
      long long typeMe;
      return action(typeMe);
    } else if (isA<unsigned long long>()) {
      unsigned long long typeMe;
      return action(typeMe);
    } else if (isA<int32_t>()) {
      int32_t typeMe;
      return action(typeMe);
    } else if (isA<uint32_t>()) {
      uint32_t typeMe;
      return action(typeMe);
    } else if (isA<int16_t>()) {
      int16_t typeMe;
      return action(typeMe);
    } else if (isA<uint16_t>()) {
      uint16_t typeMe;
      return action(typeMe);
    } else if (isA<int64_t>()) {
      int64_t typeMe;
      return action(typeMe);
    } else if (isA<uint64_t>()) {
      uint64_t typeMe;
      return action(typeMe);
    } else if (isA<long double>()) {
      long double typeMe;
      return action(typeMe);
    } else if (isA<char>()) {
      char typeMe;
      return action(typeMe);
    } else if (isA<unsigned char>()) {
      unsigned char typeMe;
      return action(typeMe);
    } else {
      std::throw_with_nested(Exception("Unsupported variable data type", ioda_Here()));
    }
  }

};
// extern template class Variable_Base<Variable>;
}  // namespace detail

/** \brief Variables store data!
 * \ingroup ioda_cxx_variable
 *
 * A variable represents a single field of data. It can be multi-dimensional and
 * usually has one or more attached **dimension scales**.
 *
 * Variables have Metadata, which describe the variable (i.e. valid_range, long_name, units).
 * Variables can have different data types (i.e. int16_t, float, double, string, datetime).
 * Variables can be resized. Depending on the backend, the data in a variable can be stored
 * using chunks, and may also be compressed.
 *
 * The backend manages how variables are stored in memory or on disk. The functions in the
 * Variable class provide methods to query and set data. The goal is to have data transfers
 * involve as few copies as possible.
 *
 * Variable objects themselves are lightweight handles that may be passed across different
 * parts of a program. Variables are always stored somewhere in a Group (or ObsSpace), so you
 * can always re-open a handle.
 *
 * \note Thread and MPI safety depend on the specific backends used to implement a variable.
 * \note A variable may be linked to multiple groups and listed under multiple names, so long as
 * the storage backends are all the same.
 **/
class IODA_DL Variable : public detail::Variable_Base<Variable> {
public:
  /// @name General Functions
  /// @{

  Variable();
  Variable(std::shared_ptr<detail::Variable_Backend> b);
  Variable(const Variable&);
  Variable& operator=(const Variable&);
  virtual ~Variable();

  /// @}
  /// @name Python compatability objects
  /// @{

  detail::python_bindings::VariableIsA<Variable> _py_isA;

  detail::python_bindings::VariableReadVector<Variable> _py_readVector;
  detail::python_bindings::VariableReadNPArray<Variable> _py_readNPArray;

  detail::python_bindings::VariableWriteVector<Variable> _py_writeVector;
  detail::python_bindings::VariableWriteNPArray<Variable> _py_writeNPArray;

  detail::python_bindings::VariableScales<Variable> _py_scales;

  /// @}
};

namespace detail {
/// \brief Variable backends inherit from this.
class IODA_DL Variable_Backend : public Variable_Base<Variable> {
public:
  virtual ~Variable_Backend();

  /// Default, trivial implementation. Customizable by backends for performance.
  std::vector<std::vector<Named_Variable>> getDimensionScaleMappings(
    const std::list<Named_Variable>& scalesToQueryAgainst,
    bool firstOnly = true) const override;

  /// Default implementation. Customizable by backends for performance.
  VariableCreationParameters getCreationParameters(bool doAtts = true,
                                                   bool doDims = true) const override;

protected:
  Variable_Backend();

  /// @brief This function de-encapsulates an Attribute's backend storage object.
  ///   This function is used by Variable_Backend's derivatives when accessing a
  ///   Variable's Attributes. IODA-internal use only.
  /// @tparam Attribute_Implementation is a dummy parameter used by Attributes.
  /// @param base is the Attribute whose backend you want.
  /// @return The encapsulated backend object.
  template <class Attribute_Implementation = Attribute>
  static std::shared_ptr<Attribute_Backend> _getAttributeBackend(
    const Attribute_Implementation& att) {
    return att.backend_;
  }

  /// @brief This function de-encapsulates a Has_Attributes backend storage object.
  ///   IODA-internal use only.
  /// @tparam Has_Attributes_Implementation is a dummy parameter for template resolution.
  /// @param hatts is the Has_Attributes whose backend you want.
  /// @return The encapsulated backend object.
  template <class Has_Attributes_Implementation = Has_Attributes>
  static std::shared_ptr<Has_Attributes_Backend> _getHasAttributesBackend(
    const Has_Attributes_Implementation& hatts) {
    return hatts.backend_;
  }
};
}  // namespace detail

/// @brief A named pair of (variable_name, ioda::Variable).
struct Named_Variable {
  std::string name;
  ioda::Variable var;
  bool operator<(const Named_Variable& rhs) const { return name < rhs.name; }

  Named_Variable() = default;
  Named_Variable(const std::string& name, const ioda::Variable& var) : name(name), var(var) {}
};

}  // namespace ioda

/// @}


#ifndef STDAFX__H
#define STDAFX__H

#if defined(_WIN32)

#include <SDKDDKVer.h>

#if !defined(_STL_EXTRA_DISABLED_WARNINGS)
#define _STL_EXTRA_DISABLED_WARNINGS 4061 4324 4365 4514 4571 4582 4583 4623 4625 4626 4710 4774 4820 4987 5026 5027 5039
#endif

#if !defined(_SCL_SECURE_NO_WARNINGS)
#define _SCL_SECURE_NO_WARNINGS 1
#endif

#if !defined(_CRT_SECURE_NO_WARNINGS)
#define _CRT_SECURE_NO_WARNINGS 1
#endif

#if !defined(_SILENCE_CXX17_ITERATOR_BASE_CLASS_DEPRECATION_WARNING)
#define _SILENCE_CXX17_ITERATOR_BASE_CLASS_DEPRECATION_WARNING 1
#endif

#if !defined(_SILENCE_CXX17_OLD_ALLOCATOR_MEMBERS_DEPRECATION_WARNING)
#define _SILENCE_CXX17_OLD_ALLOCATOR_MEMBERS_DEPRECATION_WARNING 1
#endif

#if !defined(_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING)
#define _SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING 1
#endif

#define STRICT
#define NOMINMAX

#pragma warning(disable: 4571) // warning C4571: Informational: catch(...) semantics changed since Visual C++ 7.1; structured exceptions (SEH) are no longer caught
#pragma warning(disable: 4668) // warning C4668: '%s' is not defined as a preprocessor macro, replacing with '0' for '#if/#elif'
#pragma warning(disable: 4710) // warning C4710: '%s': function not inlined
#pragma warning(disable: 4711) // warning C4711: function '%s' selected for automatic inline expansion
#pragma warning(disable: 4820) // warning C4820: '%s': '%d' bytes padding added after data member '%s'
#pragma warning(disable: 5045) // warning C5045: Compiler will insert Spectre mitigation for memory load if /Qspectre switch specified

#include <Windows.h>

#pragma warning(push)
#pragma warning(disable: 4005) // warning C4005: '%s': macro redefinition
#include <Winternl.h>
#include <ntstatus.h>
#pragma warning(pop)

#else // defined(_WIN32)

#include <cpuid.h>

#endif

#if defined(_MSC_VER)

// currently broken (triggers on iterators and other objects that are usually unnamed)
#pragma warning(disable: 26444) // warning C26444: Avoid unnamed objects with custom construction and destruction (es.84: http://go.microsoft.com/fwlink/?linkid=862923).

// disable for everything
#pragma warning(disable: 4061) // warning C4061: enumerator '%s' in switch of enum '%s' is not explicitly handled by a case label
#pragma warning(disable: 4324) // warning C4234: structure was padded due to alignment specifier
#pragma warning(disable: 4514) // warning C4514: '%s': unreferenced inline function has been removed
#pragma warning(disable: 4623) // warning C4623: '%s': default constructor was implicitly defined as deleted
#pragma warning(disable: 4625) // warning C4625: '%s': copy constructor was implicitly defined as deleted
#pragma warning(disable: 4626) // warning C4626: '%s': assignment operator was implicitly defined as deleted
#pragma warning(disable: 4710) // warning C4710: '%s': function not inlined
#pragma warning(disable: 4820) // warning C4820: '%s': '%d' bytes padding added after data member '%s'
#pragma warning(disable: 5026) // warning C5026: '%s': move constructor was implicitly defined as deleted
#pragma warning(disable: 5027) // warning C5027: '%s': move assignment operator was implicitly defined as deleted

#pragma warning(disable: 26412) // warning C26412: Do not dereference an invalid pointer (lifetimes rule 1). 'return of %s' was invalidated at line %d by 'no initialization'.
#pragma warning(disable: 26426) // warning C26426: Global initializer calls a non-constexpr function '%s' (i.22: http://go.microsoft.com/fwlink/?linkid=853919).
#pragma warning(disable: 26481) // warning C26481: Don't use pointer arithmetic. Use span instead. (bounds.1: http://go.microsoft.com/fwlink/p/?LinkID=620413)
#pragma warning(disable: 26482) // warning C26482: Only index into arrays using constant expressions (bounds.2: http://go.microsoft.com/fwlink/p/?LinkID=620414).
#pragma warning(disable: 26485) // warning C26485: Expression '%s::`vbtable'': No array to pointer decay. (bounds.3: http://go.microsoft.com/fwlink/p/?LinkID=620415)
#pragma warning(disable: 26490) // warning C26490: Don't use reinterpret_cast. (type.1: http://go.microsoft.com/fwlink/p/?LinkID=620417)
#pragma warning(disable: 26499) // warning C26499: Could not find any lifetime tracking information for '%s'

// disable for standard headers
#pragma warning(push)
#pragma warning(disable: 26400) // warning C26400: Do not assign the result of an allocation or a function call with an owner<T> return value to a raw pointer, use owner<T> instead. (i.11 http://go.microsoft.com/fwlink/?linkid=845474)
#pragma warning(disable: 26401) // warning C26401: Do not delete a raw pointer that is not an owner<T>. (i.11: http://go.microsoft.com/fwlink/?linkid=845474)
#pragma warning(disable: 26408) // warning C26408: Avoid malloc() and free(), prefer the nothrow version of new with delete. (r.10 http://go.microsoft.com/fwlink/?linkid=845483)
#pragma warning(disable: 26409) // warning C26409: Avoid calling new and delete explicitly, use std::make_unique<T> instead. (r.11 http://go.microsoft.com/fwlink/?linkid=845485)
#pragma warning(disable: 26411) // warning C26411: The parameter '%s' is a reference to unique pointer and it is never reassigned or reset, use T* or T& instead. (r.33 http://go.microsoft.com/fwlink/?linkid=845479)
#pragma warning(disable: 26412) // warning C26412: Do not dereference an invalid pointer (lifetimes rule 1). 'return of %s' was invalidated at line %d by 'end of function scope (local lifetimes end)'.
#pragma warning(disable: 26413) // warning C26413: Do not dereference nullptr (lifetimes rule 2). 'nullptr' was pointed to nullptr at line %d.
#pragma warning(disable: 26423) // warning C26423: The allocation was not directly assigned to an owner.
#pragma warning(disable: 26424) // warning C26424: Failing to delete or assign ownership of allocation at line %d.
#pragma warning(disable: 26425) // warning C26425: Assigning '%s' to a static variable.
#pragma warning(disable: 26444) // warning C26444: Avoid unnamed objects with custom construction and destruction (es.84: http://go.microsoft.com/fwlink/?linkid=862923).
#pragma warning(disable: 26461) // warning C26461: The reference argument '%s' for function %s can be marked as const. (con.3: https://go.microsoft.com/fwlink/p/?LinkID=786684)
#pragma warning(disable: 26471) // warning C26471: Don't use reinterpret_cast. A cast from void* can use static_cast. (type.1: http://go.microsoft.com/fwlink/p/?LinkID=620417).
#pragma warning(disable: 26481) // warning C26481: Don't use pointer arithmetic. Use span instead. (bounds.1: http://go.microsoft.com/fwlink/p/?LinkID=620413)
#pragma warning(disable: 26482) // warning C26482: Only index into arrays using constant expressions. (bounds.2: http://go.microsoft.com/fwlink/p/?LinkID=620414)
#pragma warning(disable: 26490) // warning C26490: Don't use reinterpret_cast. (type.1: http://go.microsoft.com/fwlink/p/?LinkID=620417)
#pragma warning(disable: 26493) // warning C26493: Don't use C-style casts that would perform a static_cast downcast, const_cast, or reinterpret_cast. (type.4: http://go.microsoft.com/fwlink/p/?LinkID=620420)
#pragma warning(disable: 26494) // warning C26494: Variable '%s' is uninitialized. Always initialize an object. (type.5: http://go.microsoft.com/fwlink/p/?LinkID=620421)
#pragma warning(disable: 26495) // warning C26495: Variable '%s' is uninitialized. Always initialize a member variable. (type.6: http://go.microsoft.com/fwlink/p/?LinkID=620422)
#pragma warning(disable: 26496) // warning C26496: Variable '%s' is assigned only once, mark it as const. (con.4: https://go.microsoft.com/fwlink/p/?LinkID=784969)
#pragma warning(disable: 26497) // warning C26497: This function %s could be marked constexpr if compile-time evaluation is desired. (f.4: https://go.microsoft.com/fwlink/p/?LinkID=784970)

#else

// linux warnings go here

#endif

#include <cstddef>
#include <map>
#include <set>
#include <vector>
#include <string>
#include <algorithm>
#include <iostream>
#include <fstream>
#include <iomanip>
#include <type_traits>
#include <utility>
#include <memory>
#include <tuple>
#include <thread>
#include <cstdlib>
#include <codecvt>

#if defined(_MSC_VER)

// disable additionally for third-party libraries
#pragma warning(push)
#pragma warning(disable: 4456) // warning C4456: declaration of '%s' hides previous local declaration
#pragma warning(disable: 4458) // warning C4458: declaration of '%s' hides class member
#pragma warning(disable: 4459) // warning C4459: declaration of '%s' hides global declaration

#pragma warning(disable: 26429) // warning C26429: Symbol '%s' is never tested for nullness, it can be marked as not_null (f.23: http://go.microsoft.com/fwlink/?linkid=853921).
#pragma warning(disable: 26432) // warning C26432: If you define or delete any default operation in the type '%s', define or delete them all (c.21: http://go.microsoft.com/fwlink/?linkid=853922).
#pragma warning(disable: 26433) // warning C26433: Function '%s' should be marked with 'override' (c.128: http://go.microsoft.com/fwlink/?linkid=853923).
#pragma warning(disable: 26434) // warning C26434: Function '%s' hides a non-virtual function '%s' (c.128: http://go.microsoft.com/fwlink/?linkid=853923).
#pragma warning(disable: 26436) // warning C26436: The type '%s' with a virtual function needs either public, virtual or protected non-virtual destructor (c.35: http://go.microsoft.com/fwlink/?linkid=853924).
#pragma warning(disable: 26439) // warning C26439: This kind of function may not throw. Declare it 'noexcept' (f.6: http://go.microsoft.com/fwlink/?linkid=853927).
#pragma warning(disable: 26440) // warning C26440: Function '%s' can be declared 'noexcept' (f.6: http://go.microsoft.com/fwlink/?linkid=853927).
#pragma warning(disable: 26443) // warning C26443: Overriding destructor should not use explicit 'override' or 'virtual' specifiers (c.128: http://go.microsoft.com/fwlink/?linkid=853923).
#pragma warning(disable: 26462) // warning C26462: The value pointed to by '%s' is assigned only once, mark it as a pointer to const (con.4: https://go.microsoft.com/fwlink/p/?LinkID=784969).
#pragma warning(disable: 26472) // warning C26472: Don't use a static_cast for arithmetic conversions. Use brace initialization, gsl::narrow_cast or gsl::narow (type.1: http://go.microsoft.com/fwlink/p/?LinkID=620417).
#pragma warning(disable: 26474) // warning C26474: Don't cast between pointer types when the conversion could be implicit (type.1: http://go.microsoft.com/fwlink/p/?LinkID=620417).
#pragma warning(disable: 26491) // warning C26491: Don't use static_cast downcasts (type.2: http://go.microsoft.com/fwlink/p/?LinkID=620418).
#pragma warning(disable: 26498) // warning C26498: The function '%s' is constexpr, mark variable '%s' constexpr if compile-time evaluation is desired (con.5: https://go.microsoft.com/fwlink/p/?LinkID=784974).

#endif

#ifndef BOOST_CONFIG_SUPPRESS_OUTDATED_MESSAGE
#define BOOST_CONFIG_SUPPRESS_OUTDATED_MESSAGE
#endif

#include <boost/algorithm/string.hpp>
#include <boost/xpressive/xpressive.hpp>

#include <gsl/gsl>

#include <fmt/format.h>

#if defined(_MSC_VER)

#pragma warning(pop)
#pragma warning(pop)

#else

// linux warning restoration goes here

#endif

#endif


#pragma once

#define NOMINMAX
#define NODRAWTEXT
#define NOGDI
#define NOBITMAP
#define NOMCX
#define NOSERVICE
#define NOHELP

#include <iostream>
#include <vector>
#include <string>
#include <string_view>
#include <sstream>
#include <variant>
#include <codecvt>
#include <locale>
#include <filesystem>
#include <fstream>
#include <chrono>
#include <functional>
#include <numeric>

#ifndef _WIN32
#include <wsl/winadapter.h>
#include "directml_guids.h"
#else
#include <Windows.h>
#endif
#include <wil/result.h>
#include <wrl/client.h>
#include <gsl/gsl>
#include <rapidjson/document.h>
#include <rapidjson/istreamwrapper.h>
#include <fmt/format.h>

#ifdef _GAMING_XBOX_SCARLETT // Xbox Series X/S - Use GDK
#include <d3d12_xs.h>
#include <d3dx12_xs.h>
#include <d3d12shader_xs.h>
#include <dxcapi_xs.h>
#include <pix3.h>
using IAdapter = IDXGIAdapter;
#elif _GAMING_XBOX_XBOXONE // XboxOne - Use GDK
#include <d3d12_x.h>
#include <d3dx12_x.h>
#include <d3d12shader_x.h>
#include <dxcapi_x.h>
#include <pix3.h>
using IAdapter = IDXGIAdapter;
#else // Desktop/PC - Use DirectX-Headers
#include <directx/d3d12.h>
#include <directx/d3dx12.h>
#include <directx/dxcore.h>
#ifndef DXCOMPILER_NONE
#include <directx/d3d12shader.h>
#endif
#include <dxcapi.h>
#include <WinPixEventRuntime/pix3.h>

#define IGraphicsUnknown IUnknown
#define IID_GRAPHICS_PPV_ARGS IID_PPV_ARGS
using IAdapter = IDXCoreAdapter;
#endif

#include <DirectML.h>
#include "DirectMLX.h"

#include "Logging.h"


/*
 * Copyright (c) 2016-2021 lymastee, All rights reserved.
 * Contact: lymastee@hotmail.com
 *
 * This file is part of the gslib project.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#pragma once

#ifndef scene_048bb735_09b4_43d2_9ff5_5e40441c230b_h
#define scene_048bb735_09b4_43d2_9ff5_5e40441c230b_h

#include <gslib/std.h>
#include <ariel/config.h>
#include <ariel/rendersys.h>
#include <ariel/framesys.h>
#include <ariel/widget.h>

__ariel_begin__

class rose;
class stage;
typedef list<stage*> stages;

class __gs_novtable stage abstract:
    public frame_listener
{
public:
    stage();
    virtual ~stage() {}
    virtual const gchar* get_name() const = 0;
    virtual bool setup() = 0;
    virtual void draw() = 0;

public:
    template<class _sty> inline
    _sty* as() { return static_cast<_sty*>(this); }
    template<class _sty> inline
    const _sty* as_const() const { return static_cast<const _sty*>(this); }

protected:
    stage*              _prev_presentation_stage;
    stage*              _next_presentation_stage;
    stage*              _prev_notification_stage;
    stage*              _next_notification_stage;

protected:
    void on_next_draw();
    void on_next_frame_start();
    void on_next_frame_end();
    bool on_next_event(const frame_event& event);

public:
    stage* prev_presentation_stage() const { return _prev_presentation_stage; }
    stage* next_presentation_stage() const { return _next_presentation_stage; }
    stage* prev_notification_stage() const { return _prev_notification_stage; }
    stage* next_notification_stage() const { return _next_notification_stage; }

public:
    static void connect_presentation_order(stage* stg1, stage* stg2);
    static void connect_notification_order(stage* stg1, stage* stg2);
    static void connect_presentation_order(stage* stg1, stage* stg2, stage* stg3);
    static void connect_notification_order(stage* stg1, stage* stg2, stage* stg3);
};

class ui_stage:
    public stage
{
public:
    ui_stage(rose* r) { _rose = r; }
    virtual const gchar* get_name() const override { return _t("ui"); }
    virtual bool setup() override;
    virtual void draw() override;
    virtual void on_frame_start() override { on_next_frame_start(); }
    virtual void on_frame_end() override { on_next_frame_end(); }
    virtual bool on_frame_event(const frame_event& event) override;

public:
    wsys_manager* get_wsys_manager() { return &_wsys_manager; }

protected:
    rose*               _rose;
    wsys_manager        _wsys_manager;

private:
    void setup_draw();
};

class scene:
    public frame_listener
{
public:
    static scene* get_singleton_ptr()
    {
        static scene inst;
        return &inst;
    }

private:
    scene();

public:
    ~scene();
    stage* get_stage(const gchar* name);
    wsys_manager* get_ui_system() const { return _uisys; }
    fontsys* get_fontsys() const { return _fontsys; }
    rendersys* get_rendersys() const { return _rendersys; }
    void set_rendersys(rendersys* rsys) { _rendersys = rsys; }
    void set_rose(rose* ptr) { _rose = ptr; }
    void set_fontsys(fontsys* fsys);
    void setup();
    void destroy();
    void destroy_all_stages();
    void add_stage(stage* stg) { _stages.push_back(stg); }
    void set_presentation_before(stage* pos, stage* tar);
    void set_presentation_after(stage* pos, stage* tar);
    void set_notification_before(stage* pos, stage* tar);
    void set_notification_after(stage* pos, stage* tar);

protected:
    rendersys*          _rendersys;
    rose*               _rose;
    stages              _stages;
    wsys_manager*       _uisys;
    fontsys*            _fontsys;
    stage*              _notify;
    stage*              _present;

protected:
    void draw();

public:
    virtual void on_frame_start() override;
    virtual void on_frame_end() override;
    virtual bool on_frame_event(const frame_event& event) override;
};

__ariel_end__

#endif


#ifndef PCH_H
#define PCH_H

#define ISOLATION_AWARE_ENABLED 1

/*
    Only put files that never change or are rarely changed in this file.
    An include graph will be provided for some headers that were not explicit
    added. VC++ is a requirement.

    We don't care about warnings for library files.
*/
#pragma warning(push)
#pragma warning(disable : 4091 4191 4365 4464 4571 4619 4623 4625 4626 4768 4774 5026 5027 5039 5045)
/* Windows Stuff */
#include "windows_nodefines.h"
#include <ShlObj.h> // CommCtrl
#include <WindowsX.h>
#include <atlbase.h> // atldef.h (Windows.h), atlcore.h (tchar.h), Shlwapi.h
#include <direct.h>
#include <io.h>
#include <wincodec.h>

#if WIN32_LEAN_AND_MEAN
#include <CommDlg.h>
#include <MMSystem.h>
#include <ShellAPI.h>
#endif // WIN32_LEAN_AND_MEAN

/*
    Nuke WinAPI #defines - anything including this file should be using the
    ra versions.
*/
#undef CreateDirectory
#undef GetMessage

/* C Stuff */
#include <cctype>

/* STL Stuff */
#include <array> // algorithm, iterator, tuple
#include <atomic>
#include <fstream>
#include <iomanip>
#include <map>
#include <memory>
#include <mutex> // chrono (time.h), functional, thread, utility
#include <queue> // deque, vector, algorithm
#include <set>
#include <sstream> // string
#include <stack>
#include <unordered_map>
#include <variant>

#pragma warning(push)
// Look at this file if you want to know what they are and what they mean
#include "pch_cppcorecheck_suppressions.h"
#include "pch_microsoft_suppressions.h"

/* RapidJSON Stuff */
#define RAPIDJSON_HAS_STDSTRING 1
#define RAPIDJSON_NOMEMBERITERATORCLASS 1
#include <rapidjson\document.h> // has reader.h
#include <rapidjson\error\en.h>
#include <rapidjson\istreamwrapper.h>
#include <rapidjson\ostreamwrapper.h>
#include <rapidjson\writer.h> // has stringbuffer.h

/* gsl stuff */
#define GSL_THROW_ON_CONTRACT_VIOLATION
#include <gsl\gsl>

// the default GSL Expects() macro throws an exception, which we don't try to catch anywhere.
// replace it with our custom handler, which will log and report the error before throwing it.
#ifndef RA_UTEST
 #ifdef NDEBUG
  extern void __gsl_contract_handler(const char* const file, unsigned int line);
  #undef Expects
  #define Expects(cond) (GSL_LIKELY(cond) ? static_cast<void>(0) : __gsl_contract_handler(__FILE__, __LINE__))
 #else
  extern void __gsl_contract_handler(const char* const file, unsigned int line, const char* const error);
  #undef Expects
  #define Expects(cond) (GSL_LIKELY(cond) ? static_cast<void>(0) : __gsl_contract_handler(__FILE__, __LINE__, GSL_STRINGIFY(cond)))
 #endif
#endif

#pragma warning(pop)

#if RA_UTEST
#include <CodeAnalysis\Warnings.h>
#pragma warning(push)
#pragma warning(disable : ALL_CPPCORECHECK_WARNINGS)
#include "CppUnitTest.h"
#pragma warning(pop)
#endif /* RA_UTEST */

/* rcheevos stuff */
#pragma warning(push)
#pragma warning(disable : 4201) // nameless struct
#include <rcheevos.h>
#pragma warning(pop)
#pragma warning(pop)

#endif /* !PCH_H */


/**
 *  Copyright (C) 2021 FISCO BCOS.
 *  SPDX-License-Identifier: Apache-2.0
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 * @brief tars implementation for Block
 * @file BlockImpl.h
 * @author: ancelmo
 * @date 2021-04-20
 */
#pragma once
#include "BlockHeaderImpl.h"
#include "TransactionImpl.h"
#include "TransactionMetaDataImpl.h"
#include "TransactionReceiptImpl.h"
#include "bcos-tars-protocol/Common.h"
#include "bcos-tars-protocol/tars/Block.h"
#include "interfaces/protocol/Transaction.h"
#include <bcos-framework/interfaces/crypto/CryptoSuite.h>
#include <bcos-framework/interfaces/protocol/Block.h>
#include <bcos-framework/interfaces/protocol/BlockHeader.h>
#include <gsl/span>
#include <memory>

namespace bcostars
{
namespace protocol
{
class BlockImpl : public bcos::protocol::Block, public std::enable_shared_from_this<BlockImpl>
{
public:
    BlockImpl(bcos::protocol::TransactionFactory::Ptr _transactionFactory,
        bcos::protocol::TransactionReceiptFactory::Ptr _receiptFactory)
      : bcos::protocol::Block(_transactionFactory, _receiptFactory),
        m_inner(std::make_shared<bcostars::Block>()),
        x_mutex(std::make_shared<std::mutex>())
    {}

    ~BlockImpl() override{};

    void decode(bcos::bytesConstRef _data, bool _calculateHash, bool _checkSig) override;
    void encode(bcos::bytes& _encodeData) const override;

    int32_t version() const override { return m_inner->blockHeader.data.version; }
    void setVersion(int32_t _version) override { m_inner->blockHeader.data.version = _version; }

    bcos::protocol::BlockType blockType() const override
    {
        return (bcos::protocol::BlockType)m_inner->type;
    }
    // FIXME: this will cause the same blockHeader calculate hash multiple times
    bcos::protocol::BlockHeader::Ptr blockHeader() override;
    bcos::protocol::BlockHeader::ConstPtr blockHeaderConst() const override;

    bcos::protocol::Transaction::ConstPtr transaction(size_t _index) const override;
    bcos::protocol::TransactionReceipt::ConstPtr receipt(size_t _index) const override;

    // get transaction metaData
    bcos::protocol::TransactionMetaData::ConstPtr transactionMetaData(size_t _index) const override;
    void setBlockType(bcos::protocol::BlockType _blockType) override
    {
        m_inner->type = (int32_t)_blockType;
    }

    // set blockHeader
    void setBlockHeader(bcos::protocol::BlockHeader::Ptr _blockHeader) override;

    void setTransaction(size_t _index, bcos::protocol::Transaction::Ptr _transaction) override
    {
        m_inner->transactions[_index] =
            std::dynamic_pointer_cast<bcostars::protocol::TransactionImpl>(_transaction)->inner();
    }
    void appendTransaction(bcos::protocol::Transaction::Ptr _transaction) override
    {
        m_inner->transactions.emplace_back(
            std::dynamic_pointer_cast<bcostars::protocol::TransactionImpl>(_transaction)->inner());
    }

    void setReceipt(size_t _index, bcos::protocol::TransactionReceipt::Ptr _receipt) override;
    void appendReceipt(bcos::protocol::TransactionReceipt::Ptr _receipt) override;

    void appendTransactionMetaData(bcos::protocol::TransactionMetaData::Ptr _txMetaData) override;

    // get transactions size
    size_t transactionsSize() const override { return m_inner->transactions.size(); }
    size_t transactionsMetaDataSize() const override;
    // get receipts size
    size_t receiptsSize() const override { return m_inner->receipts.size(); }

    void setNonceList(bcos::protocol::NonceList const& _nonceList) override;
    void setNonceList(bcos::protocol::NonceList&& _nonceList) override;
    bcos::protocol::NonceList const& nonceList() const override;

    const bcostars::Block& inner() const { return *m_inner; }
    void setInner(const bcostars::Block& inner) { *m_inner = inner; }
    void setInner(bcostars::Block&& inner) { *m_inner = std::move(inner); }

private:
    std::shared_ptr<bcostars::Block> m_inner;
    mutable bcos::protocol::NonceList m_nonceList;
    std::shared_ptr<std::mutex> x_mutex;
};
}  // namespace protocol
}  // namespace bcostars


///////////////////////////////////////////////////////////////////////////////
//
// Copyright (c) 2015 Microsoft Corporation. All rights reserved.
//
// This code is licensed under the MIT License (MIT).
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
//
///////////////////////////////////////////////////////////////////////////////

#ifndef GSL_POINTERS_H
#define GSL_POINTERS_H

#include <gsl/gsl_assert>  // for Ensures, Expects

#include <algorithm>    // for forward
#include <iosfwd>       // for ptrdiff_t, nullptr_t, ostream, size_t
#include <memory>       // for shared_ptr, unique_ptr
#include <system_error> // for hash
#include <type_traits>  // for enable_if_t, is_convertible, is_assignable

#if defined(_MSC_VER) && _MSC_VER < 1910
#pragma push_macro("constexpr")
#define constexpr /*constexpr*/

#endif                          // defined(_MSC_VER) && _MSC_VER < 1910

namespace gsl
{

//
// GSL.owner: ownership pointers
//
using std::unique_ptr;
using std::shared_ptr;

//
// owner
//
// owner<T> is designed as a bridge for code that must deal directly with owning pointers for some reason
//
// T must be a pointer type
// - disallow construction from any type other than pointer type
//
template <class T, class = std::enable_if_t<std::is_pointer<T>::value>>
using owner = T;

//
// not_null
//
// Restricts a pointer or smart pointer to only hold non-null values.
//
// Has zero size overhead over T.
//
// If T is a pointer (i.e. T == U*) then
// - allow construction from U*
// - disallow construction from nullptr_t
// - disallow default construction
// - ensure construction from null U* fails
// - allow implicit conversion to U*
//
template <class T>
class not_null
{
public:
    static_assert(std::is_assignable<T&, std::nullptr_t>::value, "T cannot be assigned nullptr.");

    template <typename U, typename = std::enable_if_t<std::is_convertible<U, T>::value>>
    constexpr explicit not_null(U&& u) : ptr_(std::forward<U>(u))
    {
        Expects(ptr_ != nullptr);
    }

    template <typename = std::enable_if_t<!std::is_same<std::nullptr_t, T>::value>>
    constexpr explicit not_null(T u) : ptr_(u)
    {
        Expects(ptr_ != nullptr);
    }

    template <typename U, typename = std::enable_if_t<std::is_convertible<U, T>::value>>
    constexpr not_null(const not_null<U>& other) : not_null(other.get())
    {
    }

    not_null(not_null&& other) = default;
    not_null(const not_null& other) = default;
    not_null& operator=(const not_null& other) = default;

    constexpr T get() const
    {
        Ensures(ptr_ != nullptr);
        return ptr_;
    }

    constexpr operator T() const { return get(); }
    constexpr T operator->() const { return get(); }
    constexpr decltype(auto) operator*() const { return *get(); }

    // prevents compilation when someone attempts to assign a null pointer constant
    not_null(std::nullptr_t) = delete;
    not_null& operator=(std::nullptr_t) = delete;

    // unwanted operators...pointers only point to single objects!
    not_null& operator++() = delete;
    not_null& operator--() = delete;
    not_null operator++(int) = delete;
    not_null operator--(int) = delete;
    not_null& operator+=(std::ptrdiff_t) = delete;
    not_null& operator-=(std::ptrdiff_t) = delete;
    void operator[](std::ptrdiff_t) const = delete;

private:
    T ptr_;
};

template <class T>
auto make_not_null(T&& t) {
    return gsl::not_null<std::remove_cv_t<std::remove_reference_t<T>>>{std::forward<T>(t)};
}

template <class T>
std::ostream& operator<<(std::ostream& os, const not_null<T>& val)
{
    os << val.get();
    return os;
}

template <class T, class U>
auto operator==(const not_null<T>& lhs, const not_null<U>& rhs) -> decltype(lhs.get() == rhs.get())
{
    return lhs.get() == rhs.get();
}

template <class T, class U>
auto operator!=(const not_null<T>& lhs, const not_null<U>& rhs) -> decltype(lhs.get() != rhs.get())
{
    return lhs.get() != rhs.get();
}

template <class T, class U>
auto operator<(const not_null<T>& lhs, const not_null<U>& rhs) -> decltype(lhs.get() < rhs.get())
{
    return lhs.get() < rhs.get();
}

template <class T, class U>
auto operator<=(const not_null<T>& lhs, const not_null<U>& rhs) -> decltype(lhs.get() <= rhs.get())
{
    return lhs.get() <= rhs.get();
}

template <class T, class U>
auto operator>(const not_null<T>& lhs, const not_null<U>& rhs) -> decltype(lhs.get() > rhs.get())
{
    return lhs.get() > rhs.get();
}

template <class T, class U>
auto operator>=(const not_null<T>& lhs, const not_null<U>& rhs) -> decltype(lhs.get() >= rhs.get())
{
    return lhs.get() >= rhs.get();
}

// more unwanted operators
template <class T, class U>
std::ptrdiff_t operator-(const not_null<T>&, const not_null<U>&) = delete;
template <class T>
not_null<T> operator-(const not_null<T>&, std::ptrdiff_t) = delete;
template <class T>
not_null<T> operator+(const not_null<T>&, std::ptrdiff_t) = delete;
template <class T>
not_null<T> operator+(std::ptrdiff_t, const not_null<T>&) = delete;

} // namespace gsl

namespace std
{
template <class T>
struct hash<gsl::not_null<T>>
{
    std::size_t operator()(const gsl::not_null<T>& value) const { return hash<T>{}(value); }
};

} // namespace std

#if defined(_MSC_VER) && _MSC_VER < 1910
#undef constexpr
#pragma pop_macro("constexpr")

#endif // defined(_MSC_VER) && _MSC_VER < 1910

#endif // GSL_POINTERS_H


/*
   Copyright [2017-2020] [IBM Corporation]
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at
       http://www.apache.org/licenses/LICENSE-2.0
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/
#ifndef __FABRIC_CONNECTION_BASE_H__
#define __FABRIC_CONNECTION_BASE_H__

#include "mcas_config.h"

#include "buffer_manager.h" /* Buffer_manager */

#include <api/fabric_itf.h> /* IFabric_memory_region, IFabric_server */
#include <gsl/pointers>
#include <algorithm>
#include <list>
#include <queue>

namespace mcas
{

struct open_connection_construct_key;

struct open_connection
{
private:
  gsl::not_null<component::IFabric_server_factory *> _factory;
  gsl::not_null<component::IFabric_server *>         _transport;
public:
  open_connection(
    gsl::not_null<component::IFabric_server_factory *> factory_
    , gsl::not_null<component::IFabric_server *> transport_
    , const open_connection_construct_key &
    )
    : _factory(factory_)
    , _transport(transport_)
  {}

  open_connection(const open_connection &) = delete;
  open_connection &operator=(const open_connection &) = delete;

  auto transport() const { return _transport; }

  ~open_connection()
  {
    /* close connection */
    _factory->close_connection(_transport);
  }
};

class Fabric_connection_base : protected common::log_source {

 public:
  friend class Connection;
  friend class Shard;
  using memory_region_t = component::IFabric_memory_region *;

 protected:
  using buffer_t = Buffer_manager<component::IFabric_memory_control>::buffer_internal;
  using pool_t   = component::IKVStore::pool_t;

  enum class action_type {
        ACTION_NONE = 0, /* unused */
        ACTION_RELEASE_VALUE_LOCK_SHARED,
#if 0
        ACTION_RELEASE_VALUE_LOCK_EXCLUSIVE, /* unused */
        ACTION_POOL_DELETE, /* unused */
#endif
  };

  /* deferred actions */
  struct action_t {
    action_type op;
    void *parm;
  };

  enum class Completion_state {
    COMPLETIONS,
    CLIENT_DISCONNECT,
    NONE,
  };

 private:
  std::unique_ptr<component::IFabric_endpoint_unconnected_server> _preconnection;
  Buffer_manager<component::IFabric_memory_control> _bm;

  /* xx_buffer_outstanding is the signal for completion,
     xx_buffer is the buffer pointer that needs to be freed (and set to null)
  */
protected:
  unsigned              _recv_buffer_posted_count;
private:
  unsigned              _send_buffer_posted_count;

protected:
  /* values for two-phase get
   */
  unsigned _send_value_posted_count;
private:

  std::list<buffer_t *> _completed_recv_buffers;

  open_connection _oc;
 protected:
  size_t                             _max_message_size;

  /* Filled by base class     check_for_posted_value_complete
   * Drained by derived class check_network_completions
   */
  std::queue<action_t> _deferred_unlock;

  /**
   * Ctor
   *
   * @param factory
   * @param fabric_connection
   */
  explicit Fabric_connection_base( //
    unsigned                           debug_level_,
    gsl::not_null<component::IFabric_server_factory *>factory,
    std::unique_ptr<component::IFabric_endpoint_unconnected_server> && preconnection);

  Fabric_connection_base(const Fabric_connection_base &) = delete;
  Fabric_connection_base &operator=(const Fabric_connection_base &) = delete;

  virtual ~Fabric_connection_base();

  static void static_send_callback(void *cnxn, buffer_t *iob) noexcept
  {
    static_cast<Fabric_connection_base *>(cnxn)->send_callback(iob);
  }

  void send_callback(buffer_t *iob) noexcept
  {
    --_send_buffer_posted_count;
    posted_count_log();
    CPLOG(2, "Completed send (%p) freeing buffer", common::p_fmt(iob));
    free_buffer(iob);
  }

  static void static_recv_callback(void *cnxn, buffer_t *iob) noexcept
  {
    static_cast<Fabric_connection_base *>(cnxn)->recv_callback(iob);
  }

  void recv_callback(buffer_t *iob) noexcept
  {
    --_recv_buffer_posted_count;
    posted_count_log();
    _completed_recv_buffers.push_front(iob);
    CPLOG(2, "Completed recv (%p) (complete %zu)", common::p_fmt(iob), _completed_recv_buffers.size());
  }

  static void completion_callback(void *   context,
                                  status_t st,
                                  std::uint64_t,  // completion_flags,
                                  std::size_t len,
                                  void *      error_data,
                                  void *      cnxn) noexcept
  {
    if (LIKELY(st == S_OK)) {
      auto iob = static_cast<buffer_t *>(context);
      iob->completion_cb(cnxn, iob);
    }
    else {
      PERR("Fabric_connection_base: fabric operation failed st != S_OK (st=%d, "
           "context=%p, len=%lu)",
           st, context, len);
      PERR("Error: %s", static_cast<char *>(error_data));
    }
  }

  bool check_for_posted_recv_complete()
  {
    /* don't free buffer (such as above); it will be used for response */
    //    return _recv_buffer_posted_outstanding == false;
    return _completed_recv_buffers.size() > 0;
  }

  void free_recv_buffer()
  {
    for (auto b : _completed_recv_buffers) {
      free_buffer(b);
    }
  }

  size_t recv_buffer_posted_count() const {
    return _recv_buffer_posted_count;
  }

  void post_recv_buffer(buffer_t *buffer)
  {
    _preconnection->post_recv(buffer->iov, buffer->iov + 1, buffer->desc, buffer);
    ++_recv_buffer_posted_count;
    posted_count_log();
    CPLOG(2, "Posted recv (%p) (complete %zu)", common::p_fmt(buffer), _completed_recv_buffers.size());
  }

  void post_send_buffer(gsl::not_null<buffer_t *> buffer)
  {
    const auto iov = buffer->iov;

    /* if packet is small enough use inject */
    if (iov->iov_len <= transport()->max_inject_size()) {
      CPLOG(2, "Fabric_connection_base: posting send with inject (iob %p %p,len=%lu)",
             common::p_fmt(buffer), iov->iov_base, iov->iov_len);

      transport()->inject_send(iov->iov_base, iov->iov_len);
      CPLOG(2, "%s: buffer %p", __func__, common::p_fmt(buffer));
      free_buffer(buffer); /* buffer is immediately complete fi_inject */
    }
    else {
      CPLOG(2, "Fabric_connection_base: posting send (%p, %p)", common::p_fmt(buffer), iov->iov_base);

      transport()->post_send(iov, iov + 1, buffer->desc, buffer);
      ++_send_buffer_posted_count;
      posted_count_log();
      CPLOG(2, "%s buffer (%p)", __func__, common::p_fmt(buffer));
    }
  }

  void post_send_buffer2(gsl::not_null<buffer_t *> buffer, const ::iovec &val_iov, void *val_desc)
  {
    buffer->iov[1] = val_iov;
    buffer->desc[1] = val_desc;

    ++_send_buffer_posted_count;
    posted_count_log();
    CPLOG(2, "Posted send (%p) ... value (%.*s) (len=%lu,ptr=%p)", common::p_fmt(buffer),
         int(val_iov.iov_len), static_cast<char *>(val_iov.iov_base), val_iov.iov_len, val_iov.iov_base);

    transport()->post_send(buffer->iov, buffer->iov + 2, buffer->desc, buffer);
  }

  buffer_t *posted_recv()
  {
    if (_completed_recv_buffers.size() == 0) return nullptr;

    auto iob = _completed_recv_buffers.back();
    _completed_recv_buffers.pop_back();
    CPLOG(2, "Presented recv (%p) (complete %zu)", common::p_fmt(iob), _completed_recv_buffers.size());
    return iob;
  }

  void posted_count_log() const
  {
    CPLOG(2, "POSTs recv %u send %u value %u", _recv_buffer_posted_count, _send_buffer_posted_count,
           _send_value_posted_count);
  }

  Completion_state poll_completions()
  {
    if (_recv_buffer_posted_count != 0 || _send_buffer_posted_count != 0 || _send_value_posted_count != 0) {
#if 0
      PLOG("%s posted_recv %u posted_send %u posted_value %u", __func__, _recv_buffer_posted_count, _send_buffer_posted_count, _send_value_posted_count);
#endif
      try {
        transport()->poll_completions(&Fabric_connection_base::completion_callback, this);
        return Completion_state::COMPLETIONS;
      }
      catch (const std::logic_error &e) {
        return Completion_state::CLIENT_DISCONNECT;
      }
    }
    return Completion_state::NONE;
  }

  /**
   * Forwarders that allow us to avoid exposing transport() and _bm
   *
   */
 public:
  inline auto register_memory(const void *base, size_t len, std::uint64_t key, std::uint64_t flags)
  {
    return transport()->register_memory(base, len, key, flags); /* flags not supported for verbs */
  }

  inline void deregister_memory(memory_region_t region) { return transport()->deregister_memory(region); }

  inline void *get_memory_descriptor(memory_region_t region) { return transport()->get_memory_descriptor(region); }

  inline uint64_t get_memory_remote_key(memory_region_t region) { return transport()->get_memory_remote_key(region); }

 protected:
  inline auto allocate(buffer_t::completion_t c) { return _bm.allocate(c); }

  inline void free_buffer(buffer_t *buffer) { _bm.free(buffer); }

  inline size_t IO_buffer_size() const { return Buffer_manager<component::IFabric_memory_control>::BUFFER_LEN; }

  gsl::not_null<component::IFabric_server *> transport() const { return _oc.transport(); }

  inline std::string get_local_addr() { return transport()->get_local_addr(); }
};

struct open_connection_construct_key
{
private:
  open_connection_construct_key() {};
public:
  friend class Fabric_connection_base;
};

}  // namespace mcas

#endif  // __FABRIC_CONNECTION_BASE_H__


#if !defined(STRIPWS_H_INCLUDED)
#define STRIPWS_H_INCLUDED

#include "FileEnumerator.h"
#include "Utils.h"

#include <filesystem>
#include <gsl/gsl>
#include <iosfwd>
#include <string>

class StripWS
{
public:
	static int usage(::std::ostream& strm, const ::std::string& progName,
		const char* pMsg);

	StripWS(::gsl::span<const char*const> args);
	int run() const;

	StripWS(const StripWS&) = delete;
	StripWS& operator=(const StripWS&) = delete;
	StripWS(StripWS&&) = delete;
	StripWS& operator=(StripWS&&) = delete;

PRIVATE_EXCEPT_IN_TEST:
	using Path = ::std::filesystem::path;

	void queryFile(const Path& p) const;
	void translateFile(const Path& p) const;

	/// \brief Scans the input file, counts the occurrences of white space at
	/// the end of a line, and optionally strips them into the provided stream.
	///
	/// If pOut is null, then the method simply counts the occurrences of white
	/// space at the end of a line.
	///
	/// If pOut is not null, then the method additionally translates the file
	/// into *pOut.
	static void scanFile(::std::istream& in, /* out */ size_t& numLinesAffected,
		/* out */ size_t& numSpacesStripped, /* out */ size_t& numTabsStripped,
		::std::ostream* pOut = nullptr);

	static void updateCounts(::std::string const& wsRun,
		/* in-out */ size_t& numLinesAffected,
		/* in-out */ size_t& numSpacesStripped,
		/* in-out */ size_t& numTabsStripped);
	static void replaceOriginalFileWithTemp(const Path& originalPath,
		const Path& tempPath);

	bool				m_isInQueryMode;
	FileEnumerator	m_fileEnumerator;
};

#endif // STRIPWS_H_INCLUDED


#if !defined(ISPLAINASCII_H_INCLUDED)
#define ISPLAINASCII_H_INCLUDED

#include "FileEnumerator.h"
#include "Utils.h"

#include <filesystem>
#include <gsl/gsl>
#include <iosfwd>
#include <string>

class IsPlainAscii
{
public:
	static int usage(::std::ostream& strm, const ::std::string& progName,
		const char* pMsg);

	IsPlainAscii(::gsl::span<const char*const> args);
	int run() const;

	IsPlainAscii(const IsPlainAscii&) = delete;
	IsPlainAscii& operator=(const IsPlainAscii&) = delete;
	IsPlainAscii(IsPlainAscii&&) = delete;
	IsPlainAscii& operator=(IsPlainAscii&&) = delete;

PRIVATE_EXCEPT_IN_TEST:
	using Path = ::std::filesystem::path;

	static void scanFile(const Path& filePath);
	static void scanFile2(const Path& filePath, ::std::istream& in,
		::std::ostream& out);
	static void reportNonAsciiRun(const Path& filePath, unsigned long lineNum,
		unsigned long approxColNum, ::std::string& nonAsciiRun, ::std::ostream& out);

	FileEnumerator m_fileEnumerator;
};

#endif // ISPLAINASCII_H_INCLUDED


// SPDX-License-Identifier: MIT
// The MIT License (MIT)
//
// Copyright (c) 2014-2018, Institute for Software & Systems Engineering
// Copyright (c) 2018-2019, Johannes Leupolz
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

#ifndef PEMC_GENERIC_TRAVERSER_PATH_TRACKER_H_
#define PEMC_GENERIC_TRAVERSER_PATH_TRACKER_H_

#include <vector>
#include <gsl/span>
#include <cstdint>
#include <atomic>
#include <stack>
#include <limits>

#include "pemc/basic/tsc_index.h"
#include "pemc/basic/label.h"
#include "pemc/basic/model_capacity.h"
#include "pemc/basic/raw_memory.h"
#include "pemc/formula/formula.h"

namespace pemc {
  struct PathFrame {
    // the offset into the states array where the index of the frame's first state is stored.
    int32_t indexOfFirstStateIndexEntry; //offset
    // the number of states the frame consists of.
    int32_t count;
  };

  ///   When enumerating all states of a model in a depth-first fashion, we have to store the next states (that are computed all
	///   at once) somewhere while also being able to generate the counter example. When check a new state, a new frame is allocated
	///   on the stack and all unknown successor states are stored in that frame. We then take the topmost state, compute its
	///   successor states, and so on. When a formula violation is detected, the counter example consists of the last states of each
	///   frame on the stack. When a frame has been fully enumerated without detecting a formula violation, the stack frame is
	///   removed, the topmost state of the topmost frame is removed, and the new topmost state is checked.
  class PathTracker {
  private:
      std::vector<PathFrame> pathFrames;
      std::vector<StateIndex> stateIndexEntries;

      ///   The lowest index of a splittable frame. -1 if no frame is splittable.
      int32_t lowestSplittableFrame = -1;

      //   Maximal number of stateIndexEntries.
      int32_t capacity = 1 << 20;

      ///   Finds the next splittable frame, if any.
      void updateLowestSplittableFrame();

  public:
      PathTracker(int32_t _capacity);

      ///   Indicates whether the pathFrames can be split.
      bool canSplit();

      ///   Gets the number of path frames.
      int32_t getPathFrameCount();

      ///   Clears all pathFrames and its stateIndexEntries.
      void clear();

      ///   Pushes a new frame onto the pathFrames stack.
      void pushFrame();

      /// Adds the stateIndex to the current path frame.
      void pushStateIndex(StateIndex stateIndex);

  		///   Tries to get the topmost stateIndex if there is one.
      ///   Returns false to indicate that the path tracker was empty and no stateIndex was returned.
      bool tryGetStateIndex(StateIndex& stateIndex);

      bool splitWork(PathTracker& other);

      ///   Gets the path the path tracker currently represents, i.e.,
      ///   returns the sequence of topmost states of each frame, starting with
      ///   the oldest one.
      std::vector<StateIndex> getCurrentPath();
  };

}

#endif  // PEMC_GENERIC_TRAVERSER_PATH_TRACKER_H_


#pragma once

#include <string>
#include <utility>
#include <vector>
#include <unordered_map>
#include <gsl/span>

#include <lmdb.h>
#include <functional>
#include "Transaction.h"
#include "LmdbExceptions.h"

namespace DeepestScatter
{
    class Dataset
    {
    public:
        struct Settings;
        class Example;

        explicit Dataset(std::shared_ptr<Settings> settings);
        ~Dataset();

        template<class T>
        size_t getRecordsCount();

        template<class T>
        T getRecord(int32_t recordId);

        template<class T>
        void dropTable();

        template<class T>
        void append(const T& example);

        template<class T>
        void batchAppend(const gsl::span<T>& examples, int32_t startId);

        struct Settings
        {
            Settings(std::string path) :
                path(std::move(path))
            {}

            std::string path;
        };

    private:
        using TableName = std::string;

        template<class T>
        void tryAppend(const T& example, int entryId);

        template<class T>
        void tryBatchAppend(const gsl::span<T>& examples, int nextExampleId);

        void increaseSizeIfNeededWhile(const std::function<void(void)>& action);
        void increaseSize();

        MDB_dbi getTable(const TableName& name);
        MDB_dbi openTable(const TableName& name);
        void closeTable(const TableName& name);

        std::unordered_map<TableName, MDB_dbi> openedTables;
        std::unordered_map<TableName, int32_t> nextIds;

        MDB_env* mdbEnv = nullptr;
    };

    template <class T>
    size_t Dataset::getRecordsCount()
    {
        const TableName tableName = T::descriptor()->name();

        MDB_dbi dbi = getTable(tableName);

        return Transaction::withTransaction<size_t>(mdbEnv, nullptr, 0, [&](Transaction& transaction)
        {
            MDB_stat stats;
            LmdbExceptions::checkError(mdb_stat(transaction, dbi, &stats));
            return stats.ms_entries;
        });
    }

    template <class T>
    T Dataset::getRecord(int32_t recordId)
    {
        const TableName tableName = T::descriptor()->name();
        MDB_dbi dbi = getTable(tableName);

        return Transaction::withTransaction<T>(mdbEnv, nullptr, 0, [&](Transaction& transaction)
        {
            MDB_val mdbKey
            {
                sizeof(int32_t),
                &recordId
            };

            MDB_val mdbVal;
            LmdbExceptions::checkError(mdb_get(transaction, dbi, &mdbKey, &mdbVal));

            T record{};
            record.ParseFromArray(mdbVal.mv_data, mdbVal.mv_size);

            return record;
        });
    }

    template <class T>
    void Dataset::dropTable()
    {
        const TableName tableName = T::descriptor()->name();
        MDB_dbi dbi = getTable(tableName);

        if (getRecordsCount<T>() == 0)
        {
            return;
        }

        while (true)
        {
            std::string tableConfirmation;

            std::cout << "YOU ARE GOING TO DELETE " << getRecordsCount<T>() << " RECORDS!!!" << std::endl;
            std::cout << "Type table name: \"" << T::descriptor()->name() << "\" to drop it" << std::endl;
            std::cin >> tableConfirmation;

            if (tableConfirmation == T::descriptor()->name())
            {
                break;
            }

            std::cout << "Mismatched table name. Try again.";
        }

        Transaction::withTransaction<void>(mdbEnv, nullptr, 0, [&](Transaction& transaction)
        {
            mdb_drop(transaction, dbi, 0);
            nextIds[tableName] = 0;
        });
    }

    template <class T>
    void Dataset::append(const T& example)
    {
        const TableName tableName = T::descriptor()->name();
        // Returns zero if not initialized;
        int entryId = nextIds[tableName];

        increaseSizeIfNeededWhile([&]()
        {
            return tryAppend(example, entryId);
        });

        nextIds[tableName] = entryId + 1;
    }

    template <class T>
    void Dataset::batchAppend(const gsl::span<T>& examples, int32_t startId)
    {
        const TableName tableName = T::descriptor()->name();

        increaseSizeIfNeededWhile([&]()
        {
            return tryBatchAppend(examples, startId);
        });

        nextIds[tableName] = startId + examples.size();
    }

    template <class T>
    void Dataset::tryAppend(const T& example, int entryId)
    {
        const TableName tableName = T::descriptor()->name();
        MDB_dbi dbi = getTable(tableName);

        Transaction::withTransaction<void>(mdbEnv, nullptr, 0, [&](Transaction& transaction)
        {

            MDB_val mdbKey
            {
                sizeof(int32_t),
                &entryId
            };

            const size_t length = example.ByteSizeLong();
            std::vector<uint8_t> serialized(length);
            example.SerializeWithCachedSizesToArray(&serialized[0]);

            MDB_val mdbVal
            {
                length * sizeof(uint8_t),
                static_cast<void*>(&serialized[0])
            };

            LmdbExceptions::checkError(mdb_put(transaction, dbi, &mdbKey, &mdbVal, 0));
        });
    }

    template <class T>
    void Dataset::tryBatchAppend(const gsl::span<T>& examples, int nextExampleId)
    {
        const TableName tableName = T::descriptor()->name();
        MDB_dbi dbi = getTable(tableName);

        Transaction::withTransaction<void>(mdbEnv, nullptr, 0, [&](Transaction& transaction)
        {
            for (const auto& example : examples)
            {
                MDB_val mdbKey
                {
                    sizeof(int32_t),
                    &nextExampleId
                };

                const size_t length = example.ByteSizeLong();
                std::vector<uint8_t> serialized(length);
                example.SerializeWithCachedSizesToArray(&serialized[0]);

                MDB_val mdbVal
                {
                    length * sizeof(uint8_t),
                    static_cast<void*>(&serialized[0])
                };

                LmdbExceptions::checkError(mdb_put(transaction, dbi, &mdbKey, &mdbVal, 0));
                nextExampleId++;
            }
        });
    }
}


//
//  Rpc.hpp
//  PretendToWork
//
//  Created by tqtifnypmb on 18/12/2017.
//  Copyright © 2017 tqtifnypmb. All rights reserved.
//

#pragma once

#include <memory>
#include <vector>

#include <gsl/gsl>
#include <uv.h>

#include "Request.h"

namespace brick
{

class Rpc {
public:
    using RpcPeer = uv_handle_t;
    using Callback = std::function<void(RpcPeer*, Request req)>;

    enum class LoopState: int {
        looping = 0,
        closing = 1,
        closed,
        ready
    };

    Rpc(const char* ip, int port, Callback req_cb);
    Rpc(const Rpc&) = delete;
    Rpc& operator=(const Rpc&) = delete;
    ~Rpc();

    void send(RpcPeer* peer, const std::string& msg);
    void loop();
    void close();
    void close(RpcPeer* peer);

private:

    void onNewConnection(uv_tcp_t* client);
    void onNewMsg(uv_stream_t* client, const std::string& msg);
    void onHandleClosed(uv_handle_t* handle);

    static void connection_cb(uv_stream_t* server, int status);
    static void read_cb(uv_stream_t* stream, ssize_t nread, const uv_buf_t* buf);
    static void close_cb(uv_handle_t*);
    static void write_cb(uv_write_t* req, int status);

    LoopState state_;

    Callback req_cb_;
    std::vector<uv_tcp_t*> clients_;
    gsl::owner<uv_loop_t*> loop_;
    gsl::owner<uv_tcp_t*> server_;
};

}   // namespace brick