// __BEGIN_LICENSE__
//  Copyright (c) 2009-2013, United States Government as represented by the
//  Administrator of the National Aeronautics and Space Administration. All
//  rights reserved.
//
//  The NGT platform is licensed under the Apache License, Version 2.0 (the
//  "License"); you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//  http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
// __END_LICENSE__

/// \file bathy_plane_calc.cc
///

#include <asp/Core/PointUtils.h>
#include <asp/Core/Macros.h>
#include <asp/Core/Common.h>
#include <asp/Core/StereoSettings.h>
#include <asp/Sessions/StereoSessionFactory.h>

#include <vw/Core/Stopwatch.h>
#include <vw/FileIO/DiskImageUtils.h>
#include <vw/Cartography/shapeFile.h>
#include <vw/Math/RANSAC.h>
#include <vw/Camera/CameraModel.h>
#include <vw/Cartography/CameraBBox.h>
#include <vw/Core/ThreadPool.h>

#include <Eigen/Dense>

#include <random>
#include <algorithm>
#include <iterator>
#include <iostream>
#include <vector>

using namespace vw;
using namespace vw::cartography;
using namespace vw::camera;
namespace po = boost::program_options;
namespace fs = boost::filesystem;

// Estimate the projection and convert point_vec to projected coordinates
void find_projection(// Inputs
                     vw::cartography::GeoReference const& dem_georef,
                     std::vector<vw::Vector3> const& llh_vec,
                     // Outputs
                     double & proj_lat, double & proj_lon,
                     vw::cartography::GeoReference & stereographic_georef,
                     std::vector<Eigen::Vector3d> & point_vec) {

  // Initialize the outputs
  proj_lat = -1.0;
  proj_lon = -1.0;

  // Find the mean water height
  Vector3 mean_llh;
  for (size_t it = 0; it < llh_vec.size(); it++)
    mean_llh += llh_vec[it];

  mean_llh /= llh_vec.size();

  stereographic_georef.set_datum(dem_georef.datum());
  double scale = 1.0;
  proj_lat = mean_llh[1];
  proj_lon = mean_llh[0];
  stereographic_georef.set_stereographic(proj_lat, proj_lon, scale);

  // Convert the points to projected coordinates with this georeference
  for (size_t it = 0; it < llh_vec.size(); it++) {
    Vector3 point = stereographic_georef.geodetic_to_point(llh_vec[it]);

    Eigen::Vector3d eigen_point;
    for (size_t coord = 0; coord < 3; coord++)
      eigen_point[coord] = point[coord];

    point_vec[it] = eigen_point;
  }
}

// This is not used for now, it could be used to make the logic
// of processing mask boundary points multi-threaded.
class MaskBoundaryTask : public vw::Task, private boost::noncopyable {
  vw::BBox2i m_bbox; // Region of image we're working in

  ImageViewRef<float>                   m_mask;
  float                                 m_mask_nodata_val;
  boost::shared_ptr<CameraModel>        m_camera_model;
  vw::cartography::GeoReference         m_shape_georef;
  vw::cartography::GeoReference         m_dem_georef;
  ImageViewRef<PixelMask<float>>        m_masked_dem;

  // Note how all of these are aliases
  Mutex                        & m_mutex;
  std::vector<Eigen::Vector3d> & m_point_vec;
  std::vector<vw::Vector3>     & m_llh_vec;
  std::vector<vw::Vector2>     & m_used_vertices;

public:
  MaskBoundaryTask(vw::BBox2i                            bbox,
                   ImageViewRef<float>                   mask,
                   float                                 mask_nodata_val,
                   boost::shared_ptr<CameraModel>        camera_model,
                   vw::cartography::GeoReference const & shape_georef,
                   vw::cartography::GeoReference const & dem_georef,
                   ImageViewRef<PixelMask<float>>        masked_dem,
                   Mutex                               & mutex,
                   std::vector<Eigen::Vector3d>        & point_vec,
                   std::vector<vw::Vector3>            & llh_vec,
                   std::vector<vw::Vector2>            & used_vertices):
    m_bbox(bbox), m_mask(mask), m_mask_nodata_val(mask_nodata_val),
    m_camera_model(camera_model), m_shape_georef(shape_georef),
    m_dem_georef(dem_georef), m_masked_dem(masked_dem),
    m_mutex(mutex), m_point_vec(point_vec),
    m_llh_vec(llh_vec), m_used_vertices(used_vertices) {}

  void operator()() {

    // Grow the box by 1 pixel as we need to look at the immediate neighbors
    BBox2i extra_box = m_bbox;
    extra_box.expand(1);
    extra_box.crop(bounding_box(m_mask));

    std::cout << "--extra box " << extra_box << std::endl;

    // Make a local copy of the tile
    ImageView<float> mask_tile = crop(m_mask, extra_box);

    for (int col = 0; col < mask_tile.cols(); col++) {
      for (int row = 0; row < mask_tile.rows(); row++) {

        // Look at pixels above threshold which have neighbors <= threshold
        if (mask_tile(col, row) <= m_mask_nodata_val)
          continue;

        // The four neighbors
        int col_vals[4] = {-1, 0, 0, 1};
        int row_vals[4] = {0, -1, 1, 0};

        bool border_pix = false;
        for (int it = 0; it < 4; it++) {

          int icol = col + col_vals[it];
          int irow = row + row_vals[it];

          if (icol < 0 || irow < 0 || icol >= mask_tile.cols() || irow >= mask_tile.rows())
            continue;

          if (mask_tile(icol, irow) <= m_mask_nodata_val) {
            border_pix = true;
            break;
          }
        }

        if (!border_pix)
          continue;

        // Create the pixel in the full image coordinates
        Vector2 pix = Vector2(col, row) + extra_box.min();

        // Only work on pixels in the current box (earlier had a
        // bigger box to be able to examine neighbors).
        if (!m_bbox.contains(pix))
          continue;

        // The ray going to the ground
        // Here we assume that the camera model is thread-safe, which is
        // true for all cameras except ISIS, and this code will be used
        // on Earth only.
        Vector3 cam_ctr = m_camera_model->camera_center(pix);
        Vector3 cam_dir = m_camera_model->pixel_to_vector(pix);

        // Intersect the ray going from the given camera pixel with a DEM.
        bool treat_nodata_as_zero = false;
        bool has_intersection = false;
        double height_error_tol = 0.001; // in meters
        double max_abs_tol = 1e-14;
        double max_rel_tol = 1e-14;
        int num_max_iter = 100;
        Vector3 xyz_guess(0, 0, 0);
        Vector3 xyz = vw::cartography::camera_pixel_to_dem_xyz
          (cam_ctr, cam_dir, m_masked_dem,
           m_dem_georef, treat_nodata_as_zero,
           has_intersection, height_error_tol, max_abs_tol, max_rel_tol,
           num_max_iter, xyz_guess);

        if (!has_intersection)
          continue;

        Vector3 llh = m_dem_georef.datum().cartesian_to_geodetic(xyz);

        Eigen::Vector3d eigen_xyz;
        for (size_t coord = 0; coord < 3; coord++)
          eigen_xyz[coord] = xyz[coord];

        // TODO(oalexan1): This is fragile due to the 360 degree
        // uncertainty in latitude
        Vector2 proj_pt = m_shape_georef.lonlat_to_point(Vector2(llh[0], llh[1]));

        {
          // Need to make sure to lock the shared resource
          Mutex::Lock lock(m_mutex);
          m_point_vec.push_back(eigen_xyz);
          m_used_vertices.push_back(proj_pt);
          m_llh_vec.push_back(llh);
        }
      }

    }

  }

};

// Find the mask boundary (points where the points in the mask have
// neighbors not in the mask), shoot points from there onto the DEM,
// and return the obtained points.
// TODO(oalexan1): Should we move the boundary a bit outward?
void find_points_at_mask_boundary(ImageViewRef<float> mask,
                                  float mask_nodata_val,
                                  boost::shared_ptr<CameraModel> camera_model,
                                  vw::cartography::GeoReference const& shape_georef,
                                  vw::cartography::GeoReference const& dem_georef,
                                  ImageViewRef<PixelMask<float>> masked_dem,
                                  int num_samples,
                                  std::vector<Eigen::Vector3d> & point_vec,
                                  std::vector<vw::Vector3> & llh_vec,
                                  std::vector<vw::Vector2> & used_vertices) {

  // Ensure that the outputs are initialized
  point_vec.clear();
  llh_vec.clear();
  used_vertices.clear();

#if 0

  // For some reason parallel processing is slower than serial processing.
  // Likely it is because the DEM is shared. Need extra heuristics to cut
  // that one as appropriate for each task.

  // Subdivide the box for parallel processing
  int block_size = vw::vw_settings().default_tile_size();
  FifoWorkQueue queue(vw_settings().default_num_threads());
  std::vector<BBox2i> bboxes = subdivide_bbox(mask, block_size, block_size);

  for (size_t it = 0; it < bboxes.size(); it++) {
    std::cout << "Box is " << bboxes[it] << std::endl;
  }

  Mutex mutex;
  for (size_t it = 0; it < bboxes.size(); it++) {
    boost::shared_ptr<MaskBoundaryTask>
      task(new MaskBoundaryTask(bboxes[it],  mask, mask_nodata_val, camera_model,
                                shape_georef, dem_georef, masked_dem, mutex,
                                point_vec, llh_vec, used_vertices));
    queue.add_task(task);
  }

  queue.join_all();

#else

  // Let the mask boundary be the mask pixels whose value
  // is above threshold and which border pixels whose values
  // is not above threshold.
  // TODO(oalexan1): Make this multi-threaded.

  vw_out() << "Processing points at mask boundary.\n";
  vw::TerminalProgressCallback tpc("asp", "\t--> ");
  double inc_amount = 1.0 / mask.cols();
  tpc.report_progress(0);

  for (int col = 0; col < mask.cols(); col++) {
    for (int row = 0; row < mask.rows(); row++) {

      // Look at pixels above threshold which have neighbors <= threshold
      if (mask(col, row) <= mask_nodata_val)
        continue;

      // The four neighbors
      int col_vals[4] = {-1, 0, 0, 1};
      int row_vals[4] = {0, -1, 1, 0};

      bool border_pix = false;
      for (int it = 0; it < 4; it++) {

        int icol = col + col_vals[it];
        int irow = row + row_vals[it];

        if (icol < 0 || irow < 0 || icol >= mask.cols() || irow >= mask.rows())
          continue;

        if (mask(icol, irow) <= mask_nodata_val) {
          border_pix = true;
          break;
        }
      }

      if (!border_pix)
        continue;

      // The ray going to the ground
      Vector2 pix(col, row);
      Vector3 cam_ctr = camera_model->camera_center(pix);
      Vector3 cam_dir = camera_model->pixel_to_vector(pix);

      // Intersect the ray going from the given camera pixel with a DEM.
      bool treat_nodata_as_zero = false;
      bool has_intersection = false;
      double height_error_tol = 0.001; // in meters
      double max_abs_tol = 1e-14;
      double max_rel_tol = 1e-14;
      int num_max_iter = 100;
      Vector3 xyz_guess(0, 0, 0);
      Vector3 xyz = vw::cartography::camera_pixel_to_dem_xyz
        (cam_ctr, cam_dir, masked_dem,
         dem_georef, treat_nodata_as_zero,
         has_intersection, height_error_tol, max_abs_tol, max_rel_tol,
         num_max_iter, xyz_guess);

      if (!has_intersection)
        continue;

      Vector3 llh = dem_georef.datum().cartesian_to_geodetic(xyz);

      Eigen::Vector3d eigen_xyz;
      for (size_t coord = 0; coord < 3; coord++)
        eigen_xyz[coord] = xyz[coord];

      // TODO(oalexan1): This is fragile due to the 360 degree
      // uncertainty in latitude
      Vector2 proj_pt = shape_georef.lonlat_to_point(Vector2(llh[0], llh[1]));

      point_vec.push_back(eigen_xyz);
      used_vertices.push_back(proj_pt);
      llh_vec.push_back(llh);
    }

    tpc.report_incremental_progress(inc_amount);
  }

  tpc.report_finished();

#endif

  // See if to select a subset
  int num_pts = point_vec.size();
  if (num_pts > num_samples) {
    vw_out() << "Found " << num_pts << " points but only " << num_samples
             << " samples are desired. Picking a random subset of this size.\n";

    std::vector<int> v(num_pts);
    for (int it = 0; it < num_pts; it++)
      v[it] = it;
    std::mt19937 g; // Each time this is run same random numbers should be produced
    std::shuffle(v.begin(), v.end(), g);

    std::vector<Eigen::Vector3d> big_point_vec     = point_vec;     point_vec.clear();
    std::vector<vw::Vector3>     big_llh_vec       = llh_vec;       llh_vec.clear();
    std::vector<vw::Vector2>     big_used_vertices = used_vertices; used_vertices.clear();

    for (int it = 0; it < num_samples; it++) {
      int random_index = v[it];
      point_vec.push_back(big_point_vec[random_index]);
      llh_vec.push_back(big_llh_vec[random_index]);
      used_vertices.push_back(big_used_vertices[random_index]);
    }
  }

  return;
}

// Compute the 3D locations at the shape corners based on interpolating
// into the DEM and converting either to ECEF or to local projected
// stereographic coordinates.
// If using a curved water surface, compute the stereographic georeference
// with the projection center being the mean lon and lat, and make the 3D locations
// in reference to this projection
void find_points_at_shape_corners(std::vector<vw::geometry::dPoly> const& polyVec,
                                  vw::cartography::GeoReference const& shape_georef,
                                  vw::cartography::GeoReference const& dem_georef,
                                  ImageViewRef< PixelMask<float> > interp_dem,
                                  std::vector<Eigen::Vector3d> & point_vec,
                                  std::vector<vw::Vector3> & llh_vec,
                                  std::vector<vw::Vector2> & used_vertices) {

  // Ensure that the outputs are initialized
  point_vec.clear();
  llh_vec.clear();
  used_vertices.clear();

  int total_num_pts = 0;

  for (size_t p = 0; p < polyVec.size(); p++){
    vw::geometry::dPoly const& poly = polyVec[p];

    const double * xv       = poly.get_xv();
    const double * yv       = poly.get_yv();
    const int    * numVerts = poly.get_numVerts();
    int numPolys            = poly.get_numPolys();

    int start = 0;
    for (int pIter = 0; pIter < numPolys; pIter++){

      if (pIter > 0) start += numVerts[pIter - 1];

      int numV = numVerts[pIter];
      for (int vIter = 0; vIter < numV; vIter++) {

        total_num_pts++;

        Vector2 proj_pt(xv[start + vIter], yv[start + vIter]);

        // Convert from projected coordinates to lonlat
        Vector2 lonlat = shape_georef.point_to_lonlat(proj_pt);

        // Convert to DEM pixel
        Vector2 pix = dem_georef.lonlat_to_pixel(lonlat);

        PixelMask<float> h = interp_dem(pix.x(), pix.y());

        if (!is_valid(h))
          continue;

        Vector3 llh;
        llh[0] = lonlat[0];
        llh[1] = lonlat[1];
        llh[2] = h.child();

        Vector3 xyz = dem_georef.datum().geodetic_to_cartesian(llh);
        Eigen::Vector3d eigen_xyz;
        for (size_t coord = 0; coord < 3; coord++)
          eigen_xyz[coord] = xyz[coord];

        point_vec.push_back(eigen_xyz);
        used_vertices.push_back(proj_pt);
        llh_vec.push_back(llh);
      }
    }
  }

  vw_out() << "Read " << total_num_pts << " vertices, with " << llh_vec.size()
            << " of them having a valid DEM height value."  << std::endl;
}

// Best fit plane without outlier removal
std::pair<Eigen::Vector3d, Eigen::Vector3d>
best_plane_from_points(const std::vector<Eigen::Vector3d> & c) {

  // Copy coordinates to a matrix in Eigen format
  size_t num_points = c.size();
  Eigen::Matrix<double, Eigen::Dynamic, Eigen::Dynamic > coord(3, num_points);
  for (size_t i = 0; i < num_points; i++)
    coord.col(i) = c[i];

  // calculate centroid
  Eigen::Vector3d centroid(coord.row(0).mean(), coord.row(1).mean(), coord.row(2).mean());

  // subtract centroid
  for (size_t i = 0; i < 3; i++)
    coord.row(i).array() -= centroid(i);

  // We only need the left-singular matrix here
  // http://math.stackexchange.com/questions/99299/best-fitting-plane-given-a-set-of-points
  auto svd = coord.jacobiSvd(Eigen::ComputeThinU | Eigen::ComputeThinV);
  Eigen::Vector3d plane_normal = svd.matrixU().rightCols<1>();
  return std::make_pair(centroid, plane_normal);
}

// A functor which returns the best fit plane a*x + b*y + c*z + d = 0
// as the vector (a, b, c, d) with a*a + b*b + c*c = 1 to be used
// with RANSAC to remove outliers.

struct BestFitPlaneFunctor {

  BestFitPlaneFunctor(bool use_proj_water_surface):
    m_use_proj_water_surface(use_proj_water_surface) {}

  typedef vw::Matrix<double, 1, 4> result_type;

  bool m_use_proj_water_surface;

  /// A best fit plane requires pairs of data points to make a fit.
  template <class ContainerT>
  size_t min_elements_needed_for_fit(ContainerT const& /*example*/) const { return 3; }

  /// This function can match points in any container that supports
  /// the size() and operator[] methods.  The container is usually a
  /// vw::Vector<>, but you could substitute other classes here as
  /// well.
  template <class ContainerT>
  vw::Matrix<double> operator() (std::vector<ContainerT> const& p1,
                                 std::vector<ContainerT> const& p2,
                                 vw::Matrix<double> const& /*seed_input*/
                                 = vw::Matrix<double>() ) const {

    // check consistency
    VW_ASSERT( p1.size() == p2.size(),
               vw::ArgumentErr() << "Cannot compute best fit plane. "
               << "p1 and p2 are not the same size." );
    VW_ASSERT( !p1.empty() && p1.size() >= min_elements_needed_for_fit(p1[0]),
               vw::ArgumentErr() << "Cannot compute best fit plane. "
               << "Insufficient data.\n");

    std::pair<Eigen::Vector3d, Eigen::Vector3d> plane = best_plane_from_points(p1);

    Eigen::Vector3d & centroid = plane.first;
    Eigen::Vector3d & normal = plane.second;

    Matrix<double> result(1, 4);
    for (int col = 0; col < 3; col++)
      result(0, col) = normal[col];

    result(0, 3) = -normal.dot(centroid);

    if (!m_use_proj_water_surface) {
      // Make the normal always point "up", away from the Earth origin,
      // which means that the free term must be negative.
      if (result(0, 3) > 0) {
        for (int col = 0; col < 4; col++)
          result(0, col) *= -1.0;
      }
    }else {
      // Make the z coefficient positive, which will make the normal
      // point "up" in the projected coordinate system.
      if (result(0, 2) < 0) {
        for (int col = 0; col < 4; col++)
          result(0, col) *= -1.0;
      }
    }

    return result;
  }
};

// Given a 1x4 matrix named 'plane', with values (a, b, c, d),
// determining the plane a * x + b * y + c * z + d = 0, find the
// distance to this plane from a given point.

template<class Vec3>
double dist_to_plane(vw::Matrix<double, 1, 4> const& plane, Vec3 const& point) {

  double ans = 0.0;
  for (unsigned col = 0; col < 3; col++) {
      ans += plane(0, col) * point[col];
  }
  ans += plane(0, 3);

  return std::abs(ans);
}

// The value p2 is needed by the interface but we don't use it
struct BestFitPlaneErrorMetric {
  template <class RelationT, class ContainerT>
  double operator() (RelationT const& plane, ContainerT const& p1, ContainerT const& p2) const {
    return dist_to_plane(plane, p1);
  }
};

void calc_plane_properties(bool use_proj_water_surface,
                           std::vector<Eigen::Vector3d> const& point_vec,
                           std::vector<size_t> const& inlier_indices,
                           vw::cartography::GeoReference & dem_georef,
                           vw::Matrix<double> const& plane) {

  double max_error = - 1.0, max_inlier_error = -1.0;
  for (size_t it = 0; it < point_vec.size(); it++) {
    max_error = std::max(max_error, dist_to_plane(plane, point_vec[it]));
  }

  // Do estimates for the mean height and angle of the plane
  Vector3 mean_point(0, 0, 0);
  double mean_height = 0.0;
  int num = 0;
  for (size_t it = 0; it < inlier_indices.size(); it++) {
    Eigen::Vector3d p = point_vec[inlier_indices[it]];
    Vector3 point(p[0], p[1], p[2]);
    max_inlier_error = std::max(max_inlier_error, dist_to_plane(plane, point));

    if (!use_proj_water_surface) {
      // the point is xyz in ecef
      Vector3 llh = dem_georef.datum().cartesian_to_geodetic(point);
      mean_height += llh[2];
    } else {
      // the point is in the stereographic projection
      mean_height += point[2];
    }

    num++;

    if (!use_proj_water_surface)
      mean_point += point;
  }

  mean_height /= num;

  if (!use_proj_water_surface)
    mean_point /= num;

  vw_out() << "Max distance to the plane (meters): " << max_error << std::endl;
  vw_out() << "Max inlier distance to the plane (meters): " << max_inlier_error << std::endl;
  vw_out() << "Mean plane height above datum (meters): " << mean_height << std::endl;

  if (!use_proj_water_surface) {
    // This does not make sense for a curved surface
    Vector3 plane_normal(plane(0, 0), plane(0, 1), plane(0, 2));
    Vector3 surface_normal = mean_point / norm_2(mean_point); // ignore the datum flattening
    double plane_angle = (180.0 / M_PI) * acos(dot_prod(plane_normal, surface_normal));
    vw_out() << "Plane inclination (degrees): " << plane_angle << std::endl;
  }
}

void save_plane(bool use_proj_water_surface, double proj_lat, double proj_lon,
                vw::Matrix<double> const& plane, std::string const& plane_file) {

  vw_out() << "Writing: " << plane_file << std::endl;
  vw::create_out_dir(plane_file);
  std::ofstream bp(plane_file.c_str());
  bp.precision(17);
  for (int col = 0; col < plane.cols(); col++) {
    bp << plane(0, col);
    if (col < plane.cols() - 1)
      bp << " ";
    else
      bp << "\n";
  }
  if (use_proj_water_surface) {
    bp << "# Latitude and longitude of the local stereographic projection with the WGS_1984 datum:\n";
    bp << proj_lat << " " << proj_lon << "\n";
  }
  bp.close();
}

// Given a DEM surface and a planar surface (with potentially some
// inclination) subtract from each DEM height the height at that
// plane. The height of the plane is obtained by considering the
// current DEM grid point, and another point at same lon-lat but with
// a height 100 meters less, converting these to the given projection
// (if m_use_proj_water_surface is true or to ECEF otherwise), tracing a ray
// through them, seeing where it intersects the plane, converting that
// point to geodetic, and taking the height difference.
class DemMinusPlaneView: public ImageViewBase<DemMinusPlaneView>{
  ImageViewRef<float> m_dem;
  vw::cartography::GeoReference m_dem_georef;
  vw::Matrix<double> m_plane;
  double m_dem_nodata_val;
  bool m_use_proj_water_surface;
  vw::cartography::GeoReference m_stereographic_georef;

  typedef float PixelT;

public:
  DemMinusPlaneView(ImageViewRef<float> const& dem,
                    vw::cartography::GeoReference const& dem_georef,
                    vw::Matrix<double> const& plane,
                    double dem_nodata_val,
                    bool use_proj_water_surface,
                    vw::cartography::GeoReference const& stereographic_georef):
    m_dem(dem), m_dem_georef(dem_georef), m_plane(plane),
    m_dem_nodata_val(dem_nodata_val),
    m_use_proj_water_surface(use_proj_water_surface),
    m_stereographic_georef(stereographic_georef){}

  typedef PixelT pixel_type;
  typedef PixelT result_type;
  typedef ProceduralPixelAccessor<DemMinusPlaneView> pixel_accessor;

  inline int32 cols() const { return m_dem.cols(); }
  inline int32 rows() const { return m_dem.rows(); }
  inline int32 planes() const { return 1; }

  inline pixel_accessor origin() const { return pixel_accessor( *this, 0, 0 ); }

  inline pixel_type operator()( double/*i*/, double/*j*/, int32/*p*/ = 0 ) const {
    vw_throw(NoImplErr() << "DemMinusPlaneView::operator()(...) is not implemented");
    return pixel_type();
  }

  typedef CropView< ImageView<pixel_type> > prerasterize_type;

  inline prerasterize_type prerasterize(BBox2i const& bbox) const {

    // Bring this portion in memory
    ImageView<result_type> cropped_dem = crop(m_dem, bbox);

    ImageView<result_type> tile(bbox.width(), bbox.height());

    for (int col = 0; col < bbox.width(); col++){
      for (int row = 0; row < bbox.height(); row++){

        // Handle the case when the DEM is not valid
        if (cropped_dem(col, row) == m_dem_nodata_val) {
          tile(col, row) = m_dem_nodata_val;
          continue;
        }

        Vector2 pix(col + bbox.min().x(), row + bbox.min().y());
        Vector2 lon_lat = m_dem_georef.pixel_to_lonlat(pix);

        Vector3 llh;
        subvector(llh, 0, 2) = lon_lat;
        llh[2] = cropped_dem(col, row);

        // Find the DEM point in local projection or ECEF.
        // Find another point on the same ray.
        Vector3 point1, point2;

        if (m_use_proj_water_surface) {
          point1 = m_stereographic_georef.geodetic_to_point(llh);
          llh[2] -= 100.0;
          point2 = m_stereographic_georef.geodetic_to_point(llh);
        } else {
          point1 = m_dem_georef.datum().geodetic_to_cartesian(llh);
          llh[2] -= 100.0;
          point2 = m_dem_georef.datum().geodetic_to_cartesian(llh);
        }

        // The ray is P = point1 + t * (point2 - point1), where t is real.
        // The plane is a*x + b*y + c*z + d = 0, where plane = (a, b, c, d).
        // Then, dot(P, plane_normal) + plane_intercept = 0.
        // Use that to find t.
        Vector3 plane_normal(m_plane(0, 0), m_plane(0, 1), m_plane(0, 2));
        double plane_intercept = m_plane(0, 3);

        double t = -(dot_prod(point1, plane_normal) + plane_intercept) /
          dot_prod(point2 - point1, plane_normal);

        Vector3 P = point1 + t * (point2 - point1);

        // Go back to llh
        if (m_use_proj_water_surface) {
          llh = m_stereographic_georef.point_to_geodetic(P);
        } else{
          llh = m_dem_georef.datum().cartesian_to_geodetic(P);
        }

        tile(col, row) = cropped_dem(col, row) - llh[2];
      }
    }

    return prerasterize_type(tile, -bbox.min().x(), -bbox.min().y(),
                             cols(), rows() );
  }

  template <class DestT>
  inline void rasterize(DestT const& dest, BBox2i bbox) const {
    vw::rasterize(prerasterize(bbox), dest, bbox);
  }
};

DemMinusPlaneView dem_minus_plane(ImageViewRef<float> const& dem,
                                  vw::cartography::GeoReference const& dem_georef,
                                  vw::Matrix<double> plane,
                                  double dem_nodata_val,
                                  bool use_proj_water_surface,
                                  vw::cartography::GeoReference const& stereographic_georef) {
  return DemMinusPlaneView(dem, dem_georef, plane, dem_nodata_val,
                           use_proj_water_surface, stereographic_georef);
}

struct Options : vw::cartography::GdalWriteOptions {
  std::string shapefile, dem, mask, camera, stereo_session, bathy_plane,
    output_inlier_shapefile, output_outlier_shapefile, dem_minus_plane;
  double outlier_threshold;
  int num_ransac_iterations, num_samples;
  bool save_shapefiles_as_polygons, use_ecef_water_surface;
  Options(): outlier_threshold(0.2), num_ransac_iterations(1000) {}
};

void handle_arguments(int argc, char *argv[], Options& opt) {

  po::options_description general_options("General Options");
  general_options.add_options()
    ("shapefile",   po::value(&opt.shapefile),
     "The shapefile with vertices whose coordinates will be looked up in the DEM.")
    ("dem",   po::value(&opt.dem),
     "The DEM to use.")
    ("mask",   po::value(&opt.mask),
     "A input mask, created from a raw camera image and hence having the same dimensions, with values of 1 on land and 0 on water, or positive values on land and no-data values on water.")
    ("camera",   po::value(&opt.camera),
     "The camera file to use with the mask.")
    ("session-type,t",   po::value(&opt.stereo_session)->default_value(""),
     "Select the stereo session type to use for processing. Usually the program can select this automatically by the file extension, except for xml cameras. See the doc for options.")
    ("bathy-plane",   po::value(&opt.bathy_plane),
     "The output file storing the computed plane as four coefficients a, b, c, d, "
     "with the plane being a*x + b*y + c*z + d = 0.")
    ("outlier-threshold",
     po::value(&opt.outlier_threshold)->default_value(0.2),
     "A value, in meters, to determine the distance from a sampled point on the DEM to the "
     "best-fit plane to determine if it will be marked as outlier and not "
     "included in the calculation of that plane.")
    ("num-ransac-iterations",
     po::value(&opt.num_ransac_iterations)->default_value(1000),
     "Number of RANSAC iterations to use to find the best-fitting plane.")
    ("output-inlier-shapefile", po::value(&opt.output_inlier_shapefile)->default_value(""),
     "If specified, save at this location the shape file with the inlier vertices.")
    ("output-outlier-shapefile", po::value(&opt.output_outlier_shapefile)->default_value(""),
     "If specified, save at this location the shape file with the outlier vertices.")
    ("num-samples",
     po::value(&opt.num_samples)->default_value(10000),
     "Number of samples to pick at the water-land interface if using a mask.")
    ("save-shapefiles-as-polygons",
     po::bool_switch(&opt.save_shapefiles_as_polygons)->default_value(false),
     "Save the inlier and outlier shapefiles as polygons, rather than "
     "discrete vertices. May be more convenient for processing in a GIS tool.")
    ("dem-minus-plane",
     po::value(&opt.dem_minus_plane),
     "If specified, subtract from the input DEM the best-fit plane and save the "
     "obtained DEM to this GeoTiff file.")
    ("use-ecef-water-surface",
     po::bool_switch(&opt.use_ecef_water_surface)->default_value(false),
     "Compute the best fit plane in ECEF coordinates rather than in a local stereographic "
     "projection. Hence don't model the Earth curvature. Not recommended.");

  general_options.add( vw::cartography::GdalWriteOptionsDescription(opt) );

  po::options_description positional("");
  //positional.add_options()
  //   ("input-files", po::value< std::vector<std::string> >(), "Input files");

  po::positional_options_description positional_desc;
  //positional_desc.add("input-files", -1);

  std::string usage("[options]");
  bool allow_unregistered = false;
  std::vector<std::string> unregistered;
  po::variables_map vm =
    asp::check_command_line(argc, argv, opt, general_options, general_options,
                            positional, positional_desc, usage,
                            allow_unregistered, unregistered);

  bool use_shapefile = (!opt.shapefile.empty());

  if (!use_shapefile && (opt.mask.empty() || opt.camera.empty()))
    vw_throw( ArgumentErr() << "A shape file must be specified, or a mask and a camera.\n"
              << usage << general_options );
  if (opt.dem == "")
    vw_throw( ArgumentErr() << "Missing the input dem.\n" << usage << general_options );
  if (opt.bathy_plane == "")
    vw_throw( ArgumentErr() << "Missing the output bathy plane file.\n"
              << usage << general_options );
}

// Given a set of polygons stored in dPoly, create a single polygon with all those vertices
void formSinglePoly(vw::geometry::dPoly const& inPoly,
                    vw::geometry::dPoly & outPoly) {

  int num_pts = inPoly.get_totalNumVerts();
  const double * xv = inPoly.get_xv();
  const double * yv = inPoly.get_yv();

  bool isPolyClosed = true;
  std::string layer = "";
  std::string poly_color = "green";
  outPoly.setPolygon(num_pts, xv, yv, isPolyClosed, poly_color, layer);
}

typedef boost::shared_ptr<asp::StereoSession> SessionPtr;

int main( int argc, char *argv[] ) {

  Options opt;
  try {
    handle_arguments(argc, argv, opt);

    // Load the camera if we use the mask and the camera
    bool use_shapefile = (!opt.shapefile.empty());
    boost::shared_ptr<CameraModel> camera_model;
    if (!use_shapefile) {
      std::string out_prefix;
      SessionPtr session(asp::StereoSessionFactory::create(opt.stereo_session, // may change
                                                           opt,
                                                           opt.mask, opt.mask,
                                                           opt.camera, opt.camera,
                                                           out_prefix));
      camera_model = session->camera_model(opt.mask, opt.camera);
    }

    // Read the DEM and its associated data
    // TODO(oalexan1): Think more about the interpolation method
    vw_out() << "Reading the DEM: " << opt.dem << std::endl;
    vw::cartography::GeoReference dem_georef;
    if (!read_georeference(dem_georef, opt.dem))
      vw_throw( ArgumentErr() << "The input DEM has no georeference.\n" );

    // We assume the WGS_1984 datum
    if (dem_georef.datum().name() != "WGS_1984")
      vw_throw( ArgumentErr() << "Only an input DEM with the "
                << "WGS_1984 datum is supported.\n"
                << "Got: " << dem_georef.datum().name() << ".\n");

    // Note we use a float nodata
    float dem_nodata_val = -std::numeric_limits<float>::max();
    if (!vw::read_nodata_val(opt.dem, dem_nodata_val))
      vw_out() << "Warning: Could not read the DEM nodata value. "
                << "Using: " << dem_nodata_val << ".\n";
    else
      vw_out() << "Read DEM nodata value: " << dem_nodata_val << ".\n";

    // Read the DEM
    DiskImageView<float> dem(opt.dem);
    ImageViewRef<PixelMask<float>> masked_dem = create_mask(dem, dem_nodata_val);
    ImageViewRef< PixelMask<float> > interp_dem
      = interpolate(masked_dem, BilinearInterpolation(), ConstantEdgeExtension());

    bool use_proj_water_surface = !opt.use_ecef_water_surface;
    std::vector<Eigen::Vector3d> point_vec;
    std::vector<vw::Vector3> llh_vec;
    double proj_lat = -1.0, proj_lon = -1.0; // only for curved water surface
    std::vector<vw::Vector2> used_vertices;
    vw::cartography::GeoReference stereographic_georef;
    bool has_shape_georef = false;
    vw::cartography::GeoReference shape_georef;
    std::string poly_color = "green";

    if (!use_shapefile) {

      // Read the mask. The no-data value is the largest of what
      // is read from the mask file and the value 0, as pixels
      // over land are supposed to be positive and be valid data.
      vw_out() << "Reading the mask: " << opt.mask << std::endl;
      float mask_nodata_val = -std::numeric_limits<float>::max();
      if (vw::read_nodata_val(opt.mask, mask_nodata_val))
        vw_out() << "Read mask nodata value: " << mask_nodata_val << ".\n";
      mask_nodata_val = std::max(0.0f, mask_nodata_val);
      vw_out() << "Pixels with values no more than " << mask_nodata_val
               << " are classified as water.\n";
      DiskImageView<float> mask(opt.mask);

      has_shape_georef = true;
      shape_georef = dem_georef;
      find_points_at_mask_boundary(mask, mask_nodata_val,
                                   camera_model, shape_georef,
                                   dem_georef, masked_dem,
                                   opt.num_samples,
                                   point_vec, llh_vec,
                                   used_vertices);
    } else {

      // Read the shapefile
      vw_out() << "Reading the shapefile: " << opt.shapefile << std::endl;
      std::vector<vw::geometry::dPoly> polyVec;
      read_shapefile(opt.shapefile, poly_color, has_shape_georef, shape_georef, polyVec);

      if (!has_shape_georef)
        vw_throw( ArgumentErr() << "The input shapefile has no georeference.\n" );

      // Find the ECEF coordinates of the shape corners
      find_points_at_shape_corners(polyVec, shape_georef, dem_georef, interp_dem, point_vec,
                                   llh_vec, used_vertices);
    }

    // See if to convert to local stereographic projection
    if (use_proj_water_surface)
      find_projection(// Inputs
                      dem_georef, llh_vec,
                      // Outputs
                      proj_lat, proj_lon,
                      stereographic_georef,
                      point_vec);

    // Compute the water surface using RANSAC
    std::vector<Eigen::Vector3d> dummy_vec(point_vec.size()); // Required by the interface
    std::vector<size_t> inlier_indices;
    double inlier_threshold = opt.outlier_threshold;
    int    min_num_output_inliers = std::max(point_vec.size()/2, size_t(3));
    bool   reduce_min_num_output_inliers_if_no_fit = true;
    vw::Matrix<double> plane;
    vw_out() << "Starting RANSAC.\n";
    try {
      // Must first create the functor and metric, then pass these to ransac. If
      // created as inline arguments to ransac, these may go go out
      // of scope prematurely, which will result in incorrect behavior.
      BestFitPlaneFunctor func(use_proj_water_surface);
      BestFitPlaneErrorMetric error_metric;
      math::RandomSampleConsensus<BestFitPlaneFunctor, BestFitPlaneErrorMetric>
        ransac(func, error_metric,
               opt.num_ransac_iterations, inlier_threshold,
               min_num_output_inliers, reduce_min_num_output_inliers_if_no_fit);

      plane = ransac(point_vec, dummy_vec);

      inlier_indices = ransac.inlier_indices(plane, point_vec, dummy_vec);
    } catch (const vw::math::RANSACErr& e ) {
      vw_out() << "RANSAC failed: " << e.what() << "\n";
    }
    vw_out() << "Found " << inlier_indices.size() << " / " << point_vec.size() << " inliers.\n";

    calc_plane_properties(use_proj_water_surface, point_vec, inlier_indices,
                           dem_georef, plane);

    save_plane(use_proj_water_surface, proj_lat, proj_lon,
               plane, opt.bathy_plane);

    // Save the shape having the inliers.
    if (opt.output_inlier_shapefile != "") {
      vw::geometry::dPoly inlierPoly;
      for (size_t inlier_it = 0; inlier_it < inlier_indices.size(); inlier_it++) {
        Vector2 p = used_vertices[inlier_indices[inlier_it]];
        std::vector<double> inlier_x, inlier_y;
        inlier_x.push_back(p.x());
        inlier_y.push_back(p.y());
        bool isPolyClosed = true;
        std::string layer = "";
        inlierPoly.appendPolygon(inlier_x.size(),
                                 vw::geometry::vecPtr(inlier_x),
                                 vw::geometry::vecPtr(inlier_y),
                                 isPolyClosed, poly_color, layer);
      }

      if (opt.save_shapefiles_as_polygons) {
        vw::geometry::dPoly localPoly;
        formSinglePoly(inlierPoly, localPoly);
        inlierPoly = localPoly;
      }

      std::vector<vw::geometry::dPoly> inlierPolyVec;
      inlierPolyVec.push_back(inlierPoly);
      vw_out() << "Writing inlier shapefile: " << opt.output_inlier_shapefile << "\n";
      write_shapefile(opt.output_inlier_shapefile, has_shape_georef, shape_georef,
                      inlierPolyVec);
    }

    // Save the shape having the outliers.
    if (opt.output_outlier_shapefile != "") {

      // First put the inliers in a set so we can exclude them
      std::set<int> inlier_set;
      for (size_t inlier_it = 0; inlier_it < inlier_indices.size(); inlier_it++)
        inlier_set.insert(inlier_indices[inlier_it]);

      vw::geometry::dPoly outlierPoly;
      for (size_t it = 0; it < used_vertices.size(); it++) {

        if (inlier_set.find(it) != inlier_set.end())
          continue; // an inlier, skip it

        Vector2 p = used_vertices[it];
        std::vector<double> outlier_x, outlier_y;
        outlier_x.push_back(p.x());
        outlier_y.push_back(p.y());
        bool isPolyClosed = true;
        std::string layer = "";
        outlierPoly.appendPolygon(outlier_x.size(),
                                 vw::geometry::vecPtr(outlier_x),
                                 vw::geometry::vecPtr(outlier_y),
                                 isPolyClosed, poly_color, layer);
      }

      if (opt.save_shapefiles_as_polygons) {
        vw::geometry::dPoly localPoly;
        formSinglePoly(outlierPoly, localPoly);
        outlierPoly = localPoly;
      }

      std::vector<vw::geometry::dPoly> outlierPolyVec;
      outlierPolyVec.push_back(outlierPoly);
      vw_out() << "Writing outlier shapefile: " << opt.output_outlier_shapefile << "\n";
      write_shapefile(opt.output_outlier_shapefile, has_shape_georef, shape_georef,
                      outlierPolyVec);
    }

    if (opt.dem_minus_plane != "") {
      bool has_nodata = true, has_georef = true;
      vw_out() << "Writing: " << opt.dem_minus_plane << std::endl;
      TerminalProgressCallback tpc("asp", ": ");
      block_write_gdal_image(opt.dem_minus_plane,
                             dem_minus_plane(dem, dem_georef, plane,
                                             dem_nodata_val, use_proj_water_surface,
                                             stereographic_georef),
                             has_georef, dem_georef, has_nodata, dem_nodata_val, opt, tpc);
    }

  } ASP_STANDARD_CATCHES;

  return 0;
}


#include <pybind11/pybind11.h>
#include <pybind11/stl.h>

#include <sstream>
#include <boost/archive/text_oarchive.hpp>
#include <boost/archive/text_iarchive.hpp>

#include <mspass/algorithms/algorithms.h>
#include <mspass/algorithms/Butterworth.h>
#include <mspass/algorithms/Taper.h>
#include <mspass/algorithms/TimeWindow.h>
#include <mspass/utility/Metadata.h>

namespace mspass {
namespace mspasspy {

namespace py=pybind11;
using namespace std;
using namespace mspass::seismic;
using namespace mspass::algorithms;

/* Trampoline class for BasicTaper - used for family of taper classes*/
class PyBasicTaper : public BasicTaper
{
public:
  int apply(TimeSeries &d)
  {
    PYBIND11_OVERLOAD_PURE(
        int,
        BasicTaper,
        TimeSeries &,
        d);
  }
  int apply(Seismogram &d)
  {
    PYBIND11_OVERLOAD_PURE(
        int,
        BasicTaper,
        Seismogram &,
        d);
  }
  double get_t0head()
  {
    PYBIND11_OVERLOAD_PURE(
      double,
      BasicTaper,
      get_t0head
    );
  }
  double get_t1head()
  {
    PYBIND11_OVERLOAD_PURE(
      double,
      BasicTaper,
      get_t1head
    );
  }
};

PYBIND11_MODULE(basic, m) {
  m.attr("__name__") = "mspasspy.ccore.algorithms.basic";
  m.doc() = "A submodule for algorithms namespace of ccore with common algorithms";

  py::class_<mspass::algorithms::Butterworth>
              (m,"Butterworth","Butterworth filter operator processing object")
    .def(py::init<>())
    .def(py::init<const bool, const bool, const bool,
        const double, const double, const double, const double,
	const double, const double, const double, const double,
	const double> ())
    .def(py::init<const mspass::utility::Metadata&>())
    .def(py::init<const bool, const bool, const bool,
        const int, const double, const int, const double, const double>())
    .def(py::init<const Butterworth&>())
    .def("impulse_response",&Butterworth::impulse_response,
         "Return impulse response")
    .def("transfer_function",&Butterworth::transfer_function,
         "Return transfer function in a complex valued array")
    .def("change_dt",&Butterworth::change_dt,
         "Change sample interval defining the operator (does not change corners) ")
    /* Note we intentionally do not overload CoreTimeSeries and CoreSeismogram.
    They do not handle errors as gracefully */
    .def("apply",py::overload_cast<mspass::seismic::TimeSeries&>
         (&Butterworth::apply),"Apply the predefined filter to a TimeSeries object")
    .def("apply",py::overload_cast<std::vector<double>&>(&Butterworth::apply),
    	"Apply the predefined filter to a vector of data")
    .def("apply",py::overload_cast<mspass::seismic::Seismogram&>
         (&Butterworth::apply),
         "Apply the predefined filter to a 3c Seismogram object")
    .def("dt",&Butterworth::current_dt,
      "Current sample interval used for nondimensionalizing frequencies")
    .def("low_corner",&Butterworth::low_corner,"Return low frequency f3d point")
    .def("high_corner",&Butterworth::high_corner,"Return high frequency 3db point")
    .def("npoles_low",&Butterworth::npoles_low,
      "Return number of poles for the low frequency (high-pass aka low-cut) filter definition")
    .def("npoles_high",&Butterworth::npoles_high,
      "Return number of poles for the high frequency (low-pass aka high-cut) filter definition")
    .def("filter_type",&Butterworth::filter_type,
      "Return a description of the filter type")
    .def("is_zerophase",&Butterworth::is_zerophase,
      "Returns True if operator defines a zerophase filter")
  ;
  m.def("ArrivalTimeReference",
      py::overload_cast<Seismogram&,std::string,TimeWindow>
          (&ArrivalTimeReference),
          "Shifts data so t=0 is a specified arrival time",
      py::return_value_policy::copy,
      py::arg("d"),
      py::arg("key"),
      py::arg("window")
  );

  m.def("ArrivalTimeReference",
      py::overload_cast<Ensemble<Seismogram>&,std::string,TimeWindow>
          (&ArrivalTimeReference),
          "Shifts data so t=0 is a specified arrival time",
      py::return_value_policy::copy,
      py::arg("d"),
      py::arg("key"),
      py::arg("window")
  );

  /* overload_cast would not work on this name because of a strange limitation with templated functions
   * used for the Ensemble definition.   */
  m.def("ExtractComponent",static_cast<TimeSeries(*)(const Seismogram&,const unsigned int)>(&ExtractComponent),
  	"Extract component as a TimeSeries object",
      py::return_value_policy::copy,
      py::arg("tcs"),
      py::arg("component")
  );

  m.def("EnsembleComponent",static_cast<Ensemble<TimeSeries>(*)(const Ensemble<Seismogram>&,const unsigned int)>(&ExtractComponent),
  	"Extract one component from a 3C ensemble",
      py::return_value_policy::copy,
      py::arg("d"),
      py::arg("component")
  );

  m.def("agc",&agc,"Automatic gain control a Seismogram",
    py::return_value_policy::copy,
    py::arg("d"),
    py::arg("twin") )
  ;
  m.def("_WindowData",py::overload_cast<const TimeSeries&,const TimeWindow&>(&WindowData),
          "Reduce data to window inside original",
    py::return_value_policy::copy,
    py::arg("d"),
    py::arg("twin") )
  ;

  m.def("_WindowData3C",py::overload_cast<const Seismogram&,const TimeWindow&>(&WindowData),
              "Reduce data to window inside original",
    py::return_value_policy::copy,
    py::arg("d"),
    py::arg("twin") )
  ;

  /* The following line is necessary for the Vectors to be recognized. Reference:
     https://pybind11.readthedocs.io/en/stable/advanced/misc.html#partitioning-code-over-multiple-extension-modules
  */
  py::module_::import("mspasspy.ccore.seismic");

  m.def("_bundle_seed_data",&bundle_seed_data,
    "Create SeismogramEnsemble from sorted TimeSeriesEnsemble",
    py::return_value_policy::copy,
    py::arg("d") )
  ;

  m.def("_BundleSEEDGroup",&BundleSEEDGroup,
    "Bundle a seed grouping of TimeSeries into one or more Seismogram objects",
    py::return_value_policy::copy,
    py::arg("d"),
    py::arg("i0"),
    py::arg("iend") )
  ;
  m.def("seed_ensemble_sort",&seed_ensemble_sort,R"mspass_doc(
      Sort a TimeSeriesEnsemble with a natural order with seed name codes.

      The seed standard tags every single miniseed record with four string keys
      that seed uses to uniquely define a single data channel.  In MsPASS the keys
      used for these name keys are:  net, sta, chan, and loc.  This function
      applies the same sort algorithm used in the bundle_seed_data algorithm to
      allow clean grouping into channels that can be assembled into
      three component (Seismogram) bundles.  That means we sort the ensemble
      data with the four keys in this order:  net, sta, loc, chan.

      We provide this function because the process of doing such a sort is far
      from trivial to do in a robust way.   A python programmer has easier tools
      for sorting BUT those standard tools cannot handle a common data problem
      that can be encountered with real data.  That is, there is a high
      probability not all the seed keys are defined.   In particular, data
      coming from a system based on the css3.0 relational data base (e.g. Antelope)
      may not have net or loc set.   The sorting algorith here handle null net or
      loc codes cleanly by treating the null case as a particular value.   Without
      those safeties the code would throw an error if net or loc were null.

      Note this algorithm alters the ensemble it receives in place.

      :param d: is the ensemble to be sorted.
    )mspass_doc",
    py::arg("d") )
  ;
  py::class_<TimeWindow>(m,"TimeWindow","Simple description of a time window")
    .def(py::init<>(),"Default constructor")
    .def(py::init<const double, const double>(),"Construct from start and end time")
    .def(py::init<const TimeWindow&>(),"Copy constuctor")
    .def("shift",&TimeWindow::shift,"Shift the reference time by a specified number of seconds")
    .def("length",&TimeWindow::length,"Return the size of the window in seconds")
    .def_readwrite("start",&TimeWindow::start,"Start time of the window")
    .def_readwrite("end",&TimeWindow::end,"End time of the window")
    .def(py::pickle(
        [](const TimeWindow& self)
        {
          return py::make_tuple(self.start,self.end);
        },
        [](py::tuple t)
        {
          return TimeWindow(t[0].cast<double>(),t[1].cast<double>());
        }
      ))
  ;
  py::class_<BasicTaper,PyBasicTaper>(m,"BasicTaper",
                    "Base class for family of taper algorithms")
      /* BasicTaper is a nearly pure abstract type.   It has some
      nonvirtual methods but we don't need them in mspass we will not define
      them in these binds.  The following is a residual from trying (incorrectly)
      to define the virtual methods.
    .def(py::init<>())
    .def("apply",py::overload_cast<TimeSeries&>(&BasicTaper::apply),
        "TimeSeries overload of base class")
    .def("apply",py::overload_cast<Seismogram&>(&BasicTaper::apply))
    .def("get_t0head",&BasicTaper::get_t0head)
    .def("get_t1head",&BasicTaper::get_t1head)
    */
  ;
  py::class_<LinearTaper,BasicTaper>(m,"LinearTaper",
      "Define a ramp taper function")
    .def(py::init<>())
    .def(py::init<const double, const double, const double, const double>())
    .def("apply",py::overload_cast<TimeSeries&>(&LinearTaper::apply),
      "Apply taper to a scalar TimeSeries object")
    .def("apply",py::overload_cast<Seismogram&>(&LinearTaper::apply),
      "Apply taper to a Seismogram (3C) object")
    .def("get_t0head",&LinearTaper::get_t0head,
      "Return time of end of zero zone - taper sets data with time < this value 0")
    .def("get_t1head",&LinearTaper::get_t1head,
      "Return time of end front end taper - point after this time are not altered until end taper zone is reached")
    .def("get_t0tail",&LinearTaper::get_t0tail,
      "Return time of taper end - data with t > this value will be zeroed")
    .def("get_t1tail",&LinearTaper::get_t1tail,
      "Return start time of end taper")
    .def(py::pickle(
        [](const LinearTaper &self) {
          double t0h,t1h,t1t,t0t;
          t0h=self.get_t0head();
          t1h=self.get_t1head();
          t0t=self.get_t0tail();
          t1t=self.get_t1tail();
          return py::make_tuple(t0h,t1h,t1t,t0t);
        },
        [](py::tuple t) {
          double t0h,t1h,t1t,t0t;
          t0h = t[0].cast<double>();
          t1h = t[1].cast<double>();
          t1t = t[2].cast<double>();
          t0t = t[3].cast<double>();
          return LinearTaper(t0h,t1h,t1t,t0t);
        }
       ))
  ;
  py::class_<CosineTaper,BasicTaper>(m,"CosineTaper",
      "Define a taper using a half period cosine function")
    .def(py::init<>())
    .def(py::init<const double, const double, const double, const double>())
    .def("apply",py::overload_cast<TimeSeries&>(&CosineTaper::apply),"Apply taper to a scalar TimeSeries object")
    .def("apply",py::overload_cast<Seismogram&>(&CosineTaper::apply),"Apply taper to a Seismogram (3C) object")
    .def("get_t0head",&CosineTaper::get_t0head,
      "Return time of end of zero zone - taper sets data with time < this value 0")
    .def("get_t1head",&CosineTaper::get_t1head,
      "Return time of end front end taper - point after this time are not altered until end taper zone is reached")
    .def("get_t0tail",&CosineTaper::get_t0tail,
      "Return time of taper end - data with t > this value will be zeroed")
    .def("get_t1tail",&CosineTaper::get_t1tail,
      "Return start time of end taper")
    .def(py::pickle(
        [](const CosineTaper &self) {
          double t0h,t1h,t1t,t0t;
          t0h=self.get_t0head();
          t1h=self.get_t1head();
          t0t=self.get_t0tail();
          t1t=self.get_t1tail();
          return py::make_tuple(t0h,t1h,t1t,t0t);
        },
        [](py::tuple t) {
          double t0h,t1h,t1t,t0t;
          t0h = t[0].cast<double>();
          t1h = t[1].cast<double>();
          t1t = t[2].cast<double>();
          t0t = t[3].cast<double>();
          return CosineTaper(t0h,t1h,t1t,t0t);
        }
       ))
  ;
  py::class_<VectorTaper,BasicTaper>(m,"VectorTaper",
      "Define generic taper function with a parallel vector of weights")
    .def(py::init<>())
    .def(py::init<const std::vector<double>>())
    .def("apply",py::overload_cast<TimeSeries&>(&VectorTaper::apply),
      "Apply taper to a scalar TimeSeries object")
    .def("apply",py::overload_cast<Seismogram&>(&VectorTaper::apply),
        "Apply taper to a Seismogram (3C) object")
    .def(py::pickle(
      [](const VectorTaper& self)
      {
        ostringstream oss;
        boost::archive::text_oarchive oa(oss);
        oa << self;
        return py::make_tuple(oss.str());
      },
      [](py::tuple t)
      {
        string serialized_taper(t[0].cast<string>());
        istringstream iss(serialized_taper);
        boost::archive::text_iarchive ia(iss);
        VectorTaper taper;
        ia >> taper;
        return taper;
      }
    ))
  ;
  py::class_<TopMute>(m,"_TopMute",
    "Defines a top mute operator (one sided taper")
    .def(py::init<>())
    .def(py::init<const double, const double, const std::string>())
    .def(py::init<const TopMute&>())
    .def("apply",py::overload_cast<TimeSeries&>(&TopMute::apply),
      "Apply to a TimeSeries object")
    .def("apply",py::overload_cast<Seismogram&>(&TopMute::apply),
      "Apply to a Seismogram object")
    .def("get_t0",&TopMute::get_t0,
      "Return the zero end time for marking the start of the mute")
    .def("get_t1",&TopMute::get_t1,
      "Return the time the mute goes to 1 - the end time of the taper")
    .def("taper_type",&TopMute::taper_type,
       "Return a string defining type of taper defining this mute")
    .def(py::pickle(
        [](const TopMute &self) {
          double t0,t1;
          t0=self.get_t0();
          t1=self.get_t1();
          string taper_type(self.taper_type());
          return py::make_tuple(t0,t1,taper_type);
        },
        [](py::tuple t) {
          double t0,t1;
          std::string taper_type;
          t0 = t[0].cast<double>();
          t1 = t[1].cast<double>();
          taper_type = t[2].cast<std::string>();
          return TopMute(t0,t1,taper_type);
        }
       ))
  ;
}

} // namespace mspasspy
} // namespace mspass


#include <iostream>

#include <graphene/chain/fund_object.hpp>
#include <graphene/chain/asset_object.hpp>
#include <graphene/chain/hardfork.hpp>
//#include <graphene/chain/settings_object.hpp>

#include <fc/uint128.hpp>
#include <boost/range.hpp>

namespace graphene { namespace chain {

double fund_object::get_rate_percent(const fund_options::fund_rate& fr_item, const database& db) const
{
   int days_passed = (db.head_block_time().sec_since_epoch() - prev_maintenance_time_on_creation.sec_since_epoch()) / 86400;

   // the further away, the 'result' is lower
   double result = db.get_percent(fr_item.day_percent) - (db.get_percent(rates_reduction_per_month) / (double)30 * (double)(days_passed - 1));

   if (result < 0) {
      result = 0;
   }

   return result;
}

optional<fund_options::fund_rate>
fund_object::get_max_fund_rate(const share_type& amnt) const
{
   optional<fund_options::fund_rate> result;
   if (fund_rates.size() == 0) return { result };

   // we need nearest (to the amnt) and maximum values
//      // suspicious variant, right?
//      const auto& iter = std::max_element(fund_rates.begin(), fund_rates.end(),
//         [&amnt](const fund_options::fund_rate& item1, const fund_options::fund_rate& item2) {
//            return ( (item1.amount > amnt) || ((amnt >= item2.amount) && (item2.amount > item1.amount)) );
//         });
//      if ( (iter != fund_rates.end()) && (amnt >= iter->amount) ) {
//         result = *iter;
//      }

   int max_idx = -1;
   for (uint32_t i = 0; i < fund_rates.size(); ++i)
   {
      if (amnt >= fund_rates[i].amount)
      {
         if (max_idx == -1) {
            max_idx = i;
         }
         else
         {
            if (fund_rates[i].amount > fund_rates[max_idx].amount) {
               max_idx = i;
            }
         }
      }
   }

   if (max_idx > -1) {
      result = fund_rates[max_idx];
   }

   return result;
}

optional<fund_options::payment_rate>
fund_object::get_payment_rate(uint32_t period) const
{
   optional<fund_options::payment_rate> result;

   auto iter = std::find_if(payment_rates.begin(), payment_rates.end(), [&period](const fund_options::payment_rate& item) {
      return (item.period == period);
   });
   if (iter != payment_rates.end()) {
      result = *iter;
   }

   return result;
}

void fund_object::process(database& db) const
{
   const dynamic_global_property_object& dpo = db.get_dynamic_global_properties();
   const global_property_object& gpo = db.get_global_properties();
   //const settings_object& settings = *db.find(settings_id_type(0));

   auto asset_itr = db.get_index_type<asset_index>().indices().get<by_id>().find(asset_id);
   assert(asset_itr != db.get_index_type<asset_index>().indices().get<by_id>().end());
   const chain::asset_object& asst = *asset_itr;

   transaction_evaluation_state eval(&db);

   // all payments to users
   share_type daily_payments_without_owner;

   // all user deposits
   share_type all_deposits = 0;
   if (db.head_block_time() < HARDFORK_640_TIME) {
      all_deposits = balance_before_hf640 - owner_balance;
   }
   else {
      all_deposits = balance - owner_balance;
   }

   /**
    * create tmp object, because below will be reducing of fund's balance if deposit is
    * overdued (in deposits loop)
    */
   share_type old_balance = 0;
   if (db.head_block_time() < HARDFORK_640_TIME) {
      old_balance = balance_before_hf640;
   }
   else {
      old_balance = balance;
   }

   fund_history_object::history_item h_item;
   h_item.create_datetime = db.head_block_time();

   const auto& users_idx = db.get_index_type<account_index>().indices().get<by_id>();
   fc::time_point_sec maint_time = dpo.next_maintenance_time - gpo.parameters.maintenance_interval;

   // find own fund deposits
   auto range = db.get_index_type<fund_deposit_index>().indices().get<by_fund_id>().equal_range(id);
   std::for_each(range.first, range.second, [&](const fund_deposit_object& dep)
   {
      auto user_ptr = users_idx.find(dep.account_id);
      const account_object& acc = *user_ptr;
      bool dep_datetime_end_reached = maint_time >= dep.datetime_end;

      // clear disabled and overdued deposits
      if ((db.head_block_time() >= HARDFORK_638_TIME) && !dep.enabled && dep_datetime_end_reached)
      {
         if (db.head_block_time() >= HARDFORK_640_TIME)
         {
            if (!dep.finished)
            {
               db.modify(dep, [&](chain::fund_deposit_object& obj) {
                  obj.finished = true;
               });
               // reduce fund balance
               db.modify(*this, [&](chain::fund_object& obj) {
                  obj.balance -= dep.amount.amount;
               });
            }
         }
         else
         {
            if (!dep.finished)
            {
               db.modify(dep, [&](chain::fund_deposit_object& obj) {
                  obj.finished = true;
               });
               db.modify(*this, [&](chain::fund_object& obj) {
                  obj.balance -= dep.amount.amount;
               });
            }
            db.modify(*this, [&](chain::fund_object& obj) {
               obj.balance_before_hf640 -= dep.amount.amount;
            });
         }
      }

      if (dep.enabled && !dep.finished && (user_ptr != users_idx.end())) // dep.enabled: important condition for full-history nodes
      {
         const optional<fund_options::payment_rate>& p_rate = get_payment_rate(dep.period);

         bool is_valid = (db.head_block_time() >= HARDFORK_626_TIME) ? (dep.daily_payment.value > 0) : p_rate.valid();

         if ( (db.head_block_time() > HARDFORK_627_TIME)
              && (asst.get_id() == EDC_ASSET)
              && !dep.can_use_percent
            ) { is_valid = false; }

         if (is_valid)
         {
            asset asst_quantity;

            if (db.head_block_time() >= HARDFORK_626_TIME) {
               asst_quantity = db.check_supply_overflow(asst.amount(dep.daily_payment));
            }
            else
            {
               share_type quantity = db.get_deposit_daily_payment(dep.percent, p_rate->period, dep.amount.amount);

               if (quantity.value > 0) {
                  asst_quantity = db.check_supply_overflow(asst.amount(quantity));
               }
            }

            if (asst_quantity.amount.value > 0)
            {
               chain::fund_payment_operation op;
               op.issuer = asst.issuer;
               op.fund_id = id;
               op.deposit_id = dep.get_id();
               op.asset_to_issue = asst_quantity;
               op.issue_to_account = dep.account_id;

               try
               {
                  op.validate();
                  db.apply_operation(eval, op);
               } catch (fc::assert_exception& e) { }

               daily_payments_without_owner += asst_quantity.amount;
            }
         }

         // return deposit amount to user and remove deposit if overdued
         if (dep_datetime_end_reached)
         {
            bool dep_was_overdued = true;

            if (acc.deposits_autorenewal_enabled && (db.head_block_time() >= HARDFORK_624_TIME))
            {
               dep_was_overdued = false;

               if (db.head_block_time() > HARDFORK_625_TIME)
               {
                  chain::deposit_renewal_operation op;
                  op.account_id = dep.account_id;
                  op.deposit_id = dep.get_id();
                  op.percent = dep.percent;

                  // fund may already have new percents, updating...
                  if (p_rate.valid() && !dep.manual_percent_enabled) {
                     op.percent = p_rate->percent;
                  }

                  if (db.head_block_time() >= HARDFORK_626_TIME)
                  {
                     op.datetime_end = db.get_dynamic_global_properties().next_maintenance_time
                                       - db.get_global_properties().parameters.maintenance_interval + (86400 * dep.period);
                  }
                  else {
                     op.datetime_end = dep.datetime_end + (86400 * dep.period);
                  }

                  try
                  {
                     op.validate();
                     db.apply_operation(eval, op);
                  } catch (fc::assert_exception& e) { }
               }
               // last_budget_time - not stable
               else
               {
                  db.modify(dep, [&](fund_deposit_object& dep)
                  {
                     if (p_rate.valid()) {
                        dep.percent = p_rate->percent;
                     }
                     dep.datetime_end = db.get_dynamic_global_properties().last_budget_time + (86400 * dep.period);
                  });
               }
            }

            if (dep_was_overdued)
            {
               // disable deposit
               db.modify(dep, [&](chain::fund_deposit_object& obj)
               {
                  obj.enabled = false;
                  obj.finished = true;
               });

               // withdraw deposit amount
               if ( (db.head_block_time() <= HARDFORK_628_TIME)
                    || ((db.head_block_time() > HARDFORK_628_TIME) && (dep.amount.amount > 0))
                    || (db.head_block_time() > HARDFORK_634_TIME) )
               {
                  if ( ((db.head_block_time() > HARDFORK_634_TIME) && (dep.amount.amount > 0))
                       || (db.head_block_time() <= HARDFORK_634_TIME) )
                  {
                     // return deposit to user
                     chain::fund_withdrawal_operation op;
                     op.issuer = asst.issuer;
                     op.fund_id = id;
                     op.asset_to_issue = asst.amount(dep.amount.amount);
                     op.issue_to_account = dep.account_id;
                     op.datetime = db.head_block_time();

                     try
                     {
                        op.validate();
                        db.apply_operation(eval, op);
                     } catch (fc::assert_exception& e) { }

                     // reduce fund balance
                     db.modify(*this, [&](chain::fund_object& obj)
                     {
                        obj.balance -= dep.amount.amount;

                        if (db.head_block_time() < HARDFORK_640_TIME) {
                           obj.balance_before_hf640 -= dep.amount.amount;
                        }
                     });
                  }
               }
            }
         }
      }
   });

   /***************** make payment to fund owner *****************/

   asset mlm_profit;

   // case 1
   if (payment_scheme == fund_payment_scheme::residual)
   {
      // MLM payment. In current scheme we're using percent from entire fund balance
      mlm_profit = asset(std::roundl((long double)old_balance.value * db.get_percent(mlm_daily_percent)), asset_id);

      const optional<fund_options::fund_rate>& p_rate = get_max_fund_rate(old_balance);
      if (p_rate.valid())
      {
         share_type fund_day_profit = std::roundl((long double)old_balance.value * get_rate_percent(*p_rate, db));
         if (fund_day_profit > 0)
         {
            share_type owner_profit = fund_day_profit - daily_payments_without_owner;

            h_item.daily_payments_total = fund_day_profit;
            h_item.daily_payments_owner = 0;
            h_item.daily_payments_without_owner = daily_payments_without_owner;

            const asset& owner_quantity = db.check_supply_overflow(asst.amount(owner_profit));

//             std::cout << "old_balance: " << old_balance.value
//                       << ", fund_day_profit: " << fund_day_profit.value
//                       << ", daily_payments_without_owner: " << daily_payments_without_owner.value
//                       << ", owner profit: " << owner_profit.value << std::endl;

            if (owner_quantity.amount.value > 0)
            {
               // monthly payment
               if ((db.head_block_time() >= HARDFORK_630_TIME) && owner_monthly_payments_enabled)
               {
                  // payment datetime has already reached
                  if (db.head_block_time() >= owner_monthly_payments_next_datetime)
                  {
                     const share_type& p = owner_monthly_payments_amount + owner_quantity.amount;
                     h_item.daily_payments_owner = p;

                     chain::fund_payment_operation op;
                     op.issuer = asst.issuer;
                     op.fund_id = id;
                     op.asset_to_issue = asset(p, asset_id);
                     op.issue_to_account = owner;

                     try
                     {
                        op.validate();
                        db.apply_operation(eval, op);
                     } catch (fc::assert_exception& e) { }

                     db.modify(*this, [&](chain::fund_object& obj)
                     {
                        obj.owner_monthly_payments_amount = 0;
                        obj.owner_monthly_payments_next_datetime = db.head_block_time() + fc::days(30);
                     });
                  }
                  // payment datetime has not reached yet
                  else
                  {
                     db.modify(*this, [&](chain::fund_object& obj) {
                        obj.owner_monthly_payments_amount += owner_quantity.amount;
                     });
                  }
               }
               // daily payment
               else if ((db.head_block_time() < HARDFORK_630_TIME) || !owner_monthly_payments_enabled)
               {
                  h_item.daily_payments_owner = owner_profit;

                  chain::fund_payment_operation op;
                  op.issuer = asst.issuer;
                  op.fund_id = id;
                  op.asset_to_issue = owner_quantity;
                  op.issue_to_account = owner;

                  try
                  {
                     op.validate();
                     db.apply_operation(eval, op);
                  } catch (fc::assert_exception& e) { }
               }
            }
         }
      }
   }
   // case 2
   else if (payment_scheme == fund_payment_scheme::fixed)
   {
      // MLM payment. In current scheme we're using percent from only user deposits
      mlm_profit = asset(std::roundl((long double)all_deposits.value * db.get_percent(mlm_daily_percent)), asset_id);

      const optional<fund_options::fund_rate>& p_rate = get_max_fund_rate(owner_balance);
      if (p_rate)
      {
         share_type quantity;
         if (db.head_block_time() >= HARDFORK_630_TIME) {
            quantity = std::roundl(db.get_percent(p_rate->day_percent) * (long double)all_deposits.value);
         }
         else {
            quantity = std::roundl(db.get_percent(p_rate->day_percent) * (long double)daily_payments_without_owner.value);
         }

         const asset& owner_quantity = db.check_supply_overflow(asst.amount(quantity));

         if (owner_quantity.amount.value > 0)
         {
            h_item.daily_payments_total = 0;
            h_item.daily_payments_owner = 0;
            h_item.daily_payments_without_owner = daily_payments_without_owner;

            // monthly payment
            if ((db.head_block_time() >= HARDFORK_630_TIME) && owner_monthly_payments_enabled)
            {
               // payment datetime has already reached
               if (db.head_block_time() >= owner_monthly_payments_next_datetime)
               {
                  const share_type& p = owner_monthly_payments_amount + owner_quantity.amount;
                  h_item.daily_payments_total = p + daily_payments_without_owner;
                  h_item.daily_payments_owner = p;

                  chain::fund_payment_operation op;
                  op.issuer = asst.issuer;
                  op.fund_id = id;
                  op.asset_to_issue = asset(p, asset_id);
                  op.issue_to_account = owner;

                  try
                  {
                     op.validate();
                     db.apply_operation(eval, op);
                  } catch (fc::assert_exception& e) { }

                  db.modify(*this, [&](chain::fund_object& obj)
                  {
                     obj.owner_monthly_payments_amount = 0;
                     obj.owner_monthly_payments_next_datetime = db.head_block_time() + fc::days(30);
                  });
               }
               // payment datetime has not reached yet
               else
               {
                  db.modify(*this, [&](chain::fund_object& obj) {
                     obj.owner_monthly_payments_amount += owner_quantity.amount;
                  });
               }
            }
            // daily payment
            else if ((db.head_block_time() < HARDFORK_630_TIME) || !owner_monthly_payments_enabled)
            {
               h_item.daily_payments_total = owner_quantity.amount + daily_payments_without_owner;
               h_item.daily_payments_owner = owner_quantity.amount;

               chain::fund_payment_operation op;
               op.issuer = asst.issuer;
               op.fund_id = id;
               op.asset_to_issue = owner_quantity;
               op.issue_to_account = owner;

               try
               {
                  op.validate();
                  db.apply_operation(eval, op);
               } catch (fc::assert_exception& e) { }
            }
         }
      }
      else {
         wlog("Not enough owner balance to make payment to owner. Owner: ${o}, balance: ${b}", ("o", owner)("b", owner_balance));
      }
   }

   /************ additionally make payment to MLM-account ************/

   mlm_profit = db.check_supply_overflow(mlm_profit);

   if ( (db.head_block_time() >= HARDFORK_630_TIME)
        && (mlm_account != account_id_type() )
        && (mlm_profit.amount.value > 0) )
   {
      // monthly payment
      if (mlm_monthly_payments_enabled)
      {
         // payment datetime has already reached
         if (db.head_block_time() >= mlm_monthly_payments_next_datetime)
         {
            const share_type& p = mlm_monthly_payments_amount + mlm_profit.amount;

            chain::fund_payment_operation op;
            op.issuer = asst.issuer;
            op.fund_id = id;
            op.asset_to_issue = asset(p, asset_id);
            op.issue_to_account = mlm_account;

            try
            {
               op.validate();
               db.apply_operation(eval, op);
            } catch (fc::assert_exception& e) { }

            db.modify(*this, [&](chain::fund_object& obj)
            {
               obj.mlm_monthly_payments_amount = 0;
               obj.mlm_monthly_payments_next_datetime = db.head_block_time() + fc::days(30);
            });
         }
         // payment datetime has not reached yet
         else
         {
            db.modify(*this, [&](chain::fund_object& obj) {
               obj.mlm_monthly_payments_amount += mlm_profit.amount;
            });
         }
      }
      // daily payment
      else
      {
         chain::fund_payment_operation op;
         op.issuer = asst.issuer;
         op.fund_id = id;
         op.asset_to_issue = mlm_profit;
         op.issue_to_account = mlm_account;

         try
         {
            op.validate();
            db.apply_operation(eval, op);
         } catch (fc::assert_exception& e) { }
      }
   }

   // erase old history items
   if ( (h_item.daily_payments_owner > 0)
        || (h_item.daily_payments_total > 0)
        || (h_item.daily_payments_without_owner > 0) )
   {
      const auto& hist_obj = history_id(db);
      db.modify(hist_obj, [&](fund_history_object& o)
      {
         o.items.emplace_back(std::move(h_item));

         if (db.get_history_size() > 0)
         {
            const time_point& tp = db.head_block_time() - fc::days(db.get_history_size());

            for (auto it = o.items.begin(); it != o.items.end();)
            {
               if (it->create_datetime < tp) {
                  it = o.items.erase(it);
               }
               else {
                  break;
                  //++it;
               }
            }
         }
      });
   }
}

void fund_object::finish(database& db) const
{
   share_type owner_deps = owner_balance;
   if (owner_deps > 0)
   {
      auto asset_itr = db.get_index_type<asset_index>().indices().get<by_id>().find(asset_id);
      assert(asset_itr != db.get_index_type<asset_index>().indices().get<by_id>().end());
      const chain::asset_object& asst = *asset_itr;
      transaction_evaluation_state eval(&db);

      // return amount to owner
      chain::fund_withdrawal_operation op;
      op.issuer           = asst.issuer;
      op.fund_id          = id;
      op.asset_to_issue   = asst.amount(owner_deps);
      op.issue_to_account = owner;
      op.datetime         = db.head_block_time();

      try
      {
         op.validate();
         db.apply_operation(eval, op);
      } catch (fc::assert_exception& e) { }
   }

   // reduce fund balance & disable
   db.modify(*this, [&](chain::fund_object& obj)
   {
      if (owner_deps > 0)
      {
         obj.balance -= owner_deps;

         if (db.head_block_time() < HARDFORK_640_TIME) {
            obj.balance_before_hf640 -= owner_deps;
         }
      }
      obj.owner_balance = 0;
      obj.enabled = false;
   });
}

} } // graphene::chain

FC_REFLECT_DERIVED_NO_TYPENAME( graphene::chain::fund_history_object,
                    (graphene::chain::object),
                    (owner)(items) )

FC_REFLECT_DERIVED_NO_TYPENAME( graphene::chain::fund_statistics_object,
                    (graphene::chain::object),
                    (owner)
                    (most_recent_op)
                    (total_ops)
                    (deposit_count) )

FC_REFLECT_DERIVED_NO_TYPENAME( graphene::chain::fund_transaction_history_object,
                    (graphene::chain::object),
                    (fund)
                    (operation_id)
                    (sequence)
                    (next)
                    (block_time) )

FC_REFLECT_DERIVED_NO_TYPENAME( graphene::chain::fund_deposit_object,
                    (graphene::db::object),
                    (fund_id)
                    (account_id)
                    (amount)
                    (enabled)
                    (finished)
                    (datetime_begin)
                    (datetime_end)
                    (prev_maintenance_time_on_creation)
                    (period)
                    (percent)
                    (daily_payment)
                    (manual_percent_enabled)
                    (can_use_percent) )

FC_REFLECT_DERIVED_NO_TYPENAME( graphene::chain::fund_object,
                    (graphene::db::object),
                    (name)
                    (description)
                    (owner)
                    (asset_id)
                    (balance_before_hf640)
                    (balance)
                    (owner_balance)
                    (max_limit_deposits_amount)
                    (owner_monthly_payments_enabled)
                    (owner_monthly_payments_next_datetime)
                    (owner_monthly_payments_amount)
                    (mlm_account)
                    (mlm_monthly_payments_enabled)
                    (mlm_monthly_payments_next_datetime)
                    (mlm_daily_percent)
                    (mlm_monthly_payments_amount)
                    (enabled)
                    (datetime_begin)
                    (datetime_end)
                    (prev_maintenance_time_on_creation)
                    (deposit_acceptance_enabled)
                    (rates_reduction_per_month)
                    (period)
                    (min_deposit)
                    (statistics_id)
                    (history_id)
                    (payment_scheme)
                    (payment_rates)
                    (fund_rates) )

GRAPHENE_IMPLEMENT_EXTERNAL_SERIALIZATION( graphene::chain::fund_history_object )
GRAPHENE_IMPLEMENT_EXTERNAL_SERIALIZATION( graphene::chain::fund_statistics_object )
GRAPHENE_IMPLEMENT_EXTERNAL_SERIALIZATION( graphene::chain::fund_transaction_history_object )
GRAPHENE_IMPLEMENT_EXTERNAL_SERIALIZATION( graphene::chain::fund_deposit_object )
GRAPHENE_IMPLEMENT_EXTERNAL_SERIALIZATION( graphene::chain::fund_object )


#include <pybind11/pybind11.h>
#include <pybind11/eigen.h>
#include <Eigen/Core>

#include "george/hodlr.h"
#include "george/kernels.h"
#include "george/parser.h"
#include "george/exceptions.h"

namespace py = pybind11;

using RowMatrixXd = Eigen::Matrix<double, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>;

class SolverMatrix {
  public:
    SolverMatrix (george::kernels::Kernel* kernel)
      : kernel_(kernel) {};
    void set_input_coordinates (RowMatrixXd x) {
      if (size_t(x.cols()) != kernel_->get_ndim()) {
        throw george::dimension_mismatch();
      }
      t_ = x;
    };
    double get_value (const int i, const int j) {
      if (i < 0 || i >= t_.rows() || j < 0 || j >= t_.rows()) {
        throw std::out_of_range("attempting to index outside of the dimension of the input coordinates");
      }
      return kernel_->value(t_.row(i).data(), t_.row(j).data());
    };

  private:
    george::kernels::Kernel* kernel_;
    RowMatrixXd t_;
};

class Solver {
public:

  Solver () {
    solver_ = NULL;
    kernel_ = NULL;
    matrix_ = NULL;
    computed_ = 0;
  };
  ~Solver () {
    if (solver_ != NULL) delete solver_;
    if (matrix_ != NULL) delete matrix_;
    if (kernel_ != NULL) delete kernel_;
  };

  int get_status () const { return 0; };
  int get_computed () const { return computed_; };
  double log_determinant () const { return log_det_; };

  int compute (
      const py::object& kernel_spec,
      const py::array_t<double>& x,
      const py::array_t<double>& yerr,
      int min_size = 100, double tol = 0.1, int seed = 0
  ) {
    computed_ = 0;
    kernel_ = george::parse_kernel_spec(kernel_spec);
    matrix_ = new SolverMatrix(kernel_);

    // Random number generator for reproducibility
    std::random_device r;
    std::mt19937 random(r());
    random.seed(seed);

    // Extract the data from the numpy arrays
    py::detail::unchecked_reference<double, 2L> x_p = x.unchecked<2>();
    py::detail::unchecked_reference<double, 1L> yerr_p = yerr.unchecked<1>();
    size_t n = x_p.shape(0), ndim = x_p.shape(1);
    RowMatrixXd X(n, ndim);
    Eigen::VectorXd diag(n);
    for (size_t i = 0; i < n; ++i) {
      diag(i) = yerr_p(i) * yerr_p(i);
      for (size_t j = 0; j < ndim; ++j) X(i, j) = x_p(i, j);
    }

    matrix_->set_input_coordinates(X);

    // Set up the solver object.
    if (solver_ != NULL) delete solver_;
    solver_ = new george::hodlr::Node<SolverMatrix> (
        diag, matrix_, 0, n, min_size, tol, random);
    solver_->compute();
    log_det_ = solver_->log_determinant();

    // Update the bookkeeping flags.
    computed_ = 1;
    size_ = n;
    return 0;
  };

  template <typename Derived>
  void apply_inverse (Eigen::MatrixBase<Derived>& x) {
    if (!computed_) throw george::not_computed();
    solver_->solve(x);
  };

  int size () const { return size_; };

private:
  double log_det_;
  int size_;
  int computed_;

  george::kernels::Kernel* kernel_;
  SolverMatrix* matrix_;
  george::hodlr::Node<SolverMatrix>* solver_;
};

PYBIND11_MODULE(_hodlr, m) {
  py::class_<Solver> solver(m, "HODLRSolver", R"delim(
A solver using `Sivaram Amambikasaran's HODLR algorithm
<http://arxiv.org/abs/1403.6015>`_ to approximately solve the GP linear
algebra in :math:`\mathcal{O}(N\,\log^2 N)`.

)delim");
  solver.def(py::init());
  solver.def_property_readonly("computed", &Solver::get_computed);
  solver.def_property_readonly("log_determinant", &Solver::log_determinant);
  solver.def("compute", &Solver::compute, R"delim(
Compute and factorize the covariance matrix.

Args:
    kernel (george.kernels.Kernel): A subclass of :class:`Kernel` specifying
        the kernel function.
    x (ndarray[nsamples, ndim]): The independent coordinates of the data
        points.
    yerr (ndarray[nsamples]): The Gaussian uncertainties on the data points at
        coordinates ``x``. These values will be added in quadrature to the
        diagonal of the covariance matrix.
    min_size (Optional[int]): The block size where the solver switches to a
        general direct factorization algorithm. This can be tuned for platform
        and problem specific performance and accuracy. As a general rule,
        larger values will be more accurate and slower, but there is some
        overhead for very small values, so we recommend choosing values in the
        hundreds. (default: ``100``)
    tol (Optional[float]): The precision tolerance for the low-rank
        approximation. This value is used as an approximate limit on the
        Frobenius norm between the low-rank approximation and the true matrix
        when reconstructing the off-diagonal blocks. Smaller values of ``tol``
        will generally give more accurate results with higher computational
        cost. (default: ``0.1``)
    seed (Optional[int]): The low-rank approximation method within the HODLR
        algorithm is not deterministic and, without a fixed seed, the method
        can give different results for the same matrix. Therefore, we require
        that the user provide a seed for the random number generator.
        (default: ``42``, obviously)
)delim",
    py::arg("kernel_spec"), py::arg("x"), py::arg("yerr"), py::arg("min_size") = 100, py::arg("tol") = 0.1, py::arg("seed") = 42
  );
  solver.def("apply_inverse", [](Solver& self, Eigen::MatrixXd& x, bool in_place = false){
    if (in_place) {
      self.apply_inverse(x);
      return x;
    }
    Eigen::MatrixXd alpha = x;
    self.apply_inverse(alpha);
    return alpha;
  }, py::arg("x"), py::arg("in_place") = false, R"delim(
Apply the inverse of the covariance matrix to the input by solving

.. math::

    K\,x = y

Args:
    y (ndarray[nsamples] or ndadrray[nsamples, nrhs]): The vector or matrix
        :math:`y`.
    in_place (Optional[bool]): Should the data in ``y`` be overwritten with
        the result :math:`x`? (default: ``False``)
)delim");

  solver.def("dot_solve", [](Solver& self, const Eigen::VectorXd& x){
    Eigen::VectorXd alpha = x;
    self.apply_inverse(alpha);
    return double(x.transpose() * alpha);
  }, R"delim(
Compute the inner product of a vector with the inverse of the covariance
matrix applied to itself:

.. math::

    y\,K^{-1}\,y

Args:
    y (ndarray[nsamples]): The vector :math:`y`.
)delim");

  solver.def("get_inverse", [](Solver& self){
    int n = self.size();
    Eigen::MatrixXd eye(n, n);
    eye.setIdentity();
    self.apply_inverse(eye);
    return eye;
  }, R"delim(
Get the dense inverse covariance matrix. This is used for computing gradients,
but it is not recommended in general.
)delim");
}


#ifndef TAMM_EIGEN_UTILS_HPP_
#define TAMM_EIGEN_UTILS_HPP_

// Eigen matrix algebra library
#include "tamm/tamm.hpp"
#include <Eigen/Dense>
#include <unsupported/Eigen/CXX11/Tensor>
#undef I

using EigenTensorType=double;
using Matrix   = Eigen::Matrix<EigenTensorType, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>;
using Tensor1D = Eigen::Tensor<EigenTensorType, 1, Eigen::RowMajor>;
using Tensor2D = Eigen::Tensor<EigenTensorType, 2, Eigen::RowMajor>;
using Tensor3D = Eigen::Tensor<EigenTensorType, 3, Eigen::RowMajor>;
using Tensor4D = Eigen::Tensor<EigenTensorType, 4, Eigen::RowMajor>;

namespace tamm {

template<typename T, int ndim>
void patch_copy(std::vector<T>& sbuf,
                Eigen::Tensor<T, ndim, Eigen::RowMajor>& etensor,
                const std::array<int, ndim>& block_dims,
                const std::array<int, ndim>& block_offset, bool t2e = true) {
    assert(0);
}

template<typename T>
void patch_copy(std::vector<T>& sbuf,
                Eigen::Tensor<T, 1, Eigen::RowMajor>& etensor,
                const std::vector<size_t>& block_dims,
                const std::vector<size_t>& block_offset, bool t2e = true) {
    size_t c = 0;
    for(size_t i = block_offset[0]; i < block_offset[0] + block_dims[0];
        i++, c++) {
        if(t2e)
            etensor(i) = sbuf[c];
        else
            sbuf[c] = etensor(i);
    }
}

template<typename T>
void patch_copy(std::vector<T>& sbuf,
                Eigen::Tensor<T, 2, Eigen::RowMajor>& etensor,
                const std::vector<size_t>& block_dims,
                const std::vector<size_t>& block_offset, bool t2e = true) {
    size_t c = 0;
    for(size_t i = block_offset[0]; i < block_offset[0] + block_dims[0]; i++) {
        for(size_t j = block_offset[1]; j < block_offset[1] + block_dims[1];
            j++, c++) {
            if(t2e)
                etensor(i, j) = sbuf[c];
            else
                sbuf[c] = etensor(i, j);
        }
    }
}

template<typename T>
void patch_copy(std::vector<T>& sbuf,
                Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic,
                              Eigen::RowMajor>& etensor,
                const std::vector<size_t>& block_dims,
                const std::vector<size_t>& block_offset, bool t2e = true) {
    size_t c = 0;
    for(size_t i = block_offset[0]; i < block_offset[0] + block_dims[0]; i++) {
        for(size_t j = block_offset[1]; j < block_offset[1] + block_dims[1];
            j++, c++) {
            if(t2e)
                etensor(i, j) = sbuf[c];
            else
                sbuf[c] = etensor(i, j);
        }
    }
}

template<typename T>
void patch_copy(std::vector<T>& sbuf,
                Eigen::Tensor<T, 3, Eigen::RowMajor>& etensor,
                const std::vector<size_t>& block_dims,
                const std::vector<size_t>& block_offset, bool t2e = true) {
    size_t c = 0;
    for(size_t i = block_offset[0]; i < block_offset[0] + block_dims[0]; i++) {
        for(size_t j = block_offset[1]; j < block_offset[1] + block_dims[1];
            j++) {
            for(size_t k = block_offset[2]; k < block_offset[2] + block_dims[2];
                k++, c++) {
                if(t2e)
                    etensor(i, j, k) = sbuf[c];
                else
                    sbuf[c] = etensor(i, j, k);
            }
        }
    }
}

template<typename T>
void patch_copy(std::vector<T>& sbuf,
                Eigen::Tensor<T, 4, Eigen::RowMajor>& etensor,
                const std::vector<size_t>& block_dims,
                const std::vector<size_t>& block_offset, bool t2e = true) {
    size_t c = 0;
    for(size_t i = block_offset[0]; i < block_offset[0] + block_dims[0]; i++) {
        for(size_t j = block_offset[1]; j < block_offset[1] + block_dims[1];
            j++) {
            for(size_t k = block_offset[2]; k < block_offset[2] + block_dims[2];
                k++) {
                for(size_t l = block_offset[3];
                    l < block_offset[3] + block_dims[3]; l++, c++) {
                    if(t2e)
                        etensor(i, j, k, l) = sbuf[c];
                    else
                        sbuf[c] = etensor(i, j, k, l);
                }
            }
        }
    }
}

template<typename T, int ndim>
Eigen::Tensor<T, ndim, Eigen::RowMajor> tamm_to_eigen_tensor(
    const tamm::Tensor<T>& tensor) {
    std::array<long, ndim> dims;
    const auto& tindices = tensor.tiled_index_spaces();
    for(int i = 0; i < ndim; i++) {
        dims[i] = tindices[i].index_space().num_indices();
    }
    Eigen::Tensor<T, ndim, Eigen::RowMajor> etensor(dims);
    etensor.setZero();

    for(const auto& blockid : tensor.loop_nest()) {
        const tamm::TAMM_SIZE size = tensor.block_size(blockid);
        std::vector<T> buf(size);
        tensor.get(blockid, buf);
        auto block_dims   = tensor.block_dims(blockid);
        auto block_offset = tensor.block_offsets(blockid);
        patch_copy<T>(buf, etensor, block_dims, block_offset, true);
    }

    return etensor;
}

template<typename T>
void tamm_to_eigen_tensor(
  const tamm::Tensor<T>& tensor,
  Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>& etensor) {
    for(const auto& blockid : tensor.loop_nest()) {
        const tamm::TAMM_SIZE size = tensor.block_size(blockid);
        std::vector<T> buf(size);
        tensor.get(blockid, buf);
        auto block_dims   = tensor.block_dims(blockid);
        auto block_offset = tensor.block_offsets(blockid);
        patch_copy<T>(buf, etensor, block_dims, block_offset, true);
    }
}

template<typename T, int ndim>
void tamm_to_eigen_tensor(const tamm::Tensor<T>& tensor,
                          Eigen::Tensor<T, ndim, Eigen::RowMajor>& etensor) {
    for(const auto& blockid : tensor.loop_nest()) {
        const tamm::TAMM_SIZE size = tensor.block_size(blockid);
        std::vector<T> buf(size);
        tensor.get(blockid, buf);
        auto block_dims   = tensor.block_dims(blockid);
        auto block_offset = tensor.block_offsets(blockid);
        patch_copy<T>(buf, etensor, block_dims, block_offset, true);
    }
}

template<typename T, int ndim>
void eigen_to_tamm_tensor(tamm::Tensor<T>& tensor,
                          Eigen::Tensor<T, ndim, Eigen::RowMajor>& etensor) {
    for(const auto& blockid : tensor.loop_nest()) {
        const tamm::TAMM_SIZE size = tensor.block_size(blockid);
        std::vector<T> buf(size);
        // tensor.get(blockid, buf);
        auto block_dims   = tensor.block_dims(blockid);
        auto block_offset = tensor.block_offsets(blockid);
        patch_copy<T>(buf, etensor, block_dims, block_offset, false);
        tensor.put(blockid, buf);
    }
}

template<typename T>
void eigen_to_tamm_tensor(
  tamm::Tensor<T>& tensor,
  Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>& etensor) {
    for(const auto& blockid : tensor.loop_nest()) {
        const tamm::TAMM_SIZE size = tensor.block_size(blockid);
        std::vector<T> buf(size);
        // tensor.get(blockid, buf);
        auto block_dims   = tensor.block_dims(blockid);
        auto block_offset = tensor.block_offsets(blockid);
        patch_copy<T>(buf, etensor, block_dims, block_offset, false);
        tensor.put(blockid, buf);
    }
}

template<typename T>
void eigen_to_tamm_tensor_acc(
  tamm::Tensor<T>& tensor,
  Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>& etensor) {
    for(const auto& blockid : tensor.loop_nest()) {
        const tamm::TAMM_SIZE size = tensor.block_size(blockid);
        std::vector<T> buf(size);
        // tensor.get(blockid, buf);
        auto block_dims   = tensor.block_dims(blockid);
        auto block_offset = tensor.block_offsets(blockid);
        patch_copy<T>(buf, etensor, block_dims, block_offset, false);
        tensor.add(blockid, buf);
    }
}

template<typename T>
void eigen_to_tamm_tensor_acc(
  tamm::Tensor<T>& tensor,
  Eigen::Tensor<T, 2, Eigen::RowMajor>& etensor) {
    for(const auto& blockid : tensor.loop_nest()) {
        const tamm::TAMM_SIZE size = tensor.block_size(blockid);
        std::vector<T> buf(size);
        // tensor.get(blockid, buf);
        auto block_dims   = tensor.block_dims(blockid);
        auto block_offset = tensor.block_offsets(blockid);
        patch_copy<T>(buf, etensor, block_dims, block_offset, false);
        tensor.add(blockid, buf);
    }
}

template<typename T>
void eigen_to_tamm_tensor_acc(
  tamm::Tensor<T>& tensor,
  Eigen::Tensor<T, 3, Eigen::RowMajor>& etensor) {
    for(const auto& blockid : tensor.loop_nest()) {
        const tamm::TAMM_SIZE size = tensor.block_size(blockid);
        std::vector<T> buf(size);
        // tensor.get(blockid, buf);
        auto block_dims   = tensor.block_dims(blockid);
        auto block_offset = tensor.block_offsets(blockid);
        patch_copy<T>(buf, etensor, block_dims, block_offset, false);
        tensor.add(blockid, buf);
    }
}

template<typename T, typename fxn_t>
void call_eigen_matrix_fxn(const tamm::Tensor<T>& tensor, fxn_t fxn) {
    tamm::Tensor<T> copy_t(tensor);
    auto eigen_t = tamm_to_eigen_tensor<T, 2>(copy_t);

    using eigen_matrix =
    Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>;
    using eigen_map = Eigen::Map<eigen_matrix>;

    const auto nrow = eigen_t.dimensions()[0];
    const auto ncol = eigen_t.dimensions()[1];

    eigen_map mapped_t(eigen_t.data(), nrow, ncol);

    fxn(mapped_t);
}

template<int nmodes, typename T, typename matrix_type>
void eigen_matrix_to_tamm(matrix_type&& mat, tamm::Tensor<T>& t){
    using tensor1_type = Eigen::Tensor<T, 1, Eigen::RowMajor>;
    using tensor2_type = Eigen::Tensor<T, 2, Eigen::RowMajor>;

    const auto nrows = mat.rows();

    if constexpr(nmodes == 1){ //Actually a vector
        tensor1_type t1(std::array<long int, 1>{nrows});
        for(long int i = 0; i < nrows; ++i) t1(i) = mat(i);
        eigen_to_tamm_tensor(t, t1);
    }
    else if constexpr(nmodes == 2){
        const auto ncols = mat.cols();
        tensor2_type t1(std::array<long int, 2>{nrows, ncols}) ;

        for(long int i=0; i < nrows; ++i)
            for(long int j =0; j< ncols; ++j)
                t1(i, j) = mat(i, j);

        eigen_to_tamm_tensor(t, t1);
    }
}

template<typename T>
tamm::Tensor<T> retile_rank2_tensor(tamm::Tensor<T>& tensor, const tamm::TiledIndexSpace &t1, const tamm::TiledIndexSpace &t2) {
   EXPECTS(tensor.num_modes() == 2);
   //auto* ec_tmp = tensor.execution_context(); //TODO: figure out why this seg faults

   tamm::ProcGroup pg = ProcGroup::create_coll(GA_MPI_Comm());
   auto mgr = tamm::MemoryManagerGA::create_coll(pg);
   tamm::Distribution_NW distribution;
   tamm::ExecutionContext ec{pg,&distribution,mgr};
   auto is1 = tensor.tiled_index_spaces()[0].index_space();
   auto is2 = tensor.tiled_index_spaces()[1].index_space();
   auto dim1 = is1.num_indices();
   auto dim2 = is2.num_indices();
   Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor> eigent(dim1,dim2);

   tamm::Tensor<T> result{t1,t2};
   tamm::Tensor<T>::allocate(&ec, result);
   tamm_to_eigen_tensor(tensor,eigent);
   eigen_to_tamm_tensor(result,eigent);
   ec.pg().barrier();

   return result;
}

}

#endif // TAMM_EIGEN_UTILS_HPP_