//
//  sall.cpp
//  Percy
//
//  Created by Linhai Zhao on 4/22/15.
//  Copyright (c) 2015 Linhai Zhao. All rights reserved.
//

#include <boost/python.hpp>
#include <math.h>
#include <map>
#include <string>
#include <vector>
#include <iostream>

using namespace std;
typedef std::vector<int> IntVector;
typedef std::vector<std::string> StringVector;
typedef std::vector<StringVector> VecStringVec;
typedef map<int, StringVector> MyMap;

/*int main(){
	double s_all;
    StringVector fallele;
    MyMap fam_allele;
    IntVector affnf;
    affnf={2,3};
    fallele={"gp0","gp1","gm0","gm1"};
    fam_allele[2]={"gp0","gm0"};
    fam_allele[3]={"gp1","gm0"};
    fam_allele[4]={"gp0","gm1"};
    s_all=sall(&affnf,&fam_allele, &fallele);
    return 0;
}*/

int factorial(int n){
    int fac=1;
    if (n==0) {
        return 1;
    }
    if (n<0) {
        return -99;
    }
    while (n>0) {
        fac = fac*n;
        n--;
    }
    return fac;
}

double sall(IntVector*aff, MyMap*fam_a,StringVector*fa, IntVector*fm,bool rv_flag){
    IntVector tmp_aff = *aff;
    int affnum = tmp_aff.size();
    double sall;
    MyMap fam_allele = *fam_a;
    StringVector founderallele=*fa;
    IntVector foundermarker=*fm;
    int total = pow(2.0,affnum);
    int total_count=0;
    for (int i=0; i<total; i++) {
        //for each unique selection of alleles
        //to binary;
        string bin;
        int number = i;
        char holder=' ';
        if (number==0) {
            bin='0';
        }
        while(number !=0){
            holder=number%2+'0';
            bin=holder+bin;
            number /=2;
        }
        //cout<<string((affnum-bin.size()),'0')<<endl;
        bin=string((affnum-bin.size()),'0')+bin;
        StringVector pick(affnum);
        //convert binary digits to founder alleles
        for (int j=0; j<bin.size(); j++) {
            int binj = bin[j]-'0'; //convert character to integer
            pick[j]=fam_allele[tmp_aff[j]][binj];
        }
        //h[i]=pick;
        int tmp_count = 1;
        //count the occurrences of founder alleles in picked alleles
        for (StringVector::iterator a=founderallele.begin(); a != founderallele.end(); ++a) {
            int occurrence=0;// = std::count(pick.begin(), pick.end(), *a);
	    if (rv_flag && foundermarker.at(a-founderallele.begin()) != 1 || !rv_flag){
			for (StringVector::iterator pit=pick.begin(); pit != pick.end();++pit){
				if (*pit == *a){occurrence += 1;}
				}
			}
            tmp_count *= factorial(occurrence);
        }
        total_count += tmp_count;
    }
    sall=double(total_count)/double(total);
	return sall;
}

IntVector toIntVec(boost::python::list aff){
	IntVector t_aff;
	for(int i =0;i<len(aff);++i){
		int tmp=boost::python::extract<int>(aff[i]);
		t_aff.push_back(tmp);
	}
	return t_aff;
}

StringVector toStrVec(boost::python::list fa){
	StringVector t_fa;
	for(int i = 0; i<len(fa);++i){
		string tmp=boost::python::extract<string>(fa[i]);
		t_fa.push_back(tmp);
	}
	return t_fa;
}

MyMap tomap(boost::python::dict nf){
	MyMap t_nf;
	boost::python::list keys = nf.keys();
	for(int i=0; i<len(keys);++i){
		int extracted_key=boost::python::extract<int>(keys[i]);
		StringVector extracted_val;
		extracted_val.push_back(boost::python::extract<string>(nf[extracted_key][0]));
		extracted_val.push_back(boost::python::extract<string>(nf[extracted_key][1]));
		t_nf[extracted_key]=extracted_val;
	}
	return t_nf;
}

double apply(boost::python::list aff, boost::python::dict fam_a, boost::python::list fa, boost::python::list fm, bool rv_flag){
	IntVector caff;
	StringVector fallele;
	IntVector fmarker;
	MyMap cfam_a;
	double s_all;
	caff = toIntVec(aff);
	fallele = toStrVec(fa);
	fmarker = toIntVec(fm);
	cfam_a = tomap(fam_a);
	s_all = sall(&caff,&cfam_a,&fallele,&fmarker,rv_flag);
	return s_all;
}

BOOST_PYTHON_MODULE(sall_cpp){
	using namespace boost::python;
	def("sall",sall);
	def("apply",apply);
}


/* ---------------------------------------------------------------------
 *
 * Copyright (C) 2019 - 2021 by the deal.II authors
 *
 * This file is part of the deal.II library.
 *
 * The deal.II library is free software; you can use it, redistribute
 * it, and/or modify it under the terms of the GNU Lesser General
 * Public License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 * The full text of the license can be found in the file LICENSE.md at
 * the top level directory of deal.II.
 *
 * ---------------------------------------------------------------------
 *
 * Author: Thomas C. Clevenger, Clemson University
 *         Timo Heister, Clemson University
 *         Guido Kanschat, Heidelberg University
 *         Martin Kronbichler, Technical University of Munich
 */

// @sect3{Include files}

// 包含文件是  step-40  ,  step-16  , 和  step-37  的组合。

#include <deal.II/base/conditional_ostream.h>
#include <deal.II/base/data_out_base.h>
#include <deal.II/base/index_set.h>
#include <deal.II/base/logstream.h>
#include <deal.II/base/quadrature_lib.h>
#include <deal.II/base/timer.h>
#include <deal.II/base/parameter_handler.h>
#include <deal.II/distributed/grid_refinement.h>
#include <deal.II/distributed/tria.h>
#include <deal.II/dofs/dof_tools.h>
#include <deal.II/fe/fe_q.h>
#include <deal.II/fe/fe_values.h>
#include <deal.II/grid/grid_generator.h>
#include <deal.II/grid/grid_refinement.h>
#include <deal.II/grid/tria.h>
#include <deal.II/lac/affine_constraints.h>
#include <deal.II/lac/dynamic_sparsity_pattern.h>
#include <deal.II/lac/solver_cg.h>

// 我们使用与 step-40 相同的策略，在PETSc和Trilinos之间进行切换。

#include <deal.II/lac/generic_linear_algebra.h>

// 如果你已经安装了PETSc和Trilinos，并且你喜欢在本例中使用PETSc，请将下面的预处理程序定义注释进去或退出。

#define FORCE_USE_OF_TRILINOS

namespace LA
{
#if defined(DEAL_II_WITH_PETSC) && !defined(DEAL_II_PETSC_WITH_COMPLEX) && \
  !(defined(DEAL_II_WITH_TRILINOS) && defined(FORCE_USE_OF_TRILINOS))
  using namespace dealii::LinearAlgebraPETSc;
#  define USE_PETSC_LA
#elif defined(DEAL_II_WITH_TRILINOS)
  using namespace dealii::LinearAlgebraTrilinos;
#else
#  error DEAL_II_WITH_PETSC or DEAL_II_WITH_TRILINOS required
#endif
} // namespace LA

#include <deal.II/matrix_free/matrix_free.h>
#include <deal.II/matrix_free/operators.h>
#include <deal.II/matrix_free/fe_evaluation.h>
#include <deal.II/multigrid/mg_coarse.h>
#include <deal.II/multigrid/mg_constrained_dofs.h>
#include <deal.II/multigrid/mg_matrix.h>
#include <deal.II/multigrid/mg_smoother.h>
#include <deal.II/multigrid/mg_tools.h>
#include <deal.II/multigrid/mg_transfer.h>
#include <deal.II/multigrid/multigrid.h>
#include <deal.II/multigrid/mg_transfer_matrix_free.h>
#include <deal.II/numerics/data_out.h>
#include <deal.II/numerics/vector_tools.h>

// 以下文件用于组装误差估计器，如  step-12  。

#include <deal.II/fe/fe_interface_values.h>
#include <deal.II/meshworker/mesh_loop.h>

using namespace dealii;
// @sect3{Coefficients and helper classes}

// MatrixFree运算符必须使用 dealii::LinearAlgebra::distributed::Vector 矢量类型。这里我们定义了复制到Trilinos向量的操作，以便与基于矩阵的代码兼容。请注意，目前PETSc矢量类型不存在这种功能，所以必须安装Trilinos来使用本教程中的MatrixFree求解器。

namespace ChangeVectorTypes
{
  template <typename number>
  void copy(LA::MPI::Vector &                                         out,
            const dealii::LinearAlgebra::distributed::Vector<number> &in)
  {
    dealii::LinearAlgebra::ReadWriteVector<double> rwv(
      out.locally_owned_elements());
    rwv.import(in, VectorOperation::insert);
#ifdef USE_PETSC_LA
    AssertThrow(false,
                ExcMessage("CopyVectorTypes::copy() not implemented for "
                           "PETSc vector types."));
#else
    out.import(rwv, VectorOperation::insert);
#endif
  }

  template <typename number>
  void copy(dealii::LinearAlgebra::distributed::Vector<number> &out,
            const LA::MPI::Vector &                             in)
  {
    dealii::LinearAlgebra::ReadWriteVector<double> rwv;
#ifdef USE_PETSC_LA
    (void)in;
    AssertThrow(false,
                ExcMessage("CopyVectorTypes::copy() not implemented for "
                           "PETSc vector types."));
#else
    rwv.reinit(in);
#endif
    out.import(rwv, VectorOperation::insert);
  }
} // namespace ChangeVectorTypes

// 让我们继续描述我们要解决的问题。我们把右边的函数设置为1.0。 @p value 函数返回一个VectorizedArray，被无矩阵代码路径所使用。

template <int dim>
class RightHandSide : public Function<dim>
{
public:
  virtual double value(const Point<dim> & /*p*/,
                       const unsigned int /*component*/ = 0) const override
  {
    return 1.0;
  }

  template <typename number>
  VectorizedArray<number>
  value(const Point<dim, VectorizedArray<number>> & /*p*/,
        const unsigned int /*component*/ = 0) const
  {
    return VectorizedArray<number>(1.0);
  }
};

// 接下来的这个类表示扩散系数。我们使用一个可变的系数，在任何一个至少有一个坐标小于-0.5的点上是100.0，在所有其他点上是1.0。如上所述，一个单独的value()返回一个VectorizedArray，用于无矩阵代码。一个 @p average()函数计算了一组点的算术平均。

template <int dim>
class Coefficient : public Function<dim>
{
public:
  virtual double value(const Point<dim> &p,
                       const unsigned int /*component*/ = 0) const override;

  template <typename number>
  VectorizedArray<number> value(const Point<dim, VectorizedArray<number>> &p,
                                const unsigned int /*component*/ = 0) const;

  template <typename number>
  number average_value(const std::vector<Point<dim, number>> &points) const;

// 当在MatrixFree框架中使用一个系数时，我们还需要一个函数，为MatrixFree运算符参数提供的一组单元格创建一个系数表。

  template <typename number>
  std::shared_ptr<Table<2, VectorizedArray<number>>> make_coefficient_table(
    const MatrixFree<dim, number, VectorizedArray<number>> &mf_storage) const;
};

template <int dim>
double Coefficient<dim>::value(const Point<dim> &p, const unsigned int) const
{
  for (int d = 0; d < dim; ++d)
    {
      if (p[d] < -0.5)
        return 100.0;
    }
  return 1.0;
}

template <int dim>
template <typename number>
VectorizedArray<number>
Coefficient<dim>::value(const Point<dim, VectorizedArray<number>> &p,
                        const unsigned int) const
{
  VectorizedArray<number> return_value = VectorizedArray<number>(1.0);
  for (unsigned int i = 0; i < VectorizedArray<number>::size(); ++i)
    {
      for (int d = 0; d < dim; ++d)
        if (p[d][i] < -0.5)
          {
            return_value[i] = 100.0;
            break;
          }
    }

  return return_value;
}

template <int dim>
template <typename number>
number Coefficient<dim>::average_value(
  const std::vector<Point<dim, number>> &points) const
{
  number average(0);
  for (unsigned int i = 0; i < points.size(); ++i)
    average += value(points[i]);
  average /= points.size();

  return average;
}

template <int dim>
template <typename number>
std::shared_ptr<Table<2, VectorizedArray<number>>>
Coefficient<dim>::make_coefficient_table(
  const MatrixFree<dim, number, VectorizedArray<number>> &mf_storage) const
{
  auto coefficient_table =
    std::make_shared<Table<2, VectorizedArray<number>>>();

  FEEvaluation<dim, -1, 0, 1, number> fe_eval(mf_storage);

  const unsigned int n_cells    = mf_storage.n_cell_batches();
  const unsigned int n_q_points = fe_eval.n_q_points;

  coefficient_table->reinit(n_cells, 1);

  for (unsigned int cell = 0; cell < n_cells; ++cell)
    {
      fe_eval.reinit(cell);

      VectorizedArray<number> average_value = 0.;
      for (unsigned int q = 0; q < n_q_points; ++q)
        average_value += value(fe_eval.quadrature_point(q));
      average_value /= n_q_points;

      (*coefficient_table)(cell, 0) = average_value;
    }

  return coefficient_table;
}

//  @sect3{Run time parameters}

// 我们将使用ParameterHandler来在运行时传入参数。 该结构 @p Settings 解析并存储这些参数，以便在整个程序中进行查询。

struct Settings
{
  bool try_parse(const std::string &prm_filename);

  enum SolverType
  {
    gmg_mb,
    gmg_mf,
    amg
  };

  SolverType solver;

  int          dimension;
  double       smoother_dampen;
  unsigned int smoother_steps;
  unsigned int n_steps;
  bool         output;
};

bool Settings::try_parse(const std::string &prm_filename)
{
  ParameterHandler prm;
  prm.declare_entry("dim", "2", Patterns::Integer(), "The problem dimension.");
  prm.declare_entry("n_steps",
                    "10",
                    Patterns::Integer(0),
                    "Number of adaptive refinement steps.");
  prm.declare_entry("smoother dampen",
                    "1.0",
                    Patterns::Double(0.0),
                    "Dampen factor for the smoother.");
  prm.declare_entry("smoother steps",
                    "1",
                    Patterns::Integer(1),
                    "Number of smoother steps.");
  prm.declare_entry("solver",
                    "MF",
                    Patterns::Selection("MF|MB|AMG"),
                    "Switch between matrix-free GMG, "
                    "matrix-based GMG, and AMG.");
  prm.declare_entry("output",
                    "false",
                    Patterns::Bool(),
                    "Output graphical results.");

  if (prm_filename.size() == 0)
    {
      std::cout << "****  Error: No input file provided!\n"
                << "****  Error: Call this program as './step-50 input.prm\n"
                << "\n"
                << "****  You may want to use one of the input files in this\n"
                << "****  directory, or use the following default values\n"
                << "****  to create an input file:\n";
      if (Utilities::MPI::this_mpi_process(MPI_COMM_WORLD) == 0)
        prm.print_parameters(std::cout, ParameterHandler::Text);
      return false;
    }

  try
    {
      prm.parse_input(prm_filename);
    }
  catch (std::exception &e)
    {
      if (Utilities::MPI::this_mpi_process(MPI_COMM_WORLD) == 0)
        std::cerr << e.what() << std::endl;
      return false;
    }

  if (prm.get("solver") == "MF")
    this->solver = gmg_mf;
  else if (prm.get("solver") == "MB")
    this->solver = gmg_mb;
  else if (prm.get("solver") == "AMG")
    this->solver = amg;
  else
    AssertThrow(false, ExcNotImplemented());

  this->dimension       = prm.get_integer("dim");
  this->n_steps         = prm.get_integer("n_steps");
  this->smoother_dampen = prm.get_double("smoother dampen");
  this->smoother_steps  = prm.get_integer("smoother steps");
  this->output          = prm.get_bool("output");

  return true;
}

//  @sect3{LaplaceProblem class}

// 这是该程序的主类。它看起来与  step-16  ,  step-37  , 和  step-40  非常相似。对于MatrixFree的设置，我们使用 MatrixFreeOperators::LaplaceOperator 类，它在内部定义了`local_apply()`, `compute_diagonal()`, 和`set_coefficient()`函数。请注意，多项式的度数是这个类的一个模板参数。这对无矩阵代码来说是必要的。

template <int dim, int degree>
class LaplaceProblem
{
public:
  LaplaceProblem(const Settings &settings);
  void run();

private:

// 我们将在整个程序中使用以下类型。首先是基于矩阵的类型，之后是无矩阵的类。对于无矩阵的实现，我们使用 @p float 作为水平运算符。

  using MatrixType         = LA::MPI::SparseMatrix;
  using VectorType         = LA::MPI::Vector;
  using PreconditionAMG    = LA::MPI::PreconditionAMG;
  using PreconditionJacobi = LA::MPI::PreconditionJacobi;

  using MatrixFreeLevelMatrix = MatrixFreeOperators::LaplaceOperator<
    dim,
    degree,
    degree + 1,
    1,
    LinearAlgebra::distributed::Vector<float>>;
  using MatrixFreeActiveMatrix = MatrixFreeOperators::LaplaceOperator<
    dim,
    degree,
    degree + 1,
    1,
    LinearAlgebra::distributed::Vector<double>>;

  using MatrixFreeLevelVector  = LinearAlgebra::distributed::Vector<float>;
  using MatrixFreeActiveVector = LinearAlgebra::distributed::Vector<double>;

  void setup_system();
  void setup_multigrid();
  void assemble_system();
  void assemble_multigrid();
  void assemble_rhs();
  void solve();
  void estimate();
  void refine_grid();
  void output_results(const unsigned int cycle);

  Settings settings;

  MPI_Comm           mpi_communicator;
  ConditionalOStream pcout;

  parallel::distributed::Triangulation<dim> triangulation;
  const MappingQ1<dim>                      mapping;
  FE_Q<dim>                                 fe;

  DoFHandler<dim> dof_handler;

  IndexSet                  locally_relevant_dofs;
  AffineConstraints<double> constraints;

  MatrixType             system_matrix;
  MatrixFreeActiveMatrix mf_system_matrix;
  VectorType             solution;
  VectorType             right_hand_side;
  Vector<double>         estimated_error_square_per_cell;

  MGLevelObject<MatrixType> mg_matrix;
  MGLevelObject<MatrixType> mg_interface_in;
  MGConstrainedDoFs         mg_constrained_dofs;

  MGLevelObject<MatrixFreeLevelMatrix> mf_mg_matrix;

  TimerOutput computing_timer;
};

// 关于构造函数的唯一有趣的部分是，除非我们使用AMG，否则我们会构造多网格的层次结构。为此，我们需要在这个构造函数完成之前解析运行时参数。

template <int dim, int degree>
LaplaceProblem<dim, degree>::LaplaceProblem(const Settings &settings)
  : settings(settings)
  , mpi_communicator(MPI_COMM_WORLD)
  , pcout(std::cout, (Utilities::MPI::this_mpi_process(mpi_communicator) == 0))
  , triangulation(mpi_communicator,
                  Triangulation<dim>::limit_level_difference_at_vertices,
                  (settings.solver == Settings::amg) ?
                    parallel::distributed::Triangulation<dim>::default_setting :
                    parallel::distributed::Triangulation<
                      dim>::construct_multigrid_hierarchy)
  , mapping()
  , fe(degree)
  , dof_handler(triangulation)
  , computing_timer(pcout, TimerOutput::never, TimerOutput::wall_times)
{
  GridGenerator::hyper_L(triangulation, -1., 1., /*colorize*/ false);
  triangulation.refine_global(1);
}

//  @sect4{LaplaceProblem::setup_system()}

// 与  step-16  和  step-37  不同，我们将设置分成两部分，setup_system() 和 setup_multigrid() 。下面是大多数教程中常见的主动网格的典型setup_system()函数。对于无矩阵，活动网格的设置类似于  step-37  ；对于基于矩阵（GMG和AMG求解器），设置类似于  step-40  。

template <int dim, int degree>
void LaplaceProblem<dim, degree>::setup_system()
{
  TimerOutput::Scope timing(computing_timer, "Setup");

  dof_handler.distribute_dofs(fe);

  DoFTools::extract_locally_relevant_dofs(dof_handler, locally_relevant_dofs);
  locally_owned_dofs = dof_handler.locally_owned_dofs();

  solution.reinit(locally_owned_dofs, mpi_communicator);
  right_hand_side.reinit(locally_owned_dofs, mpi_communicator);
  constraints.reinit(locally_relevant_dofs);
  DoFTools::make_hanging_node_constraints(dof_handler, constraints);

  VectorTools::interpolate_boundary_values(
    mapping, dof_handler, 0, Functions::ZeroFunction<dim>(), constraints);
  constraints.close();

  switch (settings.solver)
    {
      case Settings::gmg_mf:
        {
          typename MatrixFree<dim, double>::AdditionalData additional_data;
          additional_data.tasks_parallel_scheme =
            MatrixFree<dim, double>::AdditionalData::none;
          additional_data.mapping_update_flags =
            (update_gradients | update_JxW_values | update_quadrature_points);
          std::shared_ptr<MatrixFree<dim, double>> mf_storage =
            std::make_shared<MatrixFree<dim, double>>();
          mf_storage->reinit(mapping,
                             dof_handler,
                             constraints,
                             QGauss<1>(degree + 1),
                             additional_data);

          mf_system_matrix.initialize(mf_storage);

          const Coefficient<dim> coefficient;
          mf_system_matrix.set_coefficient(
            coefficient.make_coefficient_table(*mf_storage));

          break;
        }

      case Settings::gmg_mb:
      case Settings::amg:
        {
#ifdef USE_PETSC_LA
          DynamicSparsityPattern dsp(locally_relevant_dofs);
          DoFTools::make_sparsity_pattern(dof_handler, dsp, constraints);

          SparsityTools::distribute_sparsity_pattern(dsp,
                                                     locally_owned_dofs,
                                                     mpi_communicator,
                                                     locally_relevant_dofs);

          system_matrix.reinit(locally_owned_dofs,
                               locally_owned_dofs,
                               dsp,
                               mpi_communicator);
#else
          TrilinosWrappers::SparsityPattern dsp(locally_owned_dofs,
                                                locally_owned_dofs,
                                                locally_relevant_dofs,
                                                mpi_communicator);
          DoFTools::make_sparsity_pattern(dof_handler, dsp, constraints);
          dsp.compress();
          system_matrix.reinit(dsp);
#endif

          break;
        }

      default:
        Assert(false, ExcNotImplemented());
    }
}
// @sect4{LaplaceProblem::setup_multigrid()}

// 该函数为无矩阵和基于矩阵的GMG进行多级设置。无矩阵的设置类似于 step-37 ，而基于矩阵的设置类似于 step-16 ，只是我们必须使用适当的分布式稀疏度模式。

// 该函数没有被AMG方法调用，但为了安全起见，该函数的主`switch`语句还是确保了该函数只在已知的多网格设置下运行，如果该函数被调用到两种几何多网格方法以外的地方，则抛出一个断言。

template <int dim, int degree>
void LaplaceProblem<dim, degree>::setup_multigrid()
{
  TimerOutput::Scope timing(computing_timer, "Setup multigrid");

  dof_handler.distribute_mg_dofs();

  mg_constrained_dofs.clear();
  mg_constrained_dofs.initialize(dof_handler);

  const std::set<types::boundary_id> boundary_ids = {types::boundary_id(0)};
  mg_constrained_dofs.make_zero_boundary_constraints(dof_handler, boundary_ids);

  const unsigned int n_levels = triangulation.n_global_levels();

  switch (settings.solver)
    {
      case Settings::gmg_mf:
        {
          mf_mg_matrix.resize(0, n_levels - 1);

          for (unsigned int level = 0; level < n_levels; ++level)
            {
              IndexSet relevant_dofs;
              DoFTools::extract_locally_relevant_level_dofs(dof_handler,
                                                            level,
                                                            relevant_dofs);
              AffineConstraints<double> level_constraints;
              level_constraints.reinit(relevant_dofs);
              level_constraints.add_lines(
                mg_constrained_dofs.get_boundary_indices(level));
              level_constraints.close();

              typename MatrixFree<dim, float>::AdditionalData additional_data;
              additional_data.tasks_parallel_scheme =
                MatrixFree<dim, float>::AdditionalData::none;
              additional_data.mapping_update_flags =
                (update_gradients | update_JxW_values |
                 update_quadrature_points);
              additional_data.mg_level = level;
              std::shared_ptr<MatrixFree<dim, float>> mf_storage_level(
                new MatrixFree<dim, float>());
              mf_storage_level->reinit(mapping,
                                       dof_handler,
                                       level_constraints,
                                       QGauss<1>(degree + 1),
                                       additional_data);

              mf_mg_matrix[level].initialize(mf_storage_level,
                                             mg_constrained_dofs,
                                             level);

              const Coefficient<dim> coefficient;
              mf_mg_matrix[level].set_coefficient(
                coefficient.make_coefficient_table(*mf_storage_level));

              mf_mg_matrix[level].compute_diagonal();
            }

          break;
        }

      case Settings::gmg_mb:
        {
          mg_matrix.resize(0, n_levels - 1);
          mg_matrix.clear_elements();
          mg_interface_in.resize(0, n_levels - 1);
          mg_interface_in.clear_elements();

          for (unsigned int level = 0; level < n_levels; ++level)
            {
              IndexSet dof_set;
              DoFTools::extract_locally_relevant_level_dofs(dof_handler,
                                                            level,
                                                            dof_set);

              {
#ifdef USE_PETSC_LA
                DynamicSparsityPattern dsp(dof_set);
                MGTools::make_sparsity_pattern(dof_handler, dsp, level);
                dsp.compress();
                SparsityTools::distribute_sparsity_pattern(
                  dsp,
                  dof_handler.locally_owned_mg_dofs(level),
                  mpi_communicator,
                  dof_set);

                mg_matrix[level].reinit(
                  dof_handler.locally_owned_mg_dofs(level),
                  dof_handler.locally_owned_mg_dofs(level),
                  dsp,
                  mpi_communicator);
#else
                TrilinosWrappers::SparsityPattern dsp(
                  dof_handler.locally_owned_mg_dofs(level),
                  dof_handler.locally_owned_mg_dofs(level),
                  dof_set,
                  mpi_communicator);
                MGTools::make_sparsity_pattern(dof_handler, dsp, level);

                dsp.compress();
                mg_matrix[level].reinit(dsp);
#endif
              }

              {
#ifdef USE_PETSC_LA
                DynamicSparsityPattern dsp(dof_set);
                MGTools::make_interface_sparsity_pattern(dof_handler,
                                                         mg_constrained_dofs,
                                                         dsp,
                                                         level);
                dsp.compress();
                SparsityTools::distribute_sparsity_pattern(
                  dsp,
                  dof_handler.locally_owned_mg_dofs(level),
                  mpi_communicator,
                  dof_set);

                mg_interface_in[level].reinit(
                  dof_handler.locally_owned_mg_dofs(level),
                  dof_handler.locally_owned_mg_dofs(level),
                  dsp,
                  mpi_communicator);
#else
                TrilinosWrappers::SparsityPattern dsp(
                  dof_handler.locally_owned_mg_dofs(level),
                  dof_handler.locally_owned_mg_dofs(level),
                  dof_set,
                  mpi_communicator);

                MGTools::make_interface_sparsity_pattern(dof_handler,
                                                         mg_constrained_dofs,
                                                         dsp,
                                                         level);
                dsp.compress();
                mg_interface_in[level].reinit(dsp);
#endif
              }
            }
          break;
        }

      default:
        Assert(false, ExcNotImplemented());
    }
}
// @sect4{LaplaceProblem::assemble_system()}

// 汇编被分成三个部分：`assemble_system()`, `assemble_multigrid()`, 和`assemble_rhs()`。这里的`assemble_system()`函数组装并存储（全局）系统矩阵和基于矩阵的方法的右手边。它类似于  step-40  中的装配。

// 注意，无矩阵方法不执行这个函数，因为它不需要组装矩阵，而是在assemble_rhs()中组装右手边。

template <int dim, int degree>
void LaplaceProblem<dim, degree>::assemble_system()
{
  TimerOutput::Scope timing(computing_timer, "Assemble");

  const QGauss<dim> quadrature_formula(degree + 1);

  FEValues<dim> fe_values(fe,
                          quadrature_formula,
                          update_values | update_gradients |
                            update_quadrature_points | update_JxW_values);

  const unsigned int dofs_per_cell = fe.n_dofs_per_cell();
  const unsigned int n_q_points    = quadrature_formula.size();

  FullMatrix<double> cell_matrix(dofs_per_cell, dofs_per_cell);
  Vector<double>     cell_rhs(dofs_per_cell);

  std::vector<types::global_dof_index> local_dof_indices(dofs_per_cell);

  const Coefficient<dim> coefficient;
  RightHandSide<dim>     rhs;
  std::vector<double>    rhs_values(n_q_points);

  for (const auto &cell : dof_handler.active_cell_iterators())
    if (cell->is_locally_owned())
      {
        cell_matrix = 0;
        cell_rhs    = 0;

        fe_values.reinit(cell);

        const double coefficient_value =
          coefficient.average_value(fe_values.get_quadrature_points());
        rhs.value_list(fe_values.get_quadrature_points(), rhs_values);

        for (unsigned int q_point = 0; q_point < n_q_points; ++q_point)
          for (unsigned int i = 0; i < dofs_per_cell; ++i)
            {
              for (unsigned int j = 0; j < dofs_per_cell; ++j)
                cell_matrix(i, j) +=
                  coefficient_value *                // epsilon(x)
                  fe_values.shape_grad(i, q_point) * // * grad phi_i(x)
                  fe_values.shape_grad(j, q_point) * // * grad phi_j(x)
                  fe_values.JxW(q_point);            // * dx

              cell_rhs(i) +=
                fe_values.shape_value(i, q_point) * // grad phi_i(x)
                rhs_values[q_point] *               // * f(x)
                fe_values.JxW(q_point);             // * dx
            }

        cell->get_dof_indices(local_dof_indices);
        constraints.distribute_local_to_global(cell_matrix,
                                               cell_rhs,
                                               local_dof_indices,
                                               system_matrix,
                                               right_hand_side);
      }

  system_matrix.compress(VectorOperation::add);
  right_hand_side.compress(VectorOperation::add);
}
// @sect4{LaplaceProblem::assemble_multigrid()}

// 下面的函数为基于矩阵的GMG方法组装和存储多级矩阵。这个函数与 step-16 中的函数类似，只是在这里它适用于分布式网格。这个区别在于增加了一个条件，即我们只在本地拥有的水平单元上进行组装，并为每个被建立的矩阵调用压缩（）。

template <int dim, int degree>
void LaplaceProblem<dim, degree>::assemble_multigrid()
{
  TimerOutput::Scope timing(computing_timer, "Assemble multigrid");

  QGauss<dim> quadrature_formula(degree + 1);

  FEValues<dim> fe_values(fe,
                          quadrature_formula,
                          update_values | update_gradients |
                            update_quadrature_points | update_JxW_values);

  const unsigned int dofs_per_cell = fe.n_dofs_per_cell();
  const unsigned int n_q_points    = quadrature_formula.size();

  FullMatrix<double> cell_matrix(dofs_per_cell, dofs_per_cell);

  std::vector<types::global_dof_index> local_dof_indices(dofs_per_cell);

  const Coefficient<dim> coefficient;

  std::vector<AffineConstraints<double>> boundary_constraints(
    triangulation.n_global_levels());
  for (unsigned int level = 0; level < triangulation.n_global_levels(); ++level)
    {
      IndexSet dof_set;
      DoFTools::extract_locally_relevant_level_dofs(dof_handler,
                                                    level,
                                                    dof_set);
      boundary_constraints[level].reinit(dof_set);
      boundary_constraints[level].add_lines(
        mg_constrained_dofs.get_refinement_edge_indices(level));
      boundary_constraints[level].add_lines(
        mg_constrained_dofs.get_boundary_indices(level));

      boundary_constraints[level].close();
    }

  for (const auto &cell : dof_handler.cell_iterators())
    if (cell->level_subdomain_id() == triangulation.locally_owned_subdomain())
      {
        cell_matrix = 0;
        fe_values.reinit(cell);

        const double coefficient_value =
          coefficient.average_value(fe_values.get_quadrature_points());

        for (unsigned int q_point = 0; q_point < n_q_points; ++q_point)
          for (unsigned int i = 0; i < dofs_per_cell; ++i)
            for (unsigned int j = 0; j < dofs_per_cell; ++j)
              cell_matrix(i, j) +=
                coefficient_value * fe_values.shape_grad(i, q_point) *
                fe_values.shape_grad(j, q_point) * fe_values.JxW(q_point);

        cell->get_mg_dof_indices(local_dof_indices);

        boundary_constraints[cell->level()].distribute_local_to_global(
          cell_matrix, local_dof_indices, mg_matrix[cell->level()]);

        for (unsigned int i = 0; i < dofs_per_cell; ++i)
          for (unsigned int j = 0; j < dofs_per_cell; ++j)
            if (mg_constrained_dofs.is_interface_matrix_entry(
                  cell->level(), local_dof_indices[i], local_dof_indices[j]))
              mg_interface_in[cell->level()].add(local_dof_indices[i],
                                                 local_dof_indices[j],
                                                 cell_matrix(i, j));
      }

  for (unsigned int i = 0; i < triangulation.n_global_levels(); ++i)
    {
      mg_matrix[i].compress(VectorOperation::add);
      mg_interface_in[i].compress(VectorOperation::add);
    }
}

//  @sect4{LaplaceProblem::assemble_rhs()}

// 这个三要素中的最后一个函数为无矩阵方法组装右手边的向量--因为在无矩阵框架中，我们不需要组装矩阵，只需要组装右手边就可以了。我们可以通过从上面的`assemble_system()`函数中提取处理右手边的代码来做到这一点，但是我们决定完全采用无矩阵的方法，也用这种方法进行装配。

// 结果是一个类似于 step-37 中 "使用 FEEvaluation::read_dof_values_plain() 来避免解决约束 "一节中的函数。

// 这个函数的原因是MatrixFree运算符不考虑非同质的Dirichlet约束，而是将所有的Dirichlet约束视为同质的。为了说明这一点，这里的右手边被组装成残差 $r_0 = f-Au_0$ ，其中 $u_0$ 是一个零向量，除了在Dirichlet值中。然后在求解的时候，我们可以看到，解决方案是  $u = u_0 + A^{-1}r_0$  。这可以看作是对初始猜测为  $u_0$  的线性系统进行的牛顿迭代。下面`solve()`函数中的CG解计算了 $A^{-1}r_0$ ，调用`constraints.distribution()`（直接在后面）增加了 $u_0$  。

// 显然，由于我们考虑的是一个零迪里希特边界的问题，我们可以采取类似于 step-37  `assemble_rhs()`的方法，但是这个额外的工作允许我们改变问题声明，如果我们选择的话。

// 这个函数在积分循环中有两个部分：通过提交梯度的负值将矩阵  $A$  的负值应用于  $u_0$  ，并通过提交值  $f$  添加右手边的贡献。我们必须确保使用`read_dof_values_plain()`来评估 $u_0$ ，因为`read_dof_vaues()`会将所有Dirichlet值设置为0。

// 最后，system_rhs向量的类型是 LA::MPI::Vector, ，但MatrixFree类只对 dealii::LinearAlgebra::distributed::Vector. 起作用，因此我们必须使用MatrixFree功能计算右手边，然后使用`ChangeVectorType`命名空间的函数将其复制到正确的类型。

template <int dim, int degree>
void LaplaceProblem<dim, degree>::assemble_rhs()
{
  TimerOutput::Scope timing(computing_timer, "Assemble right-hand side");

  MatrixFreeActiveVector solution_copy;
  MatrixFreeActiveVector right_hand_side_copy;
  mf_system_matrix.initialize_dof_vector(solution_copy);
  mf_system_matrix.initialize_dof_vector(right_hand_side_copy);

  solution_copy = 0.;
  constraints.distribute(solution_copy);
  solution_copy.update_ghost_values();
  right_hand_side_copy = 0;
  const Table<2, VectorizedArray<double>> &coefficient =
    *(mf_system_matrix.get_coefficient());

  RightHandSide<dim> right_hand_side_function;

  FEEvaluation<dim, degree, degree + 1, 1, double> phi(
    *mf_system_matrix.get_matrix_free());

  for (unsigned int cell = 0;
       cell < mf_system_matrix.get_matrix_free()->n_cell_batches();
       ++cell)
    {
      phi.reinit(cell);
      phi.read_dof_values_plain(solution_copy);
      phi.evaluate(EvaluationFlags::gradients);

      for (unsigned int q = 0; q < phi.n_q_points; ++q)
        {
          phi.submit_gradient(-1.0 *
                                (coefficient(cell, 0) * phi.get_gradient(q)),
                              q);
          phi.submit_value(
            right_hand_side_function.value(phi.quadrature_point(q)), q);
        }

      phi.integrate_scatter(EvaluationFlags::values |
                              EvaluationFlags::gradients,
                            right_hand_side_copy);
    }

  right_hand_side_copy.compress(VectorOperation::add);

  ChangeVectorTypes::copy(right_hand_side, right_hand_side_copy);
}

//  @sect4{LaplaceProblem::solve()}

// 这里我们设置了多网格预处理程序，测试了单个V型周期的时间，并解决了线性系统。不出所料，这是三种方法差别最大的地方之一。

template <int dim, int degree>
void LaplaceProblem<dim, degree>::solve()
{
  TimerOutput::Scope timing(computing_timer, "Solve");

  SolverControl solver_control(1000, 1.e-10 * right_hand_side.l2_norm());
  solver_control.enable_history_data();

  solution = 0.;

// 无矩阵GMG方法的求解器类似于  step-37  ，除了增加一些接口矩阵，完全类似于  step-16  。

  switch (settings.solver)
    {
      case Settings::gmg_mf:
        {
          computing_timer.enter_subsection("Solve: Preconditioner setup");

          MGTransferMatrixFree<dim, float> mg_transfer(mg_constrained_dofs);
          mg_transfer.build(dof_handler);

          SolverControl coarse_solver_control(1000, 1e-12, false, false);
          SolverCG<MatrixFreeLevelVector> coarse_solver(coarse_solver_control);
          PreconditionIdentity            identity;
          MGCoarseGridIterativeSolver<MatrixFreeLevelVector,
                                      SolverCG<MatrixFreeLevelVector>,
                                      MatrixFreeLevelMatrix,
                                      PreconditionIdentity>
            coarse_grid_solver(coarse_solver, mf_mg_matrix[0], identity);

          using Smoother = dealii::PreconditionJacobi<MatrixFreeLevelMatrix>;
          MGSmootherPrecondition<MatrixFreeLevelMatrix,
                                 Smoother,
                                 MatrixFreeLevelVector>
            smoother;
          smoother.initialize(mf_mg_matrix,
                              typename Smoother::AdditionalData(
                                settings.smoother_dampen));
          smoother.set_steps(settings.smoother_steps);

          mg::Matrix<MatrixFreeLevelVector> mg_m(mf_mg_matrix);

          MGLevelObject<
            MatrixFreeOperators::MGInterfaceOperator<MatrixFreeLevelMatrix>>
            mg_interface_matrices;
          mg_interface_matrices.resize(0, triangulation.n_global_levels() - 1);
          for (unsigned int level = 0; level < triangulation.n_global_levels();
               ++level)
            mg_interface_matrices[level].initialize(mf_mg_matrix[level]);
          mg::Matrix<MatrixFreeLevelVector> mg_interface(mg_interface_matrices);

          Multigrid<MatrixFreeLevelVector> mg(
            mg_m, coarse_grid_solver, mg_transfer, smoother, smoother);
          mg.set_edge_matrices(mg_interface, mg_interface);

          PreconditionMG<dim,
                         MatrixFreeLevelVector,
                         MGTransferMatrixFree<dim, float>>
            preconditioner(dof_handler, mg, mg_transfer);

// 将求解向量和右手边从 LA::MPI::Vector 复制到 dealii::LinearAlgebra::distributed::Vector ，这样我们就可以解决了。

          MatrixFreeActiveVector solution_copy;
          MatrixFreeActiveVector right_hand_side_copy;
          mf_system_matrix.initialize_dof_vector(solution_copy);
          mf_system_matrix.initialize_dof_vector(right_hand_side_copy);

          ChangeVectorTypes::copy(solution_copy, solution);
          ChangeVectorTypes::copy(right_hand_side_copy, right_hand_side);
          computing_timer.leave_subsection("Solve: Preconditioner setup");

// 1个V型周期的时间安排。

          {
            TimerOutput::Scope timing(computing_timer,
                                      "Solve: 1 multigrid V-cycle");
            preconditioner.vmult(solution_copy, right_hand_side_copy);
          }
          solution_copy = 0.;

// 解出线性系统，更新解的鬼魂值，复制回 LA::MPI::Vector 并分配约束。

          {
            SolverCG<MatrixFreeActiveVector> solver(solver_control);

            TimerOutput::Scope timing(computing_timer, "Solve: CG");
            solver.solve(mf_system_matrix,
                         solution_copy,
                         right_hand_side_copy,
                         preconditioner);
          }

          solution_copy.update_ghost_values();
          ChangeVectorTypes::copy(solution, solution_copy);
          constraints.distribute(solution);

          break;
        }

// 基于矩阵的GMG方法的求解器，类似于  step-16  ，只是使用了雅可比平滑器，而不是SOR平滑器（该平滑器没有并行实现）。

      case Settings::gmg_mb:
        {
          computing_timer.enter_subsection("Solve: Preconditioner setup");

          MGTransferPrebuilt<VectorType> mg_transfer(mg_constrained_dofs);
          mg_transfer.build(dof_handler);

          SolverControl        coarse_solver_control(1000, 1e-12, false, false);
          SolverCG<VectorType> coarse_solver(coarse_solver_control);
          PreconditionIdentity identity;
          MGCoarseGridIterativeSolver<VectorType,
                                      SolverCG<VectorType>,
                                      MatrixType,
                                      PreconditionIdentity>
            coarse_grid_solver(coarse_solver, mg_matrix[0], identity);

          using Smoother = LA::MPI::PreconditionJacobi;
          MGSmootherPrecondition<MatrixType, Smoother, VectorType> smoother;

#ifdef USE_PETSC_LA
          smoother.initialize(mg_matrix);
          Assert(
            settings.smoother_dampen == 1.0,
            ExcNotImplemented(
              "PETSc's PreconditionJacobi has no support for a damping parameter."));
#else
          smoother.initialize(mg_matrix, settings.smoother_dampen);
#endif

          smoother.set_steps(settings.smoother_steps);

          mg::Matrix<VectorType> mg_m(mg_matrix);
          mg::Matrix<VectorType> mg_in(mg_interface_in);
          mg::Matrix<VectorType> mg_out(mg_interface_in);

          Multigrid<VectorType> mg(
            mg_m, coarse_grid_solver, mg_transfer, smoother, smoother);
          mg.set_edge_matrices(mg_out, mg_in);

          PreconditionMG<dim, VectorType, MGTransferPrebuilt<VectorType>>
            preconditioner(dof_handler, mg, mg_transfer);

          computing_timer.leave_subsection("Solve: Preconditioner setup");

// 1个V型周期的计时。

          {
            TimerOutput::Scope timing(computing_timer,
                                      "Solve: 1 multigrid V-cycle");
            preconditioner.vmult(solution, right_hand_side);
          }
          solution = 0.;

// 解决线性系统和分配约束。

          {
            SolverCG<VectorType> solver(solver_control);

            TimerOutput::Scope timing(computing_timer, "Solve: CG");
            solver.solve(system_matrix,
                         solution,
                         right_hand_side,
                         preconditioner);
          }

          constraints.distribute(solution);

        }

// AMG方法的求解器，类似于  step-40  。

      case Settings::amg:
        {
          computing_timer.enter_subsection("Solve: Preconditioner setup");

          PreconditionAMG                 preconditioner;
          PreconditionAMG::AdditionalData Amg_data;

#ifdef USE_PETSC_LA
          Amg_data.symmetric_operator = true;
#else
          Amg_data.elliptic              = true;
          Amg_data.smoother_type         = "Jacobi";
          Amg_data.higher_order_elements = true;
          Amg_data.smoother_sweeps       = settings.smoother_steps;
          Amg_data.aggregation_threshold = 0.02;
#endif

          Amg_data.output_details = false;

          preconditioner.initialize(system_matrix, Amg_data);
          computing_timer.leave_subsection("Solve: Preconditioner setup");

// 1个V型周期的计时。

          {
            TimerOutput::Scope timing(computing_timer,
                                      "Solve: 1 multigrid V-cycle");
            preconditioner.vmult(solution, right_hand_side);
          }
          solution = 0.;

// 解决线性系统和分配约束。

          {
            SolverCG<VectorType> solver(solver_control);

            TimerOutput::Scope timing(computing_timer, "Solve: CG");
            solver.solve(system_matrix,
                         solution,
                         right_hand_side,
                         preconditioner);
          }
          constraints.distribute(solution);

          break;
        }

      default:
        Assert(false, ExcInternalError());
    }

  pcout << "   Number of CG iterations:      " << solver_control.last_step()
        << std::endl;
}
// @sect3{The error estimator}

// 我们使用FEInterfaceValues类来组装一个误差估计器，以决定哪些单元需要细化。请看介绍中对单元和面积分的确切定义。为了使用该方法，我们为 MeshWorker::mesh_loop() 定义了Scratch和Copy对象，下面的大部分代码本质上与 step-12 中已经设置的一样（或者至少精神上相似）。

template <int dim>
struct ScratchData
{
  ScratchData(const Mapping<dim> &      mapping,
              const FiniteElement<dim> &fe,
              const unsigned int        quadrature_degree,
              const UpdateFlags         update_flags,
              const UpdateFlags         interface_update_flags)
    : fe_values(mapping, fe, QGauss<dim>(quadrature_degree), update_flags)
    , fe_interface_values(mapping,
                          fe,
                          QGauss<dim - 1>(quadrature_degree),
                          interface_update_flags)
  {}

  ScratchData(const ScratchData<dim> &scratch_data)
    : fe_values(scratch_data.fe_values.get_mapping(),
                scratch_data.fe_values.get_fe(),
                scratch_data.fe_values.get_quadrature(),
                scratch_data.fe_values.get_update_flags())
    , fe_interface_values(scratch_data.fe_values.get_mapping(),
                          scratch_data.fe_values.get_fe(),
                          scratch_data.fe_interface_values.get_quadrature(),
                          scratch_data.fe_interface_values.get_update_flags())
  {}

  FEValues<dim>          fe_values;
  FEInterfaceValues<dim> fe_interface_values;
};

struct CopyData
{
  CopyData()
    : cell_index(numbers::invalid_unsigned_int)
    , value(0.)
  {}

  CopyData(const CopyData &) = default;

  struct FaceData
  {
    unsigned int cell_indices[2];
    double       values[2];
  };

  unsigned int          cell_index;
  double                value;
  std::vector<FaceData> face_data;
};

template <int dim, int degree>
void LaplaceProblem<dim, degree>::estimate()
{
  TimerOutput::Scope timing(computing_timer, "Estimate");

  VectorType temp_solution;
  temp_solution.reinit(locally_owned_dofs,
                       locally_relevant_dofs,
                       mpi_communicator);
  temp_solution = solution;

  const Coefficient<dim> coefficient;

  estimated_error_square_per_cell.reinit(triangulation.n_active_cells());

  using Iterator = typename DoFHandler<dim>::active_cell_iterator;

// 剩余单元的汇编程序  $h^2 \| f + \epsilon \triangle u \|_K^2$  。
  auto cell_worker = [&](const Iterator &  cell,
                         ScratchData<dim> &scratch_data,
                         CopyData &        copy_data) {
    FEValues<dim> &fe_values = scratch_data.fe_values;
    fe_values.reinit(cell);

    RightHandSide<dim> rhs;
    const double       rhs_value = rhs.value(cell->center());

    const double nu = coefficient.value(cell->center());

    fe_values.get_function_hessians(temp_solution, hessians);

    copy_data.cell_index = cell->active_cell_index();

    double residual_norm_square = 0.;
    for (unsigned k = 0; k < fe_values.n_quadrature_points; ++k)
      {
        const double residual = (rhs_value + nu * trace(hessians[k]));
        residual_norm_square += residual * residual * fe_values.JxW(k);
      }

    copy_data.value =
      cell->diameter() * cell->diameter() * residual_norm_square;
  };

// 脸部术语的汇编器  $\sum_F h_F \| \jump{\epsilon \nabla u \cdot n} \|_F^2$  。
  auto face_worker = [&](const Iterator &    cell,
                         const unsigned int &f,
                         const unsigned int &sf,
                         const Iterator &    ncell,
                         const unsigned int &nf,
                         const unsigned int &nsf,
                         ScratchData<dim> &  scratch_data,
                         CopyData &          copy_data) {
    FEInterfaceValues<dim> &fe_interface_values =
      scratch_data.fe_interface_values;
    fe_interface_values.reinit(cell, f, sf, ncell, nf, nsf);

    copy_data.face_data.emplace_back();
    CopyData::FaceData &copy_data_face = copy_data.face_data.back();

    copy_data_face.cell_indices[0] = cell->active_cell_index();
    copy_data_face.cell_indices[1] = ncell->active_cell_index();

    const double coeff1 = coefficient.value(cell->center());
    const double coeff2 = coefficient.value(ncell->center());

    std::vector<Tensor<1, dim>> grad_u[2];

    for (unsigned int i = 0; i < 2; ++i)
      {
        grad_u[i].resize(fe_interface_values.n_quadrature_points);
        fe_interface_values.get_fe_face_values(i).get_function_gradients(
          temp_solution, grad_u[i]);
      }

    double jump_norm_square = 0.;

    for (unsigned int qpoint = 0;
         qpoint < fe_interface_values.n_quadrature_points;
         ++qpoint)
      {
        const double jump =
          coeff1 * grad_u[0][qpoint] * fe_interface_values.normal(qpoint) -
          coeff2 * grad_u[1][qpoint] * fe_interface_values.normal(qpoint);

        jump_norm_square += jump * jump * fe_interface_values.JxW(qpoint);
      }

    const double h           = cell->face(f)->measure();
    copy_data_face.values[0] = 0.5 * h * jump_norm_square;
    copy_data_face.values[1] = copy_data_face.values[0];
  };

  auto copier = [&](const CopyData &copy_data) {
    if (copy_data.cell_index != numbers::invalid_unsigned_int)
      estimated_error_square_per_cell[copy_data.cell_index] += copy_data.value;

    for (auto &cdf : copy_data.face_data)
      for (unsigned int j = 0; j < 2; ++j)
        estimated_error_square_per_cell[cdf.cell_indices[j]] += cdf.values[j];
  };

  const unsigned int n_gauss_points = degree + 1;
  ScratchData<dim>   scratch_data(mapping,
                                fe,
                                n_gauss_points,
                                update_hessians | update_quadrature_points |
                                  update_JxW_values,
                                update_values | update_gradients |
                                  update_JxW_values | update_normal_vectors);
  CopyData           copy_data;

// 我们需要对每个内部面进行一次装配，但我们需要确保两个进程都对本地拥有的单元和幽灵单元之间的面术语进行装配。这可以通过设置 MeshWorker::assemble_ghost_faces_both 标志来实现。我们需要这样做，因为我们不在这里交流误差估计器的贡献。

  MeshWorker::mesh_loop(dof_handler.begin_active(),
                        dof_handler.end(),
                        cell_worker,
                        copier,
                        scratch_data,
                        copy_data,
                        MeshWorker::assemble_own_cells |
                          MeshWorker::assemble_ghost_faces_both |
                          MeshWorker::assemble_own_interior_faces_once,
                          /*boundary_worker=*/nullptr, face_worker);

  const double global_error_estimate =
    std::sqrt(Utilities::MPI::sum(estimated_error_square_per_cell.l1_norm(),
                                  mpi_communicator));
  pcout << "   Global error estimate:        " << global_error_estimate
        << std::endl;
}
// @sect4{LaplaceProblem::refine_grid()}

// 我们使用存储在向量 @p estimate_vector 中的单元估计器，并细化固定数量的单元（这里选择的是每一步中大约两倍的DoFs数量）。

template <int dim, int degree>
void LaplaceProblem<dim, degree>::refine_grid()
{
  TimerOutput::Scope timing(computing_timer, "Refine grid");

  const double refinement_fraction = 1. / (std::pow(2.0, dim) - 1.);
  parallel::distributed::GridRefinement::refine_and_coarsen_fixed_number(
    triangulation, estimated_error_square_per_cell, refinement_fraction, 0.0);

  triangulation.execute_coarsening_and_refinement();
}
// @sect4{LaplaceProblem::output_results()}

// output_results()函数与许多教程中的函数类似（例如，见 step-40 ）。

template <int dim, int degree>
void LaplaceProblem<dim, degree>::output_results(const unsigned int cycle)
{
  TimerOutput::Scope timing(computing_timer, "Output results");

  VectorType temp_solution;
  temp_solution.reinit(locally_owned_dofs,
                       locally_relevant_dofs,
                       mpi_communicator);
  temp_solution = solution;

  DataOut<dim> data_out;
  data_out.attach_dof_handler(dof_handler);
  data_out.add_data_vector(temp_solution, "solution");

  Vector<float> subdomain(triangulation.n_active_cells());
  for (unsigned int i = 0; i < subdomain.size(); ++i)
    subdomain(i) = triangulation.locally_owned_subdomain();
  data_out.add_data_vector(subdomain, "subdomain");

  Vector<float> level(triangulation.n_active_cells());
  for (const auto &cell : triangulation.active_cell_iterators())
    level(cell->active_cell_index()) = cell->level();
  data_out.add_data_vector(level, "level");

  if (estimated_error_square_per_cell.size() > 0)
    data_out.add_data_vector(estimated_error_square_per_cell,
                             "estimated_error_square_per_cell");

  data_out.build_patches();

  const std::string pvtu_filename = data_out.write_vtu_with_pvtu_record(
    "", "solution", cycle, mpi_communicator, 2 /*n_digits*/, 1 /*n_groups*/);

  pcout << "   Wrote " << pvtu_filename << std::endl;
}
// @sect4{LaplaceProblem::run()}

// 和大多数教程一样，这个函数调用上面定义的各种函数来设置、组合、求解和输出结果。

template <int dim, int degree>
void LaplaceProblem<dim, degree>::run()
{
  for (unsigned int cycle = 0; cycle < settings.n_steps; ++cycle)
    {
      pcout << "Cycle " << cycle << ':' << std::endl;
      if (cycle > 0)
        refine_grid();

      pcout << "   Number of active cells:       "
            << triangulation.n_global_active_cells();

// 我们只为GMG方法输出层次单元数据（与下面的DoF数据相同）。请注意，对于AMG来说，分区效率是不相关的，因为在计算过程中没有分布或使用层次结构。

      if (settings.solver == Settings::gmg_mf ||
          settings.solver == Settings::gmg_mb)
        pcout << " (" << triangulation.n_global_levels() << " global levels)"
              << std::endl
              << "   Partition efficiency:         "
              << 1.0 / MGTools::workload_imbalance(triangulation);
      pcout << std::endl;

      setup_system();

// 只为GMG设置多级层次结构。

      if (settings.solver == Settings::gmg_mf ||
          settings.solver == Settings::gmg_mb)
        setup_multigrid();

      pcout << "   Number of degrees of freedom: " << dof_handler.n_dofs();
      if (settings.solver == Settings::gmg_mf ||
          settings.solver == Settings::gmg_mb)
        {
          pcout << " (by level: ";
          for (unsigned int level = 0; level < triangulation.n_global_levels();
               ++level)
            pcout << dof_handler.n_dofs(level)
                  << (level == triangulation.n_global_levels() - 1 ? ")" :
                                                                     ", ");
        }
      pcout << std::endl;

// 对于无矩阵的方法，我们只组装右手边。对于这两种基于矩阵的方法，我们同时装配主动矩阵和右手边，对于基于矩阵的GMG，我们只装配多网格矩阵。

      if (settings.solver == Settings::gmg_mf)
        assemble_rhs();
      else /*gmg_mb or amg*/
        {
          assemble_system();
          if (settings.solver == Settings::gmg_mb)
            assemble_multigrid();
        }

      solve();
      estimate();

      if (settings.output)
        output_results(cycle);

      computing_timer.print_summary();
      computing_timer.reset();
    }
}
// @sect3{The main() function}

// 这是一个类似于 step-40 的主函数，但我们要求用户传递一个.prm文件作为唯一的命令行参数（参见 step-29 和ParameterHandler类的文档，以了解关于参数文件的完整讨论）。

int main(int argc, char *argv[])
{
  using namespace dealii;
  Utilities::MPI::MPI_InitFinalize mpi_initialization(argc, argv, 1);

  Settings settings;
  if (!settings.try_parse((argc > 1) ? (argv[1]) : ""))
    return 0;

  try
    {
      constexpr unsigned int fe_degree = 2;

      switch (settings.dimension)
        {
          case 2:
            {
              LaplaceProblem<2, fe_degree> test(settings);
              test.run();

              break;
            }

          case 3:
            {
              LaplaceProblem<3, fe_degree> test(settings);
              test.run();

              break;
            }

          default:
            Assert(false, ExcMessage("This program only works in 2d and 3d."));
        }
    }
  catch (std::exception &exc)
    {
      std::cerr << std::endl
                << std::endl
                << "----------------------------------------------------"
                << std::endl;
      std::cerr << "Exception on processing: " << std::endl
                << exc.what() << std::endl
                << "Aborting!" << std::endl
                << "----------------------------------------------------"
                << std::endl;
      MPI_Abort(MPI_COMM_WORLD, 1);
      return 1;
    }
  catch (...)
    {
      std::cerr << std::endl
                << std::endl
                << "----------------------------------------------------"
                << std::endl;
      std::cerr << "Unknown exception!" << std::endl
                << "Aborting!" << std::endl
                << "----------------------------------------------------"
                << std::endl;
      MPI_Abort(MPI_COMM_WORLD, 2);
      return 1;
    }

  return 0;
}


/**
 * $Id$
 *
 * Copyright (C)
 * 2015 - $Date$
 *     Martin Wolf <ndhist@martin-wolf.org>
 *
 * This file is distributed under the BSD 2-Clause Open Source License
 * (See LICENSE file).
 *
 */
#include <boost/python.hpp>

#include <ndhist/stats/skewness.hpp>

namespace bp = boost::python;

namespace ndhist {
namespace stats {

void register_skewness()
{
    bp::def("skewness"
      , &py::skewness
      , ( bp::arg("hist")
        , bp::arg("axis")=bp::object()
        )
      , "Calculates the skewness along the given axis of the given                \n"
        "ndhist object. As in statistics, the skewness is defined as              \n"
        ":math:`Skewness[x] = ( E[x^3] - 3 V[x] E[x] - E[x]^3 ) / \\sqrt{V[x]^3}`.\n"
        "This function generates a projection along the given axis and then       \n"
        "calculates the skewness.                                                 \n"
        "If ``None`` is given as axis argument (the default), the skewness        \n"
        "for all individual axes of the ndhist object is calculated and           \n"
        "returned as a tuple. But if the dimensionality of the histogram is       \n"
        "1, a scalar value is returned.                                           \n"
        "                                                                         \n"
        ".. note:: This function is only defined for ndhist objects with POD      \n"
        "          type axis values AND POD type weight values.                   \n"
    );
}

}// namespace stats
}// namespace ndhist


/**
 * @cond ___LICENSE___
 *
 * Copyright (c) 2016-2018 Zefiros Software.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 *
 * @endcond
 */
#include "sync/bench/mpiPingpong.h"
#include "sync/bench/partitioning.h"
#include "sync/sync.h"
#include "sync/util/algorithm.h"
#include "sync/util/json.h"

#include "bench.h"

#include "args/args.h"
#include "preproc/preproc.h"

#include <armadillo>

#include <numeric>

auto LeastSquares(const size_t h0, const size_t h1, const size_t multiplier, const std::vector<double> &t)
{
    /* This function computes the parameters g and l of the
    linear function T(h)= g*h+l that best fits
    the data points (h,t[h]) with h0<= h<= h1. */

    /* Compute sums:
    sumt  =  sum of t[h] over h0<= h<= h1
    sumth =         t[h]*h
    nh    =         1
    sumh  =         h
    sumhh =         h*h */

    const arma::vec subT = arma::vec(t).rows(h0, h1);
    const double sumt = arma::sum(subT);
    const double sumth = arma::sum(subT % SyncLib::Util::ArmaRange(h0, h1)) * multiplier;

    const size_t nh = h1 - h0 + 1;
    const double sumh = multiplier * (h1 * (h1 + 1.0) - (h0 - 1.0) * h0) / 2;
    const double sumhh = multiplier * multiplier * (h1 * (h1 + 1) * (2.0 * h1 + 1) - (h0 - 1) * h0 * (2.0 * h0 - 1)) / 6;

    /* Solve    nh*l +  sumh*g =  sumt
    sumh*l + sumhh*g =  sumth */

    const double a = nh / sumh; // nh<= sumh

    /* subtract a times second eqn from first eqn to obtain g */
    double g = (sumt - a * sumth) / (sumh - a * sumhh);

    /* use second eqn to obtain l */
    double l = (sumth - sumhh * g) / sumh;

    return std::make_tuple(g, l);
}

template <typename tTimer>
NOINLINE double MeasureR(tTimer &timer, size_t n, arma::vec &y, double alpha, const arma::vec &x, arma::vec &z, double beta,
                         size_t repetitions)
{
    /* Measure time of 2*repetitions DAXPY operations of length n */
    timer.Tic();

    constexpr size_t rep = 50;

    for (size_t iter = 0; iter < rep * repetitions; ++iter)
    {
        for (size_t i = 0; i < n; ++i)
        {
            y[i] += alpha * x[i];
        }

        for (size_t i = 0; i < n; ++i)
        {
            z[i] -= beta * x[i];
        }
    }

    return timer.Toc() / rep;
}

struct BenchReporter
{
    static void ReportR(const size_t n, const double minR, const double maxR, const double avR, const double fool)
    {
        fmt::print("n= {:>5} min= {:>8.3f} max= {:>8.3f} av= {:>8.3f} Mflop/s "
                   "fool={:>7.1f}\n",
                   n, minR, maxR, avR, fool);
    }

    static void ReportR0()
    {
        fmt::print("minimum time is 0\n");
    }

    static void ReportRelationH(const size_t h, const double time, const double flops, const bool printH)
    {
        if (printH)
        {
            fmt::print("Time of {:>5}-relation = {:>7.2f} microsec = {:>8.0f} flops\n", h, time, flops);
        }
    }

    template <typename tT>
    static void ReportSize()
    {
        fmt::print("size of {} = {} bytes\n", typeid(tT).name(), sizeof(tT));
    }

    template <typename tH0, typename tH1>
    static void ReportGL(tH0 h0, tH1 h1, double g, double l)
    {
        fmt::print("Range h={:>4} to {:>5}: g= {:>7.1f}, l= {:>7.1f}\n", h0, h1, g, l);
    }

    static void ReportBottomLine(const size_t p, const double rDivMega, const double g, const double l)
    {
        fmt::print("The bottom line for this BSP computer is:\n");
        fmt::print("p= {}, r= {:.3f} Mflop/s, g= {:.1f}, l= {:.1f}\n", p, rDivMega, g, l);
    }

    static size_t RequestP(const size_t maxP, const bool restrictP = true)
    {
        fmt::print("How many processors do you want to use?\n");
        const size_t p = 4;
        // std::cin>>p;

        if (restrictP && p > maxP)
        {
            fmt::print("Sorry, your requested {}, but only {} processors available.\n", p, maxP);
            exit(EXIT_FAILURE);
        }

        return p;
    }
};

template <typename tEnv>
void BspBench(tEnv &env, const size_t maxH, const size_t maxN, const size_t repetitions, const size_t batchSize,
              const std::string &output, const bool printH)
{

    SyncLib::Util::Timer<> timer;
    /**** Determine p ****/
    const size_t p = env.Size();
    const size_t s = env.Rank();

    typename tEnv::template SendQueue<double> timeQueue(env);
    typename tEnv::template SharedArray<double> dest(env, 2 * maxH + p);
    std::vector<double> hTime;

    /**** Determine r ****/
    double r = 0.0;

    arma::vec x(maxN), y(maxN), z(maxN);

    for (size_t n = 1; n <= maxN; n *= 2)
    {
        /* Initialize scalars and vectors */
        const double alpha = 1.0 / 3.0;
        const double beta = 4.0 / 9.0;

        for (size_t i = 0; i < n; ++i)
        {
            const auto xyz = static_cast<double>(i);
            z[i] = xyz;
            y[i] = xyz;
            x[i] = xyz;
        }

        double time = MeasureR(timer, n, y, alpha, x, z, beta, repetitions);

        timeQueue.Send(0, time);
        env.Sync();

        /* Processor 0 determines minimum, maximum, average computing rate */
        if (s == 0)
        {
            std::vector<double> timeCopy = timeQueue.ToVector();

            auto [mintime, maxtime] = SyncLib::Util::MinMax(timeCopy.begin(), timeCopy.end());

            if (mintime > 0.0)
            {
                /* Compute r = average computing rate in flop/s */
                const size_t nflops = 4 * repetitions * n;
                mintime = nflops / (mintime * std::mega::num);
                maxtime = nflops / (maxtime * std::mega::num);

                r = static_cast<double>(nflops) / arma::mean(arma::vec(timeCopy));

                BenchReporter::ReportR(n, mintime, maxtime, r / std::mega::num, y[n - 1] + z[n - 1]);
            }
            else
            {
                BenchReporter::ReportR0();
            }
        }
    }

    /* r is taken as the value at length maxN */

    /**** Determine g and l ****/
    std::vector<size_t> destproc(maxH), destindex(maxH);
    std::vector<double> src(maxH), t(maxH + 1);

    typename tEnv::template SendQueue<size_t, double> queue(env);

    for (size_t h = 0; h <= maxH; ++h)
    {
        /* Initialize communication pattern */
        for (size_t i = 0; i < h; ++i)
        {
            src[i] = static_cast<double>(i);

            if (p == 1)
            {
                destproc[i] = 0;
                destindex[i] = i;
            }
            else
            {
                /* destination processor is one of the p-1 others */
                destproc[i] = (s + 1 + (i % (p - 1))) % p;
                /* destination index is in my own part of dest */
                destindex[i] = s + (i / (p - 1)) * p;
            }
        }

        /* Measure time of repetitions h-relations */
        env.Sync();
        timer.Tic();

        for (size_t iter = 0; iter < repetitions; iter++)
        {
            for (size_t i = 0; i < h; ++i)
            {
                for (size_t l = 0; l < batchSize; ++l)
                {
                    // dest.PutValue(destproc[i], src[i], destindex[i]);
                    queue.Send(destproc[i], destindex[i], src[i]);
                    // env.Print("Send: {}, {}, {}\n", destproc[i], destindex[i], src[i]);
                }
            }

            env.Sync();

            for (auto [index, val] : queue)
            {
                dest.Value()[index] = val;
                // env.Print("Recv: {}, {}, {}\n", s, index, val);
            }
        }

        const double time = timer.Toc();
        hTime.push_back(time * std::mega::num / repetitions);

        /* Compute time of one h-relation */
        if (s == 0)
        {
            // time in flop units
            t[h] = (time * r) / repetitions;
            BenchReporter::ReportRelationH(h, (time * std::mega::num) / repetitions, t[h], printH || h == maxH);
        }
    }

    if (s == 0)
    {
        const auto [g0, l0] = LeastSquares(0, p, batchSize, t);
        const auto [g, l] = LeastSquares(p, maxH, batchSize, t);

        const double flops = maxH * batchSize * g + l;
        BenchReporter::ReportRelationH(maxH, flops * std::mega::num / r, flops, true);
        BenchReporter::ReportSize<double>();
        BenchReporter::ReportGL(0, "p", g0, l0);
        BenchReporter::ReportGL("p", maxH, g, l);
        BenchReporter::ReportBottomLine(p, r / std::mega::num, g, l);

        if (!output.empty())
        {
            std::ofstream benchResults(output);
            nlohmann::json dump;
            dump["timings"] = arma::vec(t);
            dump["r"] = r;
            dump["g"] = g;
            dump["l"] = l;
            benchResults << dump;
            benchResults.flush();
        }
    }
}

#ifdef IS_WINDOWS
void Pause(bool pause = true)
{

    if (pause)
    {
        system("pause");
    }
}
#else
void Pause(bool)
{
}
#endif

template <typename tEnv/*, typename... tArgs*/>
void RunBenchmark(Args &parser, tEnv &env /*bool pause, tArgs &&... args*/)
{
    Pause(env.Rank() == 0 && parser.GetOption("start-paused"));
    uint32_t maxH = parser.GetOption("h");
    uint32_t maxN = parser.GetOption("n");
    uint32_t niters = parser.GetOption("i");
    uint32_t batchSize = parser.GetOption("b");
    uint32_t parts = parser.GetOption("parts");
    auto outputOption = parser.GetOption("o");
    std::string output;

    if (outputOption.Count() > 0)
    {
        output = outputOption.Get<std::string>();
    }

    auto printOptions = parser.GetOption("print");
    bool printH = false;

    if (printOptions.Count() > 0)
    {
        std::vector<std::string> prints = printOptions.Get<std::vector<std::string>>();

        for (auto &o : prints)
        {
            if (o == "h" || o == "h-relation")
            {
                printH = true;
            }
        }
    }

    // tEnv env(std::forward<tArgs>(args)...);

    if (parts > 1)
    {
        const auto &splitBench = [&](tEnv & env0)
        {
            size_t partSize = (env0.Size() + parts - 1) / parts;
            auto &env1 = env0.Split(env0.Rank() / partSize, env0.Rank());
            BspBench<tEnv>(env1, maxH, maxN, niters, batchSize, output, printH);
        };
        env.Run(splitBench);
    }
    else
    {
        env.Run(BspBench<tEnv>, maxH, maxN, niters, batchSize, output, printH);
    }

    Pause(env.Rank() == 0 && parser.GetOption("exit-paused"));
}

#if defined(_DEBUG)
constexpr bool IsDebugBuild = true;
#elif defined(NDEBUG)
constexpr bool IsDebugBuild = false;
#else
constexpr bool IsDebugBuild = true;
#endif

template <typename tEnv>
double InnerProductPut(tEnv &env, std::vector<double> &x, std::vector<double> &y)
{
    size_t p = env.Size();
    size_t s = env.Rank();

    // using shared array with put for partial inner products
    typename tEnv::template SharedArray<double> partialInnerProducts(env, p);

    double alpha = 0.0;

    for (size_t i = 0, iEnd = x.size(); i < iEnd; ++i)
    {
        alpha += x[i] * y[i];
    }

    for (size_t t = 0; t < p; ++t)
    {
        partialInnerProducts.PutValue(t, alpha, s);
    }

    env.Sync();

    // Sum the partial inner products
    return std::accumulate(partialInnerProducts.begin(), partialInnerProducts.end(), 0.0);
}

template <typename tEnv>
double InnerProduct(tEnv &env, std::vector<double> &x, std::vector<double> &y)
{
    const size_t p = env.Size();

    // using shared array with put for partial inner products
    typename tEnv::template SendQueue<double> partialInnerProducts(env);

    double alpha = 0.0;

    for (size_t i = 0, iEnd = x.size(); i < iEnd; ++i)
    {
        alpha += x[i] * y[i];
    }

    for (size_t t = 0; t < p; ++t)
    {
        partialInnerProducts.Send(t, alpha);
    }

    env.Sync();

    // Sum the partial inner products
    return std::accumulate(partialInnerProducts.begin(), partialInnerProducts.end(), 0.0);
}

template <typename tEnv>
void BaselProblem(tEnv &env, size_t n)
{
    const size_t p = env.Size();
    const size_t s = env.Rank();

    const size_t nl = (n + p - s - 1) / p;
    std::vector<double> x(nl);

    for (size_t i = 0; i < nl; ++i)
    {
        const size_t iGlob = i * p + s;
        x[i] = 1.0 / (iGlob + 1);
    }

    SyncLib::Util::Timer<std::chrono::microseconds> timer;

    // Let everyone start at the same time
    env.Barrier();
    timer.Tic();

    double alpha = 0.0;

    for (size_t j = 0; j < 1000; ++j)
    {
        alpha = InnerProduct(env, x, x);
    }

    // Measure the time when everyone is done
    env.Barrier();
    const double elapsed = timer.Toc() / 1000;

    printf("Processor %zd: solution to the Basel problem "
           "with reciprocals up to 1/%zd^2 is %.6f\n",
           s, n, alpha);

    if (s == 0)
    {
        printf("This took only %.6lf microseconds.\n", elapsed);
    }
}

// int main(int argc, char **argv)
// {
//     using tEnv = SyncLib::Environments::SharedMemoryBSP;
//     tEnv env(4);
//
//     // n can be requested from the user using your favorite method.
//     // You can also request if for s=0 inside the function.
//     size_t n = 1000000;
//
//     env.Run(BaselProblem<tEnv>, n);
//
//     return 0;
// }

int main(int argc, char **argv)
{
    Args args("bench", "Edupack benchmark for SyncLib");

    std::string iterations(IsDebugBuild ? "100" : "1000");

    args.AddOptions(
    {
        {
            "start-paused",
            "Whether we want to pause before the benchmark starts",
            Option::Boolean(),
            "true",
            "false",
        },
        {
            "exit-paused",
            "Whether we want to pause before the benchmark exits",
            Option::Boolean(),
            "true",
            "false",
        },
        {
            { "i", "iterations" },
            "The number of iterations for each h-relation",
            Option::U32(),
            iterations,
            iterations,
        },
        {
            { "h", "maximum-h" },
            "The maximum h-relation",
            Option::U32(),
            "256",
            "256",
        },
        {
            "print",
            "Fields to print",
            Option::StringList(),
            {},
            {},
        },
        {
            { "n", "maximum-n" },
            "The maximum number of DAXPYs for measuring r",
            Option::U32(),
            "1024",
            "1024",
        },
        {
            { "o", "output-file" },
            "The file to store the output in",
            Option::String(),
        },
        {
            "parts",
            "The number of parts to split the environment in",
            Option::U32(),
            "1",
            "1",
        },
    });

    SyncLib::MPI::Comm comm(argc, argv);

    {
        std::string batchSize = comm.Size() > 1 ? "8" : "1";

        args.AddOptions(
        {
            {
                { "b", "batch-size" },
                "Size of each communication packet",
                Option::U32(),
                batchSize,
                batchSize,
            },
        });
    }

    if (comm.Size() > 1)
    {
        //         char parsed = false;
        //
        //         if (comm.Rank() == 0)
        //         {
        //             args.Parse(argc, argv);
        //             parsed = true;
        //         }
        //
        //         comm.Broadcast(parsed, 0);
        //
        //         if (parsed && comm.Rank() != 0)
        //         {
        //             args.Parse(argc, argv);
        //         }
        //
        //         // SyncLib::Internal::PinThread(comm.Rank());
        //
        //         // const uint32_t parts = args.GetOption("parts");
        //         //         Pause(comm.Rank() == 0);
        //         //         comm.Barrier();
        using tEnv = SyncLib::Environments::DistributedBSP;
        //         tEnv env(argc, argv);
        //         env.Split(0, env.Size() - env.Rank());
        //
        //         if (parts > 1)
        //         {
        //             const size_t partSize = (comm.Size() + parts - 1) / parts;
        //             auto comm2 = comm.Split(comm.Rank() / partSize, comm.Rank() % partSize);
        //             RunBenchmark<tEnv>(args, comm2.Rank() == 0, comm2);
        //         }
        //         else
        {
            tEnv env(comm);
            args.Parse(argc, argv);
            RunBenchmark<tEnv>(args, env);
        }
    }
    else
    {
        using tEnv = SyncLib::Environments::SharedMemoryBSP;
        {
            std::string p = std::to_string(tEnv::MaxSize());

            args.AddOptions(
            {
                {
                    { "p", "processors" },
                    "The number of processors",
                    Option::U32(),
                    p,
                    p,
                },
            });
        }

        args.Parse(argc, argv);
        uint32_t p = args.GetOption("processors");
        tEnv env(p);

        //const uint32_t parts = args.GetOption("parts");

        RunBenchmark<tEnv>(args, env);
    }

    // fflush(stdout);
    comm.Barrier();

    return 0;
}


//  Copyright John Maddock 2007.
//  Copyright Paul a. Bristow 2010
//  Use, modification and distribution are subject to the
//  Boost Software License, Version 1.0. (See accompanying file
//  LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)

// Note that this file contains quickbook mark-up as well as code
// and comments, don't change any of the special comment mark-ups!

#ifdef _MSC_VER
# pragma warning (disable : 4100) // unreferenced formal parameters
#endif

#include <iostream>
using std::cout;  using std::endl; using std::cerr;

//[policy_eg_8

/*`
Suppose we want our own user-defined error handlers rather than the
any of the default ones supplied by the library to be used.
If we set the policy for a specific type of error to `user_error`
then the library will call a user-supplied error handler.
These are forward declared, but not defined in
boost/math/policies/error_handling.hpp like this:

   namespace boost{ namespace math{ namespace policies{

   template <class T>
   T user_domain_error(const char* function, const char* message, const T& val);
   template <class T>
   T user_pole_error(const char* function, const char* message, const T& val);
   template <class T>
   T user_overflow_error(const char* function, const char* message, const T& val);
   template <class T>
   T user_underflow_error(const char* function, const char* message, const T& val);
   template <class T>
   T user_denorm_error(const char* function, const char* message, const T& val);
   template <class T>
   T user_evaluation_error(const char* function, const char* message, const T& val);
   template <class T, class TargetType>
   T user_rounding_error(const char* function, const char* message, const T& val, const TargetType& t);
   template <class T>
   T user_indeterminate_result_error(const char* function, const char* message, const T& val);

   }}} // namespaces

So out first job is to include the header we want to use, and then
provide definitions for our user-defined error handlers that we want to use.
We only provide our special domain and pole error handlers;
other errors like overflow and underflow use the default.
*/

#include <boost/math/special_functions.hpp>

namespace boost{ namespace math
{
  namespace policies
  {
    template <class T>
    T user_domain_error(const char* function, const char* message, const T& val)
    { // Ignoring function, message and val for this example, perhaps unhelpfully.
       cerr << "Domain Error!" << endl;
       return std::numeric_limits<T>::quiet_NaN();
    }

    template <class T>
    T user_pole_error(const char* function, const char* message, const T& val)
    { // Ignoring function, message and val for this example, perhaps unhelpfully.
       cerr << "Pole Error!" << endl;
       return std::numeric_limits<T>::quiet_NaN();
    }
  } // namespace policies
}} // namespace boost{ namespace math

/*`
Now we'll need to define a suitable policy that will call these handlers,
and define some forwarding functions that make use of the policy:
*/

namespace{

using namespace boost::math::policies;

typedef policy<
   domain_error<user_error>,
   pole_error<user_error>
> user_error_policy;

BOOST_MATH_DECLARE_SPECIAL_FUNCTIONS(user_error_policy)

} // close unnamed namespace

/*`
We now have a set of forwarding functions defined in an unnamed namespace
that all look something like this:

``
template <class RealType>
inline typename boost::math::tools::promote_args<RT>::type
   tgamma(RT z)
{
   return boost::math::tgamma(z, user_error_policy());
}
``

So that when we call `tgamma(z)` we really end up calling
`boost::math::tgamma(z, user_error_policy())`, and any
errors will get directed to our own error handlers.
*/

int main()
{
   cout << "Result of erf_inv(-10) is: "
      << erf_inv(-10) << endl;
   cout << "Result of tgamma(-10) is: "
      << tgamma(-10) << endl;
}

/*`

Which outputs:

[pre
  Domain Error!
  Pole Error!
  Result of erf_inv(-10) is: 1.#QNAN
  Result of tgamma(-10) is: 1.#QNAN
]
*/

//] // //[/policy_eg_8]


/// \file inner_product_grad.cc
/// \author David Stutz
/// \brief Implementation of the gradient of a inner product operation, see
/// inner_product.cc.

#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/shape_inference.h"

#include <sophus/so3.hpp>
#include <sophus/se3.hpp>
#include <Eigen/Dense>
#include <cmath>

using namespace Sophus;
using namespace Eigen;

// the gradients are simply passed as additional arguments as
// they are available in the Python function for registering the gradient operation.
REGISTER_OP("SE3toMatrixRtGrad")
  .Input("grad: float32")
  .Input("se3_vector: float32")
  .Output("grad_se3: float32");

namespace tensorflow{

using namespace tensorflow;

/// \brief Implementation of an inner product gradient operation.
/// Note that this operation is used in Python to register the gradient as
/// this is not possible in C*+ right now.
/// \param context
/// \author David Stutz
class SE3toMatrixRtGradOp : public OpKernel {
public:
  /// \brief Constructor.
  /// \param context
  explicit SE3toMatrixRtGradOp(OpKernelConstruction* context) : OpKernel(context) {

  }

  Eigen::Matrix<float,3,3> skew_symmetric(Eigen::Matrix<float,3,1>v)
  {
    Eigen::Matrix<float, 3,3>v_cross;
        v_cross << 0, -v(2,0), v(1,0),
                   v(2,0), 0, -v(0,0),
                  -v(1,0), v(0,0), 0;

    return v_cross;

  }

  /// \brief Compute the inner product gradients.
  /// \param context
  void Compute(OpKernelContext* context) override {

    // output and grad is provided as input
    DCHECK_EQ(2, context->num_inputs());

    // get the gradient tensor
    const Tensor& grad = context->input(0);

    // get the original input tensor
    const Tensor& se3_vector = context->input(1);

    // create input shape (inferred from the additional attribute `n`)
    TensorShape se3_vector_shape = se3_vector.shape();

    DCHECK_EQ(se3_vector_shape.dim_size(0), grad.shape().dim_size(0));

    // create output tensors
    Tensor* grad_se3 = NULL;
    OP_REQUIRES_OK(context, context->allocate_output(0, se3_vector_shape, &grad_se3));

    // get the Eigen tensors for data access
    auto grad_tensor = grad.tensor<float, 3>();
    auto se3_vector_tensor = se3_vector.tensor<float,2>();
    auto grad_se3_tensor = grad_se3->tensor<float,2>();

    for(int b = 0; b < se3_vector_shape.dim_size(0); b++)
    {
        auto v1 = se3_vector_tensor(b,0);
        auto v2 = se3_vector_tensor(b,1);
        auto v3 = se3_vector_tensor(b,2);

        auto v_mag = sqrt(v1*v1 + v2*v2 + v3*v3);

        Eigen::Matrix<float,3,1>vec;
        vec<< v1, v2, v3;

        Eigen::Matrix<float,3,3> R = SO3<float>::exp(vec).matrix();
        Eigen::Matrix<float, 3,3>v_cross = skew_symmetric(vec);
        Eigen::Matrix<float,3,3> I = Eigen::Matrix<float,3,3>::Identity(3,3);

        Eigen::Matrix<float, 3, 3>v_cross_I_minus_R = v_cross  * (I-R);

        Eigen::Matrix<float,3,1> e_i;
        e_i << 1, 0, 0;

        Eigen::Matrix<float, 3, 3>dR_dv1 = v1 * v_cross;
        Eigen::Matrix<float, 3, 3>v_cross_I_minus_R_times_e1 = skew_symmetric(v_cross_I_minus_R * e_i);

        dR_dv1 += v_cross_I_minus_R_times_e1;
        dR_dv1 *= R;
        dR_dv1 /= v_mag;

        e_i << 0, 1, 0;

        Eigen::Matrix<float, 3, 3>dR_dv2 = v2 * v_cross;
        Eigen::Matrix<float, 3, 3>v_cross_I_minus_R_times_e2 = skew_symmetric(v_cross_I_minus_R * e_i);

        dR_dv2 += v_cross_I_minus_R_times_e2;
        dR_dv2 *= R;
        dR_dv2 /= v_mag;

        e_i << 0, 0, 1;

        Eigen::Matrix<float, 3, 3>dR_dv3 = v3 * v_cross;
        Eigen::Matrix<float, 3, 3>v_cross_I_minus_R_times_e3 = skew_symmetric(v_cross_I_minus_R * e_i);

        dR_dv3 += v_cross_I_minus_R_times_e3;
        dR_dv3 *= R;
        dR_dv3 /= v_mag;

    }

  }
};

REGISTER_KERNEL_BUILDER(Name("SE3toMatrixRtGrad").Device(DEVICE_CPU), SE3toMatrixRtGradOp);
};