// Copyright (c) 2020 Chris Richardson & Matthew Scroggs
// FEniCS Project
// SPDX-License-Identifier:    MIT

#include "nce-rtc.h"
#include "core/dof-permutations.h"
#include "core/element-families.h"
#include "core/log.h"
#include "core/mappings.h"
#include "core/moments.h"
#include "core/polyset.h"
#include "core/quadrature.h"
#include "lagrange.h"
#include <Eigen/Dense>
#include <numeric>
#include <vector>

using namespace basix;

//----------------------------------------------------------------------------
FiniteElement basix::create_rtc(cell::type celltype, int degree)
{
  if (celltype != cell::type::quadrilateral
      and celltype != cell::type::hexahedron)
    throw std::runtime_error("Unsupported cell type");

  if (degree > 4)
  {
    // TODO: suggest alternative with non-uniform points once implemented
    LOG(WARNING) << "RTC spaces with high degree using equally spaced"
                 << " points are unstable.";
  }

  const int tdim = cell::topological_dimension(celltype);

  const cell::type facettype
      = (tdim == 2) ? cell::type::interval : cell::type::quadrilateral;

  // Evaluate the expansion polynomials at the quadrature points
  auto [Qpts, Qwts]
      = quadrature::make_quadrature("default", celltype, 2 * degree);
  Eigen::ArrayXXd polyset_at_Qpts
      = polyset::tabulate(celltype, degree, 0, Qpts)[0];

  // The number of order (degree) polynomials
  const int psize = polyset_at_Qpts.cols();

  const int facet_count = tdim == 2 ? 4 : 6;
  const int facet_dofs = polyset::dim(facettype, degree - 1);
  const int internal_dofs = tdim == 2 ? 2 * degree * (degree - 1)
                                      : 3 * degree * degree * (degree - 1);
  const int ndofs = facet_count * facet_dofs + internal_dofs;

  // Create coefficients for order (degree-1) vector polynomials
  Eigen::MatrixXd wcoeffs = Eigen::MatrixXd::Zero(ndofs, psize * tdim);

  const int nv_interval = polyset::dim(cell::type::interval, degree);
  const int ns_interval = polyset::dim(cell::type::interval, degree - 1);
  int dof = 0;
  if (tdim == 2)
  {
    for (int d = 0; d < tdim; ++d)
      for (int i = 0; i < ns_interval; ++i)
        for (int j = 0; j < ns_interval; ++j)
          wcoeffs(dof++, psize * d + i * nv_interval + j) = 1;
  }
  else
  {
    for (int d = 0; d < tdim; ++d)
      for (int i = 0; i < ns_interval; ++i)
        for (int j = 0; j < ns_interval; ++j)
          for (int k = 0; k < ns_interval; ++k)
            wcoeffs(dof++, psize * d + i * nv_interval * nv_interval
                               + j * nv_interval + k)
                = 1;
  }

  // Create coefficients for additional polynomials in the div space
  for (int i = 0; i < pow(degree, tdim - 1); ++i)
  {
    std::vector<int> indices(tdim - 1);
    if (tdim == 2)
      indices[0] = i;
    else
    {
      indices[0] = i / degree;
      indices[1] = i % degree;
    }
    for (int d = 0; d < tdim; ++d)
    {
      int n = 0;
      Eigen::ArrayXd integrand = Qpts.col(d);
      for (int j = 1; j < degree; ++j)
        integrand *= Qpts.col(d);
      for (int c = 0; c < tdim; ++c)
      {
        if (c != d)
        {
          for (int j = 0; j < indices[n]; ++j)
            integrand *= Qpts.col(c);
          ++n;
        }
      }
      for (int k = 0; k < psize; ++k)
      {
        const double w_sum = (Qwts * integrand * polyset_at_Qpts.col(k)).sum();
        wcoeffs(dof, k + psize * d) = w_sum;
      }
      ++dof;
    }
  }

  // Dual space
  Eigen::MatrixXd dual = Eigen::MatrixXd::Zero(ndofs, psize * tdim);

  // quadrature degree
  int quad_deg = 2 * degree;

  // Add rows to dualmat for integral moments on facets
  dual.block(0, 0, facet_count * facet_dofs, psize * tdim)
      = moments::make_normal_integral_moments(
          create_dlagrange(facettype, degree - 1), celltype, tdim, degree,
          quad_deg);

  // Add rows to dualmat for integral moments on interior
  if (degree > 1)
  {
    // Interior integral moment
    dual.block(facet_count * facet_dofs, 0, internal_dofs, psize * tdim)
        = moments::make_dot_integral_moments(create_nce(celltype, degree - 1),
                                             celltype, tdim, degree, quad_deg);
  }

  const std::vector<std::vector<std::vector<int>>> topology
      = cell::topology(celltype);

  int perm_count = 0;
  for (int i = 1; i < tdim; ++i)
    perm_count += topology[i].size() * i;

  std::vector<Eigen::MatrixXd> base_permutations(
      perm_count, Eigen::MatrixXd::Identity(ndofs, ndofs));
  if (tdim == 2)
  {
    Eigen::ArrayXi edge_ref = dofperms::interval_reflection(degree);
    Eigen::ArrayXXd edge_dir
        = dofperms::interval_reflection_tangent_directions(degree);
    for (int edge = 0; edge < facet_count; ++edge)
    {
      const int start = edge_ref.size() * edge;
      for (int i = 0; i < edge_ref.size(); ++i)
      {
        base_permutations[edge](start + i, start + i) = 0;
        base_permutations[edge](start + i, start + edge_ref[i]) = 1;
      }
      Eigen::MatrixXd directions = Eigen::MatrixXd::Identity(ndofs, ndofs);
      directions.block(edge_dir.rows() * edge, edge_dir.cols() * edge,
                       edge_dir.rows(), edge_dir.cols())
          = edge_dir;
      base_permutations[edge] *= directions;
    }
  }
  else if (tdim == 3)
  {
    Eigen::ArrayXi face_ref = dofperms::quadrilateral_reflection(degree);
    Eigen::ArrayXi face_rot = dofperms::quadrilateral_rotation(degree);

    for (int face = 0; face < facet_count; ++face)
    {
      const int start = face_ref.size() * face;
      for (int i = 0; i < face_rot.size(); ++i)
      {
        base_permutations[12 + 2 * face](start + i, start + i) = 0;
        base_permutations[12 + 2 * face](start + i, start + face_rot[i]) = 1;
        base_permutations[12 + 2 * face + 1](start + i, start + i) = 0;
        base_permutations[12 + 2 * face + 1](start + i, start + face_ref[i])
            = -1;
      }
    }
  }

  std::vector<std::vector<int>> entity_dofs(topology.size());
  for (int i = 0; i < tdim - 1; ++i)
    entity_dofs[i].resize(topology[i].size(), 0);
  entity_dofs[tdim - 1].resize(topology[tdim - 1].size(), facet_dofs);
  entity_dofs[tdim] = {internal_dofs};

  Eigen::MatrixXd coeffs = compute_expansion_coefficients(wcoeffs, dual);
  return FiniteElement(element::family::RT, celltype, degree, {tdim}, coeffs,
                       entity_dofs, base_permutations, {}, {},
                       mapping::type::contravariantPiola);
}
//-----------------------------------------------------------------------------
FiniteElement basix::create_nce(cell::type celltype, int degree)
{
  if (celltype != cell::type::quadrilateral
      and celltype != cell::type::hexahedron)
    throw std::runtime_error("Unsupported cell type");

  if (degree > 4)
  {
    // TODO: suggest alternative with non-uniform points once implemented
    LOG(WARNING) << "NC spaces with high degree using equally spaced"
                 << " points are unstable.";
  }

  const int tdim = cell::topological_dimension(celltype);

  // Evaluate the expansion polynomials at the quadrature points
  auto [Qpts, Qwts]
      = quadrature::make_quadrature("default", celltype, 2 * degree);
  Eigen::ArrayXXd polyset_at_Qpts
      = polyset::tabulate(celltype, degree, 0, Qpts)[0];

  // The number of order (degree) polynomials
  const int psize = polyset_at_Qpts.cols();

  const int edge_count = tdim == 2 ? 4 : 12;
  const int edge_dofs = polyset::dim(cell::type::interval, degree - 1);
  const int face_count = tdim == 2 ? 1 : 6;
  const int face_dofs = 2 * degree * (degree - 1);
  const int volume_count = tdim == 2 ? 0 : 1;
  const int volume_dofs = 3 * degree * (degree - 1) * (degree - 1);

  const int ndofs = edge_count * edge_dofs + face_count * face_dofs
                    + volume_count * volume_dofs;

  // Create coefficients for order (degree-1) vector polynomials
  Eigen::MatrixXd wcoeffs = Eigen::MatrixXd::Zero(ndofs, psize * tdim);

  const int nv_interval = polyset::dim(cell::type::interval, degree);
  const int ns_interval = polyset::dim(cell::type::interval, degree - 1);
  int dof = 0;
  if (tdim == 2)
  {
    for (int d = 0; d < tdim; ++d)
      for (int i = 0; i < ns_interval; ++i)
        for (int j = 0; j < ns_interval; ++j)
          wcoeffs(dof++, psize * d + i * nv_interval + j) = 1;
  }
  else
  {
    for (int d = 0; d < tdim; ++d)
      for (int i = 0; i < ns_interval; ++i)
        for (int j = 0; j < ns_interval; ++j)
          for (int k = 0; k < ns_interval; ++k)
            wcoeffs(dof++, psize * d + i * nv_interval * nv_interval
                               + j * nv_interval + k)
                = 1;
  }

  // Create coefficients for additional polynomials in the curl space
  if (tdim == 2)
  {
    for (int i = 0; i < degree; ++i)
    {
      for (int d = 0; d < tdim; ++d)
      {
        Eigen::ArrayXd integrand = Qpts.col(1 - d);
        for (int j = 1; j < degree; ++j)
          integrand *= Qpts.col(1 - d);
        for (int j = 0; j < i; ++j)
          integrand *= Qpts.col(d);

        for (int k = 0; k < psize; ++k)
        {
          const double w_sum
              = (Qwts * integrand * polyset_at_Qpts.col(k)).sum();
          wcoeffs(dof, k + psize * d) = w_sum;
        }
        ++dof;
      }
    }
  }
  else
  {
    for (int i = 0; i < degree; ++i)
    {
      for (int j = 0; j < degree + 1; ++j)
      {
        for (int c = 0; c < tdim; ++c)
        {
          for (int d = 0; d < tdim; ++d)
          {
            if (d != c)
            {
              const int e
                  = (c == 0 || d == 0) ? ((c == 1 || d == 1) ? 2 : 1) : 0;
              if (c < e and j == degree)
                continue;
              Eigen::ArrayXd integrand = Qpts.col(e);
              for (int k = 1; k < degree; ++k)
                integrand *= Qpts.col(e);
              for (int k = 0; k < i; ++k)
                integrand *= Qpts.col(d);
              for (int k = 0; k < j; ++k)
                integrand *= Qpts.col(c);

              for (int k = 0; k < psize; ++k)
              {
                const double w_sum
                    = (Qwts * integrand * polyset_at_Qpts.col(k)).sum();
                wcoeffs(dof, k + psize * d) = w_sum;
              }
              ++dof;
            }
          }
        }
      }
    }
  }

  // Dual space
  Eigen::MatrixXd dual = Eigen::MatrixXd::Zero(ndofs, psize * tdim);
  Eigen::ArrayXXd points;
  Eigen::MatrixXd matrix;

  // quadrature degree
  int quad_deg = 2 * degree;

  // Add rows to dualmat for integral moments on facets
  dual.block(0, 0, edge_count * edge_dofs, psize * tdim)
      = moments::make_tangent_integral_moments(
          create_dlagrange(cell::type::interval, degree - 1), celltype, tdim,
          degree, quad_deg);

  Eigen::ArrayXXd points_1d;
  Eigen::MatrixXd matrix_1d;
  std::tie(points_1d, matrix_1d)
      = moments::make_tangent_integral_moments_interpolation(
          create_dlagrange(cell::type::interval, degree - 1), celltype, tdim,
          degree, quad_deg);

  if (degree == 1)
  {
    points = points_1d;
    matrix = matrix_1d;
  }

  // Add rows to dualmat for integral moments on interior
  if (degree > 1)
  {
    // Face integral moment
    dual.block(edge_count * edge_dofs, 0, face_count * face_dofs, psize * tdim)
        = moments::make_dot_integral_moments(
            create_rtc(cell::type::quadrilateral, degree - 1), celltype, tdim,
            degree, quad_deg);

    Eigen::ArrayXXd points_2d;
    Eigen::MatrixXd matrix_2d;
    std::tie(points_2d, matrix_2d)
        = moments::make_dot_integral_moments_interpolation(
            create_rtc(cell::type::quadrilateral, degree - 1), celltype, tdim,
            degree, quad_deg);

    if (tdim == 2)
    {
      points.resize(points_1d.rows() + points_2d.rows(), tdim);
      matrix.resize(matrix_1d.rows() + matrix_2d.rows(),
                    matrix_1d.cols() + matrix_2d.cols());
      matrix.setZero();

      points.block(0, 0, points_1d.rows(), tdim) = points_1d;
      points.block(points_1d.rows(), 0, points_2d.rows(), tdim) = points_2d;

      const int r1d = matrix_1d.rows();
      const int r2d = matrix_2d.rows();
      const int c1d = matrix_1d.cols() / tdim;
      const int c2d = matrix_2d.cols() / tdim;
      for (int i = 0; i < tdim; ++i)
      {
        matrix.block(0, i * (c1d + c2d), r1d, c1d)
            = matrix_1d.block(0, i * c1d, r1d, c1d);
        matrix.block(r1d, i * (c1d + c2d) + c1d, r2d, c2d)
            = matrix_2d.block(0, i * c2d, r2d, c2d);
      }
    }

    if (tdim == 3)
    {
      // Interior integral moment
      dual.block(edge_count * edge_dofs + face_count * face_dofs, 0,
                 volume_dofs, psize * tdim)
          = moments::make_dot_integral_moments(
              create_rtc(cell::type::hexahedron, degree - 1), celltype, tdim,
              degree, quad_deg);

      Eigen::ArrayXXd points_3d;
      Eigen::MatrixXd matrix_3d;
      std::tie(points_3d, matrix_3d)
          = moments::make_dot_integral_moments_interpolation(
              create_rtc(cell::type::hexahedron, degree - 1), celltype, tdim,
              degree, quad_deg);

      points.resize(points_1d.rows() + points_2d.rows() + points_3d.rows(),
                    tdim);
      matrix.resize(matrix_1d.rows() + matrix_2d.rows() + matrix_3d.rows(),
                    matrix_1d.cols() + matrix_2d.cols() + matrix_3d.cols());
      matrix.setZero();

      points.block(0, 0, points_1d.rows(), tdim) = points_1d;
      points.block(points_1d.rows(), 0, points_2d.rows(), tdim) = points_2d;
      points.block(points_1d.rows() + points_2d.rows(), 0, points_3d.rows(),
                   tdim)
          = points_3d;

      const int r1d = matrix_1d.rows();
      const int r2d = matrix_2d.rows();
      const int r3d = matrix_3d.rows();
      const int c1d = matrix_1d.cols() / tdim;
      const int c2d = matrix_2d.cols() / tdim;
      const int c3d = matrix_3d.cols() / tdim;
      for (int i = 0; i < tdim; ++i)
      {
        matrix.block(0, i * (c1d + c2d + c3d), r1d, c1d)
            = matrix_1d.block(0, i * c1d, r1d, c1d);
        matrix.block(r1d, i * (c1d + c2d + c3d) + c1d, r2d, c2d)
            = matrix_2d.block(0, i * c2d, r2d, c2d);
        matrix.block(r1d + r2d, i * (c1d + c2d + c3d) + c1d + c2d, r3d, c3d)
            = matrix_3d.block(0, i * c3d, r3d, c3d);
      }
    }
  }

  const std::vector<std::vector<std::vector<int>>> topology
      = cell::topology(celltype);

  int perm_count = 0;
  for (int i = 1; i < tdim; ++i)
    perm_count += topology[i].size() * i;

  std::vector<Eigen::MatrixXd> base_permutations(
      perm_count, Eigen::MatrixXd::Identity(ndofs, ndofs));

  Eigen::ArrayXi edge_ref = dofperms::interval_reflection(degree);
  Eigen::ArrayXXd edge_dir
      = dofperms::interval_reflection_tangent_directions(degree);

  for (int edge = 0; edge < edge_count; ++edge)
  {
    const int start = edge_ref.size() * edge;
    for (int i = 0; i < edge_ref.size(); ++i)
    {
      base_permutations[edge](start + i, start + i) = 0;
      base_permutations[edge](start + i, start + edge_ref[i]) = 1;
    }
    Eigen::MatrixXd directions = Eigen::MatrixXd::Identity(ndofs, ndofs);
    directions.block(edge_dir.rows() * edge, edge_dir.cols() * edge,
                     edge_dir.rows(), edge_dir.cols())
        = edge_dir;
    base_permutations[edge] *= directions;
  }

  if (tdim == 3 and degree > 1)
  {
    Eigen::MatrixXd face_ref
        = dofperms::quadrilateral_rtc_reflection(degree - 1);
    Eigen::MatrixXd face_rot = dofperms::quadrilateral_rtc_rotation(degree - 1);

    for (int face = 0; face < face_count; ++face)
    {
      const int start = edge_ref.size() * edge_count + face_ref.rows() * face;
      const int p = edge_count + 2 * face;

      base_permutations[p].block(start, start, face_rot.rows(), face_rot.cols())
          = face_rot;
      base_permutations[p + 1].block(start, start, face_ref.rows(),
                                     face_ref.cols())
          = face_ref;
    }
  }

  std::vector<std::vector<int>> entity_dofs(topology.size());
  entity_dofs[0].resize(topology[0].size(), 0);
  entity_dofs[1].resize(topology[1].size(), edge_dofs);
  entity_dofs[2].resize(topology[2].size(), face_dofs);
  if (tdim == 3)
    entity_dofs[3].resize(topology[3].size(), volume_dofs);

  Eigen::MatrixXd coeffs = compute_expansion_coefficients(wcoeffs, dual);
  return FiniteElement(element::family::N1E, celltype, degree, {tdim}, coeffs,
                       entity_dofs, base_permutations, points, matrix,
                       mapping::type::covariantPiola);
}
//-----------------------------------------------------------------------------
Eigen::MatrixXd basix::dofperms::quadrilateral_rtc_rotation(int degree)
{
  const int n = 2 * degree * (degree + 1);
  Eigen::MatrixXd perm = Eigen::MatrixXd::Zero(n, n);

  // Permute functions on edges
  for (int i = 0; i < degree; ++i)
  {
    perm(i, 2 * degree - 1 - i) = -1;
    perm(degree + i, 3 * degree + i) = 1;
    perm(2 * degree + i, i) = 1;
    perm(3 * degree + i, 3 * degree - 1 - i) = -1;
  }
  if (degree > 1)
    perm.block(4 * degree, 4 * degree, n - 4 * degree, n - 4 * degree)
        = quadrilateral_nce_rotation(degree - 1);
  return perm;
}
//-----------------------------------------------------------------------------
Eigen::MatrixXd basix::dofperms::quadrilateral_nce_rotation(int degree)
{
  const int n = 2 * degree * (degree + 1);
  Eigen::MatrixXd perm = Eigen::MatrixXd::Zero(n, n);

  // Permute functions on edges
  for (int i = 0; i < degree; ++i)
  {
    perm(i, 2 * degree - 1 - i) = -1;
    perm(degree + i, 3 * degree + i) = 1;
    perm(2 * degree + i, i) = 1;
    perm(3 * degree + i, 3 * degree - 1 - i) = -1;
  }
  if (degree > 1)
    perm.block(4 * degree, 4 * degree, n - 4 * degree, n - 4 * degree)
        = quadrilateral_rtc_rotation(degree - 1);
  return perm;
}
//-----------------------------------------------------------------------------
Eigen::MatrixXd basix::dofperms::quadrilateral_rtc_reflection(int degree)
{
  const int n = 2 * degree * (degree + 1);
  Eigen::MatrixXd perm = Eigen::MatrixXd::Zero(n, n);

  // Permute functions on edges
  for (int i = 0; i < degree; ++i)
  {
    perm(i, degree + i) = -1;
    perm(degree + i, i) = -1;
    perm(2 * degree + i, 3 * degree + i) = -1;
    perm(3 * degree + i, 2 * degree + i) = -1;
  }
  if (degree > 1)
    perm.block(4 * degree, 4 * degree, n - 4 * degree, n - 4 * degree)
        = quadrilateral_nce_reflection(degree - 1);

  return perm;
}
//-----------------------------------------------------------------------------
Eigen::MatrixXd basix::dofperms::quadrilateral_nce_reflection(int degree)
{
  const int n = 2 * degree * (degree + 1);
  Eigen::MatrixXd perm = Eigen::MatrixXd::Zero(n, n);

  // Permute functions on edges
  for (int i = 0; i < degree; ++i)
  {
    perm(i, degree + i) = 1;
    perm(degree + i, i) = 1;
    perm(2 * degree + i, 3 * degree + i) = 1;
    perm(3 * degree + i, 2 * degree + i) = 1;
  }
  if (degree > 1)
    perm.block(4 * degree, 4 * degree, n - 4 * degree, n - 4 * degree)
        = quadrilateral_rtc_reflection(degree - 1);

  return perm;
}
//-----------------------------------------------------------------------------


/* vim: set tabstop=4 expandtab shiftwidth=4 softtabstop=4: */

/**
 * \file boost/numeric/ublasx/operation/dot.hpp
 *
 * \brief Vector dot product.
 *
 * \author Marco Guazzone (marco.guazzone@gmail.com)
 *
 * <hr/>
 *
 * Copyright (c) 2011, Marco Guazzone
 *
 * Distributed under the Boost Software License, Version 1.0. (See
 * accompanying file LICENSE_1_0.txt or copy at
 * http://www.boost.org/LICENSE_1_0.txt)
 */

#ifndef BOOST_NUMERIC_UBLASX_OPERATION_DOT_HPP
#define BOOST_NUMERIC_UBLASX_OPERATION_DOT_HPP

#include <boost/numeric/ublas/expression_types.hpp>
#include <boost/numeric/ublas/traits.hpp>
#include <boost/numeric/ublas/vector_expression.hpp>
#include <boost/numeric/ublasx/operation/sum.hpp>

namespace boost { namespace numeric { namespace ublasx {

using namespace ::boost::numeric::ublas;

template <typename VecExpr1T, typename VecExpr2T>
struct vdot_traits_type
{
    typedef typename vector_scalar_binary_traits<
                    VecExpr1T,
                    VecExpr2T,
                    vector_inner_prod<
                        VecExpr1T,
                        VecExpr2T,
                        typename promote_traits<
                            typename vector_traits<VecExpr1T>::value_type,
                            typename vector_traits<VecExpr2T>::value_type
                        >::promote_type
                    >
            >::result_type result_type;
};

template <typename MatExpr1T, typename MatExpr2T>
struct mdot_traits_type
{
    typedef vector<
                typename promote_traits<
                    typename matrix_traits<MatExpr1T>::value_type,
                    typename matrix_traits<MatExpr2T>::value_type
                >::promote_type
//              typename vector_traits<
//                  typename matrix_binary_traits<
//                          MatExpr1T,
//                          MatExpr2T,
//                          scalar_multiplies<
//                                  typename MatExpr1T::value_type,
//                                  typename MatExpr2T::value_type
//                          >
//                  >::result_type
//              >::value_type
            > result_type;
};

/**
 * \brief Scalar product of two vectors.
 *
 * \tparam VecExpr1T The type of the first vector.
 * \tparam VecExpr2T The type of the second vector.
 * \param v1 The first vector.
 * \param v2 The second vector.
 * \return The scalar product of vectors \a v1 and \a v2.
 *
 * The scalar product of two vectors \f$u\f$ and \f$v\f$ is defined as:
 * \f[
 *   \sum_{i} u_{i}v_{i}
 * \f]
 */
template <typename VecExpr1T, typename VecExpr2T>
BOOST_UBLAS_INLINE
typename vdot_traits_type<VecExpr1T,VecExpr2T>::result_type dot(vector_expression<VecExpr1T> const& v1,
                                                                vector_expression<VecExpr2T> const& v2)
{
    return inner_prod(v1, v2);
}

/**
 * \brief Scalar product of two matrices along a given dimension.
 *
 * \tparam Dim The dimension used for computing the scalar product.
 * \tparam MatExpr1T The type of the first matrix.
 * \tparam MatExpr2T The type of the second matrix.
 * \param M1 The first matrix.
 * \param M2 The second matrix.
 * \return A vector representing the scalar product of vectors \a M1 and \a M2
 *  along dimension \a Dim.
 *
 * The scalar product of two matrices \f$A\f$ and \f$B\f$ along dimensino
 * \f$d\f$ is defined as:
 * - If \f$d=1\f$:
 *  \f[
 *    \sum_{j} A_{ij}B_{ij}, i=1,2,\ldots
 *  \f]
 * - If \f$d=2\f$:
 *  \f[
 *    \sum_{i} A_{ij}B_{ij}, j=1,2,\ldots
 *  \f]
 * .
 */
template <std::size_t Dim, typename MatExpr1T, typename MatExpr2T>
typename mdot_traits_type<MatExpr1T,MatExpr2T>::result_type dot(matrix_expression<MatExpr1T> const& M1,
                                                                matrix_expression<MatExpr2T> const& M2)
{
    return sum<Dim>(element_prod(M1, M2));
}

}}} // Namespace boost::numeric::ublas

#endif // BOOST_NUMERIC_UBLASX_OPERATION_DOT_HPP


//Author: Dr. Shantanu Shahane
#ifndef class_H_ /* Include guard */
#define class_H_
#include <stdio.h>
#include <math.h>
#include <stdlib.h>
#include <time.h>
#include <float.h>
#include <string.h>
#include <iostream>
#include <vector>
#include <Eigen/Dense>
#include <unsupported/Eigen/SparseExtra>
#include <Eigen/SparseLU>
#include <Eigen/OrderingMethods>
#include <Eigen/Core>
#include "_hypre_utilities.h"
#include "HYPRE_krylov.h"
#include "HYPRE.h"
#include "HYPRE_parcsr_ls.h"
#include "general_functions.hpp"
using namespace std;

const int INTERIOR_TAG = -1;
// const int INLET_BC_TAG = 1;
// const int OUTLET_BC_TAG = 2;
// const int WALL_BC_TAG = 3;
// const int SYMMETRY_BC_TAG = 4;

template <typename T>
struct PointCloud
{ //taken from Nanoflann library: "utils.h": https://github.com/jlblancoc/nanoflann/blob/master/examples/utils.h
    struct Point
    {
        T x, y, z;
    };
    std::vector<Point> pts;
    inline size_t kdtree_get_point_count() const { return pts.size(); } // Must return the number of data points
    // Returns the dim'th component of the idx'th point in the class:
    // Since this is inlined and the "dim" argument is typically an immediate value, the
    //  "if/else's" are actually solved at compile time.
    inline T kdtree_get_pt(const size_t idx, const size_t dim) const
    {
        if (dim == 0)
            return pts[idx].x;
        else if (dim == 1)
            return pts[idx].y;
        else
            return pts[idx].z;
    }
    // Optional bounding-box computation: return false to default to a standard bbox computation loop.
    //   Return true if the BBOX was already computed by the class and returned in "bb" so it can be avoided to redo it again.
    //   Look at bb.size() to find out the expected dimensionality (e.g. 2 or 3 for point clouds)
    template <class BBOX>
    bool kdtree_get_bbox(BBOX & /* bb */) const { return false; }
};

class PARAMETERS
{
public:
    int cloud_size;                            //cloud size
    double cloud_size_multiplier;              //cloud size: cloud_size_multiplier * num_poly_terms
    int phs_deg;                               //degree of the polyharmonic spline: r^(phs_deg)
    int poly_deg;                              //degree of the polynomial to be appended
    int num_poly_terms;                        //number of the polynomial terms to be appended
    int dimension;                             //problem dimension (can be either 2 or 3 only)
    Eigen::MatrixXi polynomial_term_exponents; //exponents of the polynomial terms
    string meshfile;                           //meshfile name
    string output_file_prefix;                 //used to write all output files
    double max_dx, min_dx, avg_dx;             //characteristic length scales of mesh
    vector<int> periodic_bc_index;             //periodic BC wrt 0:'x', 1:'y', 2:'z'; : size [no. of periodic axes]; ex. ['z','x']:[2,0]; ['y']:[1]
    double dt;
    double rho = -10.0, mu = rho; //should be set in the main file
    double steady_tolerance, solver_tolerance, Courant, precond_droptol;
    int nt, euclid_precond_level_hypre, gmres_kdim, n_iter;
    string solver_type; //hypre_ilu_gmres, eigen_direct, eigen_ilu_bicgstab

    double grad_x_eigval_real, grad_x_eigval_imag, grad_y_eigval_real, grad_y_eigval_imag, grad_z_eigval_real, grad_z_eigval_imag, laplace_eigval_real, laplace_eigval_imag;
    double cloud_id_timer, rcm_timer, cloud_misc_timer, points_timer, grad_laplace_coeff_timer, factoring_timer, solve_timer, total_timer;
    int nt_actual = 0;

    vector<double> rel_res_log, abs_res_log, regul_alpha_log, steady_error_log; //logs of size nt_actual
    vector<int> n_iter_actual;                                                  //logs of size nt_actual

public:
    PARAMETERS(string parameter_file, string gmsh_file);
    void read_calc_parameters(string parameter_file);
    void verify_parameters();
    void calc_cloud_num_points();
    void get_problem_dimension_msh();
    void calc_polynomial_term_exponents();
    void calc_dt(Eigen::SparseMatrix<double, Eigen::RowMajor> &grad_x, Eigen::SparseMatrix<double, Eigen::RowMajor> &grad_y, Eigen::SparseMatrix<double, Eigen::RowMajor> &grad_z, Eigen::SparseMatrix<double, Eigen::RowMajor> &laplacian, double u0, double v0, double w0, double alpha);
};

class POINTS
{ //uses Compressed Row Format of Sparse Matrices
public:
    vector<bool> corner_edge_vertices;                                   //true at corner (for 2D, 3D problems) and edge (for 3D problems only): size [nv]
    int nv;                                                              //Number of vertices in original msh file
    int nv_original;                                                     //Number of vertices
    vector<double> xyz_min, xyz_max, xyz_length;                         //each of size:[dim]
    vector<double> xyz;                                                  //vertex co-ordinates: size [nv X dim]
    vector<double> xyz_original;                                         //all vertex co-ordinates in original msh file: size [nv_original X dim]; nv_original can be greater than nv
    int nelem_original;                                                  //no. of elements in the original file
    vector<bool> elem_boundary_flag_original;                            //boundary_flag=1 if elem is on boundary; else boundary_flag=0: size [nelem_original]
    vector<vector<int>> elem_vert_original;                              //connectivity in original msh file
    vector<double> boundary_face_area_original;                          //area or length of boundary elements for 3D or 2D (used to compute fluxes at boundaries): size [nelem_original]
    vector<int> elem_bc_tag_original;                                    //bc_tag associated with each element (helps to identify various boundary areas and internal region): size [nelem_original]
    vector<int> iv_original_nearest_vert;                                //nearest vertex no for vertices in xyz_original: size [nv_original]; nv_original can be greater than nv
    vector<double> normal;                                               //vertex co-ordinates (relavant only for boundary vertices): size [nv X dim]
    vector<bool> boundary_flag;                                          //boundary_flag=1 if it is on boundary; else boundary_flag=0: size [nv]
    vector<vector<bool>> periodic_bc_flag;                               //periodic_bc_flag=1 if it is on periodic boundary; else periodic_bc_flag=0: size [nv][no. of periodic axes]
    vector<vector<int>> periodic_bc_section;                             //takes values [-1,0,1] for [near_min,middle,near_max] sections respectively: size [nv][no. of periodic axes]
    vector<int> bc_tag;                                                  //bc_tag associated with each vertex (helps to identify various boundary areas and internal region): size [nv]
    Eigen::SparseMatrix<double, Eigen::RowMajor> grad_x_matrix_EIGEN;    //used for convection source term size [points.nv X points.nv]
    Eigen::SparseMatrix<double, Eigen::RowMajor> grad_y_matrix_EIGEN;    //used for convection source term size [points.nv X points.nv]
    Eigen::SparseMatrix<double, Eigen::RowMajor> grad_z_matrix_EIGEN;    //used for convection source term size [points.nv X points.nv]
    Eigen::SparseMatrix<double, Eigen::RowMajor> laplacian_matrix_EIGEN; //used for diffusion source term size [points.nv X points.nv]

    Eigen::SparseMatrix<double, Eigen::RowMajor> grad_x_matrix_EIGEN_boundary, grad_x_matrix_EIGEN_internal;       //[points.nv X points.nv]
    Eigen::SparseMatrix<double, Eigen::RowMajor> grad_y_matrix_EIGEN_boundary, grad_y_matrix_EIGEN_internal;       //[points.nv X points.nv]
    Eigen::SparseMatrix<double, Eigen::RowMajor> grad_z_matrix_EIGEN_boundary, grad_z_matrix_EIGEN_internal;       //[points.nv X points.nv]
    Eigen::SparseMatrix<double, Eigen::RowMajor> laplacian_matrix_EIGEN_boundary, laplacian_matrix_EIGEN_internal; //[points.nv X points.nv]

public:
    POINTS(PARAMETERS &parameters);
    void read_points_xyz_msh(PARAMETERS &parameters);
    void read_points_flag_msh(PARAMETERS &parameters);
    void calc_vert_normal(PARAMETERS &parameters);
    void calc_boundary_face_area(PARAMETERS &parameters);
    void calc_elem_bc_tag(PARAMETERS &parameters);
    void read_elem_vert_complete_msh(PARAMETERS &parameters, vector<vector<int>> &elem_vert, vector<bool> &elem_boundary_flag);
    void calc_vert_nb_cv(PARAMETERS &parameters, vector<vector<int>> &vert_nb_cv, vector<vector<int>> &elem_vert);
    void calc_elem_normal_2D(vector<double> &elem_normal, vector<vector<int>> &vert_nb_cv, vector<vector<int>> &elem_vert, vector<bool> &elem_boundary_flag);
    void calc_elem_normal_3D(vector<double> &elem_normal, vector<vector<int>> &vert_nb_cv, vector<vector<int>> &elem_vert, vector<bool> &elem_boundary_flag);
    void delete_corner_edge_vertices(PARAMETERS &parameters);
    void delete_periodic_bc_vertices(PARAMETERS &parameters);
    void set_periodic_bc(PARAMETERS &parameters, vector<string> periodic_axis);
};

class CLOUD
{
public:
    vector<int> nb_points_row;      //neighboring points of each point: size: nv+1
    vector<int> nb_points_col;      //neighboring points of each point: size: nb_points_row[nv]
    vector<double> grad_x_coeff;    //coefficient for grad_x at each point: base: (CLOUD.nb_points_row, CLOUD.nb_points_col)
    vector<double> grad_y_coeff;    //coefficient for grad_y at each point: base: (CLOUD.nb_points_row, CLOUD.nb_points_col)
    vector<double> grad_z_coeff;    //coefficient for grad_z at each point: base: (CLOUD.nb_points_row, CLOUD.nb_points_col)
    vector<double> laplacian_coeff; //coefficient for laplacian at each point: base: (CLOUD.nb_points_row, CLOUD.nb_points_col)
    vector<double> cond_num_RBF;    //condition number of RBF A matrix for each point: size [nv]
    double cond_num_RBF_max;        //statistics of cond_num_RBF
    double cond_num_RBF_min;        //statistics of cond_num_RBF
    double cond_num_RBF_avg;        //statistics of cond_num_RBF
    vector<int> rcm_points_order;   //reordering list obtained by RCM algorithm
public:
    CLOUD(POINTS &points, PARAMETERS &parameters);
    void calc_cloud_points_slow(POINTS &points, PARAMETERS &parameters);
    void calc_cloud_points_fast(POINTS &points, PARAMETERS &parameters);
    void calc_cloud_points_fast_periodic_bc(POINTS &points, PARAMETERS &parameters);
    void calc_cloud_points_fast_periodic_bc_shifted(POINTS &points, PARAMETERS &parameters, vector<double> &xyz_shifted, vector<int> &periodic_bc_section_value);
    void re_order_points_reverse_cuthill_mckee(POINTS &points, PARAMETERS &parameters);
    void re_order_points(POINTS &points, PARAMETERS &parameters);
    void calc_iv_original_nearest_vert(POINTS &points, PARAMETERS &parameters);
    void calc_charac_dx(POINTS &points, PARAMETERS &parameters);
    void calc_grad_laplace_coeffs(POINTS &points, PARAMETERS &parameters);
    void EIGEN_set_grad_laplace_matrix(POINTS &points, PARAMETERS &parameters);
    void EIGEN_set_grad_laplace_matrix_separate(POINTS &points, PARAMETERS &parameters);
};

class SOLVER
{
public:
    // Functions
    void init(POINTS &points, CLOUD &cloud, PARAMETERS &parameters, vector<bool> &dirichlet_flag, double unsteady_term_coeff, double conv_term_coeff, double diff_term_coeff, bool log_flag);
    void HYPRE_set_coeff(PARAMETERS &parameters);
    void EIGEN_set_coeff();
    void general_solve(POINTS &points, PARAMETERS &parameters, Eigen::VectorXd &field_new, Eigen::VectorXd &field_old, Eigen::VectorXd &rhs);
    void set_solve_parameters();
    void calc_coeff_matrix(POINTS &points, CLOUD &cloud, PARAMETERS &parameters);
    void scale_coeff(POINTS &points);

    // Variables
    HYPRE_IJMatrix coeff_HYPRE;
    HYPRE_ParCSRMatrix parcsr_coeff_HYPRE;
    HYPRE_IJVector source_HYPRE;
    HYPRE_ParVector par_source_HYPRE;
    HYPRE_IJVector X_HYPRE;
    HYPRE_ParVector par_X_HYPRE;
    HYPRE_Solver solver_X_HYPRE, precond_X_HYPRE;
    Eigen::SparseMatrix<double, Eigen::RowMajor> coeff_EIGEN;
    Eigen::SparseLU<Eigen::SparseMatrix<double, Eigen::RowMajor>, Eigen::COLAMDOrdering<int>> solver_eigen_direct;
    Eigen::BiCGSTAB<Eigen::SparseMatrix<double, Eigen::RowMajor>, Eigen::IncompleteLUT<double>> solver_eigen_ilu_bicgstab;

    vector<tuple<int, int, double>> coeff_matrix; //store coefficient matrix with sparsity structure of (row, col, value)
    vector<double> scale;

    int *rows_HYPRE; //array of size ncv going from 0 to ncv-1
    double *X;       //unknown vector
    double *source;  //source term

    string solver_type; //hypre_ilu_gmres, eigen_direct, eigen_ilu_bicgstab
    double unsteady_term_coeff_1, conv_term_coeff_1, diff_term_coeff_1;
    int print_flag = 0;             //decide how much to print during solution
    int n_iter;                     //max solver iterations
    int euclid_precond_level_hypre; //level setting in Euclid pre-conditioner
    double precond_droptol;         //setting in Euclid pre-conditioner
    int gmres_kdim;                 //GMRES max size of Krylov subspace
    double solver_tolerance;        //solver tolerance
    double l2_norm;
    vector<bool> dirichlet_flag_1; //true: use Dirichlet BC, else use Neumann BC
    int system_size;               //no. of unknowns
    bool log_flag_1 = true;
};

class FRACTIONAL_STEP_1
{ //hat velocity formulation
public:
    // Functions
    FRACTIONAL_STEP_1(POINTS &points, CLOUD &cloud, PARAMETERS &parameters, vector<bool> &u_dirichlet_flag1, vector<bool> &v_dirichlet_flag1, vector<bool> &p_dirichlet_flag1, int temporal_order1);
    void check_bc(POINTS &points, PARAMETERS &parameters);
    void single_timestep_2d(POINTS &points, CLOUD &cloud, PARAMETERS &parameters, Eigen::VectorXd &u_new, Eigen::VectorXd &v_new, Eigen::VectorXd &p_new, Eigen::VectorXd &u_old, Eigen::VectorXd &v_old, Eigen::VectorXd &p_old, int it1);
    void single_timestep_2d(POINTS &points, CLOUD &cloud, PARAMETERS &parameters, Eigen::VectorXd &u_new, Eigen::VectorXd &v_new, Eigen::VectorXd &p_new, Eigen::VectorXd &u_old, Eigen::VectorXd &v_old, Eigen::VectorXd &p_old, Eigen::VectorXd &body_force_x, Eigen::VectorXd &body_force_y, int it1);
    void calc_vel_hat(POINTS &points, PARAMETERS &parameters, Eigen::VectorXd &u_old, Eigen::VectorXd &v_old, Eigen::VectorXd &p_old);
    void calc_vel_hat(POINTS &points, PARAMETERS &parameters, Eigen::VectorXd &u_old, Eigen::VectorXd &v_old, Eigen::VectorXd &p_old, Eigen::VectorXd &body_force_x, Eigen::VectorXd &body_force_y);
    void calc_pressure(POINTS &points, PARAMETERS &parameters, Eigen::VectorXd &u_new, Eigen::VectorXd &v_new, Eigen::VectorXd &p_new, Eigen::VectorXd &u_old, Eigen::VectorXd &v_old, Eigen::VectorXd &p_old);
    void calc_vel_corr(POINTS &points, CLOUD &cloud, PARAMETERS &parameters, Eigen::VectorXd &u_new, Eigen::VectorXd &v_new, Eigen::VectorXd &p_new, Eigen::VectorXd &u_old, Eigen::VectorXd &v_old);
    void extras(POINTS &points, PARAMETERS &parameters, Eigen::VectorXd &u_new, Eigen::VectorXd &v_new, Eigen::VectorXd &p_new, Eigen::VectorXd &u_old, Eigen::VectorXd &v_old, Eigen::VectorXd &p_old);

    // Variables
    SOLVER solver_p;
    Eigen::VectorXd zero_vector, zero_vector_1;
    Eigen::VectorXd uh, vh;
    Eigen::VectorXd p_source;
    Eigen::VectorXd u_source_old, v_source_old;
    Eigen::VectorXd u_source_old_old, v_source_old_old; //used only for multi-step method
    vector<bool> u_dirichlet_flag, v_dirichlet_flag, p_dirichlet_flag;
    bool p_bc_full_neumann;
    int temporal_order = -1, it;
};

class IMPLICIT_SCALAR_TRANSPORT_SOLVER
{ //scalar transport with spatially varying velocity field: BDF2
public:
    // Functions
    IMPLICIT_SCALAR_TRANSPORT_SOLVER(POINTS &points, CLOUD &cloud, PARAMETERS &parameters, vector<bool> &dirichlet_flag1, int precond_freq_it1, double unsteady_coeff1, double conv_coeff1, double diff_coeff1, bool solver_log_flag1);
    void single_timestep_2d(POINTS &points, CLOUD &cloud, PARAMETERS &parameters, Eigen::VectorXd &phi_new, Eigen::VectorXd &phi_old, Eigen::VectorXd &u_new, Eigen::VectorXd &v_new, int it1);
    void set_matrix(POINTS &points, CLOUD &cloud, PARAMETERS &parameters, Eigen::VectorXd &u_new, Eigen::VectorXd &v_new);
    void modify_matrix(POINTS &points, CLOUD &cloud, PARAMETERS &parameters, Eigen::VectorXd &u_new, Eigen::VectorXd &v_new);
    void calc_nb_points_col_matrix(POINTS &points, CLOUD &cloud, PARAMETERS &parameters);

    // Variables
    Eigen::SparseMatrix<double, Eigen::RowMajor> matrix;
    Eigen::BiCGSTAB<Eigen::SparseMatrix<double, Eigen::RowMajor>, Eigen::IncompleteLUT<double>> solver_eigen;
    Eigen::VectorXd zero_vector, source, phi_old_old;
    vector<bool> dirichlet_flag;
    vector<int> nb_points_col_matrix;
    bool bc_full_neumann, solver_log_flag;
    int it, precond_freq_it;
    double unsteady_coeff, conv_coeff, diff_coeff;
    double bdf2_alpha_1 = 1.5, bdf2_alpha_2 = -2.0, bdf2_alpha_3 = 0.5; //https://en.wikipedia.org/wiki/Backward_differentiation_formula
};

class SEMI_IMPLICIT_SPLIT_SOLVER
{ //iterative split solver for Navier-Stokes equations
public:
    // Functions
    SEMI_IMPLICIT_SPLIT_SOLVER(POINTS &points, CLOUD &cloud, PARAMETERS &parameters, vector<bool> &u_dirichlet_flag1, vector<bool> &v_dirichlet_flag1, vector<bool> &p_dirichlet_flag1, int n_outer_iter1, double iterative_tolerance1, int precond_freq_it1);
    void check_bc(POINTS &points, PARAMETERS &parameters);
    void single_timestep_2d(POINTS &points, CLOUD &cloud, PARAMETERS &parameters, Eigen::VectorXd &u_new, Eigen::VectorXd &v_new, Eigen::VectorXd &p_new, Eigen::VectorXd &u_old, Eigen::VectorXd &v_old, Eigen::VectorXd &p_old, int it1, vector<int> &n_outer_iter_log, vector<double> &iterative_l1_err_log, vector<double> &iterative_max_err_log);
    void single_timestep_2d(POINTS &points, CLOUD &cloud, PARAMETERS &parameters, Eigen::VectorXd &u_new, Eigen::VectorXd &v_new, Eigen::VectorXd &p_new, Eigen::VectorXd &u_old, Eigen::VectorXd &v_old, Eigen::VectorXd &p_old, Eigen::VectorXd &body_force_x, Eigen::VectorXd &body_force_y, int it1, vector<int> &n_outer_iter_log, vector<double> &iterative_l1_err_log, vector<double> &iterative_max_err_log);
    void set_vel_matrix(POINTS &points, CLOUD &cloud, PARAMETERS &parameters, Eigen::VectorXd &u_new, Eigen::VectorXd &v_new);
    void calc_nb_points_col_matrix(POINTS &points, CLOUD &cloud, PARAMETERS &parameters);
    void modify_vel_matrix(POINTS &points, CLOUD &cloud, PARAMETERS &parameters, Eigen::VectorXd &u_new, Eigen::VectorXd &v_new);
    void calc_vel(POINTS &points, PARAMETERS &parameters, Eigen::VectorXd &u_new, Eigen::VectorXd &v_new, Eigen::VectorXd &p_new, Eigen::VectorXd &u_old, Eigen::VectorXd &v_old, Eigen::VectorXd &p_old);
    void calc_vel(POINTS &points, PARAMETERS &parameters, Eigen::VectorXd &u_new, Eigen::VectorXd &v_new, Eigen::VectorXd &p_new, Eigen::VectorXd &u_old, Eigen::VectorXd &v_old, Eigen::VectorXd &p_old, Eigen::VectorXd &body_force_x, Eigen::VectorXd &body_force_y);
    void calc_pressure(POINTS &points, CLOUD &cloud, PARAMETERS &parameters, Eigen::VectorXd &u_new, Eigen::VectorXd &v_new, Eigen::VectorXd &p_new, Eigen::VectorXd &u_old, Eigen::VectorXd &v_old, Eigen::VectorXd &p_old);
    void calc_vel_corr(POINTS &points, CLOUD &cloud, PARAMETERS &parameters, Eigen::VectorXd &u_new, Eigen::VectorXd &v_new);
    void extras(POINTS &points, PARAMETERS &parameters, Eigen::VectorXd &u_new, Eigen::VectorXd &v_new, Eigen::VectorXd &p_new, Eigen::VectorXd &u_old, Eigen::VectorXd &v_old, Eigen::VectorXd &p_old);

    // Variables
    SOLVER solver_p;
    Eigen::SparseMatrix<double, Eigen::RowMajor> matrix_u, matrix_v;
    Eigen::BiCGSTAB<Eigen::SparseMatrix<double, Eigen::RowMajor>, Eigen::IncompleteLUT<double>> solver_eigen_u, solver_eigen_v;
    Eigen::VectorXd zero_vector, zero_vector_1;
    Eigen::VectorXd normal_mom_x, normal_mom_y;
    Eigen::VectorXd p_prime, p_source, u_source, v_source, u_prime, v_prime, u_iter_old, v_iter_old, u_old_old, v_old_old;
    vector<bool> u_dirichlet_flag, v_dirichlet_flag, p_dirichlet_flag;
    vector<int> nb_points_col_matrix_u, nb_points_col_matrix_v;
    bool p_bc_full_neumann;
    double iterative_tolerance, iterative_l1_err, iterative_max_err;
    int it, n_outer_iter, outer_iter, precond_freq_it;
    double bdf2_alpha_1 = 1.5, bdf2_alpha_2 = -2.0, bdf2_alpha_3 = 0.5; //https://en.wikipedia.org/wiki/Backward_differentiation_formula
};

class SOLIDIFICATION
{
public:
    // Functions
    SOLIDIFICATION(POINTS &points, CLOUD &cloud, PARAMETERS &parameters, int temporal_order1);
    void single_timestep_2d(POINTS &points, CLOUD &cloud, PARAMETERS &parameters, Eigen::VectorXd &T_new, Eigen::VectorXd &T_old, Eigen::VectorXd &fs_new, Eigen::VectorXd &fs_old, int it);

    // Variables
    Eigen::VectorXd dfs_dT, T_source;
    Eigen::VectorXd T_old_old, fs_old_old; //used only for multi-step method
    double Tliq = 915.0;
    double Tsol = 850.0;
    double Tf = 935.2;
    double Teps = 2.0;
    double k_partition = 0.13;
    double Lf = 390000.0;
    double Cp = 1006.0;
    double alpha = 5E-5;
    int temporal_order = -1;
};

#endif


/* ---------------------------------------------------------------------
 *
 * Copyright (C) 2018 - 2021 by the deal.II authors
 *
 * This file is part of the deal.II library.
 *
 * The deal.II library is free software; you can use it, redistribute
 * it, and/or modify it under the terms of the GNU Lesser General
 * Public License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 * The full text of the license can be found in the file LICENSE.md at
 * the top level directory of deal.II.
 *
 * ---------------------------------------------------------------------
 *
 * Authors: Katharina Kormann, Martin Kronbichler, 2018
 */

// 包含的文件与  step-37  中的基本相同，只是用有限元类FE_DGQHermite代替了FE_Q。所有对面积分进行无矩阵计算的功能已经包含在`fe_evaluation.h`中。

#include <deal.II/base/quadrature_lib.h>
#include <deal.II/base/function.h>
#include <deal.II/base/logstream.h>
#include <deal.II/base/timer.h>

#include <deal.II/lac/affine_constraints.h>
#include <deal.II/lac/full_matrix.h>
#include <deal.II/lac/solver_cg.h>
#include <deal.II/lac/la_parallel_vector.h>
#include <deal.II/lac/precondition.h>
#include <deal.II/lac/tensor_product_matrix.h>

#include <deal.II/fe/fe_dgq.h>
#include <deal.II/fe/fe_tools.h>

#include <deal.II/grid/tria.h>
#include <deal.II/grid/grid_generator.h>
#include <deal.II/grid/grid_tools.h>

#include <deal.II/multigrid/multigrid.h>
#include <deal.II/multigrid/mg_transfer_matrix_free.h>
#include <deal.II/multigrid/mg_tools.h>
#include <deal.II/multigrid/mg_coarse.h>
#include <deal.II/multigrid/mg_smoother.h>
#include <deal.II/multigrid/mg_matrix.h>

#include <deal.II/numerics/vector_tools.h>

#include <deal.II/matrix_free/matrix_free.h>
#include <deal.II/matrix_free/fe_evaluation.h>

#include <iostream>
#include <fstream>

namespace Step59
{
  using namespace dealii;

// 和 step-37 一样，为了简单起见，我们在程序顶部将维数和多项式程度收集为常数。与 step-37 不同的是，这次我们选择了一个真正的高阶方法，度数为8，任何不使用和因式分解的实现都会变得非常慢，而使用MatrixFree的实现则提供了与度数为2或3时基本相同的效率。此外，本教程程序中的所有类都是模板化的，因此，通过在`main()`函数中添加适当度数的实例，可以很容易地在运行时从输入文件或命令行参数中选择度数。

  const unsigned int degree_finite_element = 8;
  const unsigned int dimension             = 3;
// @sect3{Equation data}

// 与 step-7 相类似，我们定义了一个分析解，我们试图用离散化重现这个分析解。由于本教程的目的是展示无矩阵方法，我们选择了一个最简单的可能性，即一个余弦函数，其导数对我们来说足够简单，可以通过分析计算。再往下看，我们在这里选择的波数2.4将与 $x$ -方向的域范围即2.5相匹配，这样我们在 $x = 2.5$ 得到一个周期性的解，包括 $6pi$ 或余弦的三个整波转。第一个函数定义了解和它的梯度，分别用于表达Dirichlet和Neumann边界条件的解析解。此外，一个代表解的负拉普拉斯的类被用来表示右手边（强制）函数，我们用它来匹配离散化版本中的给定分析解（制造解）。

  template <int dim>
  class Solution : public Function<dim>
  {
  public:
    virtual double value(const Point<dim> &p,
                         const unsigned int = 0) const override final
    {
      double val = 1.;
      for (unsigned int d = 0; d < dim; ++d)
        val *= std::cos(numbers::PI * 2.4 * p[d]);
      return val;
    }

    virtual Tensor<1, dim> gradient(const Point<dim> &p,
                                    const unsigned int = 0) const override final
    {
      const double   arg = numbers::PI * 2.4;
      Tensor<1, dim> grad;
      for (unsigned int d = 0; d < dim; ++d)
        {
          grad[d] = 1.;
          for (unsigned int e = 0; e < dim; ++e)
            if (d == e)
              grad[d] *= -arg * std::sin(arg * p[e]);
            else
              grad[d] *= std::cos(arg * p[e]);
        }
      return grad;
    }
  };

  template <int dim>
  class RightHandSide : public Function<dim>
  {
  public:
    virtual double value(const Point<dim> &p,
                         const unsigned int = 0) const override final
    {
      const double arg = numbers::PI * 2.4;
      double       val = 1.;
      for (unsigned int d = 0; d < dim; ++d)
        val *= std::cos(arg * p[d]);
      return dim * arg * arg * val;
    }
  };

//  @sect3{Matrix-free implementation}

// `LaplaceOperator`类与  step-37  中的相应类类似。一个重要的区别是，我们没有从  MatrixFreeOperators::Base  派生出这个类，因为我们想呈现  MatrixFree::loop()  的一些额外特性，这些特性在通用类  MatrixFreeOperators::Base.  中是不可用的。我们从Subscriptor类派生出这个类，以便能够在Chebyshev预处理程序中使用该操作符，因为该预处理程序通过SmartPointer存储基础矩阵。

// 鉴于我们手工实现了一个完整的矩阵接口，我们需要添加一个`initialize()`函数，一个`m()`函数，一个`vmult()`函数和一个`Tvmult()`函数，这些都是之前由  MatrixFreeOperators::Base.  我们的LaplaceOperator还包含一个成员函数`get_penalty_factor()`，根据  step-39  集中选择对称内部惩罚方法中的惩罚参数 。

  template <int dim, int fe_degree, typename number>
  class LaplaceOperator : public Subscriptor
  {
  public:
    using value_type = number;

    LaplaceOperator() = default;

    void initialize(std::shared_ptr<const MatrixFree<dim, number>> data);

    void clear();

    types::global_dof_index m() const;

    void initialize_dof_vector(
      LinearAlgebra::distributed::Vector<number> &vec) const;

    std::shared_ptr<const MatrixFree<dim, number>> get_matrix_free() const;

    void vmult(LinearAlgebra::distributed::Vector<number> &      dst,
               const LinearAlgebra::distributed::Vector<number> &src) const;

    void Tvmult(LinearAlgebra::distributed::Vector<number> &      dst,
                const LinearAlgebra::distributed::Vector<number> &src) const;

    number get_penalty_factor() const
    {
      return 1.0 * fe_degree * (fe_degree + 1);
    }

  private:
    void
    apply_cell(const MatrixFree<dim, number> &                   data,
               LinearAlgebra::distributed::Vector<number> &      dst,
               const LinearAlgebra::distributed::Vector<number> &src,
               const std::pair<unsigned int, unsigned int> &cell_range) const;

    void
    apply_face(const MatrixFree<dim, number> &                   data,
               LinearAlgebra::distributed::Vector<number> &      dst,
               const LinearAlgebra::distributed::Vector<number> &src,
               const std::pair<unsigned int, unsigned int> &face_range) const;

    void apply_boundary(
      const MatrixFree<dim, number> &                   data,
      LinearAlgebra::distributed::Vector<number> &      dst,
      const LinearAlgebra::distributed::Vector<number> &src,
      const std::pair<unsigned int, unsigned int> &     face_range) const;

    std::shared_ptr<const MatrixFree<dim, number>> data;
  };

// `%PreconditionBlockJacobi`类定义了我们对这个问题的自定义预处理程序。与基于矩阵对角线的 step-37 不同，我们在这里通过使用介绍中讨论的所谓快速对角线化方法来计算非连续Galerkin方法中对角线块的近似反演。

  template <int dim, int fe_degree, typename number>
  class PreconditionBlockJacobi
  {
  public:
    using value_type = number;

    void clear()
    {
      cell_matrices.clear();
    }

    void initialize(const LaplaceOperator<dim, fe_degree, number> &op);

    void vmult(LinearAlgebra::distributed::Vector<number> &      dst,
               const LinearAlgebra::distributed::Vector<number> &src) const;

    void Tvmult(LinearAlgebra::distributed::Vector<number> &      dst,
                const LinearAlgebra::distributed::Vector<number> &src) const
    {
      vmult(dst, src);
    }

  private:
    std::shared_ptr<const MatrixFree<dim, number>> data;
    std::vector<TensorProductMatrixSymmetricSum<dim,
                                                VectorizedArray<number>,
                                                fe_degree + 1>>
      cell_matrices;
  };

//这个独立的函数在`LaplaceOperator'和`%PreconditionBlockJacobi'类中都被用来调整鬼魂范围。这个函数是必要的，因为`vmult()`函数所提供的一些向量没有用包括正确的鬼魂条目布局的 `LaplaceOperator::initialize_dof_vector` 来正确初始化，而是来自MGTransferMatrixFree类，该类对无矩阵类的鬼魂选择没有概念。为了避免索引混乱，我们必须在对这些向量进行实际操作之前调整鬼域。由于向量在多网格平滑器和传输类中被保留下来，一个曾经被调整过重影范围的向量在对象的整个生命周期中都会保持这种状态，所以我们可以在函数的开始使用一个快捷方式来查看分布式向量的分区器对象（以共享指针的形式存储）是否与MatrixFree所期望的布局相同，它被存储在一个由 MatrixFree::get_dof_info(0),  访问的数据结构中 ]，其中的0表示从中提取的DoFHandler编号；我们在MatrixFree中只使用一个DoFHandler，所以这里唯一有效的编号是0。

  template <int dim, typename number>
  void adjust_ghost_range_if_necessary(
    const MatrixFree<dim, number> &                   data,
    const LinearAlgebra::distributed::Vector<number> &vec)
  {
    if (vec.get_partitioner().get() ==
        data.get_dof_info(0).vector_partitioner.get())
      return;

    LinearAlgebra::distributed::Vector<number> copy_vec(vec);
    const_cast<LinearAlgebra::distributed::Vector<number> &>(vec).reinit(
      data.get_dof_info(0).vector_partitioner);
    const_cast<LinearAlgebra::distributed::Vector<number> &>(vec)
      .copy_locally_owned_data_from(copy_vec);
  }

// 接下来的五个函数用于清除和初始化`LaplaceOperator`类，返回持有MatrixFree数据容器的共享指针，以及正确初始化向量和运算符大小，与 step-37 或者说 MatrixFreeOperators::Base. 的内容相同。
  template <int dim, int fe_degree, typename number>
  void LaplaceOperator<dim, fe_degree, number>::clear()
  {
    data.reset();
  }

  template <int dim, int fe_degree, typename number>
  void LaplaceOperator<dim, fe_degree, number>::initialize(
    std::shared_ptr<const MatrixFree<dim, number>> data)
  {
    this->data = data;
  }

  template <int dim, int fe_degree, typename number>
  std::shared_ptr<const MatrixFree<dim, number>>
  LaplaceOperator<dim, fe_degree, number>::get_matrix_free() const
  {
    return data;
  }

  template <int dim, int fe_degree, typename number>
  void LaplaceOperator<dim, fe_degree, number>::initialize_dof_vector(
    LinearAlgebra::distributed::Vector<number> &vec) const
  {
    data->initialize_dof_vector(vec);
  }

  template <int dim, int fe_degree, typename number>
  types::global_dof_index LaplaceOperator<dim, fe_degree, number>::m() const
  {
    Assert(data.get() != nullptr, ExcNotInitialized());
    return data->get_dof_handler().n_dofs();
  }

// 这个函数在向量`src`上实现了LaplaceOperator的动作，并将结果存储在向量`dst`中。与 step-37 相比，这个调用有四个新特性。

// 第一个新特性是上面提到的`adjust_ghost_range_if_necessary`函数，该函数需要使向量符合单元和面函数中FEEvaluation和FEFaceEvaluation所期望的布局。

// 第二个新特征是我们没有像 step-37 中那样实现`vmult_add()`函数（通过虚拟函数 MatrixFreeOperators::Base::vmult_add()), ，而是直接实现`vmult()`功能。由于单元和面的积分都将和到目的向量中，我们当然必须在某处将向量归零。对于DG元素，我们有两个选择&ndash；一个是使用 FEEvaluation::set_dof_values() 而不是下面`apply_cell`函数中的 FEEvaluation::distribute_local_to_global() 。这是因为MatrixFree中的循环布局是这样的：单元积分总是在面积分之前接触到一个给定的向量条目。然而，这实际上只适用于完全不连续的基数，其中每个单元都有自己的自由度，不与邻近的结果共享。另一种设置，即这里选择的设置，是让 MatrixFree::loop() 来处理向量的归零问题。这可以被认为是在代码的某个地方简单地调用`dst = 0;`。对于像 `LinearAlgebra::distributed::Vector`, 这样的支持性向量来说，实现起来就比较麻烦了，因为我们的目标是不要一次性将整个向量清零。在足够小的几千个向量项上进行归零操作的好处是，在 FEEvaluation::distribute_local_to_global() 和 FEFaceEvaluation::distribute_local_to_global(). 中再次访问之前，被归零的向量项会保留在缓存中，因为无矩阵运算符的评估真的很快，仅仅归零一个大的向量就会相当于运算符评估时间的25%，我们显然希望避免这种代价。对于 MatrixFree::cell_loop 和连续基数来说，也可以使用这种将向量归零的选项，尽管在 step-37 或 step-48 的教程程序中没有使用它。

// 第三个新特征是我们提供了在单元格、内面和边界面进行计算的函数方式。MatrixFree类有一个叫做`loop`的函数，它接收三个函数指针，用于三种情况，允许分开实现不同的东西。正如在 step-37 中所解释的，这些函数指针可以是 `std::function` 对象或类的成员函数。在这种情况下，我们使用指向成员函数的指针。

// 最后的新特征是可以给 MatrixFree::DataAccessOnFaces 类型的最后两个参数，这个类将面积分的数据访问类型传递给并行向量的MPI数据交换例程 LinearAlgebra::distributed::Vector::update_ghost_values() 和 LinearAlgebra::distributed::Vector::compress() 。其目的是不发送相邻元素的所有自由度，而是将数据量减少到手头计算真正需要的程度。数据交换是一个真正的瓶颈，特别是对于高自由度的DG方法来说，因此一个更严格的交换方式往往是有益的。枚举字段 MatrixFree::DataAccessOnFaces 可以取值`none`，这意味着根本不做面的积分，这类似于 MatrixFree::cell_loop(), 的值`values`，意味着只使用面的形状函数值（但不使用导数），而值`gradients`时，除了值之外还访问面的第一导数。值`unspecified`意味着所有的自由度将被交换给位于处理器边界的面，并指定在本地处理器上进行处理。

// 为了了解数据是如何被减少的，想想节点元素FE_DGQ的情况，节点点在元素表面，在一个单元的 $(k+1)^d$ 个自由度中，只有 $(k+1)^{d-1}$ 个自由度对 $d$ 个空间维度的多项式程度的面的值有贡献。类似的减少也可以用于内部惩罚方法，该方法评估面的值和一导数。当在一维中使用类Hermite基时，最多只有两个基函数对数值和导数有贡献。FE_DGQHermite类实现了这一概念的张量乘积，在介绍中已经讨论过。因此，每个面只需交换 $2(k+1)^{d-1}$ 个自由度，一旦 $k$ 个自由度大于4或5个，这显然是一种胜利。请注意，FE_DGQHermite的这种减少的交换在具有弯曲边界的网格上也是有效的，因为导数是在参考元素上取的，而几何体只在内部混合它们。因此，这与试图用连续的Hermite型形状函数获得 $C^1$ 的连续性不同，在这种情况下，非笛卡尔的情况会大大改变情况。显然，在非笛卡尔网格上，导数还包括超出法向导数的形状函数的切向导数，但这些也只需要元素表面的函数值。如果元素不提供任何压缩，循环会自动交换受影响单元的所有条目。

  template <int dim, int fe_degree, typename number>
  void LaplaceOperator<dim, fe_degree, number>::vmult(
    LinearAlgebra::distributed::Vector<number> &      dst,
    const LinearAlgebra::distributed::Vector<number> &src) const
  {
    adjust_ghost_range_if_necessary(*data, dst);
    adjust_ghost_range_if_necessary(*data, src);
    data->loop(&LaplaceOperator::apply_cell,
               &LaplaceOperator::apply_face,
               &LaplaceOperator::apply_boundary,
               this,
               dst,
               src,
               /* zero_dst =  */ true,
               MatrixFree<dim, number>::DataAccessOnFaces::gradients,
               MatrixFree<dim, number>::DataAccessOnFaces::gradients);
  }

// 由于拉普拉斯是对称的，`Tvmult()`（多网格平滑界面需要）操作被简单地转发给`vmult()`的情况。

  template <int dim, int fe_degree, typename number>
  void LaplaceOperator<dim, fe_degree, number>::Tvmult(
    LinearAlgebra::distributed::Vector<number> &      dst,
    const LinearAlgebra::distributed::Vector<number> &src) const
  {
    vmult(dst, src);
  }

//单元格的操作与 step-37 非常相似。不过我们在这里没有使用系数。第二个区别是，我们用一个单一的函数调用 FEEvaluation::gather_evaluate() 代替了 FEEvaluation::read_dof_values() 后面的 FEEvaluation::evaluate() 这两个步骤，在内部调用这两个单独方法的序列。同样， FEEvaluation::integrate_scatter() 实现了 FEEvaluation::integrate() 之后的 FEEvaluation::distribute_local_to_global(). 的序列。 在这种情况下，这些新函数只是节省了两行代码。然而，我们用它们来与FEFaceEvaluation进行类比，在那里它们更重要，如下所述。

  template <int dim, int fe_degree, typename number>
  void LaplaceOperator<dim, fe_degree, number>::apply_cell(
    const MatrixFree<dim, number> &                   data,
    LinearAlgebra::distributed::Vector<number> &      dst,
    const LinearAlgebra::distributed::Vector<number> &src,
    const std::pair<unsigned int, unsigned int> &     cell_range) const
  {
    FEEvaluation<dim, fe_degree, fe_degree + 1, 1, number> phi(data);
    for (unsigned int cell = cell_range.first; cell < cell_range.second; ++cell)
      {
        phi.reinit(cell);
        phi.gather_evaluate(src, EvaluationFlags::gradients);
        for (unsigned int q = 0; q < phi.n_q_points; ++q)
          phi.submit_gradient(phi.get_gradient(q), q);
        phi.integrate_scatter(EvaluationFlags::gradients, dst);
      }
  }

// 面部操作实现了与 step-39 类似的内部惩罚方法的条款，正如介绍中所解释的。我们需要两个评估器对象来完成这个任务，一个用于处理来自内部面的两边之一的单元格的解，另一个用于处理来自另一边的解。面积分的评价器被称为FEFaceEvaluation，并在构造函数的第二个槽中接受一个布尔参数，以指示评价器应属于两边中的哪一边。在FEFaceEvaluation和MatrixFree中，我们称两边中的一边为 "内部"，另一边为 "外部"。`外部'这个名字是指两边的评价器将返回相同的法向量。对于 "内部 "一侧，法向量指向外部，而另一侧则指向内部，并且与该单元的外部法向量相对应。除了新的类名之外，我们再次得到了一系列的项目，与 step-37 中讨论的类似，但在这种情况下是针对内部面的。请注意，MatrixFree的数据结构形成了面的批次，类似于单元积分的单元批次。一批中的所有面涉及不同的单元格编号，但在参考单元格中具有相同的面编号，具有相同的细化配置（无细化或相同的子面）和相同的方向，以保持SIMD操作的简单和高效。

// 注意，除了法线方向的逻辑决定外，内部与外部没有任何隐含的意义，这在内部是相当随机的。我们绝对不能依赖分配内部与外部标志的某种模式，因为这个决定是为了MatrixFree设置例程中的访问规则性和统一性而做出的。由于大多数正常的DG方法都是保守的，也就是说，通量在接口的两边看起来都是一样的，所以如果内部/外部标志被调换，法线向量得到相反的符号，那么数学就不会有任何改变。

  template <int dim, int fe_degree, typename number>
  void LaplaceOperator<dim, fe_degree, number>::apply_face(
    const MatrixFree<dim, number> &                   data,
    LinearAlgebra::distributed::Vector<number> &      dst,
    const LinearAlgebra::distributed::Vector<number> &src,
    const std::pair<unsigned int, unsigned int> &     face_range) const
  {
    FEFaceEvaluation<dim, fe_degree, fe_degree + 1, 1, number> phi_inner(data,
                                                                         true);
    FEFaceEvaluation<dim, fe_degree, fe_degree + 1, 1, number> phi_outer(data,
                                                                         false);
    for (unsigned int face = face_range.first; face < face_range.second; ++face)
      {

// 在给定的一批面孔上，我们首先更新指向当前面孔的指针，然后访问矢量。如上所述，我们把访问向量和评估结合起来。在面积分的情况下，对于FE_DGQHermite基础的特殊情况，可以减少对向量的数据访问，正如上面解释的数据交换。由于 $2(k+1)^{d-1}$ 个单元自由度中只有 $(k+1)^d$ 个单元自由度被非零值或形状函数的导数所乘，这种结构可以被用于评估，大大减少了数据访问。减少数据访问不仅是有益的，因为它减少了飞行中的数据，从而有助于缓存，而且当从单元格索引列表中相距较远的单元格收集数值时，对面的数据访问往往比单元格积分更不规则。

        phi_inner.reinit(face);
        phi_inner.gather_evaluate(src,
                                  EvaluationFlags::values |
                                    EvaluationFlags::gradients);
        phi_outer.reinit(face);
        phi_outer.gather_evaluate(src,
                                  EvaluationFlags::values |
                                    EvaluationFlags::gradients);

// 接下来的两个语句是计算内部惩罚法的惩罚参数。正如在介绍中所解释的，我们希望有一个像 $\frac{1}{h_\text{i}}$ 这样的长度 $h_\text{i}$ 法线到面的缩放比例。对于一般的非笛卡尔网格，这个长度必须由反雅各布系数乘以实坐标的法向量的乘积来计算。从这个 "dim "分量的向量中，我们必须最终挑选出与参考单元的法线方向一致的分量。在MatrixFree中存储的几何数据中，雅各布式中的分量被应用，使得后一个方向总是最后一个分量`dim-1`（这很有利，因为参考单元的导数排序可以与面的方向无关）。这意味着我们可以简单地访问最后一个分量`dim-1`，而不必在`data.get_face_info(face).internal_face_no`和`data.get_face_info(face).exterior_face_no`中查找局部面的编号。最后，我们还必须取这些因素的绝对值，因为法线可能指向正或负的方向。

        const VectorizedArray<number> inverse_length_normal_to_face =
          0.5 * (std::abs((phi_inner.get_normal_vector(0) *
                           phi_inner.inverse_jacobian(0))[dim - 1]) +
                 std::abs((phi_outer.get_normal_vector(0) *
                           phi_outer.inverse_jacobian(0))[dim - 1]));
        const VectorizedArray<number> sigma =
          inverse_length_normal_to_face * get_penalty_factor();

// 在正交点的循环中，我们最终计算了对内部惩罚方案的所有贡献。根据介绍中的公式，测试函数的值被乘以解决方案中的跳跃乘以惩罚参数和实空间中的法向导数的平均值的差值。由于内侧和外侧的两个评估器由于跳跃而得到不同的符号，我们在这里用不同的符号传递结果。测试函数的正态导数会被内侧和外侧的解决方案中的负跳跃所乘。这个术语，被称为邻接一致性术语，根据其与原始一致性术语的关系，在代码中还必须包括 $\frac{1}{2}$ 的系数，由于测试函数槽中的平均数，它得到了二分之一的系数。

        for (unsigned int q = 0; q < phi_inner.n_q_points; ++q)
          {
            const VectorizedArray<number> solution_jump =
              (phi_inner.get_value(q) - phi_outer.get_value(q));
            const VectorizedArray<number> average_normal_derivative =
              (phi_inner.get_normal_derivative(q) +
               phi_outer.get_normal_derivative(q)) *
              number(0.5);
            const VectorizedArray<number> test_by_value =
              solution_jump * sigma - average_normal_derivative;

            phi_inner.submit_value(test_by_value, q);
            phi_outer.submit_value(-test_by_value, q);

            phi_inner.submit_normal_derivative(-solution_jump * number(0.5), q);
            phi_outer.submit_normal_derivative(-solution_jump * number(0.5), q);
          }

// 一旦我们完成了正交点的循环，我们就可以对面的积分循环进行和因子化操作，并将结果加到结果向量中，使用`integrate_scatter`函数。`scatter'这个名字反映了使用与`gather_evaluate'相同的模式将矢量数据分布到矢量中的分散位置。像以前一样，整合+写操作的组合允许我们减少数据访问。

        phi_inner.integrate_scatter(EvaluationFlags::values |
                                      EvaluationFlags::gradients,
                                    dst);
        phi_outer.integrate_scatter(EvaluationFlags::values |
                                      EvaluationFlags::gradients,
                                    dst);
      }
  }

// 边界面函数大体上沿用了内部面函数。唯一的区别是，我们没有一个单独的FEFaceEvaluation对象为我们提供外部值  $u^+$  ，但我们必须从边界条件和内部值  $u^-$  来定义它们。正如介绍中所解释的，我们在Dirichlet边界上使用 $u^+ = -u^- + 2 g_\text{D}$ 和 $\mathbf{n}^-\cdot \nabla u^+ = \mathbf{n}^-\cdot \nabla u^-$ ，在Neumann边界上使用 $u^+=u^-$ 和 $\mathbf{n}^-\cdot \nabla u^+ = -\mathbf{n}^-\cdot \nabla u^- + 2 g_\text{N}$  。由于这个操作实现了同质部分，即矩阵-向量乘积，我们必须在这里忽略边界函数 $g_\text{D}$ 和 $g_\text{N}$ ，并在 `LaplaceProblem::compute_rhs()`. 中把它们加到右侧。 ] 注意，由于通过 $u^+$ 将解 $u^-$ 扩展到外部，我们可以保持所有因子 $0.5$ 与内面函数相同，也可参见 step-39 中的讨论。

// 在这一点上有一个问题。下面的实现使用一个布尔变量`is_dirichlet`来切换Dirichlet和Neumann情况。然而，我们解决了一个问题，我们还想在一些边界上施加周期性的边界条件，即沿着 $x$ 方向的边界。人们可能会问，这里应该如何处理这些条件。答案是MatrixFree会自动将周期性边界视为技术上的边界，即两个相邻单元的解值相遇的内面，必须用适当的数值通量来处理。因此，周期性边界上的所有面将出现在`apply_face()`函数中，而不是这个函数中。

  template <int dim, int fe_degree, typename number>
  void LaplaceOperator<dim, fe_degree, number>::apply_boundary(
    const MatrixFree<dim, number> &                   data,
    LinearAlgebra::distributed::Vector<number> &      dst,
    const LinearAlgebra::distributed::Vector<number> &src,
    const std::pair<unsigned int, unsigned int> &     face_range) const
  {
    FEFaceEvaluation<dim, fe_degree, fe_degree + 1, 1, number> phi_inner(data,
                                                                         true);
    for (unsigned int face = face_range.first; face < face_range.second; ++face)
      {
        phi_inner.reinit(face);
        phi_inner.gather_evaluate(src,
                                  EvaluationFlags::values |
                                    EvaluationFlags::gradients);

        const VectorizedArray<number> inverse_length_normal_to_face =
          std::abs((phi_inner.get_normal_vector(0) *
                    phi_inner.inverse_jacobian(0))[dim - 1]);
        const VectorizedArray<number> sigma =
          inverse_length_normal_to_face * get_penalty_factor();

        const bool is_dirichlet = (data.get_boundary_id(face) == 0);

        for (unsigned int q = 0; q < phi_inner.n_q_points; ++q)
          {
            const VectorizedArray<number> u_inner = phi_inner.get_value(q);
            const VectorizedArray<number> u_outer =
              is_dirichlet ? -u_inner : u_inner;
            const VectorizedArray<number> normal_derivative_inner =
              phi_inner.get_normal_derivative(q);
            const VectorizedArray<number> normal_derivative_outer =
              is_dirichlet ? normal_derivative_inner : -normal_derivative_inner;
            const VectorizedArray<number> solution_jump = (u_inner - u_outer);
            const VectorizedArray<number> average_normal_derivative =
              (normal_derivative_inner + normal_derivative_outer) * number(0.5);
            const VectorizedArray<number> test_by_value =
              solution_jump * sigma - average_normal_derivative;
            phi_inner.submit_normal_derivative(-solution_jump * number(0.5), q);
            phi_inner.submit_value(test_by_value, q);
          }
        phi_inner.integrate_scatter(EvaluationFlags::values |
                                      EvaluationFlags::gradients,
                                    dst);
      }
  }

// 接下来我们来看看预处理程序的初始化。正如介绍中所解释的，我们想从一维质量和拉普拉斯矩阵的乘积中构造一个（近似的）单元矩阵的逆。我们的首要任务是计算一维矩阵，我们通过首先创建一个一维有限元来实现。在这里，我们没有预见到FE_DGQHermite<1>，而是从DoFHandler获得有限元的名称，用1替换 @p dim 参数（2或3）来创建一个一维名称，并通过使用FETools来构造一维元素。

  template <int dim, int fe_degree, typename number>
  void PreconditionBlockJacobi<dim, fe_degree, number>::initialize(
    const LaplaceOperator<dim, fe_degree, number> &op)
  {
    data = op.get_matrix_free();

    std::string name = data->get_dof_handler().get_fe().get_name();
    name.replace(name.find('<') + 1, 1, "1");
    std::unique_ptr<FiniteElement<1>> fe_1d = FETools::get_fe_by_name<1>(name);

// 至于在单位元素上计算一维矩阵，我们简单地写下在矩阵的行和列以及正交点上的典型装配程序会做什么。我们一劳永逸地选择相同的拉普拉斯矩阵，对内部面使用系数0.5（但可能由于网格的原因，在不同方向上的缩放比例不同）。因此，我们在Dirichlet边界（正确的系数应该是导数项为1，惩罚项为2，见 step-39 ）或在Neumann边界（系数应该为0）犯了一个小错误。由于我们只在多网格方案中使用这个类作为平滑器，这个错误不会有任何重大影响，只是影响平滑质量。

    const unsigned int                                 N = fe_degree + 1;
    FullMatrix<double>                                 laplace_unscaled(N, N);
    std::array<Table<2, VectorizedArray<number>>, dim> mass_matrices;
    std::array<Table<2, VectorizedArray<number>>, dim> laplace_matrices;
    for (unsigned int d = 0; d < dim; ++d)
      {
        mass_matrices[d].reinit(N, N);
        laplace_matrices[d].reinit(N, N);
      }

    QGauss<1> quadrature(N);
    for (unsigned int i = 0; i < N; ++i)
      for (unsigned int j = 0; j < N; ++j)
        {
          double sum_mass = 0, sum_laplace = 0;
          for (unsigned int q = 0; q < quadrature.size(); ++q)
            {
              sum_mass += (fe_1d->shape_value(i, quadrature.point(q)) *
                           fe_1d->shape_value(j, quadrature.point(q))) *
                          quadrature.weight(q);
              sum_laplace += (fe_1d->shape_grad(i, quadrature.point(q))[0] *
                              fe_1d->shape_grad(j, quadrature.point(q))[0]) *
                             quadrature.weight(q);
            }
          for (unsigned int d = 0; d < dim; ++d)
            mass_matrices[d](i, j) = sum_mass;

// 接下来两个语句组装的左右边界项似乎有一些任意的符号，但这些都是正确的，可以通过查看 step-39 并插入1D情况下的法向量的值-1和1来验证。

          sum_laplace +=
            (1. * fe_1d->shape_value(i, Point<1>()) *
               fe_1d->shape_value(j, Point<1>()) * op.get_penalty_factor() +
             0.5 * fe_1d->shape_grad(i, Point<1>())[0] *
               fe_1d->shape_value(j, Point<1>()) +
             0.5 * fe_1d->shape_grad(j, Point<1>())[0] *
               fe_1d->shape_value(i, Point<1>()));

          sum_laplace +=
            (1. * fe_1d->shape_value(i, Point<1>(1.0)) *
               fe_1d->shape_value(j, Point<1>(1.0)) * op.get_penalty_factor() -
             0.5 * fe_1d->shape_grad(i, Point<1>(1.0))[0] *
               fe_1d->shape_value(j, Point<1>(1.0)) -
             0.5 * fe_1d->shape_grad(j, Point<1>(1.0))[0] *
               fe_1d->shape_value(i, Point<1>(1.0)));

          laplace_unscaled(i, j) = sum_laplace;
        }

// 接下来，我们通过单元格，将缩放后的矩阵传递给TensorProductMatrixSymmetricSum，以实际计算代表逆的广义特征值问题。由于矩阵的近似构造为 $A\otimes M + M\otimes A$ ，并且每个元素的权重是恒定的，我们可以在拉普拉斯矩阵上应用所有的权重，并简单地保持质量矩阵不被缩放。在单元格的循环中，我们要利用MatrixFree类提供的几何体压缩，并检查当前的几何体是否与上一批单元格上的几何体相同，在这种情况下就没有什么可做的。一旦调用了`reinit()`，就可以通过 FEEvaluation::get_mapping_data_index_offset() 访问这种压缩。

// 一旦我们通过FEEvaluation访问函数访问了反雅各布系数（我们取第4个正交点的，因为它们在笛卡尔单元的所有正交点上都应该是一样的），我们检查它是对角线的，然后提取原始雅各布系数的行列式，即反雅各布系数的行列式，并根据质量矩阵的一维拉普拉斯乘以 $d-1$ 份，设置权重为 $\text{det}(J) / h_d^2$  。

    cell_matrices.clear();
    FEEvaluation<dim, fe_degree, fe_degree + 1, 1, number> phi(*data);
    unsigned int old_mapping_data_index = numbers::invalid_unsigned_int;
    for (unsigned int cell = 0; cell < data->n_cell_batches(); ++cell)
      {
        phi.reinit(cell);

        if (phi.get_mapping_data_index_offset() == old_mapping_data_index)
          continue;

        Tensor<2, dim, VectorizedArray<number>> inverse_jacobian =
          phi.inverse_jacobian(0);

        for (unsigned int d = 0; d < dim; ++d)
          for (unsigned int e = 0; e < dim; ++e)
            if (d != e)
              for (unsigned int v = 0; v < VectorizedArray<number>::size(); ++v)
                AssertThrow(inverse_jacobian[d][e][v] == 0.,
                            ExcNotImplemented());

        VectorizedArray<number> jacobian_determinant = inverse_jacobian[0][0];
        for (unsigned int e = 1; e < dim; ++e)
          jacobian_determinant *= inverse_jacobian[e][e];
        jacobian_determinant = 1. / jacobian_determinant;

        for (unsigned int d = 0; d < dim; ++d)
          {
            const VectorizedArray<number> scaling_factor =
              inverse_jacobian[d][d] * inverse_jacobian[d][d] *
              jacobian_determinant;

// 一旦我们知道了拉普拉斯矩阵的比例系数，我们就将这个权重应用于未被缩放的DG拉普拉斯矩阵，并将数组发送到TensorProductMatrixSymmetricSum类，以计算介绍中提到的广义特征值问题。

            for (unsigned int i = 0; i < N; ++i)
              for (unsigned int j = 0; j < N; ++j)
                laplace_matrices[d](i, j) =
                  scaling_factor * laplace_unscaled(i, j);
          }
        if (cell_matrices.size() <= phi.get_mapping_data_index_offset())
          cell_matrices.resize(phi.get_mapping_data_index_offset() + 1);
        cell_matrices[phi.get_mapping_data_index_offset()].reinit(
          mass_matrices, laplace_matrices);
      }
  }

// 在DG背景下，用于近似块状Jacobi预处理的vmult函数非常简单。我们只需要读取当前单元格批次的值，对张量积矩阵阵列中的给定条目进行逆运算，并将结果写回。在这个循环中，我们覆盖了`dst`中的内容，而不是首先将条目设置为零。这对于DG方法来说是合法的，因为每个单元都有独立的自由度。此外，我们手动写出所有单元批的循环，而不是通过 MatrixFree::cell_loop(). 我们这样做是因为我们知道我们在这里不需要通过MPI网络进行数据交换，因为所有的计算都是在每个处理器上的本地单元上完成的。

  template <int dim, int fe_degree, typename number>
  void PreconditionBlockJacobi<dim, fe_degree, number>::vmult(
    LinearAlgebra::distributed::Vector<number> &      dst,
    const LinearAlgebra::distributed::Vector<number> &src) const
  {
    adjust_ghost_range_if_necessary(*data, dst);
    adjust_ghost_range_if_necessary(*data, src);

    FEEvaluation<dim, fe_degree, fe_degree + 1, 1, number> phi(*data);
    for (unsigned int cell = 0; cell < data->n_cell_batches(); ++cell)
      {
        phi.reinit(cell);
        phi.read_dof_values(src);
        cell_matrices[phi.get_mapping_data_index_offset()].apply_inverse(
          ArrayView<VectorizedArray<number>>(phi.begin_dof_values(),
                                             phi.dofs_per_cell),
          ArrayView<const VectorizedArray<number>>(phi.begin_dof_values(),
                                                   phi.dofs_per_cell));
        phi.set_dof_values(dst);
      }
  }

// LaplaceProblem类的定义与  step-37  非常相似。一个区别是我们将元素度作为模板参数添加到类中，这将允许我们通过在`main()`函数中创建不同的实例，更容易在同一个程序中包含多个度。第二个区别是选择了FE_DGQHermite这个元素，它是专门用于这种方程的。

  template <int dim, int fe_degree>
  class LaplaceProblem
  {
  public:
    LaplaceProblem();
    void run();

  private:
    void setup_system();
    void compute_rhs();
    void solve();
    void analyze_results() const;

#ifdef DEAL_II_WITH_P4EST
    parallel::distributed::Triangulation<dim> triangulation;
#else
    Triangulation<dim> triangulation;
#endif

    FE_DGQHermite<dim> fe;
    DoFHandler<dim>    dof_handler;

    MappingQ1<dim> mapping;

    using SystemMatrixType = LaplaceOperator<dim, fe_degree, double>;
    SystemMatrixType system_matrix;

    using LevelMatrixType = LaplaceOperator<dim, fe_degree, float>;
    MGLevelObject<LevelMatrixType> mg_matrices;

    LinearAlgebra::distributed::Vector<double> solution;
    LinearAlgebra::distributed::Vector<double> system_rhs;

    double             setup_time;
    ConditionalOStream pcout;
    ConditionalOStream time_details;
  };

  template <int dim, int fe_degree>
  LaplaceProblem<dim, fe_degree>::LaplaceProblem()
    :
#ifdef DEAL_II_WITH_P4EST
    triangulation(
      MPI_COMM_WORLD,
      Triangulation<dim>::limit_level_difference_at_vertices,
      parallel::distributed::Triangulation<dim>::construct_multigrid_hierarchy)
    ,
#else
    triangulation(Triangulation<dim>::limit_level_difference_at_vertices)
    ,
#endif
    fe(fe_degree)
    , dof_handler(triangulation)
    , setup_time(0.)
    , pcout(std::cout, Utilities::MPI::this_mpi_process(MPI_COMM_WORLD) == 0)
    , time_details(std::cout,
                   false &&
                     Utilities::MPI::this_mpi_process(MPI_COMM_WORLD) == 0)
  {}

// 设置函数在两个方面与  step-37  不同。首先是我们不需要为不连续的Ansatz空间插值任何约束，而只是将一个假的AffineConstraints对象传入 Matrixfree::reinit().  第二个变化是因为我们需要告诉MatrixFree同时初始化面的数据结构。我们通过为内部面和边界面分别设置更新标志来做到这一点。在边界面，我们需要函数值、它们的梯度、JxW值（用于积分）、法向量和正交点（用于边界条件的评估），而对于内部面，我们只需要形状函数值、梯度、JxW值和法向量。只要`mapping_update_flags_inner_faces`或`mapping_update_flags_boundary_faces`中的一个与UpdateFlags的默认值`update_default`不同，MatrixFree中的面数据结构总是被建立的。

  template <int dim, int fe_degree>
  void LaplaceProblem<dim, fe_degree>::setup_system()
  {
    Timer time;
    setup_time = 0;

    system_matrix.clear();
    mg_matrices.clear_elements();

    dof_handler.distribute_dofs(fe);
    dof_handler.distribute_mg_dofs();

    pcout << "Number of degrees of freedom: " << dof_handler.n_dofs()
          << std::endl;

    setup_time += time.wall_time();
    time_details << "Distribute DoFs               " << time.wall_time() << " s"
                 << std::endl;
    time.restart();

    AffineConstraints<double> dummy;
    dummy.close();

    {
      typename MatrixFree<dim, double>::AdditionalData additional_data;
      additional_data.tasks_parallel_scheme =
        MatrixFree<dim, double>::AdditionalData::none;
      additional_data.mapping_update_flags =
        (update_gradients | update_JxW_values | update_quadrature_points);
      additional_data.mapping_update_flags_inner_faces =
        (update_gradients | update_JxW_values | update_normal_vectors);
      additional_data.mapping_update_flags_boundary_faces =
        (update_gradients | update_JxW_values | update_normal_vectors |
         update_quadrature_points);
      const auto system_mf_storage =
        std::make_shared<MatrixFree<dim, double>>();
      system_mf_storage->reinit(
        mapping, dof_handler, dummy, QGauss<1>(fe.degree + 1), additional_data);
      system_matrix.initialize(system_mf_storage);
    }

    system_matrix.initialize_dof_vector(solution);
    system_matrix.initialize_dof_vector(system_rhs);

    setup_time += time.wall_time();
    time_details << "Setup matrix-free system      " << time.wall_time() << " s"
                 << std::endl;
    time.restart();

    const unsigned int nlevels = triangulation.n_global_levels();
    mg_matrices.resize(0, nlevels - 1);

    for (unsigned int level = 0; level < nlevels; ++level)
      {
        typename MatrixFree<dim, float>::AdditionalData additional_data;
        additional_data.tasks_parallel_scheme =
          MatrixFree<dim, float>::AdditionalData::none;
        additional_data.mapping_update_flags =
          (update_gradients | update_JxW_values);
        additional_data.mapping_update_flags_inner_faces =
          (update_gradients | update_JxW_values);
        additional_data.mapping_update_flags_boundary_faces =
          (update_gradients | update_JxW_values);
        additional_data.mg_level = level;
        const auto mg_mf_storage_level =
          std::make_shared<MatrixFree<dim, float>>();
        mg_mf_storage_level->reinit(mapping,
                                    dof_handler,
                                    dummy,
                                    QGauss<1>(fe.degree + 1),
                                    additional_data);

        mg_matrices[level].initialize(mg_mf_storage_level);
      }
    setup_time += time.wall_time();
    time_details << "Setup matrix-free levels      " << time.wall_time() << " s"
                 << std::endl;
  }

// 右手边的计算比  step-37  中的计算要复杂一些。现在的单元项包括分析解的负拉普拉斯，`RightHandSide'，为此我们需要首先将VectorizedArray字段的Point，即一批点，通过分别评估VectorizedArray中的所有通道，拆成一个点。请记住，通道的数量取决于硬件；对于不提供矢量化的系统（或deal.II没有本征），它可能是1，但在最近的Intel架构的AVX-512上也可能是8或16。

  template <int dim, int fe_degree>
  void LaplaceProblem<dim, fe_degree>::compute_rhs()
  {
    Timer time;
    system_rhs                          = 0;
    const MatrixFree<dim, double> &data = *system_matrix.get_matrix_free();
    FEEvaluation<dim, fe_degree>   phi(data);
    RightHandSide<dim>             rhs_func;
    Solution<dim>                  exact_solution;
    for (unsigned int cell = 0; cell < data.n_cell_batches(); ++cell)
      {
        phi.reinit(cell);
        for (unsigned int q = 0; q < phi.n_q_points; ++q)
          {
            VectorizedArray<double> rhs_val = VectorizedArray<double>();
            Point<dim, VectorizedArray<double>> point_batch =
              phi.quadrature_point(q);
            for (unsigned int v = 0; v < VectorizedArray<double>::size(); ++v)
              {
                Point<dim> single_point;
                for (unsigned int d = 0; d < dim; ++d)
                  single_point[d] = point_batch[d][v];
                rhs_val[v] = rhs_func.value(single_point);
              }
            phi.submit_value(rhs_val, q);
          }
        phi.integrate_scatter(EvaluationFlags::values, system_rhs);
      }

// 其次，我们还需要应用Dirichlet和Neumann边界条件。一旦Dirichlet边界上的外部求解值 $u^+ = -u^- + 2 g_\text{D}$ 和 $\mathbf{n}^-\cdot \nabla u^+ = \mathbf{n}^-\cdot \nabla u^-$ 以及Neumann边界上的 $u^+=u^-$ 和 $\mathbf{n}^-\cdot \nabla u^+ = -\mathbf{n}^-\cdot \nabla u^- + 2 g_\text{N}$ 被插入并以边界函数 $g_\text{D}$ 和 $g_\text{N}$ 展开，这个函数就是到函数 `LaplaceOperator::apply_boundary()` 所缺的部分。需要记住的一点是，我们把边界条件移到右手边，所以符号与我们强加在解的部分相反。

// 我们可以通过 MatrixFree::loop 部分发出单元格和边界部分，但我们选择手动写出所有面的完整循环，以了解MatrixFree中面的索引布局是如何设置的：内部面和边界面都共享索引范围，所有批次的内部面的数字都比批次的边界单元格低。两种变体的单一索引使我们可以很容易地在两种情况下使用相同的数据结构FEFaceEvaluation，它附着在同一个数据域上，只是位置不同。内层面的批次（其中一个批次是由于将几个面合并成一个面进行矢量化）的数量由 MatrixFree::n_inner_face_batches(), 给出，而边界面的批次数量由 MatrixFree::n_boundary_face_batches(). 给出。
    FEFaceEvaluation<dim, fe_degree> phi_face(data, true);
    for (unsigned int face = data.n_inner_face_batches();
         face < data.n_inner_face_batches() + data.n_boundary_face_batches();
         ++face)
      {
        phi_face.reinit(face);

        const VectorizedArray<double> inverse_length_normal_to_face =
          std::abs((phi_face.get_normal_vector(0) *
                    phi_face.inverse_jacobian(0))[dim - 1]);
        const VectorizedArray<double> sigma =
          inverse_length_normal_to_face * system_matrix.get_penalty_factor();

        for (unsigned int q = 0; q < phi_face.n_q_points; ++q)
          {
            VectorizedArray<double> test_value = VectorizedArray<double>(),
                                    test_normal_derivative =
                                      VectorizedArray<double>();
            Point<dim, VectorizedArray<double>> point_batch =
              phi_face.quadrature_point(q);

            for (unsigned int v = 0; v < VectorizedArray<double>::size(); ++v)
              {
                Point<dim> single_point;
                for (unsigned int d = 0; d < dim; ++d)
                  single_point[d] = point_batch[d][v];

// MatrixFree类让我们查询当前面批的边界_id。请记住，MatrixFree为矢量化设置了批次，使一个批次中的所有面孔都有相同的属性，其中包括它们的`边界_id'。因此，我们可以在这里为当前面的索引`face`查询该id，并在Dirichlet情况下（我们在函数值上添加一些东西）或Neumann情况下（我们在法线导数上添加一些东西）施加。

                if (data.get_boundary_id(face) == 0)
                  test_value[v] = 2.0 * exact_solution.value(single_point);
                else
                  {
                    Tensor<1, dim> normal;
                    for (unsigned int d = 0; d < dim; ++d)
                      normal[d] = phi_face.get_normal_vector(q)[d][v];
                    test_normal_derivative[v] =
                      -normal * exact_solution.gradient(single_point);
                  }
              }
            phi_face.submit_value(test_value * sigma - test_normal_derivative,
                                  q);
            phi_face.submit_normal_derivative(-0.5 * test_value, q);
          }
        phi_face.integrate_scatter(EvaluationFlags::values |
                                     EvaluationFlags::gradients,
                                   system_rhs);
      }

// 由于我们手动运行了单元格的循环，而不是使用 MatrixFree::loop(), ，我们不能忘记与MPI进行数据交换。

// 或者说，对于DG元素来说，我们不需要这样做，因为每个单元都有自己的自由度，单元和边界积分只对本地拥有的单元进行评估。与相邻子域的耦合只通过内表面积分来实现，我们在这里没有做这个。也就是说，在这里调用这个函数并没有什么坏处，所以我们这样做是为了提醒大家在 MatrixFree::loop(). 里面发生了什么。
    system_rhs.compress(VectorOperation::add);
    setup_time += time.wall_time();
    time_details << "Compute right hand side       " << time.wall_time()
                 << " s\n";
  }

// `solve()`函数几乎逐字复制自  step-37  。我们设置了相同的多网格成分，即水平转移、平滑器和粗略的网格求解器。唯一不同的是，我们没有使用拉普拉斯的对角线作为用于平滑的切比雪夫迭代的预处理，而是使用我们新解决的类`%PreconditionBlockJacobi`。不过，机制是一样的。

  template <int dim, int fe_degree>
  void LaplaceProblem<dim, fe_degree>::solve()
  {
    Timer                            time;
    MGTransferMatrixFree<dim, float> mg_transfer;
    mg_transfer.build(dof_handler);
    setup_time += time.wall_time();
    time_details << "MG build transfer time        " << time.wall_time()
                 << " s\n";
    time.restart();

    using SmootherType =
      PreconditionChebyshev<LevelMatrixType,
                            LinearAlgebra::distributed::Vector<float>,
                            PreconditionBlockJacobi<dim, fe_degree, float>>;
    mg::SmootherRelaxation<SmootherType,
                           LinearAlgebra::distributed::Vector<float>>
                                                         mg_smoother;
    MGLevelObject<typename SmootherType::AdditionalData> smoother_data;
    smoother_data.resize(0, triangulation.n_global_levels() - 1);
    for (unsigned int level = 0; level < triangulation.n_global_levels();
         ++level)
      {
        if (level > 0)
          {
            smoother_data[level].smoothing_range     = 15.;
            smoother_data[level].degree              = 3;
            smoother_data[level].eig_cg_n_iterations = 10;
          }
        else
          {
            smoother_data[0].smoothing_range = 2e-2;
            smoother_data[0].degree          = numbers::invalid_unsigned_int;
            smoother_data[0].eig_cg_n_iterations = mg_matrices[0].m();
          }
        smoother_data[level].preconditioner =
          std::make_shared<PreconditionBlockJacobi<dim, fe_degree, float>>();
        smoother_data[level].preconditioner->initialize(mg_matrices[level]);
      }
    mg_smoother.initialize(mg_matrices, smoother_data);

    MGCoarseGridApplySmoother<LinearAlgebra::distributed::Vector<float>>
      mg_coarse;
    mg_coarse.initialize(mg_smoother);

    mg::Matrix<LinearAlgebra::distributed::Vector<float>> mg_matrix(
      mg_matrices);

    Multigrid<LinearAlgebra::distributed::Vector<float>> mg(
      mg_matrix, mg_coarse, mg_transfer, mg_smoother, mg_smoother);

    PreconditionMG<dim,
                   LinearAlgebra::distributed::Vector<float>,
                   MGTransferMatrixFree<dim, float>>
      preconditioner(dof_handler, mg, mg_transfer);

    SolverControl solver_control(10000, 1e-12 * system_rhs.l2_norm());
    SolverCG<LinearAlgebra::distributed::Vector<double>> cg(solver_control);
    setup_time += time.wall_time();
    time_details << "MG build smoother time        " << time.wall_time()
                 << "s\n";
    pcout << "Total setup time              " << setup_time << " s\n";

    time.reset();
    time.start();
    cg.solve(system_matrix, solution, system_rhs, preconditioner);

    pcout << "Time solve (" << solver_control.last_step() << " iterations)    "
          << time.wall_time() << " s" << std::endl;
  }

// 由于我们已经用分析法解决了一个问题，我们想通过计算数值结果与分析法的L2误差来验证我们实现的正确性。

  template <int dim, int fe_degree>
  void LaplaceProblem<dim, fe_degree>::analyze_results() const
  {
    Vector<float> error_per_cell(triangulation.n_active_cells());
    VectorTools::integrate_difference(mapping,
                                      dof_handler,
                                      solution,
                                      Solution<dim>(),
                                      error_per_cell,
                                      QGauss<dim>(fe.degree + 2),
                                      VectorTools::L2_norm);
    pcout << "Verification via L2 error:    "
          << std::sqrt(
               Utilities::MPI::sum(error_per_cell.norm_sqr(), MPI_COMM_WORLD))
          << std::endl;
  }

// `run()`函数设置了初始网格，然后以常规方式运行多网格程序。作为一个域，我们选择一个矩形，在 $x$ -方向上有周期性的边界条件，在 $y$ 方向上的正面（即索引号为2的面，边界id等于0）有迪里希特条件，在背面以及 $z$ 方向上的两个面为三维情况（边界id等于1）有纽曼条件。与 $y$ 和 $z$ 方向相比， $x$ 方向的域的范围有些不同（考虑到 "解决方案 "的定义，我们希望在这里实现周期性的解决方案）。

  template <int dim, int fe_degree>
  void LaplaceProblem<dim, fe_degree>::run()
  {
    const unsigned int n_ranks =
      Utilities::MPI::n_mpi_processes(MPI_COMM_WORLD);
    pcout << "Running with " << n_ranks << " MPI process"
          << (n_ranks > 1 ? "es" : "") << ", element " << fe.get_name()
          << std::endl
          << std::endl;
    for (unsigned int cycle = 0; cycle < 9 - dim; ++cycle)
      {
        pcout << "Cycle " << cycle << std::endl;

        if (cycle == 0)
          {
            Point<dim> upper_right;
            upper_right[0] = 2.5;
            for (unsigned int d = 1; d < dim; ++d)
              upper_right[d] = 2.8;
            GridGenerator::hyper_rectangle(triangulation,
                                           Point<dim>(),
                                           upper_right);
            triangulation.begin_active()->face(0)->set_boundary_id(10);
            triangulation.begin_active()->face(1)->set_boundary_id(11);
            triangulation.begin_active()->face(2)->set_boundary_id(0);
            for (unsigned int f = 3;
                 f < triangulation.begin_active()->n_faces();
                 ++f)
              triangulation.begin_active()->face(f)->set_boundary_id(1);

            std::vector<GridTools::PeriodicFacePair<
              typename Triangulation<dim>::cell_iterator>>
              periodic_faces;
            GridTools::collect_periodic_faces(
              triangulation, 10, 11, 0, periodic_faces);
            triangulation.add_periodicity(periodic_faces);

            triangulation.refine_global(6 - 2 * dim);
          }
        triangulation.refine_global(1);
        setup_system();
        compute_rhs();
        solve();
        analyze_results();
        pcout << std::endl;
      };
  }
} // namespace Step59

// `main()`函数中没有任何意外。我们通过`MPI_Init()`类调用`MPI_InitFinalize`，传入文件顶部设置的关于维度和度的两个参数，然后运行拉普拉斯问题。

int main(int argc, char *argv[])
{
  try
    {
      using namespace Step59;

      Utilities::MPI::MPI_InitFinalize mpi_init(argc, argv, 1);

      LaplaceProblem<dimension, degree_finite_element> laplace_problem;
      laplace_problem.run();
    }
  catch (std::exception &exc)
    {
      std::cerr << std::endl
                << std::endl
                << "----------------------------------------------------"
                << std::endl;
      std::cerr << "Exception on processing: " << std::endl
                << exc.what() << std::endl
                << "Aborting!" << std::endl
                << "----------------------------------------------------"
                << std::endl;
      return 1;
    }
  catch (...)
    {
      std::cerr << std::endl
                << std::endl
                << "----------------------------------------------------"
                << std::endl;
      std::cerr << "Unknown exception!" << std::endl
                << "Aborting!" << std::endl
                << "----------------------------------------------------"
                << std::endl;
      return 1;
    }

  return 0;
}