/* Siconos is a program dedicated to modeling, simulation and control
 * of non smooth dynamical systems.
 *
 * Copyright 2018 INRIA.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
*/
/*! \file JointStopR.cpp

*/

#include "JointStopR.hpp"
#include <NewtonEulerDS.hpp>
#include <Interaction.hpp>
#include <boost/math/quaternion.hpp>
#include <BlockVector.hpp>
#include <cfloat>
#include <iostream>

// #define DEBUG_BEGIN_END_ONLY
// #define DEBUG_STDOUT
// #define DEBUG_MESSAGES
#include "debug.h"

/** Initialize a joint stop for a common case: a single axis with a
 * single stop, either positive or negative. For use with
 * NewtonImpactNSL. */
JointStopR::JointStopR(SP::NewtonEulerJointR joint, double pos, bool dir,
                       unsigned int axis)
  : NewtonEulerR()
  , _joint(joint)
  , _axis(std11::make_shared< std::vector<unsigned int> >())
  , _pos(std11::make_shared<SiconosVector>(1))
  , _dir(std11::make_shared<SiconosVector>(1))
{
  _axis->push_back(axis);
  _pos->setValue(0, pos);
  _dir->setValue(0, dir ? -1 : 1);
  _axisMin = axis;
  _axisMax = axis;
  assert( (_axisMax - _axisMin + 1) <= _joint->numberOfDoF() );
}

/** Initialize a multidimensional joint stop, e.g. the cone stop on
 * a ball joint. For use with NewtonImpactFrictionNSL size 2 or 3. */
JointStopR::JointStopR(SP::NewtonEulerJointR joint, SP::SiconosVector pos,
                       SP::SiconosVector dir, SP::UnsignedIntVector axes)
  : NewtonEulerR()
  , _joint(joint)
  , _axis(axes)
  , _pos(pos)
  , _dir(dir)
{
  _axisMin = 100;
  _axisMax = 0;
  for (unsigned int i=0; i < _axis->size(); i++)
  {
    if ((*_axis)[i] > _axisMax) _axisMax = (*_axis)[i];
    if ((*_axis)[i] < _axisMin) _axisMin = (*_axis)[i];
  }
  assert( (_axisMax - _axisMin + 1) <= _joint->numberOfDoF() );
}

#if 0  // Disabled, see JointStopR.hpp.  Use multiple JointStopR instead.
/** Initialize a joint stop for a common case: a single axis with a
 * double stop, one positive and one negative. */
JointStopR::JointStopR(SP::NewtonEulerJointR joint, double pos, double neg,
                       unsigned int axis)
  : NewtonEulerR()
  , _joint(joint)
  , _axis(std11::make_shared< std::vector<unsigned int> >())
  , _pos(std11::make_shared<SiconosVector>(2))
  , _dir(std11::make_shared<SiconosVector>(2))
{
  _axis->push_back(axis);
  _axis->push_back(axis);
  _pos->setValue(0, pos);
  _pos->setValue(1, neg);
  _dir->setValue(0, 1);
  _dir->setValue(1, -1);
  _axisMin = axis;
  _axisMax = axis;
  assert( (_axisMax - _axisMin + 1) <= _joint->numberOfDoF() );
}
#endif

void JointStopR::computeh(double time, BlockVector& q0, SiconosVector& y)
{
  // Common cases optimisation
  bool case_onestop = y.size()==1;
  bool case_posneg = y.size()==2 && (*_axis)[0] == (*_axis)[1];
  if (case_onestop || case_posneg)
  {
    _joint->computehDoF(time, q0, y, (*_axis)[0]);

    y.setValue(0, (y.getValue(0) - _pos->getValue(0)) * _dir->getValue(0));
    if (case_posneg)
      y.setValue(1, (y.getValue(0) - _pos->getValue(1)) * _dir->getValue(1));
    return;
  }

  // Get h for each relevant axis
  SiconosVector tmp_y(_axisMax - _axisMin + 1);
  _joint->computehDoF(time, q0, tmp_y, _axisMin);

  // Copy and scale each stop for its axis/position/direction
  for (unsigned int i=0; i < y.size(); i++) {
    y.setValue(i, (tmp_y.getValue((*_axis)[i])
                   - _pos->getValue(i))*_dir->getValue(i));
  }
}

void JointStopR::computeJachq(double time, Interaction& inter, SP::BlockVector q0)
{
  unsigned int n = _axisMax - _axisMin + 1;

  if (!_jachqTmp || !(_jachqTmp->size(1) == q0->size() &&
                      _jachqTmp->size(0) == n))
  {
    _jachqTmp = std11::make_shared<SimpleMatrix>(n, q0->size());
  }

  // Compute the jacobian for the required range of axes
  _joint->computeJachqDoF(time, inter, q0, *_jachqTmp, _axisMin);

  // Copy indicated axes into the stop jacobian, possibly flipped for negative stops
  for (unsigned int i=0; i<_jachq->size(0); i++)
    for (unsigned int j=0; j<_jachq->size(1); j++)
      _jachq->setValue(i,j,
                       _jachqTmp->getValue((*_axis)[i]-_axisMin,j) * _dir->getValue(i));
}

unsigned int JointStopR::numberOfConstraints()
{
  return _axis->size();
}


/**
 * \file   MeshSurfaceExtraction.cpp
 * \author Karsten Rink
 * \date   2013-04-04
 * \brief  Implementation of the MeshSurfaceExtraction class.
 *
 * \copyright
 * Copyright (c) 2012-2016, OpenGeoSys Community (http://www.opengeosys.org)
 *            Distributed under a Modified BSD License.
 *              See accompanying file LICENSE.txt or
 *              http://www.opengeosys.org/project/license
 *
 */

#include "MeshSurfaceExtraction.h"

#include <boost/math/constants/constants.hpp>

#include "logog/include/logog.hpp"

#include "GeoLib/Point.h"

#include "MeshLib/Mesh.h"
#include "MeshLib/Elements/Line.h"
#include "MeshLib/Elements/Tri.h"
#include "MeshLib/Elements/Quad.h"
#include "MeshLib/MeshEditing/DuplicateMeshComponents.h"
#include "MeshLib/MeshQuality/MeshValidation.h"
#include "MeshLib/MeshSearch/NodeSearch.h"
#include "MeshLib/MeshEditing/RemoveMeshComponents.h"

namespace MeshLib {

std::vector<double> MeshSurfaceExtraction::getSurfaceAreaForNodes(const MeshLib::Mesh &mesh)
{
    std::vector<double> node_area_vec;
    if (mesh.getDimension() != 2)
    {
        ERR ("Error in MeshSurfaceExtraction::getSurfaceAreaForNodes() - Given mesh is no surface mesh (dimension != 2).");
        return node_area_vec;
    }

    double total_area (0);

    // for each node, a vector containing all the element idget every element
    const std::vector<MeshLib::Node*> &nodes = mesh.getNodes();
    const std::size_t nNodes ( mesh.getNumberOfNodes() );
    for (std::size_t n=0; n<nNodes; ++n)
    {
        double node_area (0);

        std::vector<MeshLib::Element*> conn_elems = nodes[n]->getElements();
        const std::size_t nConnElems (conn_elems.size());

        for (std::size_t i=0; i<nConnElems; ++i)
        {
            const MeshLib::Element* elem (conn_elems[i]);
            const unsigned nElemParts = (elem->getGeomType() == MeshElemType::TRIANGLE) ? 3 : 4;
            const double area = conn_elems[i]->getContent() / nElemParts;
            node_area += area;
            total_area += area;
        }

        node_area_vec.push_back(node_area);
    }

    INFO ("Total surface Area: %f", total_area);

    return node_area_vec;
}

MeshLib::Mesh* MeshSurfaceExtraction::getMeshSurface(
    const MeshLib::Mesh& mesh, const MathLib::Vector3& dir, double angle,
    std::string const& subsfc_node_id_backup_prop_name)
{
    if (angle < 0 || angle > 90)
    {
        ERR ("Supported angle between 0 and 90 degrees only.");
        return nullptr;
    }

    INFO ("Extracting mesh surface...");
    std::vector<MeshLib::Element*> sfc_elements;
    get2DSurfaceElements(mesh.getElements(), sfc_elements, dir, angle, mesh.getDimension());

    if (sfc_elements.empty())
        return nullptr;

    std::vector<MeshLib::Node*> sfc_nodes;
    std::vector<std::size_t> node_id_map(mesh.getNumberOfNodes());
    get2DSurfaceNodes(sfc_nodes, mesh.getNumberOfNodes(), sfc_elements, node_id_map);

    // create new elements vector with newly created nodes
    std::vector<MeshLib::Element*> new_elements;
    new_elements.reserve(sfc_elements.size());
    for (auto elem = sfc_elements.cbegin(); elem != sfc_elements.cend(); ++elem)
    {
        unsigned const n_elem_nodes ((*elem)->getNumberOfBaseNodes());
        MeshLib::Node** new_nodes = new MeshLib::Node*[n_elem_nodes];
        for (unsigned k(0); k<n_elem_nodes; k++)
            new_nodes[k] = sfc_nodes[node_id_map[(*elem)->getNode(k)->getID()]];
        if ((*elem)->getGeomType() == MeshElemType::TRIANGLE)
            new_elements.push_back(new MeshLib::Tri(new_nodes));
        else {
            assert((*elem)->getGeomType() == MeshElemType::QUAD);
            new_elements.push_back(new MeshLib::Quad(new_nodes));
        }
        delete *elem;
    }

    std::vector<std::size_t> id_map;
    if (!subsfc_node_id_backup_prop_name.empty())
    {
        id_map.reserve(sfc_nodes.size());
        for (auto node = sfc_nodes.cbegin(); node != sfc_nodes.cend(); ++node)
            id_map.push_back((*node)->getID());
    }
    MeshLib::Mesh* result (new Mesh(mesh.getName()+"-Surface", sfc_nodes, new_elements));
    // transmit the original node ids of the subsurface mesh as a property
    if (!subsfc_node_id_backup_prop_name.empty()) {
        boost::optional<MeshLib::PropertyVector<std::size_t>&> orig_node_ids(
            result->getProperties().createNewPropertyVector<std::size_t>(
                subsfc_node_id_backup_prop_name , MeshLib::MeshItemType::Node, 1));
        if (orig_node_ids) {
            orig_node_ids->resize(id_map.size());
            std::copy(id_map.cbegin(), id_map.cend(), orig_node_ids->begin());
        }
    }
    return result;
}

MeshLib::Mesh* MeshSurfaceExtraction::getMeshBoundary(const MeshLib::Mesh &mesh)
{
    if (mesh.getDimension()==1)
        return nullptr;

    // For 3D meshes return the 2D surface
    if (mesh.getDimension()==3)
    {
        MathLib::Vector3 dir(0,0,0);
        return getMeshSurface(mesh, dir, 90);
    }

    // For 2D meshes return the boundary lines
    std::vector<MeshLib::Node*> nodes = MeshLib::copyNodeVector(mesh.getNodes());
    std::vector<MeshLib::Element*> boundary_elements;

    std::vector<MeshLib::Element*> const& org_elems (mesh.getElements());
    for (auto it=org_elems.begin(); it!=org_elems.end(); ++it)
    {
        MeshLib::Element* elem (*it);
        std::size_t const n_edges (elem->getNumberOfEdges());
        for (std::size_t i=0; i<n_edges; ++i)
            if (elem->getNeighbor(i) == nullptr)
            {
                MeshLib::Element const*const edge (elem->getEdge(i));
                boundary_elements.push_back(MeshLib::copyElement(edge, nodes));
                delete edge;
            }
    }
    MeshLib::Mesh* result = new MeshLib::Mesh("Boundary Mesh", nodes, boundary_elements);
    MeshLib::NodeSearch ns(*result);
    if (ns.searchUnused() == 0) {
        return result;
    } else {
        auto removed = MeshLib::removeNodes(*result, ns.getSearchedNodeIDs(), result->getName());
        delete result;
        return removed;
    }
}

void MeshSurfaceExtraction::get2DSurfaceElements(const std::vector<MeshLib::Element*> &all_elements, std::vector<MeshLib::Element*> &sfc_elements, const MathLib::Vector3 &dir, double angle, unsigned mesh_dimension)
{
    if (mesh_dimension<2 || mesh_dimension>3)
        ERR("Cannot handle meshes of dimension %i", mesh_dimension);

    bool const complete_surface = (MathLib::scalarProduct(dir, dir) == 0);

    double const pi (boost::math::constants::pi<double>());
    double const cos_theta (std::cos(angle * pi / 180.0));
    MathLib::Vector3 const norm_dir (dir.getNormalizedVector());

    for (auto elem = all_elements.cbegin(); elem != all_elements.cend(); ++elem)
    {
        const unsigned element_dimension ((*elem)->getDimension());
        if (element_dimension < mesh_dimension)
            continue;

        if (element_dimension == 2)
        {
            if (!complete_surface)
            {
                MeshLib::Element* face = *elem;
                if (MathLib::scalarProduct(FaceRule::getSurfaceNormal(face).getNormalizedVector(), norm_dir) > cos_theta)
                    continue;
            }
            sfc_elements.push_back(*elem);
        }
        else
        {
            if (!(*elem)->isBoundaryElement())
                continue;
            const unsigned nFaces ((*elem)->getNumberOfFaces());
            for (unsigned j=0; j<nFaces; ++j)
            {
                if ((*elem)->getNeighbor(j) != nullptr)
                    continue;

                auto const face = std::unique_ptr<MeshLib::Element const>{(*elem)->getFace(j)};
                if (!complete_surface)
                {
                    if (MathLib::scalarProduct(FaceRule::getSurfaceNormal(face.get()).getNormalizedVector(), norm_dir) < cos_theta)
                    {
                        continue;
                    }
                }
                if (face->getGeomType() == MeshElemType::TRIANGLE)
                    sfc_elements.push_back(new MeshLib::Tri(*static_cast<const MeshLib::Tri*>(face.get())));
                else
                    sfc_elements.push_back(new MeshLib::Quad(*static_cast<const MeshLib::Quad*>(face.get())));
            }
        }
    }
}

void MeshSurfaceExtraction::get2DSurfaceNodes(std::vector<MeshLib::Node*> &sfc_nodes, std::size_t n_all_nodes, const std::vector<MeshLib::Element*> &sfc_elements, std::vector<std::size_t> &node_id_map)
{
    const std::size_t nNewElements (sfc_elements.size());
    std::vector<const MeshLib::Node*> tmp_nodes(n_all_nodes, nullptr);
    for (std::size_t i=0; i<nNewElements; ++i)
    {
        const MeshLib::Element* elem (sfc_elements[i]);
        for (unsigned j=0; j<elem->getNumberOfBaseNodes(); ++j)
        {
            const MeshLib::Node* node (elem->getNode(j));
            tmp_nodes[node->getID()] = node;
        }
    }
    const std::size_t nNodes (tmp_nodes.size());
    for (unsigned i=0; i<nNodes; ++i)
    {
        if (tmp_nodes[i])
        {
            node_id_map[i] = sfc_nodes.size();
            sfc_nodes.push_back(new MeshLib::Node(tmp_nodes[i]->getCoords(), tmp_nodes[i]->getID()));
        }
    }
}

std::vector<GeoLib::Point*> MeshSurfaceExtraction::getSurfaceNodes(const MeshLib::Mesh &mesh, const MathLib::Vector3 &dir, double angle)
{
    INFO ("Extracting surface nodes...");
    std::vector<MeshLib::Element*> sfc_elements;
    get2DSurfaceElements(mesh.getElements(), sfc_elements, dir, angle, mesh.getDimension());

    std::vector<MeshLib::Node*> sfc_nodes;
    std::vector<std::size_t> node_id_map(mesh.getNumberOfNodes());
    get2DSurfaceNodes(sfc_nodes, mesh.getNumberOfNodes(), sfc_elements, node_id_map);

    for (auto e : sfc_elements)
        delete e;

    const std::size_t nNodes (sfc_nodes.size());
    std::vector<GeoLib::Point*> surface_pnts(nNodes);
    for (std::size_t i=0; i<nNodes; ++i)
    {
        surface_pnts[i] = new GeoLib::Point(*(sfc_nodes[i]), sfc_nodes[i]->getID());
        delete sfc_nodes[i];
    }
    return surface_pnts;
}

} // end namespace MeshLib


/**
MIT License

Copyright (c) [2022] [Michel Kakulphimp]

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
**/

// C/C++ includes
#include <iostream>
#include <fstream>
#include <cmath>
#include <chrono>
#include <ctime>
#include <locale>
#include <vector>

// Eigen includes
#define EIGEN_USE_BLAS // use external BLAS routines
#define EIGEN_USE_LAPACKE_STRICT // use LAPACKE C interface to use LAPACK routines (STRICT: disable
                                 // less numerically robust routines)
#include <Eigen/Dense>
// This is a workaround needed to enable the use of EIGEN_USE_LAPACKE
// https://github.com/libigl/libigl/issues/651 for more info
#undef I

// Boost includes
#include <boost/format.hpp>
#include <boost/program_options.hpp>
namespace po = boost::program_options;

// OMP includes
#include <omp.h>

// BLAS includes
#include <cblas.h>

// CUDA
#include <cuda.h>

// Program includes
#include "main.hpp"
#include "config.hpp"
#include "console.hpp"
#include "kernel.h"
#include "version.hpp"
#include "perfmon.hpp"

// Clean up code a bit by using aliases and typedefs
using namespace std;
using namespace boost;
typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic> EigenFloatMatrix_t;
typedef Eigen::Matrix<float, Eigen::Dynamic, 1> EigenFloatColVector_t;
typedef Eigen::Matrix<float, 1, Eigen::Dynamic> EigenFloatRowVector_t;
typedef Eigen::Map<EigenFloatMatrix_t> EigenFloatMatrixMap_t;
typedef Eigen::Map<EigenFloatColVector_t> EigenFloatColVectorMap_t;

// Naughty naughty global!!!
int program_verbosity;

void parse_program_options(int argc, char **argv, Cfg_t &config)
{
    po::variables_map vm;
    po::options_description desc("EHFS options");
    stringstream ss;
    desc.add_options()
        ("help", "produce help message")
        ("iterations", po::value<int>()->default_value(20), "set maximum number of iterations")
        ("partitions", po::value<int>()->default_value(12), "set number of partitions to divide solution space")
        ("limit", po::value<int>()->default_value(4), "set the solution space maximum x=y=z limit")
        ("convergence", po::value<float>()->default_value(0.01), "set the convergence condition (%)")
        ("structure", po::value<int>()->default_value(0), "set the atomic structure: (0:He, 1:H2)")
        ("verbosity", po::value<int>()->default_value(2), "set the verbosity of the program")
        ("max-threads", po::value<int>()->default_value(16), "set the maximum number of threads that CPU operations can spawn")
        ("use-gpu-int", po::value<bool>()->default_value(1), "enable CUDA GPU acceleration for numerical integration")
        ("use-gpu-eig", po::value<bool>()->default_value(1), "enable CUDA GPU acceleration for the eigensolver")
        ("csv-header", po::value<bool>()->default_value(0), "enable CSV output (header) of simulation run for piping to file (disables other messages)")
        ("csv-data-all", po::value<bool>()->default_value(0), "enable CSV output (all data) of simulation run for piping to file (disables other messages)")
        ("csv-data-avg", po::value<bool>()->default_value(0), "enable CSV output (average data) of simulation run for piping to file (disables other messages)")
    ;
    po::store(po::parse_command_line(argc, argv, desc), vm);
    po::notify(vm);
    if (vm.count("help"))
    {
        ss << desc << "\n";
        console_print(0, ss.str(), CLIENT_SIM);
        exit(1);
    }

    if (vm["structure"].as<int>() >= ATOMIC_STRUCTURE_NUM)
    {
        ss << "Invalid atomic structure selection: " << vm["structure"].as<int>() << endl;
        ss << "Valid options: (0:He, 1:H2)" << endl;
        console_print(0, ss.str(), CLIENT_SIM);
        exit(0);
    }
    else
    {
        config.atomic_structure = (atomic_structure_e)(vm["structure"].as<int>());
    }

    // Program configuration from CLI
    config.num_partitions = vm["partitions"].as<int>();
    config.limit = vm["limit"].as<int>();
    config.max_iterations = vm["iterations"].as<int>();
    config.num_solutions = 6;
    config.convergence_percentage = vm["convergence"].as<float>();
    config.verbosity = vm["verbosity"].as<int>();
    config.max_num_threads = vm["max-threads"].as<int>();
    config.enable_cuda_integration = vm["use-gpu-int"].as<bool>();
    config.enable_cuda_eigensolver = vm["use-gpu-eig"].as<bool>();
    config.enable_csv_header_output = vm["csv-header"].as<bool>();
    config.enable_csv_data_all_output = vm["csv-data-all"].as<bool>();
    config.enable_csv_data_average_output = vm["csv-data-avg"].as<bool>();
    if (config.enable_csv_header_output || config.enable_csv_data_all_output || config.enable_csv_data_average_output)
    {
        config.verbosity = -1;
    }
    program_verbosity = config.verbosity;
}

void print_header(void)
{
    stringstream ss;

    ss  << ANSI_FG_COLOR_MAGENTA "                                           " ANSI_COLOR_RESET << endl
        << ANSI_FG_COLOR_MAGENTA "     ██████╗ ███████╗██╗  ██╗███████╗      " ANSI_COLOR_RESET << endl
        << ANSI_FG_COLOR_MAGENTA "     ██╔══██╗██╔════╝██║  ██║██╔════╝      " ANSI_COLOR_RESET << endl
        << ANSI_FG_COLOR_MAGENTA "     ██║  ██║███████╗███████║█████╗        " ANSI_COLOR_RESET << endl
        << ANSI_FG_COLOR_MAGENTA "     ██║  ██║╚════██║██╔══██║██╔══╝        " ANSI_COLOR_RESET << endl
        << ANSI_FG_COLOR_MAGENTA "     ██████╔╝███████║██║  ██║██║           " ANSI_COLOR_RESET << endl
        << ANSI_FG_COLOR_MAGENTA "     ╚═════╝ ╚══════╝╚═╝  ╚═╝╚═╝           " ANSI_COLOR_RESET << endl
        << ANSI_FG_COLOR_MAGENTA "                                           " ANSI_COLOR_RESET << endl
        <<                       " < Discrete Space Hartree-Fock Simulator > " << endl << endl
        << "Author: Michel Kakulphimp" << endl
        << "Build Date: " __DATE__ " " __TIME__ << endl
        << "Git Branch: " << GIT_BRANCH << endl
        << "Git Hash: " << GIT_COMMIT_HASH << endl << endl;

    console_print_hr(0, CLIENT_SIM);
    console_print(0, ss.str(), CLIENT_SIM);
}

void populate_lookup_values(Cfg_t &config, Lut_t &lut)
{
    // Memory for the arrays filled out in this function should have been
    // allocated already. Maybe check for null later. TODO.

    // Populate LUTs
    for (size_t coordinate_index = 0; coordinate_index < lut.matrix_dim; coordinate_index++)
    {
        // convert coordinate index to x,y,z coordinate indices
        int base_10_num = coordinate_index;
        for (int i = 0; i < IDX_NUM; i++)
        {
            lut.coordinate_index_array[i * lut.matrix_dim + coordinate_index] =
                base_10_num % config.num_partitions;
            base_10_num /= config.num_partitions;

            lut.coordinate_value_array[i * lut.matrix_dim + coordinate_index] =
                (float)(-config.limit) + ((float)lut.coordinate_index_array[i * lut.matrix_dim + coordinate_index]
                    * lut.step_size);
        }
    }
}

// Generate the 3D Laplacian matrix for the given number of partitions
void generate_laplacian_matrix(Lut_t lut, float *matrix)
{
    size_t col_index_x;
    size_t col_index_y;
    size_t col_index_z;
    size_t row_index_x;
    size_t row_index_y;
    size_t row_index_z;

    for (size_t row_coordinate_index = 0; row_coordinate_index < lut.matrix_dim; row_coordinate_index++)
    {
        for (size_t col_coordinate_index = 0; col_coordinate_index < lut.matrix_dim; col_coordinate_index++)
        {
            col_index_x = lut.coordinate_index_array[IDX_X * lut.matrix_dim + col_coordinate_index];
            col_index_y = lut.coordinate_index_array[IDX_Y * lut.matrix_dim + col_coordinate_index];
            col_index_z = lut.coordinate_index_array[IDX_Z * lut.matrix_dim + col_coordinate_index];

            row_index_x = lut.coordinate_index_array[IDX_X * lut.matrix_dim + row_coordinate_index];
            row_index_y = lut.coordinate_index_array[IDX_Y * lut.matrix_dim + row_coordinate_index];
            row_index_z = lut.coordinate_index_array[IDX_Z * lut.matrix_dim + row_coordinate_index];

            // U(x,y,z)
            if (row_coordinate_index == col_coordinate_index)
            {
                matrix[row_coordinate_index + col_coordinate_index*lut.matrix_dim] = -6.0;
            }

            if ((row_index_y == col_index_y) && (row_index_z == col_index_z))
            {
                // U(x-1,y,z)
                if (row_index_x == col_index_x + 1)
                {
                    matrix[row_coordinate_index + col_coordinate_index*lut.matrix_dim] = 1.0;
                }
                // U(x+1,y,z)
                if (row_index_x == col_index_x - 1)
                {
                    matrix[row_coordinate_index + col_coordinate_index*lut.matrix_dim] = 1.0;
                }
            }

            if ((row_index_x == col_index_x) && (row_index_z == col_index_z))
            {
                // U(x,y-1,z)
                if (row_index_y == col_index_y + 1)
                {
                    matrix[row_coordinate_index + col_coordinate_index*lut.matrix_dim] = 1.0;
                }
                // U(x,y+1,z)
                if (row_index_y == col_index_y - 1)
                {
                    matrix[row_coordinate_index + col_coordinate_index*lut.matrix_dim] = 1.0;
                }
            }

            if ((row_index_x == col_index_x) && (row_index_y == col_index_y))
            {
                // U(x,y,z-1)
                if (row_index_z == col_index_z + 1)
                {
                    matrix[row_coordinate_index + col_coordinate_index*lut.matrix_dim] = 1.0;
                }
                // U(x,y,z+1)
                if (row_index_z == col_index_z - 1)
                {
                    matrix[row_coordinate_index + col_coordinate_index*lut.matrix_dim] = 1.0;
                }
            }
        }
    }
}

// Helium atom nucleus-electron Coulombic attraction function
float attraction_function_helium(Lut_t lut, int linear_coordinates)
{
    const float epsilon = EPSILON;

    float x = lut.coordinate_value_array[IDX_X * lut.matrix_dim + linear_coordinates];
    float y = lut.coordinate_value_array[IDX_Y * lut.matrix_dim + linear_coordinates];
    float z = lut.coordinate_value_array[IDX_Z * lut.matrix_dim + linear_coordinates];

    float denominator = sqrt(pow(x, 2.0) + pow(y + Y_OFFSET, 2.0) + pow(z + Z_OFFSET, 2.0));

    if (abs(denominator) < epsilon)
    {
        denominator = sqrt(TINY_NUMBER);
    }

    return (2.0/(denominator));
}

// Hydrogen molecule nucleus-electron Coulombic attraction function
float attraction_function_hydrogen(Lut_t lut, int linear_coordinates)
{
    const float epsilon = EPSILON;

    float x = lut.coordinate_value_array[IDX_X * lut.matrix_dim + linear_coordinates];
    float y = lut.coordinate_value_array[IDX_Y * lut.matrix_dim + linear_coordinates];
    float z = lut.coordinate_value_array[IDX_Z * lut.matrix_dim + linear_coordinates];

    float denominator_1 = sqrt(pow((H2_BOND_LENGTH_ATOMIC_UNITS/2.0) - x, 2.0) + pow(y + Y_OFFSET, 2.0) + pow(z + Z_OFFSET, 2.0));
    float denominator_2 = sqrt(pow((-H2_BOND_LENGTH_ATOMIC_UNITS/2.0) - x, 2.0) + pow(y + Y_OFFSET, 2.0) + pow(z + Z_OFFSET, 2.0));

    if (abs(denominator_1) < epsilon)
    {
        denominator_1 = sqrt(TINY_NUMBER);
    }

    if (abs(denominator_2) < epsilon)
    {
        denominator_2 = sqrt(TINY_NUMBER);
    }

    return ((1.0/(denominator_1)) + (1.0/(denominator_2)));
}

// Generate the attraction matrix
void generate_attraction_matrix(Lut_t lut, atomic_structure_e atomic_structure, float *matrix)
{
    if (atomic_structure == HELIUM_ATOM)
    {
        for (size_t diagonal_index = 0; diagonal_index < lut.matrix_dim; diagonal_index++)
        {
            matrix[diagonal_index + diagonal_index*lut.matrix_dim] = attraction_function_helium(lut, diagonal_index);
        }
    }
    else if (atomic_structure == HYDROGEN_MOLECULE)
    {
        for (size_t diagonal_index = 0; diagonal_index < lut.matrix_dim; diagonal_index++)
        {
            matrix[diagonal_index + diagonal_index*lut.matrix_dim] = attraction_function_hydrogen(lut, diagonal_index);
        }
    }
}

// Electron-electron Coulombic repulsion function
float repulsion_function(Lut_t lut, int linear_coordinates_1, int linear_coordinates_2)
{
    const float epsilon = EPSILON;

    float x1 = lut.coordinate_value_array[IDX_X * lut.matrix_dim + linear_coordinates_1];
    float y1 = lut.coordinate_value_array[IDX_Y * lut.matrix_dim + linear_coordinates_1];
    float z1 = lut.coordinate_value_array[IDX_Z * lut.matrix_dim + linear_coordinates_1];

    float x2 = lut.coordinate_value_array[IDX_X * lut.matrix_dim + linear_coordinates_2];
    float y2 = lut.coordinate_value_array[IDX_Y * lut.matrix_dim + linear_coordinates_2];
    float z2 = lut.coordinate_value_array[IDX_Z * lut.matrix_dim + linear_coordinates_2];

    float denominator = sqrt(pow(x2 - x1, 2.0) + pow(y2 - y1, 2.0) + pow(z2 - z1, 2.0));

    if (abs(denominator) < epsilon)
    {
        denominator = sqrt(TINY_NUMBER);
    }

    return (1.0/(denominator));
}

float repulsion_diagonal_integrand_function(Lut_t lut, float *orbital_values, int linear_coords_1, int linear_coords_2)
{
    return pow(orbital_values[linear_coords_2], 2.0)*repulsion_function(lut, linear_coords_1, linear_coords_2);
}

float exchange_diagonal_integrand_function(Lut_t lut, float *orbital_values, int linear_coords_1, int linear_coords_2)
{
    return orbital_values[linear_coords_1]*orbital_values[linear_coords_2]*repulsion_function(lut, linear_coords_1, linear_coords_2);
}

void generate_repulsion_diagonal(Lut_t lut, DynamicDataPointers_t ddp)
{
    for (size_t electron_one_coordinate_index = 0; electron_one_coordinate_index < lut.matrix_dim; electron_one_coordinate_index++)
    {
        float sum = 0;
        for (size_t electron_two_coordinate_index = 0; electron_two_coordinate_index < lut.matrix_dim; electron_two_coordinate_index++)
        {
            sum += repulsion_diagonal_integrand_function(lut, ddp.orbital_values_data, electron_one_coordinate_index, electron_two_coordinate_index);
        }
        ddp.repulsion_diagonal_data[electron_one_coordinate_index] = sum*lut.step_size_cubed;
    }
}

void generate_exchange_diagonal(Lut_t lut, DynamicDataPointers_t ddp)
{
    for (size_t electron_one_coordinate_index = 0; electron_one_coordinate_index < lut.matrix_dim; electron_one_coordinate_index++)
    {
        float sum = 0;
        for (size_t electron_two_coordinate_index = 0; electron_two_coordinate_index < lut.matrix_dim; electron_two_coordinate_index++)
        {
            sum += exchange_diagonal_integrand_function(lut, ddp.orbital_values_data, electron_one_coordinate_index, electron_two_coordinate_index);
        }
        ddp.exchange_diagonal_data[electron_one_coordinate_index] = sum*lut.step_size_cubed;
    }
}

template <typename A, typename B, typename C, typename D, typename E>
float calculate_total_energy(Eigen::MatrixBase<A> &orbital_values, Eigen::MatrixBase<B> &kinetic_matrix, Eigen::MatrixBase<C> &attraction_matrix, Eigen::MatrixBase<D> &repulsion_diagonal, Eigen::MatrixBase<E> &exchange_diagonal)
{
    // orbital values are real, so no need to take conjugate
    EigenFloatRowVector_t psi_prime = orbital_values.transpose();
    EigenFloatColVector_t psi = orbital_values;

    EigenFloatMatrix_t repulsion_matrix = repulsion_diagonal.asDiagonal();
    EigenFloatMatrix_t exchange_matrix = exchange_diagonal.asDiagonal();

    float energy_sum = 0;

    console_print_hr(1, CLIENT_SIM);
    console_print(1, "Calculating total energy", CLIENT_SIM);

    // sum for the total number of electrons in the systems: 2
    for (int i = 0; i < 2; i++)
    {
        auto hermitian_term = psi_prime * (-kinetic_matrix - attraction_matrix) * psi;
        auto hf_term = 0.5 * psi_prime * (2.0 * repulsion_matrix - exchange_matrix) * psi;
        energy_sum += (hermitian_term(0) + hf_term(0));
    }

    return energy_sum;
}

// Using the LAPACK routines directly by reimplementing the LAPACKE wrapper
// without querying for sizes. I was originally using LAPACKE_ssyevd directly
// from LAPACKE.h, but for some reason, the automatic work size was returning an
// invalid number. I'm re-implementing the core of that function in this
// version, but I'm using manual lwork and liwork calculations instead. From the
// LAPACK documentation on ssyevd(): SSYEVD computes all eigenvalues and,
// optionally, eigenvectors of a real symmetric matrix A. If eigenvectors are
// desired, it uses a divide and conquer algorithm.The divide and conquer
// algorithm makes very mild assumptions about floating point arithmetic.
//
// @param[in]  lut     Program constants
// @param      matrix       The matrix originally containing the matrix to be
//                          solved and the resulting eigenvectors
// @param      eigenvalues  The eigenvalues of the solution
//
// @return     True if the solver found solutions, false if the solver failed
//             for some reason
//
bool lapack_solve_eigh(Lut_t &lut, DynamicDataPointers_t ddp)
{
    char jobz = 'V'; // compute eigenvalues and eigenvectors.
    char uplo = 'U'; // perform calculation on upper triangle of matrix
    lapack_int n = lut.matrix_dim; // order of the matrix (size)
    lapack_int lda = lut.matrix_dim; // the leading dimension of the array A. LDA >= max(1,N).
    lapack_int info = 0;
    lapack_int liwork;
    lapack_int lwork;
    lapack_int* iwork = nullptr;
    float* work = nullptr;

    auto lapack_start = chrono::system_clock::now();

    console_print(0, "LAPACK ssyevd start", CLIENT_LAPACK);
    console_print(2, TAB1 "LAPACK solver debug", CLIENT_LAPACK);
    console_print(2, str(format(TAB2 "n = %d") % n), CLIENT_LAPACK);
    console_print(2, str(format(TAB2 "lda = %d") % lda), CLIENT_LAPACK);

    // Setting lwork and liwork manually based on the LAPACK documentation
    lwork = 1 + 6*n + 2*n*n;
    liwork = 3 + 5*n;

    console_print(2, str(format(TAB2 "liwork = %d") % liwork), CLIENT_LAPACK);
    console_print(2, str(format(TAB2 "lwork = %d") % lwork), CLIENT_LAPACK);

    // Allocate memory for work arrays
    iwork = (lapack_int*)malloc(sizeof(lapack_int) * liwork);
    if(iwork == nullptr)
    {
        info = LAPACK_WORK_MEMORY_ERROR;
        console_print_err(2, TAB2 "FATAL! Could not allocate iwork array", CLIENT_LAPACK);
    }
    work = (float*)malloc(sizeof(float) * lwork);
    if(work == nullptr)
    {
        info = LAPACK_WORK_MEMORY_ERROR;
        console_print_err(2, TAB2 "FATAL! Could not allocate work array", CLIENT_LAPACK);
    }

    // Call LAPACK function and adjust info if our work areas are OK
    if ((iwork != nullptr) && (work != nullptr))
    {
        console_print(2, TAB2 "calling LAPACK function", CLIENT_LAPACK);
        LAPACK_ssyevd(&jobz, &uplo, &n, ddp.eigenvectors_data, &lda, ddp.eigenvalues_data, work, &lwork, iwork, &liwork, &info);
        if( info < 0 ) {
            info = info - 1;
        }
    }

    // Release memory and exit
    LAPACKE_free(iwork);
    LAPACKE_free(work);

    console_print(2, str(format(TAB2 "info = %d") % info), CLIENT_LAPACK);

    auto lapack_end = chrono::system_clock::now();
    auto lapack_time = chrono::duration<float>(lapack_end - lapack_start);

    console_print(0, str(format("LAPACK ssyevd took: %0.3f seconds") % (float)(lapack_time.count())), CLIENT_LAPACK);

    return (info==0);
}

int cpu_allocate_integration_memory(Lut_t &lut, DynamicDataPointers_t &ddp)
{
    int rv = 0;

    console_print_hr(2, CLIENT_SIM);
    console_print(2, "Allocating memory for CPU integration", CLIENT_SIM);

    size_t orbital_vector_size_bytes = lut.matrix_dim;
    size_t repulsion_exchange_matrices_size_bytes = lut.matrix_dim;
    size_t coordinate_luts_size_bytes = IDX_NUM * lut.matrix_dim;

    ddp.orbital_values_data = (float*)(calloc(orbital_vector_size_bytes, sizeof(float)));
    ddp.repulsion_diagonal_data = (float*)(calloc(repulsion_exchange_matrices_size_bytes, sizeof(float)));
    ddp.exchange_diagonal_data = (float*)(calloc(repulsion_exchange_matrices_size_bytes, sizeof(float)));

    lut.coordinate_value_array = (float*)(calloc(coordinate_luts_size_bytes, sizeof(float)));
    lut.coordinate_index_array = (float*)(calloc(coordinate_luts_size_bytes, sizeof(float)));

    if ( (ddp.orbital_values_data == nullptr) || (ddp.repulsion_diagonal_data == nullptr) || (ddp.exchange_diagonal_data == nullptr) ||
         (lut.coordinate_value_array == nullptr) || (lut.coordinate_index_array == nullptr) )
    {
        console_print_err(0, "Memory allocation error!", CLIENT_SIM);
        rv = 1;
    }
    else
    {
        console_print(2, str(format("Allocated %d bytes for orbital values vector") % orbital_vector_size_bytes), CLIENT_SIM);
        console_print(2, str(format("Allocated %d bytes for repulsion matrix diagonal") % repulsion_exchange_matrices_size_bytes), CLIENT_SIM);
        console_print(2, str(format("Allocated %d bytes for exchange matrix diagonal") % repulsion_exchange_matrices_size_bytes), CLIENT_SIM);
        console_print(2, str(format("Allocated 3x %d bytes for coordinate LUTs") % coordinate_luts_size_bytes), CLIENT_SIM);
    }

    return rv;
}

int cpu_allocate_eigensolver_memory(Lut_t &lut, DynamicDataPointers_t &ddp)
{
    int rv = 0;

    console_print_hr(2, CLIENT_SIM);
    console_print(2, "Allocating memory for CPU eigensolver", CLIENT_SIM);

    size_t eigenvectors_size_bytes = sizeof(float) * lut.matrix_dim * lut.matrix_dim;
    size_t eigenvalues_size_bytes = sizeof(float) * lut.matrix_dim;

    ddp.eigenvectors_data = (float*)(malloc(eigenvectors_size_bytes));
    ddp.eigenvalues_data = (float*)(malloc(eigenvalues_size_bytes));

    if ((ddp.eigenvectors_data == nullptr) || (ddp.eigenvalues_data == nullptr))
    {
        console_print_err(0, "Memory allocation error!", CLIENT_SIM);
        rv = 1;
    }
    else
    {
        console_print(2, str(format("Allocated %d bytes for eigenvector matrix") % eigenvectors_size_bytes), CLIENT_SIM);
        console_print(2, str(format("Allocated %d bytes for eigenvalue vector") % eigenvalues_size_bytes), CLIENT_SIM);
    }

    return rv;
}

int cpu_free_integration_memory(Lut_t &lut, DynamicDataPointers_t &ddp)
{
    int rv = 0;

    console_print(2, "Freeing allocated integration memory...", CLIENT_SIM);

    free(ddp.orbital_values_data);
    free(ddp.repulsion_diagonal_data);
    free(ddp.exchange_diagonal_data);
    free(lut.coordinate_value_array);
    free(lut.coordinate_index_array);

    // null the pointers
    (ddp.orbital_values_data) = nullptr;
    (ddp.repulsion_diagonal_data) = nullptr;
    (ddp.exchange_diagonal_data) = nullptr;
    (lut.coordinate_value_array) = nullptr;
    (lut.coordinate_index_array) = nullptr;

    console_print(2, "Successfully freed allocated integration memory", CLIENT_SIM);

    return rv;
}

int cpu_free_eigensolver_memory(DynamicDataPointers_t &ddp)
{
    int rv = 0;

    console_print(2, "Freeing allocated eigensolver memory...", CLIENT_SIM);

    free(ddp.eigenvectors_data);
    free(ddp.eigenvalues_data);

    // null the pointers
    ddp.eigenvectors_data = nullptr;
    ddp.eigenvalues_data = nullptr;

    console_print(2, "Successfully freed allocated eigensolver memory", CLIENT_SIM);

    return rv;
}

void print_program_configurations(Cfg_t &config, Lut_t &lut)
{
    // Print program information
    console_print_hr(0, CLIENT_SIM);
    console_print(0, "Program Configurations:\n", CLIENT_SIM);
    console_print(0, str(format(TAB1 "Verbosity = %d") % config.verbosity), CLIENT_SIM);
    console_print(0, str(format(TAB1 "Iterations = %d") % config.max_iterations), CLIENT_SIM);
    console_print(0, str(format(TAB1 "Num Partitions = %d") % config.num_partitions), CLIENT_SIM);
    console_print(0, str(format(TAB1 "Limits = %d") % config.limit), CLIENT_SIM);
    console_print(0, str(format(TAB1 "Matrix Dimension = %d") % lut.matrix_dim), CLIENT_SIM);
    console_print(0, str(format(TAB1 "Step Size = %f") % lut.step_size), CLIENT_SIM);

    if (config.atomic_structure == HELIUM_ATOM)
    {
        console_print(0, TAB1 "Atomic Structure: Helium Atom", CLIENT_SIM);
    }
    else if (config.atomic_structure == HYDROGEN_MOLECULE)
    {
        console_print(0, TAB1 "Atomic Structure: Hydrogen Molecule", CLIENT_SIM);
    }
    console_print(0, str(format(TAB1 "Maximum CPU Threads = %d") % config.max_num_threads), CLIENT_SIM);
}

void config_cuda(Cfg_t &config)
{
    if (config.enable_cuda_eigensolver || config.enable_cuda_integration)
    {
        console_print(0, "Checking for available CUDA devices", CLIENT_SIM);
        int num_cuda_devices = cuda_get_device_info();
        if ((num_cuda_devices == 0) && (config.enable_cuda_eigensolver || config.enable_cuda_integration))
        {
            console_print_warn(0, "Disabling CUDA acceleration since no device is available", CLIENT_SIM);
            config.enable_cuda_integration = false;
            config.enable_cuda_eigensolver = false;
        }
    }
}

// Solve for Fock matrix eigenvectors
void eigensolver(Cfg_t config, Lut_t &lut, PerformanceMonitor &perfmon, DynamicDataPointers_t ddp)
{
    console_print(1, "Obtaining eigenvalues and eigenvectors", CLIENT_SIM);

    auto eig_start = chrono::system_clock::now();

    if (config.enable_cuda_eigensolver)
    {
        // CUDA cuSOLVER
        if (!cuda_eigensolver(lut, ddp))
        {
            console_print_err(0, "Something went horribly wrong with the CUDA solver, aborting", CLIENT_SIM);
            exit(EXIT_FAILURE);
        }
    }
    else
    {
        // Eigen solver, slow.
        // EigenFloatMatrixMap_t eigenvectors(eigenvectors_data, lut.matrix_dim, lut.matrix_dim);
        // EigenFloatColVectorMap_t eigenvalues(eigenvalues_data, lut.matrix_dim, 1);
        // Eigen::SelfAdjointEigenSolver<Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic>> solver(eigenvectors);
        // eigenvectors = solver.eigenvectors();
        // eigenvalues = solver.eigenvalues();

        // CPU LAPACK solver
        if (!lapack_solve_eigh(lut, ddp))
        {
            console_print_err(0, "Something went horribly wrong with the LAPACK solver, aborting", CLIENT_SIM);
            exit(EXIT_FAILURE);
        }
    }

    auto eig_end = chrono::system_clock::now();
    auto eig_time = chrono::duration<float>(eig_end - eig_start);
    perfmon.record(PerformanceMonitor::ITERATION_EIGENSOLVER_TIME, (float)(eig_time.count()));
    console_print(0, str(format("Eigenvalues and eigenvectors computed in: %0.3f seconds") % (float)(eig_time.count())), CLIENT_SIM);
}

void generate_repulsion_and_integration_matrices(Cfg_t &config, Lut_t &lut, PerformanceMonitor &perfmon, DynamicDataPointers_t ddp)
{
    auto int_start = chrono::system_clock::now();

    if (config.enable_cuda_integration)
    {
        console_print(0, "Generating repulsion and exchange matrices on GPU", CLIENT_SIM);

        // generate repulsion and exchange matrices on GPU
        cuda_numerical_integration(lut, ddp);
    }
    else
    {
        console_print(0, "Generating repulsion and exchange matrices on CPU", CLIENT_SIM);

        // generate repulsion and exchange matrices on CPU
        generate_repulsion_diagonal(lut, ddp);
        generate_exchange_diagonal(lut, ddp);
    }

    auto int_end = chrono::system_clock::now();
    auto int_time = chrono::duration<float>(int_end - int_start);
    perfmon.record(PerformanceMonitor::ITERATION_INTEGRATION_TIME, (float)(int_time.count()));
    console_print(0, str(format("Repulsion and exchange matrix computed in: %0.3f seconds") % (float)(int_time.count())), CLIENT_SIM);
}

void print_csv_header(PerformanceMonitor &perfmon)
{
    cout << perfmon.str_csv_header();
}

void print_csv_data_all(Cfg_t &config, Lut_t &lut, PerformanceMonitor &perfmon)
{
    cout << perfmon.str_csv_data_all(config, lut);
}

void print_csv_data_average(Cfg_t &config, Lut_t &lut, PerformanceMonitor &perfmon)
{
    cout << perfmon.str_csv_data_average(config, lut);
}

int main(int argc, char *argv[])
{
    // Performance monitor object
    PerformanceMonitor perfmon;
    // Program config struct
    Cfg_t config;
    // Program lookup table values
    Lut_t lut;

    // Boost Program options
    parse_program_options(argc, argv, config);

    // Print the CSV header if requested, and exit the program
    if (config.enable_csv_header_output)
    {
        print_csv_header(perfmon);
        exit(EXIT_SUCCESS);
    }

    // Set the maximum number of thread to use for OMP and Eigen. This doesn't
    // appear to modulate the number of threads the LAPACK solver is using, so
    // something isn't right here. Leaving here anyways just in case I can fix
    // it sometime later.
    omp_set_dynamic(0); // Disable dynamic teams in OpenMP
    omp_set_num_threads(config.max_num_threads); // Set the number of maximum threads to use for OMP (Eigen will use this value)

    // Print the header
    print_header();

    // Fill out lut (the rest is done in populate_lookup_values once memory
    // has been allocated for the arrays inside)
    lut.matrix_dim = pow(config.num_partitions, 3.0);
    lut.step_size = (float)(4<<1)/(float)(config.num_partitions - 1);
    lut.step_size_cubed = pow(lut.step_size, 3.0);

    // Print program configuration
    print_program_configurations(config, lut);

    // Get and print CUDA information
    config_cuda(config);

    // Matrix declarations
    EigenFloatMatrix_t laplacian_matrix;
    EigenFloatMatrix_t kinetic_matrix;
    EigenFloatMatrix_t attraction_matrix;
    EigenFloatColVectorMap_t repulsion_diagonal(nullptr, lut.matrix_dim, 1);
    EigenFloatColVectorMap_t exchange_diagonal(nullptr, lut.matrix_dim, 1);
    EigenFloatMatrix_t fock_matrix;
    // Orbital values
    EigenFloatColVectorMap_t orbital_values(nullptr, lut.matrix_dim, 1);
    // Eigenvectors and eigenvalues
    EigenFloatColVectorMap_t eigenvalues(nullptr, lut.matrix_dim, 1);
    EigenFloatMatrixMap_t eigenvectors(nullptr, lut.matrix_dim, 1);

    // For the repulsion, exchange, and orbital values, we will overlay the
    // Eigen objects over unified memory that is allocated via CUDA (if
    // enabled). Otherwise we will malloc data normally. This overlay is
    // performed using the Eigen::Map object, which allows one to map a pointer
    // to data allocated outside of Eigen to an Eigen object, like a matrix or
    // vector. kernel.h externs the three pointers that are used to store the
    // addresses and the following call will allocate memory for them. We will
    // also allocate memory for the lookup tables used.
    DynamicDataPointers_t ddp;
    int allocate_error = 0;

    allocate_error |= cpu_allocate_integration_memory(lut, ddp);
    allocate_error |= cpu_allocate_eigensolver_memory(lut, ddp);

    if (!allocate_error)
    {
        new (&orbital_values) EigenFloatColVectorMap_t(ddp.orbital_values_data, lut.matrix_dim, 1);
        new (&repulsion_diagonal) EigenFloatColVectorMap_t(ddp.repulsion_diagonal_data, lut.matrix_dim, 1);
        new (&exchange_diagonal) EigenFloatColVectorMap_t(ddp.exchange_diagonal_data, lut.matrix_dim, 1);
        new (&eigenvectors) EigenFloatMatrixMap_t(ddp.eigenvectors_data, lut.matrix_dim, lut.matrix_dim);
        new (&eigenvalues) EigenFloatColVectorMap_t(ddp.eigenvalues_data, lut.matrix_dim, 1);
        // Populate LUTs
        populate_lookup_values(config, lut);
    }
    else
    {
        console_print_err(0, "Something went horribly wrong when allocating memory, aborting", CLIENT_SIM);
        exit(EXIT_FAILURE);
    }

    // Instantiate matrices and vectors
    new (&laplacian_matrix) EigenFloatMatrix_t(lut.matrix_dim, lut.matrix_dim);
    new (&kinetic_matrix) EigenFloatMatrix_t(lut.matrix_dim, lut.matrix_dim);
    new (&attraction_matrix) EigenFloatMatrix_t(lut.matrix_dim, lut.matrix_dim);
    new (&fock_matrix) EigenFloatMatrix_t(lut.matrix_dim, lut.matrix_dim);

    // Trimmed eigenvectors and eigenvalues
    EigenFloatColVector_t trimmed_eigenvalues(config.num_solutions, 1);
    EigenFloatMatrix_t trimmed_eigenvectors(lut.matrix_dim, config.num_solutions);

    console_print_hr(0, CLIENT_SIM);
    console_print(0, "Simulation start!", CLIENT_SIM);
    console_print_hr(0, CLIENT_SIM);
    auto sim_start = chrono::system_clock::now();

    // generate the second order Laplacian matrix for 3D space
    console_print(1, "Generating Laplacian matrix", CLIENT_SIM);
    generate_laplacian_matrix(lut, laplacian_matrix.data());
    // generate the kinetic energy matrix
    console_print(1, "Generating kinetic energy matrix", CLIENT_SIM);
    kinetic_matrix = (laplacian_matrix/(2.0*lut.step_size*lut.step_size));
    // generate the Coulombic attraction matrix
    console_print(1, "Generating electron-nucleus Coulombic attraction matrix", CLIENT_SIM);
    generate_attraction_matrix(lut, config.atomic_structure, attraction_matrix.data());

    // Main HF loop
    float last_total_energy = 0;
    float total_energy;
    float total_energy_percent_diff;
    int interation_count = 0;

    // initial solution
    fock_matrix = -kinetic_matrix - attraction_matrix;

    console_print(1, "Solving for initial solution", CLIENT_SIM);
    // Solve for Fock matrix eigenvectors
    eigenvectors = fock_matrix;
    eigensolver(config, lut, perfmon, ddp);
    orbital_values = eigenvectors.col(0);

    do
    {
        console_print_hr(0, CLIENT_SIM);
        console_print(0, str(format("Iteration Start: %d") % interation_count), CLIENT_SIM);
        console_print_hr(0, CLIENT_SIM);

        auto iteration_start = chrono::system_clock::now();

        // Generate repulsion and integraion matrices
        generate_repulsion_and_integration_matrices(config, lut, perfmon, ddp);

        // form fock matrix
        console_print(1, "Generating Fock matrix", CLIENT_SIM);
        fock_matrix = -kinetic_matrix - attraction_matrix + 2.0 * EigenFloatMatrix_t(repulsion_diagonal.asDiagonal()) - EigenFloatMatrix_t(exchange_diagonal.asDiagonal());

        // Solve for Fock matrix eigenvectors and eigenvalues
        eigenvectors = fock_matrix;
        eigensolver(config, lut, perfmon, ddp);
        orbital_values = eigenvectors.col(0);

        // Extract num_solutions eigenvalues
        trimmed_eigenvalues = eigenvalues.block(0, 0, config.num_solutions, 1);
        // Extract num_solutions eigenvectors
        trimmed_eigenvectors = eigenvectors.block(0, 0, lut.matrix_dim, config.num_solutions);

        total_energy = calculate_total_energy(orbital_values, kinetic_matrix, attraction_matrix, repulsion_diagonal, exchange_diagonal);
        perfmon.record(PerformanceMonitor::ITERATION_TOTAL_ENERGY, total_energy);
        total_energy_percent_diff = abs((total_energy - last_total_energy)/((total_energy + last_total_energy) / 2.0));

        console_print_hr(0, CLIENT_SIM);
        console_print(0, str(format("Total energy: %.3f") % (total_energy)), CLIENT_SIM);
        console_print(0, str(format("Energy %% diff: %.3f%%") % (total_energy_percent_diff * 100.0)), CLIENT_SIM);

        // update last value
        last_total_energy = total_energy;
        // update iteration count
        interation_count++;

        auto iteration_end = chrono::system_clock::now();
        auto iteration_time = chrono::duration<float>(iteration_end - iteration_start);
        perfmon.record(PerformanceMonitor::ITERATION_TOTAL_TIME, (float)(iteration_time.count()));
        console_print(0, str(format("Iteration end! Iteration time: %0.3f seconds") % (float)(iteration_time.count())), CLIENT_SIM);

        // Increment performance monitor data
        perfmon.next_iteration();

        // check if we've hit the maximum iteration limit
        if (interation_count == config.max_iterations)
        {
            break;
        }

        // check if we meet convergence condition
        if (abs(total_energy_percent_diff) < (config.convergence_percentage/100.0))
        {
            break;
        }
    }
    while(1);

    cpu_free_integration_memory(lut, ddp);
    cpu_free_eigensolver_memory(ddp);

    console_print_hr(0, CLIENT_SIM);
    console_print(0, "Final Eigenvalues:", CLIENT_SIM);
    stringstream ss;
    ss << trimmed_eigenvalues.transpose();
    console_print(0, ss.str(), CLIENT_SIM);
    ss.str(string()); // clear ss
    console_print_hr(0, CLIENT_SIM);
    console_print(0, str(format("Final Total energy: %.3f") % (total_energy)), CLIENT_SIM);
    console_print_hr(0, CLIENT_SIM);

    auto sim_end = chrono::system_clock::now();
    auto sim_time = chrono::duration<float>(sim_end - sim_start);
    perfmon.total_time = (float)(sim_time.count());
    console_print(0, str(format("Simulation end! Total time: %0.3f seconds") % (float)(sim_time.count())), CLIENT_SIM);

    console_print_hr(0, CLIENT_SIM);
    console_print(0, "Performance monitor records:", CLIENT_SIM);
    console_print(0, perfmon.str(), CLIENT_SIM);
    console_print_hr(0, CLIENT_SIM);

    if (config.enable_csv_data_all_output)
    {
        print_csv_data_all(config, lut, perfmon);
    }
    else if (config.enable_csv_data_average_output)
    {
        print_csv_data_average(config, lut, perfmon);
    }

    return 0;
}


// Software License for MTL
//
// Copyright (c) 2007 The Trustees of Indiana University.
//               2008 Dresden University of Technology and the Trustees of Indiana University.
//               2010 SimuNova UG (haftungsbeschränkt), www.simunova.com.
// All rights reserved.
// Authors: Peter Gottschling and Andrew Lumsdaine
//
// This file is part of the Matrix Template Library
//
// See also license.mtl.txt in the distribution.

// Adapted from GLAS implementation by Karl Meerbergen and Toon Knappen

#ifndef MTL_DENSE_VECTOR_INCLUDE
#define MTL_DENSE_VECTOR_INCLUDE

#include <iostream>
#include <cassert>
#include <vector>
#include <algorithm>
#include <boost/static_assert.hpp>
#include <boost/utility/enable_if.hpp>
#include <boost/type_traits/is_integral.hpp>

#include <boost/numeric/mtl/utility/assert.hpp>
#include <boost/numeric/mtl/utility/exception.hpp>
#include <boost/numeric/mtl/utility/ashape.hpp>
#include <boost/numeric/mtl/vector/all_vec_expr.hpp>
#include <boost/numeric/mtl/vector/parameter.hpp>
#include <boost/numeric/mtl/detail/contiguous_memory_block.hpp>
#include <boost/numeric/mtl/vector/crtp_base_vector.hpp>
#include <boost/numeric/mtl/utility/dense_el_cursor.hpp>
#include <boost/numeric/mtl/utility/range_generator.hpp>
#include <boost/numeric/mtl/utility/irange.hpp>
#include <boost/numeric/mtl/utility/is_static.hpp>
#include <boost/numeric/mtl/utility/is_row_major.hpp>
#include <boost/numeric/mtl/utility/transposed_orientation.hpp>
#include <boost/numeric/mtl/operation/is_negative.hpp>
#include <boost/numeric/mtl/interface/vpt.hpp>

#ifdef MTL_WITH_INITLIST
# include <initializer_list>
#endif

namespace mtl { namespace vec {

/// Dense vector
template <class Value, typename Parameters = parameters<> >
class dense_vector
  : public vec_expr<dense_vector<Value, Parameters> >,
    public ::mtl::detail::contiguous_memory_block< Value, Parameters::on_stack, Parameters::dimension::value >,
    public crtp_base_vector< dense_vector<Value, Parameters>, Value, std::size_t >
{
  public:
    typedef dense_vector<Value, Parameters>                                          self;
    typedef ::mtl::detail::contiguous_memory_block< Value, Parameters::on_stack,
                                                    Parameters::dimension::value >   memory_base;
    typedef crtp_base_vector< self, Value, std::size_t >                             crtp_base;
    typedef crtp_vector_assign< self, Value, std::size_t >                           assign_base;
    typedef vec_expr<dense_vector<Value, Parameters> >                               expr_base;
    typedef Value             value_type ;
    typedef std::size_t       size_type ;
    typedef value_type&       reference ;
    typedef value_type const& const_reference ;
    typedef Value*            pointer ;
    typedef Value const*      const_pointer ;
    typedef typename Parameters::orientation  orientation;

    typedef const_pointer     key_type;

    /// Check whether index is non-negative and less than size
    void check_index( size_type MTL_DEBUG_ARG(i) ) const
    {
	MTL_CRASH_IF( is_negative(i) ||  i >= this->used_memory(), "Index out of range!");
    }

    /// Check for a given vector if the sizes are equal or this has size 0 (and can take the size of source)
    void check_dim( size_type MTL_DEBUG_ARG(s) ) const
    {
	MTL_CRASH_IF( this->used_memory() != 0 && this->used_memory() != s, "Incompatible size!");
    }

    /// Check at compile time for a given vector if the sizes are equal
    void static_check( size_type MTL_DEBUG_ARG(s) ) const
    {
	assert(!traits::is_static<self>::value || s == size(typename Parameters::dimension()));
    }

    /// Check if a given vector expression whether it has the same shape as the object
    /** Both must be row vectors or column vectors. The elements must have the same
	algebraic shape, e.g. a row vector of scalars is not compatible with a row
	vector of matrices. **/
    template <class E>
    void check_consistent_shape( vec_expr<E> const& ) const
    {
	MTL_CRASH_IF((!boost::is_same<
			        typename ashape::ashape<self>::type
			      , typename ashape::ashape<E>::type
			    >::value),
			   "Incompatible shape!");
    }

    /// Default constructor
    dense_vector( ) : memory_base( Parameters::dimension::value ) {}

    /// Constructor for size \p n
    explicit dense_vector( size_type n ) : memory_base( n ) { static_check( n ); }

    /// Constructor for size \p n and value \p value that is set in all entries
    explicit dense_vector( size_type n, value_type value )
      : memory_base( n )
    {
	static_check( n );
	std::fill(begin(), end(), value);
    }

    /// Constructor for size \p n and \p address
    /** Can be used to handle vectors from other libraries, even in other languages like Fortran **/
    explicit dense_vector( size_type n, value_type *address )
      : memory_base( address, n, true )
    { static_check( n ); }

    /// Copy constructor
    dense_vector( const self& src )
      : memory_base( src )
    {
	vampir_trace<2042> tracer;
	using std::copy;
	copy(src.begin(), src.end(), this->begin());
    }

#ifdef MTL_WITH_MOVE
    dense_vector(self&& src) : memory_base(std::move(src)) {}
#endif

    /// Clone constructor
    /** Copies every vector, even those that refer to external data, sub-vectors, or rows and columns in a matrix **/
    dense_vector( const self& src, clone_ctor )
      : memory_base( src, clone_ctor()) {}

    struct dummy_type {};

    /// Constructor from vector expressions
    template <typename VectorSrc>
    explicit dense_vector(const VectorSrc& src,
			  typename boost::disable_if<boost::is_integral<VectorSrc>, dummy_type>::type= dummy_type())
    {	vampir_trace<2043> tracer; *this= src;    }

#ifdef MTL_WITH_INITLIST
    /// Constructor for initializer list \p values
    template <typename Value2>
    dense_vector(std::initializer_list<Value2> values)
      : memory_base(values.size())
    {
	static_check(values.size());
	std::copy(values.begin(), values.end(), begin());
    }

    /// Assignment from initializer list \p values
    template <typename Value2>
    self& operator=(std::initializer_list<Value2> values)
    {
	checked_change_dim(values.size());
	std::copy(values.begin(), values.end(), begin());
	return *this;
    }
#endif

    /// Constructor from std::vector; value_type must be identic
    explicit dense_vector(const std::vector<value_type>& src)
      : memory_base(src.size())
    {	std::copy(src.begin(), src.end(), this->begin());    }

    /// Stride is always 1
    size_type stride() const { return 1 ; }

    /// i-th entry
    reference operator()( size_type i )
    {
        check_index(i);
        return this->value_n( i ) ;
    }

    /// i-th entry (constant)
    const_reference operator()( size_type i ) const
    {
        check_index(i);
        return this->value_n( i ) ;
    }

    reference operator[]( size_type i ) { return (*this)( i ) ; } ///< i-th entry
    const_reference operator[]( size_type i ) const { return (*this)( i ) ;  } ///< i-th entry (constant)

    self operator[]( irange r ) { return sub_vector(*this, r.start(), r.finish()); } ///< sub-vector
    const self  operator[]( irange r ) const { return sub_vector(*this, r.start(), r.finish());  } ///< sub-vector

    void delay_assign() const {}

    // Compatibility with STL
    const_pointer begin() const { return this->elements() ; }
    const_pointer end() const { return this->elements() + this->used_memory(); }
    pointer begin() { return this->elements() ; }
    pointer end() { return this->elements() + this->used_memory(); }
    bool empty() const { return this->used_memory() == 0; } ///< Whether it is empty

    /// Address of first data entry; to be used with care.
    value_type* address_data() { return begin(); }
    const value_type* address_data() const { return begin(); }

#ifdef MTL_VECTOR_MOVE_EMULATION
    /// Move assignment
    self& operator=(self src)
    {
	// Self-copy would be an indication of an error
	assert(this != &src);

	checked_change_dim(src.used_memory());
	memory_base::move_assignment(src);
	return *this;
    }
#else
// #if defined(_MSC_VER) && !defined(MTL_VECTOR_MOVE_EMULATION)
    /// Copy assignment
    self& operator=(const self& src)
    {
	if (this == &src)
	    return *this;

	checked_change_dim(src.used_memory());
	memory_base::operator=(src);
	return *this;
    }
#endif

#ifdef MTL_WITH_MOVE
    self& operator=(self&& src)
    {
	checked_change_dim(src.used_memory());
	memory_base::move_assignment(src);
	return *this;
    }
#endif

#if 0 // def __PGI
    using crtp_base::operator=;
#else
    using assign_base::operator=;
#endif

    template <typename Value2> friend void fill(self&, const Value2&);

    /// swap two vectors
    friend void swap(self& vector1, self& vector2)
    {
	swap(static_cast<memory_base&>(vector1), static_cast<memory_base&>(vector2));
    }

    void change_resource(size_type n) { this->realloc(n); } ///< Change resource, like \ref change_dim
    void change_dim(size_type n) { this->realloc(n); } ///< Change dimension of vector
    void checked_change_dim(size_type n) { check_dim(n); change_dim(n); } ///< Only change dim if it was empty before

    void crop() {} ///< Delete structural zeros, only dummy here for completeness

} ; // dense_vector

/// Size of v
template <typename Value, typename Parameters>
inline typename dense_vector<Value, Parameters>::size_type
size(const dense_vector<Value, Parameters>& v)
{ return v.used_memory() ; }

// ================
// Free functions
// ================

/// Fill \p vector with \p value
template <typename Value, typename Parameters, typename Value2>
inline void fill(dense_vector<Value, Parameters>& vector, const Value2& value)
{
    std::fill(vector.begin(), vector.end(), value);
}

template <typename Value, typename Parameters>
typename dense_vector<Value, Parameters>::size_type
inline num_rows_aux(const dense_vector<Value, Parameters>& , tag::row_major)
{
    return 1;
}

template <typename Value, typename Parameters>
typename dense_vector<Value, Parameters>::size_type
inline num_rows_aux(const dense_vector<Value, Parameters>& vector, tag::col_major)
{
    return vector.used_memory();
}

/// Number of rows: is size for column vectors and 1 for row vectors
template <typename Value, typename Parameters>
typename dense_vector<Value, Parameters>::size_type
inline num_rows(const dense_vector<Value, Parameters>& vector)
{
    return num_rows_aux(vector, typename Parameters::orientation());
}

/// Number of columns: is size for row vectors and 1 for column vectors
template <typename Value, typename Parameters>
typename dense_vector<Value, Parameters>::size_type
inline num_cols(const dense_vector<Value, Parameters>& vector)
{
    return num_rows_aux(vector, typename mtl::traits::transposed_orientation<typename Parameters::orientation>::type());
}

/// Sub-vector function, more convenient with irange
template <typename Value, typename Parameters>
dense_vector<Value, Parameters>
inline sub_vector(dense_vector<Value, Parameters>& v,
		  typename dense_vector<Value, Parameters>::size_type start,
		  typename dense_vector<Value, Parameters>::size_type finish)
{
    typedef dense_vector<Value, Parameters>    Vector;

    MTL_CRASH_IF( is_negative(start) || is_negative(finish), "Index out of range!");
    irange r= intersection(irange(start, finish), irange(0, mtl::vec::size(v)));
    return r.empty() ? Vector() : Vector(r.size(), &v[r.start()]);

#if 0
    finish= min(finish, size(v));
    start= min(start, finish); // implies min(start, size(v))
    return start < finish ? Vector(finish - start, &v[start]) : Vector();
#endif
}

template <typename Value, typename Parameters>
const dense_vector<Value, Parameters>
inline sub_vector(const dense_vector<Value, Parameters>& v,
		  typename dense_vector<Value, Parameters>::size_type start,
		  typename dense_vector<Value, Parameters>::size_type finish)
{
    typedef dense_vector<Value, Parameters>    Vector;
    return sub_vector(const_cast<Vector&>(v), start, finish);
}

}} // namespace mtl::vector

namespace mtl {

    // Enable cloning of dense vectors
    template <typename Value, typename Parameters>
    struct is_clonable< vec::dense_vector<Value, Parameters> > : boost::mpl::true_ {};

} // namespace mtl

namespace mtl { namespace traits {

// ================
// Range generators
// For cursors
// ================

    template <typename Value, class Parameters>
    struct range_generator<tag::all, mtl::vec::dense_vector<Value, Parameters> >
      : public detail::dense_element_range_generator<mtl::vec::dense_vector<Value, Parameters>,
						     dense_el_cursor<Value>, complexity_classes::linear_cached>
    {};

    template <typename Value, class Parameters>
    struct range_generator<tag::nz, mtl::vec::dense_vector<Value, Parameters> >
	: public range_generator<tag::all, mtl::vec::dense_vector<Value, Parameters> >
    {};

    template <typename Value, class Parameters>
    struct range_generator<tag::iter::all, mtl::vec::dense_vector<Value, Parameters> >
    {
	typedef mtl::vec::dense_vector<Value, Parameters>   collection_t;
	typedef complexity_classes::linear_cached complexity;
	static int const                          level = 1;
	typedef typename collection_t::pointer    type;

	type begin(collection_t& collection)
	{
	    return collection.begin();
	}
	type end(collection_t& collection)
	{
	    return collection.end();
	}
    };

    template <typename Value, class Parameters>
    struct range_generator<tag::iter::nz, mtl::vec::dense_vector<Value, Parameters> >
	: public range_generator<tag::iter::all, mtl::vec::dense_vector<Value, Parameters> >
    {};

    template <typename Value, class Parameters>
    struct range_generator<tag::const_iter::all, mtl::vec::dense_vector<Value, Parameters> >
    {
	typedef mtl::vec::dense_vector<Value, Parameters>   collection_t;
	typedef complexity_classes::linear_cached complexity;
	static int const                          level = 1;
	typedef typename collection_t::const_pointer type;

	type begin(const collection_t& collection) const
	{
	    return collection.begin();
	}
	type end(const collection_t& collection) const
	{
	    return collection.end();
	}
    };

    template <typename Value, class Parameters>
    struct range_generator<tag::const_iter::nz, mtl::vec::dense_vector<Value, Parameters> >
	: public range_generator<tag::const_iter::all, mtl::vec::dense_vector<Value, Parameters> >
    {};

}} // namespace mtl::traits

namespace mtl {
    using vec::dense_vector;
}

#endif // MTL_DENSE_VECTOR_INCLUDE


//| Copyright Inria May 2015
//| This project has received funding from the European Research Council (ERC) under
//| the European Union's Horizon 2020 research and innovation programme (grant
//| agreement No 637972) - see http://www.resibots.eu
//|
//| Contributor(s):
//|   - Jean-Baptiste Mouret (jean-baptiste.mouret@inria.fr)
//|   - Antoine Cully (antoinecully@gmail.com)
//|   - Konstantinos Chatzilygeroudis (konstantinos.chatzilygeroudis@inria.fr)
//|   - Federico Allocati (fede.allocati@gmail.com)
//|   - Vaios Papaspyros (b.papaspyros@gmail.com)
//|   - Roberto Rama (bertoski@gmail.com)
//|
//| This software is a computer library whose purpose is to optimize continuous,
//| black-box functions. It mainly implements Gaussian processes and Bayesian
//| optimization.
//| Main repository: http://github.com/resibots/limbo
//| Documentation: http://www.resibots.eu/limbo
//|
//| This software is governed by the CeCILL-C license under French law and
//| abiding by the rules of distribution of free software.  You can  use,
//| modify and/ or redistribute the software under the terms of the CeCILL-C
//| license as circulated by CEA, CNRS and INRIA at the following URL
//| "http://www.cecill.info".
//|
//| As a counterpart to the access to the source code and  rights to copy,
//| modify and redistribute granted by the license, users are provided only
//| with a limited warranty  and the software's author,  the holder of the
//| economic rights,  and the successive licensors  have only  limited
//| liability.
//|
//| In this respect, the user's attention is drawn to the risks associated
//| with loading,  using,  modifying and/or developing or reproducing the
//| software by the user in light of its specific status of free software,
//| that may mean  that it is complicated to manipulate,  and  that  also
//| therefore means  that it is reserved for developers  and  experienced
//| professionals having in-depth computer knowledge. Users are therefore
//| encouraged to load and test the software's suitability as regards their
//| requirements in conditions enabling the security of their systems and/or
//| data to be ensured and,  more generally, to use and operate it in the
//| same conditions as regards security.
//|
//| The fact that you are presently reading this means that you have had
//| knowledge of the CeCILL-C license and that you accept its terms.
//|
#ifndef LIMBO_OPT_NLOPT_BASE_HPP
#define LIMBO_OPT_NLOPT_BASE_HPP

#ifndef USE_NLOPT
#warning No NLOpt
#else
#include <Eigen/Core>

#include <vector>

#include <nlopt.hpp>

#include <limbo/opt/optimizer.hpp>
#include <limbo/tools/macros.hpp>

namespace limbo {
    namespace opt {
        /**
          @ingroup opt
        Base class for NLOpt wrappers
        */
        template <typename Params, nlopt::algorithm Algorithm>
        struct NLOptBase {
        public:
            virtual void initialize(int dim)
            {
                _opt = nlopt::opt(Algorithm, dim);
                _initialized = true;
            }

            template <typename F>
            Eigen::VectorXd operator()(const F& f, const Eigen::VectorXd& init, bool bounded)
            {
                int dim = init.size();
                if (!_initialized)
                    initialize(dim);

                _opt.set_max_objective(nlopt_func<F>, (void*)&f);

                std::vector<double> x(dim);
                Eigen::VectorXd::Map(&x[0], dim) = init;

                if (bounded) {
                    _opt.set_lower_bounds(std::vector<double>(dim, 0));
                    _opt.set_upper_bounds(std::vector<double>(dim, 1));
                }

                double max;

                try {
                    _opt.optimize(x, max);
                }
                catch (nlopt::roundoff_limited& e) {
                    // In theory it's ok to ignore this error
                    std::cerr << "[NLOptNoGrad]: " << e.what() << std::endl;
                }
                catch (std::invalid_argument& e) {
                    // In theory it's ok to ignore this error
                    std::cerr << "[NLOptNoGrad]: " << e.what() << std::endl;
                }
                catch (std::runtime_error& e) {
                    // In theory it's ok to ignore this error
                    std::cerr << "[NLOptGrad]: " << e.what() << std::endl;
                }

                return Eigen::VectorXd::Map(x.data(), x.size());
            }

            // Inequality constraints of the form f(x) <= 0
            template <typename F>
            void add_inequality_constraint(const F& f, double tolerance = 1e-8)
            {
                if (_initialized)
                    _opt.add_inequality_constraint(nlopt_func<F>, (void*)&f, tolerance);
                else
                    std::cerr << "[NLOptNoGrad]: Trying to set an inequality constraint without having initialized the optimizer! Nothing will be done!" << std::endl;
            }

            // Equality constraints of the form f(x) = 0
            template <typename F>
            void add_equality_constraint(const F& f, double tolerance = 1e-8)
            {
                if (_initialized)
                    _opt.add_equality_constraint(nlopt_func<F>, (void*)&f, tolerance);
                else
                    std::cerr << "[NLOptNoGrad]: Trying to set an equality constraint without having initialized the optimizer! Nothing will be done!" << std::endl;
            }

        protected:
            nlopt::opt _opt;
            bool _initialized = false;

            template <typename F>
            static double nlopt_func(const std::vector<double>& x, std::vector<double>& grad, void* my_func_data)
            {
                F* f = (F*)(my_func_data);
                Eigen::VectorXd params = Eigen::VectorXd::Map(x.data(), x.size());
                double v;
                if (!grad.empty()) {
                    auto r = eval_grad(*f, params);
                    v = opt::fun(r);
                    Eigen::VectorXd g = opt::grad(r);
                    Eigen::VectorXd::Map(&grad[0], g.size()) = g;
                }
                else {
                    v = eval(*f, params);
                }
                return v;
            }
        };
    } // namespace opt
} // namespace limbo

#endif
#endif


/**
 * \file der_snr.cpp
 * \brief An C++ implementation of the der_snr fortran code from:
 * F. Stoehr et al: DER_SNR: A Simple & General Spectroscopic Signal-to-Noise Measurement Algorithm,\n
    394, Astronomical Data Analysis Software and Systems (ADASS) XVII\n
    2008ASPC..394..505S
 * \author Audric Lemonnier
 * \version 0.2
 * \date 18/04/2020
 */

#include <iostream>
#include <cstdlib>
#include <cstdio>
#include <fstream>
#include <vector>
#include <algorithm>
#include <string>
#include <cmath>
#include <functional>
#include <thread>
#include <future>
#include <tuple>
#include <chrono>

#include <boost/program_options.hpp>

#if __has_include (<boost/timer/timer.hpp>)
#include <boost/timer/timer.hpp>
#define HAS_BOOST_TIMER /**< boost::timer availability */
#endif

#if __has_include (<sys/syscall.h>)
#include <sys/syscall.h>
#define HAS_SYSCALL /**< linux syscall availability */
#endif

#if __has_include (<filesystem>)
#include <filesystem>
#define FS_STD /**< std::filesystem availability (C++17) */
namespace fs = std::filesystem;
#elif __has_include (<experimental/filesystem>) && !__has_include (<filesystem>)
#include <experimental/filesystem>
#define FS_STDEXP /**< std::experimental::filesystem availability */
namespace fs = std::experimental::filesystem;
#elif __has_include(<boost/filesystem.hpp>) && !__has_include (<filesystem>) && !__has_include (<experimental/filesystem>)
#include <boost/filesystem.hpp>
#define FS_BOOST /**< boost::filesystem availability */
namespace fs = boost::filesystem;
#else
#error "No filesystem header found"
#endif

#include <csv.h>
#include <msg.h>
#include <log.h>
#include <der_snr.h>

// Reference
// ----------------------------------------------------
// F. Stoehr et al: DER_SNR: A Simple & General Spectroscopic Signal-to-Noise Measurement Algorithm
// 394, Astronomical Data Analysis Software and Systems (ADASS) XVII
// 2008ASPC..394..505S
// ----------------------------------------------------

int main(int argc, char** argv) {

#ifdef HAS_BOOST_TIMER
    boost::timer::cpu_timer btTimer;
#endif

    _msg msgM;
    msgM.set_name("der_snr");
    msgM.set_log(LOGFILE);

// Parse cmd line
// ----------------------------------------------------
    namespace po = boost::program_options;
    po::options_description description("Usage");

    description.add_options()
    ("help,h", "Display this help message")
    ("filename,f",  po::value<std::string>(),"Filename of the spectrum")
    ("directory,d",  po::value<std::string>(),"Directory where compute the S/N")
    ("output,o",  po::value<std::string>()->default_value("output.csv"),"Filename of results")
    ("separator,s",  po::value<char>()->default_value('\t'),"The column separator. Do not set this option for \\tab.")
    ("exclude,e",  po::value<std::string>(),"Exclude a string in filenames");

    po::variables_map vm;
    po::store(po::command_line_parser(argc, argv).options(description).run(), vm);
    po::notify(vm);

     if (vm.count("help") || vm.size()<1 || !(vm.count("filename") ^ vm.count("directory"))) {
        msgM.enable_log(false);
        std::cout << description;
        std::cout << std::endl;
        std::cout << "Examples:\n";
        std::cout << "./der_snr -f CPD-591792.obs\n";
        msgM.msg(_msg::eMsg::START);
        msgM.msg(_msg::eMsg::MID, "check command line");
        msgM.msg(_msg::eMsg::MID, "compute S/N for 1 file");
        msgM.msg(_msg::eMsg::MID, "CPD-591792.obs: S/N = 95.68");
        msgM.msg(_msg::eMsg::MID, "output: output.csv");
        msgM.msg(_msg::eMsg::END, " 0.039347s wall, 0.040000s user + 0.000000s system = 0.040000s CPU (101.7%)\n");
        std::cout << std::endl;

        std::cout << "./der_snr -d data\n";
        msgM.msg(_msg::eMsg::START);
        msgM.msg(_msg::eMsg::MID, "check command line");
        msgM.msg(_msg::eMsg::MID, "CPU utilization: 0 %");
        msgM.msg(_msg::eMsg::MID, "starting 8 async threads");
        msgM.set_threadname("compute");
        msgM.msg(_msg::eMsg::THREADS, "S/N for 521 files");
        msgM.msg(_msg::eMsg::THREADS, "S/N for 521 files");
        msgM.msg(_msg::eMsg::THREADS, "S/N for 521 files");
        msgM.msg(_msg::eMsg::THREADS, "S/N for 521 files");
        msgM.msg(_msg::eMsg::THREADS, "S/N for 521 files");
        msgM.msg(_msg::eMsg::THREADS, "S/N for 521 files");
        msgM.msg(_msg::eMsg::THREADS, "S/N for 521 files");
        msgM.msg(_msg::eMsg::THREADS, "S/N for 524 files");
        msgM.set_name("merge()");
        msgM.msg(_msg::eMsg::MID, "merge file part1_snr.csv");
        msgM.msg(_msg::eMsg::MID, "merge file part2_snr.csv");
        msgM.msg(_msg::eMsg::MID, "merge file part3_snr.csv");
        msgM.msg(_msg::eMsg::MID, "merge file part4_snr.csv");
        msgM.msg(_msg::eMsg::MID, "merge file part5_snr.csv");
        msgM.msg(_msg::eMsg::MID, "merge file part6_snr.csv");
        msgM.msg(_msg::eMsg::MID, "merge file part7_snr.csv");
        msgM.msg(_msg::eMsg::MID, "merge file part8_snr.csv");
        msgM.set_name("der_snr");
        msgM.msg(_msg::eMsg::MID, "output: output.csv");
        msgM.msg(_msg::eMsg::END, " 56.395825s wall, 323.050000s user + 0.450000s system = 323.500000s CPU (573.6%)\n");

        std::cout << "\nRef:\n";
        std::cout << "F. Stoehr et al: DER_SNR: A Simple & General Spectroscopic Signal-to-Noise Measurement Algorithm\n";
        std::cout << "394, Astronomical Data Analysis Software and Systems (ADASS) XVII\n";
        std::cout << "2008ASPC..394..505S\n";
        return EXIT_SUCCESS;
    }

    bool bDefExclude=false;
    bool bDefSep=false;

    std::string sOutput;
    std::string sExclude;

    char cSep;

    fs::path pDirectory;
    fs::path pFilename;
    fs::path pOutput;

    // ----------------------------------------------------
    msgM.msg(_msg::eMsg::START);

    _log log;
    log.set_execname(argv);
    log.set_historyname(HISTFILE);
    log.set_logname(LOGFILE);

    // Write history
    // ----------------------------------------------------
    msgM.msg(_msg::eMsg::MID, "write history");
    if (!log.write_history(vm))
        msgM.msg(_msg::eMsg::ERROR, "cannot open history");
    // ----------------------------------------------------

    // Remove duplicates
    // ----------------------------------------------------
    msgM.msg(_msg::eMsg::MID, "remove duplicates in history");
    if (!log.remove_duplicate())
        msgM.msg(_msg::eMsg::ERROR, "cannot open history");
    // ----------------------------------------------------

    msgM.msg(_msg::eMsg::MID, "check command line");

    if (vm.count("directory") && vm.count("filename")) {
        msgM.msg(_msg::eMsg::ERROR, "ambiguous entries");
        return EXIT_FAILURE;
    }

    if (vm.count("directory")) {
        pDirectory=fs::path(vm["directory"].as<std::string>());
        if (!fs::is_directory(pDirectory)) {
            msgM.msg(_msg::eMsg::ERROR, pDirectory.string(),"does not exist");
            return EXIT_FAILURE;
        }
    }

    if (vm.count("filename")) {
        pFilename=fs::path(vm["filename"].as<std::string>());
        if (!fs::exists(pFilename)) {
            msgM.msg(_msg::eMsg::ERROR, "error:", pFilename.string(),"does not exist");
            return EXIT_FAILURE;
        }
    }

    if (vm.count("output")) {
        pOutput=fs::path(vm["output"].as<std::string>());
        if (fs::exists(pOutput)) {
            msgM.msg(_msg::eMsg::MID, pOutput.string(), " exists: deleting");
            fs::remove(pOutput);
        }
        sOutput=vm["output"].as<std::string>();
    }

    if (vm.count("separator")) {
        cSep=vm["separator"].as<char>();
        bDefSep=true;
    }

    if (vm.count("exclude")) {
        sExclude=vm["exclude"].as<std::string>();
        bDefExclude=true;
    }

    if (vm.count("filename")) {
        msgM.msg(_msg::eMsg::MID, "compute S/N for 1 file");

        std::string sFilename=vm["filename"].as<std::string>();

        _csv<float> csv(sFilename, '\t');

        if(csv.read()) {
            csv.set_verbose(_csv<float>::eVerbose::QUIET);
            msgM.msg(_msg::eMsg::MID,sFilename,": S/N =", der_snr(csv.select_column(1)));
        }
    }

    if (vm.count("directory")) {

        fs::recursive_directory_iterator step0(pDirectory);
        std::vector<std::string> list;

        if (bDefExclude) {
            for(auto &file: boost::make_iterator_range(step0,{}) )
                // csv<> unable to parse non csv file
                if (!fs::is_directory(file) &&
                    file.path().has_extension() &&
                    file.path().filename().string().find(std::string(".tar"))==std::string::npos &&
                    file.path().filename().string().find(std::string(".tar.gz"))==std::string::npos &&
                    file.path().filename().string().find(std::string(".tgz"))==std::string::npos &&
                    file.path().filename().string().find(std::string(".zip"))==std::string::npos &&
                    file.path().filename().string().find(std::string(".txt"))==std::string::npos &&
                    file.path().filename().string().find(std::string(".directory"))==std::string::npos &&
                    file.path().filename().string().find(sExclude)==std::string::npos )
                    list.emplace_back(file.path().relative_path().string());
        }
        else {
            for(auto &file: boost::make_iterator_range(step0,{}) )
                // csv<> unable to parse non csv file
                if (!fs::is_directory(file) &&
                    file.path().has_extension() &&
                    file.path().filename().string().find(std::string(".tar"))==std::string::npos &&
                    file.path().filename().string().find(std::string(".tar.gz"))==std::string::npos &&
                    file.path().filename().string().find(std::string(".tgz"))==std::string::npos &&
                    file.path().filename().string().find(std::string(".zip"))==std::string::npos &&
                    file.path().filename().string().find(std::string(".txt"))==std::string::npos &&
                    file.path().filename().string().find(std::string(".directory"))==std::string::npos )
                    list.emplace_back(file.path().relative_path().string());
        }
        int iMax_thread=std::thread::hardware_concurrency();

// Limit CPU usage -------------------------------------
#if defined (__linux__)
    msgM.msg(_msg::eMsg::MID, "CPU utilization:", static_cast<int>(CPU_utilization()), "%");
    if (CPU_utilization()>50) {
        iMax_thread=1;
        msgM.msg(_msg::eMsg::MID, "Reduce max thread");
    }
#endif
// -----------------------------------------------------

        if (iMax_thread>1) {
            const std::size_t stSize_divided=list.size()/iMax_thread;

            std::vector<std::vector<std::string> > vvsList_divided;

            for(std::size_t i=0; i< iMax_thread-1; i++)
                vvsList_divided.emplace_back(std::vector<std::string>(list.begin()+i*stSize_divided, list.begin() + stSize_divided*(i+1)));

            // 1 extra thread for extra files : rest of division
            vvsList_divided.emplace_back(std::vector<std::string>(list.begin()+(iMax_thread-1)*stSize_divided, list.end()));

            msgM.msg(_msg::eMsg::MID, "starting", iMax_thread, " async threads");

            // let's don't decide the flag
            std::launch lFlag=std::launch::async | std::launch::deferred;

            std::vector<std::future<void> > vfThread;
            std::vector<std::vector<std::string> > vvsRes;

            if (bDefSep) {
                int iCount=0;
                for(auto t_list: vvsList_divided) {
                    iCount++;
                    std::string sNewoutput="part"+std::to_string(iCount)+"_"+sOutput;
                    vfThread.emplace_back(std::async(lFlag, compute_sep, t_list, sNewoutput, cSep));
                }
            }
            else {
                int iCount=0;
                for(auto t_list: vvsList_divided) {
                    iCount++;
                    std::string sNewoutput="part"+std::to_string(iCount)+"_"+sOutput;
                    vfThread.emplace_back(std::async(lFlag, compute, t_list, sNewoutput));
                }
            }
            std::for_each(vfThread.begin(), vfThread.end(), [](std::future<void> &th) { th.get(); });
            merge(sOutput);
        }
        else {
            msgM.msg(_msg::eMsg::MID, "multi-threading disabled");
            compute(list, sOutput);
        }
    }

    msgM.msg(_msg::eMsg::MID, "output:", sOutput);

#ifdef HAS_BOOST_TIMER
    msgM.msg(_msg::eMsg::END, btTimer.format());
#endif

    return EXIT_SUCCESS;
}