/* ------------------------------------------------------
 *
 * @file marginal.cpp
 *
 * @brief Marginal effects for regression functions
 *
 *
 *//* ----------------------------------------------------------------------- */
#include <limits>
#include <dbconnector/dbconnector.hpp>
#include <modules/shared/HandleTraits.hpp>
#include <modules/prob/boost.hpp>
#include <boost/math/distributions.hpp>
#include <modules/prob/student.hpp>
#include "marginal.hpp"

namespace madlib {

// Use Eigen
using namespace dbal::eigen_integration;

namespace modules {

// Import names from other MADlib modules
using dbal::NoSolutionFoundException;

namespace regress {

inline double logistic(double x) {
    return 1. / (1. + std::exp(-x));
}

/**
 * @brief Helper function that computes the final statistics for the marginal variance
 */

AnyType margins_stateToResult(
    const Allocator &inAllocator,
    const ColumnVector &diagonal_of_variance_matrix,
    const ColumnVector &inmarginal_effects_per_observation,
    const double numRows) {

    uint16_t n_basis_terms = static_cast<uint16_t>(inmarginal_effects_per_observation.size());
    MutableNativeColumnVector marginal_effects(
        inAllocator.allocateArray<double>(n_basis_terms));
    MutableNativeColumnVector stdErr(
        inAllocator.allocateArray<double>(n_basis_terms));
    MutableNativeColumnVector tStats(
        inAllocator.allocateArray<double>(n_basis_terms));
    MutableNativeColumnVector pValues(
        inAllocator.allocateArray<double>(n_basis_terms));

    for (Index i = 0; i < n_basis_terms; ++i) {
        marginal_effects(i) = inmarginal_effects_per_observation(i) / numRows;
        stdErr(i) = std::sqrt(diagonal_of_variance_matrix(i));
        tStats(i) = marginal_effects(i) / stdErr(i);

        // p-values only make sense if numRows > coef.size()
        if (numRows > n_basis_terms)
            pValues(i) = 2. * prob::cdf( prob::normal(),
                                         -std::abs(tStats(i)));
    }

    // Return all coefficients, standard errors, etc. in a tuple
    // Note: p-values will return NULL if numRows <= coef.size
    AnyType tuple;
    tuple << marginal_effects
          << stdErr
          << tStats
          << (numRows > n_basis_terms? pValues: Null());
    return tuple;
}
// -------------------------------------------------------------------------

// ---------------------------------------------------------------------------
//             Marginal Effects Linear Regression States
// ---------------------------------------------------------------------------

/**
 * @brief State for marginal effects calculation for logistic regression
 *
 * TransitionState encapsualtes the transition state during the
 * marginal effects calculation for the logistic-regression aggregate function.
 * To the database, the state is exposed as a single DOUBLE PRECISION array,
 * to the C++ code it is a proper object containing scalars and vectors.
 *
 * Note: We assume that the DOUBLE PRECISION array is initialized by the
 * database with length at least 5, and all elemenets are 0.
 *
 */
template <class Handle>
class MarginsLinregrInteractionState {
    template <class OtherHandle>
    friend class MarginsLinregrInteractionState;

  public:
    MarginsLinregrInteractionState(const AnyType &inArray)
        : mStorage(inArray.getAs<Handle>()) {
        rebind(static_cast<uint16_t>(mStorage[1]),
               static_cast<uint16_t>(mStorage[2]));
    }

    /**
     * @brief Convert to backend representation
     *
     * We define this function so that we can use State in the
     * argument list and as a return type.
     */
    inline operator AnyType() const {
        return mStorage;
    }

    /**
     * @brief Initialize the marginal variance calculation state.
     *
     * This function is only called for the first iteration, for the first row.
     */
    inline void initialize(const Allocator &inAllocator,
                           const uint16_t inWidthOfX,
                           const uint16_t inNumBasis) {
        mStorage = inAllocator.allocateArray<double, dbal::AggregateContext,
                                             dbal::DoZero, dbal::ThrowBadAlloc>(
                arraySize(inWidthOfX, inNumBasis));
        rebind(inWidthOfX, inNumBasis);
        widthOfX = inWidthOfX;
        numBasis = inNumBasis;
    }

    /**
     * @brief We need to support assigning the previous state
     */
    template <class OtherHandle>
    MarginsLinregrInteractionState &operator=(
        const MarginsLinregrInteractionState<OtherHandle> &inOtherState) {

        for (size_t i = 0; i < mStorage.size(); i++)
            mStorage[i] = inOtherState.mStorage[i];
        return *this;
    }

    /**
     * @brief Merge with another State object by copying the intra-iteration
     *     fields
     */
    template <class OtherHandle>
    MarginsLinregrInteractionState &operator+=(
        const MarginsLinregrInteractionState<OtherHandle> &inOtherState) {

        if (mStorage.size() != inOtherState.mStorage.size() ||
            widthOfX != inOtherState.widthOfX)
            throw std::logic_error("Internal error: Incompatible transition "
                                   "states");
        numRows += inOtherState.numRows;
        marginal_effects += inOtherState.marginal_effects;
        delta += inOtherState.delta;
        return *this;
    }

    /**
     * @brief Reset the inter-iteration fields.
     */
    inline void reset() {
        numRows = 0;
        training_data_vcov.fill(0);
        marginal_effects.fill(0);
        delta.fill(0);
    }

  private:
    static inline size_t arraySize(const uint16_t inWidthOfX,
                                   const uint16_t inNumBasis) {
        return 4 + inNumBasis + (inWidthOfX + inNumBasis) * inWidthOfX;
    }

    /**
     * @brief Rebind to a new storage array
     *
     * @param inWidthOfX The number of independent variables.
     *
     */
    void rebind(uint16_t inWidthOfX, uint16_t inNumBasis) {
        iteration.rebind(&mStorage[0]);
        widthOfX.rebind(&mStorage[1]);
        numBasis.rebind(&mStorage[2]);
        numRows.rebind(&mStorage[3]);
        marginal_effects.rebind(&mStorage[4], inNumBasis);
        training_data_vcov.rebind(&mStorage[4 + inNumBasis], inWidthOfX, inWidthOfX);
        delta.rebind(&mStorage[4 + inNumBasis + inWidthOfX * inWidthOfX],
                     inNumBasis, inWidthOfX);
    }
    Handle mStorage;

  public:
    typename HandleTraits<Handle>::ReferenceToUInt32 iteration;
    typename HandleTraits<Handle>::ReferenceToUInt16 widthOfX;
    typename HandleTraits<Handle>::ReferenceToUInt16 numBasis;
    typename HandleTraits<Handle>::ReferenceToUInt64 numRows;
    typename HandleTraits<Handle>::ColumnVectorTransparentHandleMap marginal_effects;
    typename HandleTraits<Handle>::MatrixTransparentHandleMap training_data_vcov;
    typename HandleTraits<Handle>::MatrixTransparentHandleMap delta;
};
// ----------------------------------------------------------------------

/**
 * @brief Perform the marginal effects transition step
 */
AnyType
margins_linregr_int_transition::run(AnyType &args) {
    // Early return because of an exception has been "thrown"
    // (actually "warning") in the previous invocations
    if (args[0].isNull())
        return Null();
    MarginsLinregrInteractionState<MutableArrayHandle<double> > state = args[0];
    if (args[1].isNull() || args[2].isNull() ||
            args[3].isNull() || args[4].isNull()) {
        return args[0];
    }
    MappedColumnVector x;
    try {
        // an exception is raised in the backend if args[2] contains nulls
        MappedColumnVector xx = args[1].getAs<MappedColumnVector>();
        // x is a const reference, we can only rebind to change its pointer
        x.rebind(xx.memoryHandle(), xx.size());
    } catch (const ArrayWithNullException &e) {
        return args[0];
    }

    // The following check was added with MADLIB-138.
    if (!dbal::eigen_integration::isfinite(x)) {
        //throw std::domain_error("Design matrix is not finite.");
        warning("Design matrix is not finite.");
        return Null();
     }

    MappedColumnVector beta = args[2].getAs<MappedColumnVector>();

    Matrix J_trans = args[4].getAs<MappedMatrix>();
    J_trans.transposeInPlace(); // we actually pass-in J but transpose since
                                // we only require J^T in our equations

    if (state.numRows == 0) {
        if (x.size() > std::numeric_limits<uint16_t>::max()) {
            //throw std::domain_error("Number of independent variables cannot be "
            //                        "larger than 65535.");
            warning("Number of independent variables cannot be larger than 65535.");
            return Null();
        }
        state.initialize(*this,
                         static_cast<uint16_t>(beta.size()),
                         static_cast<uint16_t>(J_trans.rows()));
        Matrix training_data_vcov = args[3].getAs<MappedMatrix>();
        state.training_data_vcov = training_data_vcov;
    }

    // Now do the transition step
    state.numRows++;

    // compute marginal effects and delta using 1st and 2nd derivatives
    state.marginal_effects += J_trans * beta;
    state.delta += J_trans;
    return state;
}

/**
 * @brief Marginal effects: Merge transition states
 */
AnyType
margins_linregr_int_merge::run(AnyType &args) {
    // In case the aggregator should be terminated because
    // an exception has been "thrown" in the transition function
    if (args[0].isNull() || args[1].isNull())
        return Null();
    MarginsLinregrInteractionState<MutableArrayHandle<double> > stateLeft = args[0];
    MarginsLinregrInteractionState<ArrayHandle<double> > stateRight = args[1];
    // We first handle the trivial case where this function is called with one
    // of the states being the initial state
    if (stateLeft.numRows == 0)
        return stateRight;
    else if (stateRight.numRows == 0)
        return stateLeft;

    // Merge states together and return
    stateLeft += stateRight;
    return stateLeft;
}

/**
 * @brief Marginal effects: Final step
 */
AnyType
margins_linregr_int_final::run(AnyType &args) {
    // In case the aggregator should be terminated because
    // an exception has been "thrown" in the transition function
    if (args[0].isNull())
        return Null();
    // We request a mutable object.
    // Depending on the backend, this might perform a deep copy.
    MarginsLinregrInteractionState<ArrayHandle<double> > state = args[0];
    // Aggregates that haven't seen any data just return Null.
    if (state.numRows == 0)
        return Null();

    // Variance of the marginal effects (computed by delta method)
    Matrix variance;
    variance = state.delta * state.training_data_vcov;
    // we only need the diagonal elements of the variance, so we perform a dot
    // product of each row with itself to compute each diagonal element.
    ColumnVector variance_diagonal =
        variance.cwiseProduct(state.delta).rowwise().sum() / static_cast<double>(state.numRows * state.numRows);

    // Computing the marginal effects
    return margins_stateToResult(*this, variance_diagonal,
                                 state.marginal_effects, static_cast<double>(state.numRows));
}

// ---------------------------------------------------------------------------
//             Marginal Effects Logistic Regression States
// ---------------------------------------------------------------------------
/**
 * @brief State for marginal effects calculation for logistic regression
 *
 * TransitionState encapsualtes the transition state during the
 * marginal effects calculation for the logistic-regression aggregate function.
 * To the database, the state is exposed as a single DOUBLE PRECISION array,
 * to the C++ code it is a proper object containing scalars and vectors.
 *
 * Note: We assume that the DOUBLE PRECISION array is initialized by the
 * database with length at least 5, and all elemenets are 0.
 *
 */
template <class Handle>
class MarginsLogregrInteractionState {
    template <class OtherHandle>
    friend class MarginsLogregrInteractionState;

  public:
    MarginsLogregrInteractionState(const AnyType &inArray)
        : mStorage(inArray.getAs<Handle>()) {

        rebind(static_cast<uint16_t>(mStorage[1]),
               static_cast<uint16_t>(mStorage[2]),
               static_cast<uint16_t>(mStorage[3]));
    }

    /**
     * @brief Convert to backend representation
     *
     * We define this function so that we can use State in the
     * argument list and as a return type.
     */
    inline operator AnyType() const {
        return mStorage;
    }

    /**
     * @brief Initialize the marginal variance calculation state.
     *
     * This function is only called for the first iteration, for the first row.
     */
    inline void initialize(const Allocator &inAllocator,
                           const uint16_t inWidthOfX,
                           const uint16_t inNumBasis,
                           const uint16_t inNumCategoricals) {
        mStorage = inAllocator.allocateArray<double, dbal::AggregateContext,
                                             dbal::DoZero, dbal::ThrowBadAlloc>(
                arraySize(inWidthOfX, inNumBasis, inNumCategoricals));
        rebind(inWidthOfX, inNumBasis, inNumCategoricals);
        widthOfX = inWidthOfX;
        numBasis = inNumBasis;
        numCategoricalVarsInSubset = inNumCategoricals;
    }

    /**
     * @brief We need to support assigning the previous state
     */
    template <class OtherHandle>
    MarginsLogregrInteractionState &operator=(
        const MarginsLogregrInteractionState<OtherHandle> &inOtherState) {

        for (size_t i = 0; i < mStorage.size(); i++)
            mStorage[i] = inOtherState.mStorage[i];
        return *this;
    }

    /**
     * @brief Merge with another State object by copying the intra-iteration
     *     fields
     */
    template <class OtherHandle>
    MarginsLogregrInteractionState &operator+=(
        const MarginsLogregrInteractionState<OtherHandle> &inOtherState) {

        if (mStorage.size() != inOtherState.mStorage.size() ||
            widthOfX != inOtherState.widthOfX)
            throw std::logic_error("Internal error: Incompatible transition "
                                   "states");

        numRows += inOtherState.numRows;
        marginal_effects += inOtherState.marginal_effects;
        delta += inOtherState.delta;
        return *this;
    }

    /**
     * @brief Reset the inter-iteration fields.
     */
    inline void reset() {
        numRows = 0;
        marginal_effects.fill(0);
        categorical_basis_indices.fill(0);
        training_data_vcov.fill(0);
        delta.fill(0);
    }

  private:
    static inline size_t arraySize(const uint16_t inWidthOfX,
                                   const uint16_t inNumBasis,
                                   const uint16_t inNumCategoricals) {
        return 5 + inNumBasis + inNumCategoricals + (inWidthOfX + inNumBasis) * inWidthOfX;
    }

    /**
     * @brief Rebind to a new storage array
     *
     * @param inWidthOfX The number of independent variables.
     *
     */
    void rebind(uint16_t inWidthOfX, uint16_t inNumBasis, uint16_t inNumCategoricals) {
        iteration.rebind(&mStorage[0]);
        widthOfX.rebind(&mStorage[1]);
        numBasis.rebind(&mStorage[2]);
        numCategoricalVarsInSubset.rebind(&mStorage[3]);
        numRows.rebind(&mStorage[4]);
        marginal_effects.rebind(&mStorage[5], inNumBasis);
        training_data_vcov.rebind(&mStorage[5 + inNumBasis], inWidthOfX, inWidthOfX);
        delta.rebind(&mStorage[5 + inNumBasis + inWidthOfX * inWidthOfX],
                     inNumBasis, inWidthOfX);
        if (inNumCategoricals > 0)
            categorical_basis_indices.rebind(&mStorage[5 + inNumBasis +
                                            (inWidthOfX + inNumBasis) * inWidthOfX],
                                            inNumCategoricals);
    }
    Handle mStorage;

  public:

    typename HandleTraits<Handle>::ReferenceToUInt32 iteration;
    typename HandleTraits<Handle>::ReferenceToUInt16 widthOfX;
    typename HandleTraits<Handle>::ReferenceToUInt16 numBasis;
    typename HandleTraits<Handle>::ReferenceToUInt16 numCategoricalVarsInSubset;
    typename HandleTraits<Handle>::ReferenceToUInt64 numRows;
    typename HandleTraits<Handle>::ColumnVectorTransparentHandleMap marginal_effects;
    typename HandleTraits<Handle>::ColumnVectorTransparentHandleMap categorical_basis_indices;
    typename HandleTraits<Handle>::MatrixTransparentHandleMap training_data_vcov;
    typename HandleTraits<Handle>::MatrixTransparentHandleMap delta;
};
// ----------------------------------------------------------------------

/**
 * @brief Perform the marginal effects transition step
 */
AnyType
margins_logregr_int_transition::run(AnyType &args) {
    // Early return because of an exception has been "thrown"
    // (actually "warning") in the previous invocations
    if (args[0].isNull())
        return Null();
    MarginsLogregrInteractionState<MutableArrayHandle<double> > state = args[0];
    if (args[1].isNull() || args[2].isNull() ||
            args[3].isNull() || args[4].isNull()) {
        return args[0];
    }    MappedColumnVector f;
    try {
        // an exception is raised in the backend if args[2] contains nulls
        MappedColumnVector xx = args[1].getAs<MappedColumnVector>();
        // x is a const reference, we can only rebind to change its pointer
        f.rebind(xx.memoryHandle(), xx.size());
    } catch (const ArrayWithNullException &e) {
        return args[0];
    }

    // The following check was added with MADLIB-138.
    if (!dbal::eigen_integration::isfinite(f)) {
        //throw std::domain_error("Design matrix is not finite.");
        warning("Design matrix is not finite.");
        return Null();
    }

    // beta is the coefficient vector from logistic regression
    MappedColumnVector beta = args[2].getAs<MappedColumnVector>();

    // basis_indices represents the indices (from beta) of which we want to
    // compute the marginal effect. We don't need to compute the ME for all
    // variables, thus basis_indices could be a subset of all indices.
    MappedColumnVector basis_indices = args[4].getAs<MappedColumnVector>();

    // below symbols match the ones used in the design doc
    const uint16_t N = static_cast<uint16_t>(beta.size());
    const uint16_t M = static_cast<uint16_t>(basis_indices.size());
    assert(N >= M);

    Matrix J;  // J: N * M
    if (args[5].isNull()){
        J = Matrix::Zero(N, M);
        for (Index i = 0; i < M; ++i)
            J(static_cast<Index>(basis_indices(i)), i) = 1;
    } else{
        J = args[5].getAs<MappedMatrix>();
    }
    assert(J.rows() == N && J.cols() == M);

    MappedColumnVector categorical_indices;
    uint16_t numCategoricalVars = 0;

    if (!args[6].isNull()) {
        // categorical_indices represents which indices (from beta) are
        // categorical variables
        try {
            MappedColumnVector xx = args[6].getAs<MappedColumnVector>();
            categorical_indices.rebind(xx.memoryHandle(), xx.size());
        } catch (const ArrayWithNullException &e) {
             //throw std::runtime_error("The categorical indices contain NULL values");
             warning("The categorical indices contain NULL values");
             return Null();
        }
        numCategoricalVars = static_cast<uint16_t>(categorical_indices.size());
    }

    if (state.numRows == 0) {
        if (f.size() > std::numeric_limits<uint16_t>::max()) {
            //throw std::domain_error("Number of independent variables cannot be "
            //                        "larger than 65535.");
            warning("Number of independent variables cannot be larger than 65535.");
            return Null();
        }
        std::vector<uint16_t> tmp_cat_basis_indices;
        if (numCategoricalVars > 0){
            // find which of the variables in basis_indices are categorical
            // and store only the indices for these variables in our state
            for (Index i = 0; i < basis_indices.size(); ++i){
                for (Index j = 0; j < categorical_indices.size(); ++j){
                    if (basis_indices(i) == categorical_indices(j)){
                        tmp_cat_basis_indices.push_back(static_cast<uint16_t>(i));
                        continue;
                    }
                }
            }
            state.numCategoricalVarsInSubset = static_cast<uint16_t>(tmp_cat_basis_indices.size());
        }
        state.initialize(*this,
                         static_cast<uint16_t>(N),
                         static_cast<uint16_t>(M),
                         static_cast<uint16_t>(state.numCategoricalVarsInSubset));

        Matrix training_data_vcov = args[3].getAs<MappedMatrix>();
        state.training_data_vcov = training_data_vcov;

        if (state.numCategoricalVarsInSubset > 0){
            for (int i=0; i < state.numCategoricalVarsInSubset; ++i){
                state.categorical_basis_indices(i) = tmp_cat_basis_indices[i];
            }
        }
    }

    // Now do the transition step
    state.numRows++;
    double f_beta = dot(f, beta);
    double p = std::exp(f_beta)/ (1 + std::exp(f_beta));

    // compute marginal effects and delta using 1st and 2nd derivatives
    ColumnVector J_trans_beta;
    J_trans_beta = trans(J) * beta;
    ColumnVector curr_margins = J_trans_beta * p * (1 - p);

    Matrix curr_delta;
    curr_delta = p * (1 - p) * (trans(J) +
                                (1 - 2 * p) * J_trans_beta * trans(f));

    // margins and delta using discrete differences for categoricals variables

    // f_set_mat and f_unset_mat are matrices where each row corresponds to a
    //  categorical basis variable and the columns correspond to all terms
    // (basis and interaction)
    Matrix f_set_mat;  // numCategoricalVarsInSubset x N
    Matrix f_unset_mat;  // numCategoricalVarsInSubset x N
    if (!args[7].isNull() && !args[8].isNull()){
        // the matrix is read in column-order but passed in row-order
        f_set_mat = args[7].getAs<MappedMatrix>();
        f_set_mat.transposeInPlace();

        f_unset_mat = args[8].getAs<MappedMatrix>();
        f_unset_mat.transposeInPlace();
    }

    // PERFORMANCE TWEAK: for the no interaction case, f_set_mat and f_unset_mat
    // only need column entries for the categorical variables (others are same
    // as f). Since, passing a smaller matrix into the transition function is
    // faster, for the no interaction case, we don't need to input the whole
    // matrix into this function. For the interaction case, since it is unknown
    // which indices have interaction, we require entries for all columns in the
    // matrices.
    bool no_interactions = (f_set_mat.cols() < N);
    for (Index i = 0; i < state.numCategoricalVarsInSubset; ++i) {
        // Note: categorical_indices are assumed to be zero-based
        ColumnVector f_set;
        ColumnVector f_unset;
        ColumnVector shortened_f_set = f_set_mat.row(i);
        ColumnVector shortened_f_unset = f_unset_mat.row(i);

        if (no_interactions){
            f_set = f;
            f_unset = f;
            for (Index j=0; j < shortened_f_set.size(); ++j){
                f_set(static_cast<Index>(categorical_indices(j))) = shortened_f_set(j);
                f_unset(static_cast<Index>(categorical_indices(j))) = shortened_f_unset(j);
            }
        } else {
            f_set = shortened_f_set;
            f_unset = shortened_f_unset;
        }
        double p_set = logistic(dot(f_set, beta));
        double p_unset = logistic(dot(f_unset, beta));

        curr_margins(static_cast<uint16_t>(state.categorical_basis_indices(i))) = p_set - p_unset;
        curr_delta.row(static_cast<uint16_t>(state.categorical_basis_indices(i))) = (
            p_set * (1 - p_set) * f_set - p_unset * (1 - p_unset) * f_unset);
    }
    state.marginal_effects += curr_margins;
    state.delta += curr_delta;
    return state;
}

/**
 * @brief Marginal effects: Merge transition states
 */
AnyType
margins_logregr_int_merge::run(AnyType &args) {
    // In case the aggregator should be terminated because
    // an exception has been "thrown" in the transition function
    if (args[0].isNull() || args[1].isNull())
        return Null();
    MarginsLogregrInteractionState<MutableArrayHandle<double> > stateLeft = args[0];
    MarginsLogregrInteractionState<ArrayHandle<double> > stateRight = args[1];
    // We first handle the trivial case where this function is called with one
    // of the states being the initial state
    if (stateLeft.numRows == 0)
        return stateRight;
    else if (stateRight.numRows == 0)
        return stateLeft;

    // Merge states together and return
    stateLeft += stateRight;
    return stateLeft;
}

/**
 * @brief Marginal effects: Final step
 */
AnyType
margins_logregr_int_final::run(AnyType &args) {
    // In case the aggregator should be terminated because
    // an exception has been "thrown" in the transition function
    if (args[0].isNull())
        return Null();
    // We request a mutable object.
    // Depending on the backend, this might perform a deep copy.
    MarginsLogregrInteractionState<MutableArrayHandle<double> > state = args[0];
    // Aggregates that haven't seen any data just return Null.
    if (state.numRows == 0)
        return Null();

    // Variance for marginal effects according to the delta method
    Matrix variance;
    variance = state.delta * state.training_data_vcov;

    // we only need the diagonal elements of the variance, so we perform a dot
    // product of each row with state.delta to compute each diagonal element.
    // We divide by numRows^2 since we need the average variance
    ColumnVector variance_diagonal =
        variance.cwiseProduct(state.delta).rowwise().sum() / static_cast<double>(state.numRows * state.numRows);

    // Computing the final results
    return margins_stateToResult(*this, variance_diagonal,
                                 state.marginal_effects, static_cast<double>(state.numRows));
}
// ------------------------ End of Logistic Marginal ---------------------------

// ---------------------------------------------------------------------------
//             Marginal Effects Multilogistic Regression
// ---------------------------------------------------------------------------
/**
 * @brief State for marginal effects calculation for multilogistic regression
 *
 * TransitionState encapsualtes the transition state during the
 * marginal effects calculation for the logistic-regression aggregate function.
 * To the database, the state is exposed as a single DOUBLE PRECISION array,
 * to the C++ code it is a proper object containing scalars and vectors.
 *
 * Note: We assume that the DOUBLE PRECISION array is initialized by the
 * database with length at least 5, and all elemenets are 0.
 *
 */
template <class Handle>
class MarginsMLogregrInteractionState {
    template <class OtherHandle>
    friend class MarginsMLogregrInteractionState;

  public:
    MarginsMLogregrInteractionState(const AnyType &inArray)
        : mStorage(inArray.getAs<Handle>()) {

        rebind(static_cast<uint16_t>(mStorage[0]),
               static_cast<uint16_t>(mStorage[1]),
               static_cast<uint16_t>(mStorage[2]),
               static_cast<uint16_t>(mStorage[3]));
    }

    /**
     * @brief Convert to backend representation
     *
     * We define this function so that we can use State in the
     * argument list and as a return type.
     */
    inline operator AnyType() const {
        return mStorage;
    }

    /**
     * @brief Initialize the marginal variance calculation state.
     *
     * This function is only called for the first iteration, for the first row.
     */
    inline void initialize(const Allocator &inAllocator,
                           const uint16_t inWidthOfX,
                           const uint16_t inNumCategories,
                           const uint16_t inNumBasis,
                           const uint16_t inNumCategoricalVars) {
        mStorage = inAllocator.allocateArray<double, dbal::AggregateContext,
                                             dbal::DoZero, dbal::ThrowBadAlloc>(
                arraySize(inWidthOfX, inNumCategories, inNumBasis, inNumCategoricalVars));
        rebind(inWidthOfX,  inNumCategories, inNumBasis, inNumCategoricalVars);
        widthOfX = inWidthOfX;
        numCategories = inNumCategories;
        numBasis = inNumBasis;
        numCategoricalVarsInSubset = inNumCategoricalVars;
    }

    /**
     * @brief We need to support assigning the previous state
     */
    template <class OtherHandle>
    MarginsMLogregrInteractionState &operator=(
        const MarginsMLogregrInteractionState<OtherHandle> &inOtherState) {

        for (size_t i = 0; i < mStorage.size(); i++)
            mStorage[i] = inOtherState.mStorage[i];
        return *this;
    }

    /**
     * @brief Merge with another State object by copying the intra-iteration
     *     fields
     */
    template <class OtherHandle>
    MarginsMLogregrInteractionState &operator+=(
        const MarginsMLogregrInteractionState<OtherHandle> &inOtherState) {

        if (mStorage.size() != inOtherState.mStorage.size() ||
            widthOfX != inOtherState.widthOfX)
            throw std::logic_error("Internal error: Incompatible transition "
                                   "states");

        numRows += inOtherState.numRows;
        marginal_effects += inOtherState.marginal_effects;
        delta += inOtherState.delta;
        return *this;
    }

    /**
     * @brief Reset the inter-iteration fields.
     */
    inline void reset() {
        numRows = 0;
        marginal_effects.fill(0);
        categorical_basis_indices.fill(0);
        training_data_vcov.fill(0);
        delta.fill(0);
    }

  private:
    static inline size_t arraySize(const uint16_t inWidthOfX,
                                   const uint16_t inNumCategories,
                                   const uint16_t inNumBasis,
                                   const uint16_t inNumCategoricalVars) {
        return 5 + (inNumCategories - 1) * (inNumBasis +
                    inNumBasis*inWidthOfX*(inNumCategories - 1) +
                    inWidthOfX*inWidthOfX*(inNumCategories - 1)) +
                inNumCategoricalVars;
    }

    /**
     * @brief Rebind to a new storage array
     *
     * @param inWidthOfX The number of independent variables.
     *
     */
    void rebind(const uint16_t inWidthOfX,
                const uint16_t inNumCategories,
                const uint16_t inNumBasis,
                const uint16_t inNumCategoricalVars) {

        const uint16_t & L = inNumCategories;
        const uint16_t & N = inWidthOfX;
        const uint16_t & M = inNumBasis;

        widthOfX.rebind(&mStorage[0]);
        numCategories.rebind(&mStorage[1]);
        numBasis.rebind(&mStorage[2]);
        numCategoricalVarsInSubset.rebind(&mStorage[3]);
        numRows.rebind(&mStorage[4]);

        if (L == 0) { return; }

        marginal_effects.rebind(&mStorage[5], M, L-1);

        int current_length = 5 + M * (L - 1);

        training_data_vcov.rebind(&mStorage[current_length], N*(L-1), N*(L-1));
        current_length += N * (L - 1) * N * (L - 1);

        delta.rebind(&mStorage[current_length], M*(L-1), N*(L-1));
        current_length += N * (L - 1) * M * (L - 1);

        if (inNumCategoricalVars > 0)
            categorical_basis_indices.rebind(&mStorage[static_cast<uint16_t>(current_length)], inNumCategoricalVars);
    }
    Handle mStorage;

  public:
    // symbols in comments correspond to the design document
    typename HandleTraits<Handle>::ReferenceToUInt16 widthOfX;       // N
    typename HandleTraits<Handle>::ReferenceToUInt16 numCategories;  // L
    typename HandleTraits<Handle>::ReferenceToUInt16 numBasis;       // M
    typename HandleTraits<Handle>::ReferenceToUInt16 numCategoricalVarsInSubset;
    typename HandleTraits<Handle>::ReferenceToUInt64 numRows;
    typename HandleTraits<Handle>::MatrixTransparentHandleMap marginal_effects; // ME
    typename HandleTraits<Handle>::ColumnVectorTransparentHandleMap categorical_basis_indices;
    typename HandleTraits<Handle>::MatrixTransparentHandleMap training_data_vcov;
    typename HandleTraits<Handle>::MatrixTransparentHandleMap delta;   // S
};
// ----------------------------------------------------------------------

namespace {
inline Index
reindex(Index outer, Index inner, Index block) { return outer * block + inner; }
}

/**
 * @brief Perform the marginal effects transition step
 */
AnyType
margins_mlogregr_int_transition::run(AnyType &args) {
    // Early return because of an exception has been "thrown"
    // (actually "warning") in the previous invocations
    if (args[0].isNull())
        return Null();
    MarginsMLogregrInteractionState<MutableArrayHandle<double> > state = args[0];
    if (args[1].isNull() || args[2].isNull() || args[3].isNull() ||
        args[4].isNull()) {
        return args[0];
    }

    MappedColumnVector f;
    try {
        // an exception is raised in the backend if args[1] contains nulls
        MappedColumnVector xx = args[1].getAs<MappedColumnVector>();
        // x is a const reference, we can only rebind to change its pointer
        f.rebind(xx.memoryHandle(), xx.size());
    } catch (const ArrayWithNullException &e) {
        return args[0];
    }

    // The following check was added with MADLIB-138.
    if (!dbal::eigen_integration::isfinite(f)) {
        //throw std::domain_error("Design matrix is not finite.");
        warning("Design matrix is not finite.");
        return Null();
    }

    // coefficients are arranged in a matrix
    MappedMatrix beta = args[2].getAs<MappedMatrix>();  // beta: N x (L - 1)

    // basis_indices represents the indices (from beta) of which we want to
    // compute the marginal effect. We don't need to compute the ME for all
    // variables, thus basis_indices could be a subset of all indices.
    MappedColumnVector basis_indices = args[4].getAs<MappedColumnVector>();

    // all variable symbols correspond to the design document
    const uint16_t N = static_cast<uint16_t>(beta.rows());
    const uint16_t M = static_cast<uint16_t>(basis_indices.size());
    assert(N >= M);

    Matrix J;  // J: N x M
    if (args[5].isNull()){
        J = Matrix::Zero(N, M);
        for (Index i = 0; i < M; ++i)
            J(static_cast<Index>(basis_indices(i)), i) = 1;
    } else{
        J = args[5].getAs<MappedMatrix>();
    }
    assert(J.rows() == N && J.cols() == M);

    MappedColumnVector categorical_indices;
    uint16_t numCategoricalVars = 0;

    if (!args[6].isNull()) {
        // categorical_indices represents which indices (from beta) are
        // categorical variables
        try {
            MappedColumnVector xx = args[6].getAs<MappedColumnVector>();
            categorical_indices.rebind(xx.memoryHandle(), xx.size());
        } catch (const ArrayWithNullException &e) {
             //throw std::runtime_error("The categorical indices contain NULL values");
             warning("The categorical indices contain NULL values");
             return Null();
        }
        numCategoricalVars = static_cast<uint16_t>(categorical_indices.size());
    }

    if (state.numRows == 0) {
        if (f.size() > std::numeric_limits<uint16_t>::max()) {
            //throw std::domain_error("Number of independent variables cannot be "
            //                        "larger than 65535.");
            warning("Number of independent variables cannot be larger than 65535.");
            return Null();
        }
        std::vector<uint16_t> tmp_cat_basis_indices;
        if (numCategoricalVars > 0){
            // find which of the variables in basis_indices are categorical
            // and store only the indices for these variables in our state
            for (Index i = 0; i < basis_indices.size(); ++i){
                for (Index j = 0; j < categorical_indices.size(); ++j){
                    if (basis_indices(i) == categorical_indices(j)){
                        tmp_cat_basis_indices.push_back(static_cast<uint16_t>(i));
                        continue;
                    }
                }
            }
            state.numCategoricalVarsInSubset = static_cast<uint16_t>(tmp_cat_basis_indices.size());
        }
        state.initialize(*this,
                         static_cast<uint16_t>(J.rows()),
                         static_cast<uint16_t>(beta.cols() + 1),
                         static_cast<uint16_t>(J.cols()),
                         static_cast<uint16_t>(state.numCategoricalVarsInSubset));

        Matrix training_data_vcov = args[3].getAs<MappedMatrix>();
        state.training_data_vcov = training_data_vcov;
        if (state.numCategoricalVarsInSubset > 0){
            for (int i=0; i < state.numCategoricalVarsInSubset; ++i){
                state.categorical_basis_indices(i) = tmp_cat_basis_indices[i];
            }
        }
    }

    state.numRows++;

    // all variable symbols correspond to the design document
    const uint16_t & L = state.numCategories;
    ColumnVector prob(trans(beta) * f);
    Matrix J_trans_beta(trans(J) * beta);

    // Calculate the odds ratio
    prob = prob.array().exp();
    double prob_sum = prob.sum();
    prob = prob / (1 + prob_sum);

    ColumnVector JBP(J_trans_beta * prob);
    Matrix curr_margins = J_trans_beta * prob.asDiagonal() - JBP * trans(prob);

    // compute delta using 2nd derivatives
    // delta matrix is 2-D of size (L-1)M x (L-1)N:
    //      row_index = [0, (L-1)M), col_index = [0, (L-1)N)
    // row_index(m, l) = m * (L-1) + l
    // col_index(n, l1) = n * (L-1) + l1
    Index row_index, col_index;
    int delta_l_l1;
    for (int m = 0; m < M; m++){
        // Skip the categorical variables
        if (state.numCategoricalVarsInSubset > 0) {
            bool is_categorical = false;
            for(int i = 0; i < state.categorical_basis_indices.size(); i++)
                if (m == state.categorical_basis_indices(i)) {
                    is_categorical = true;
                    break;
                }
             if (is_categorical)
                continue;
        }

        for (int l=0; l < (L-1); l++){
            row_index = reindex(m, l, L-1);
            for (int n=0; n < N; n++){
                for (int l1=0; l1 < (L-1); l1++){
                    delta_l_l1 = (l==l1) ? 1 : 0;
                    col_index = reindex(n, l1, L-1);
                    state.delta(row_index, col_index) +=
                        f(n)*(delta_l_l1 - prob(l1)) * curr_margins(m, l) +
                        prob(l) * (delta_l_l1 * J(n, m) -
                                   f(n) * curr_margins(m, l1) -
                                   prob(l1) * J(n, m));
                }
            }
        }
    }

    // update marginal effects and delta using discrete differences just for
    // categorical variables
    Matrix f_set_mat;   // numCategoricalVarsInSubset * N
    Matrix f_unset_mat; // numCategoricalVarsInSubset * N
    // the above matrices contain the f_set and f_unset for all categorical variables
    if (!args[7].isNull() && !args[8].isNull()){
        // the matrix is read in column-order but passed in row-order
        f_set_mat = args[7].getAs<MappedMatrix>();
        f_set_mat.transposeInPlace();

        f_unset_mat = args[8].getAs<MappedMatrix>();
        f_unset_mat.transposeInPlace();
    }

    // PERFORMANCE TWEAK: for the no interaction case, f_set_mat and f_unset_mat
    // only need column entries for the categorical variables (others are same
    // as f). Since, passing a smaller matrix into the transition function is
    // faster, for the no interaction case, we don't need to input the whole
    // matrix into this function. For the interaction case, since it is unknown
    // which indices have interaction, we require entries for all columns in the
    // matrices.
    bool no_interactions = (f_set_mat.cols() < N);
    for (Index i = 0; i < state.numCategoricalVarsInSubset; ++i) {
        // Note: categorical_indices are assumed to be zero-based
        ColumnVector f_set;
        ColumnVector f_unset;
        ColumnVector shortened_f_set = f_set_mat.row(i);
        ColumnVector shortened_f_unset = f_unset_mat.row(i);

        if (no_interactions){
            f_set = f;
            f_unset = f;
            for (Index j=0; j < shortened_f_set.size(); ++j){
                f_set(static_cast<Index>(categorical_indices(j))) = shortened_f_set(j);
                f_unset(static_cast<Index>(categorical_indices(j))) = shortened_f_unset(j);
            }
        } else {
            f_set = shortened_f_set;
            f_unset = shortened_f_unset;
        }

        RowVector p_set(trans(f_set) * beta);
        {
            p_set = p_set.array().exp();
            double p_sum = p_set.sum();
            p_set = p_set / (1 + p_sum);
        }

        RowVector p_unset(trans(f_unset) * beta);
        {
            p_unset = p_unset.array().exp();
            double p_sum = p_unset.sum();
            p_unset = p_unset / (1 + p_sum);
        }
        // Compute the marginal effect using difference method
        curr_margins.row(static_cast<uint16_t>(state.categorical_basis_indices(i))) = p_set - p_unset;

        // Compute the delta using difference method
        int m = static_cast<uint16_t>(state.categorical_basis_indices(i));
        for (int l = 0; l < L - 1; l++) {
            row_index = reindex(m, l, L - 1);
            for (int n = 0; n < N; n++) {
                for (int l1 = 0; l1 < L - 1; l1++) {
                    double delta = - p_set(l) * p_set(l1) * f_set(n) + p_unset(l) * p_unset(l1) * f_unset(n);
                    if (l1 == l)
                        delta += p_set(l) * f_set(n) - p_unset(l) * f_unset(n);
                    col_index = reindex(n, l1, L - 1);
                    state.delta(row_index, col_index) += delta;
                }
            }
        }

    }
    state.marginal_effects += curr_margins;
    return state;
}

/**
 * @brief Marginal effects: Merge transition states
 */
AnyType
margins_mlogregr_int_merge::run(AnyType &args) {
    // In case the aggregator should be terminated because
    // an exception has been "thrown" in the transition function
    if (args[0].isNull() || args[1].isNull())
        return Null();
    MarginsMLogregrInteractionState<MutableArrayHandle<double> > stateLeft = args[0];
    MarginsMLogregrInteractionState<ArrayHandle<double> > stateRight = args[1];
    // We first handle the trivial case where this function is called with one
    // of the states being the initial state
    if (stateLeft.numRows == 0)
        return stateRight;
    else if (stateRight.numRows == 0)
        return stateLeft;

    // Merge states together and return
    stateLeft += stateRight;
    return stateLeft;
}

/**
 * @brief Marginal effects: Final step
 */
AnyType
margins_mlogregr_int_final::run(AnyType &args) {
    // In case the aggregator should be terminated because
    // an exception has been "thrown" in the transition function
    if (args[0].isNull())
        return Null();
    // We request a mutable object.
    // Depending on the backend, this might perform a deep copy.
    MarginsMLogregrInteractionState<MutableArrayHandle<double> > state = args[0];
    // Aggregates that haven't seen any data just return Null.
    if (state.numRows == 0)
        return Null();

    state.marginal_effects /= static_cast<double>(state.numRows);
    Matrix marginal_effects_trans = trans(state.marginal_effects);
    AnyType tuple;
    tuple << marginal_effects_trans;

    // Variance for marginal effects according to the delta method
    Matrix variance(state.delta * state.training_data_vcov);
    // // we only need the diagonal elements of the variance, so we perform a dot
    // // product of each row with itself to compute each diagonal element.
    // // We divide by numRows^2 since we need the average variance
    Matrix std_err = variance.cwiseProduct(state.delta).rowwise().sum() / static_cast<double>(state.numRows * state.numRows);
    std_err = std_err.array().sqrt();
    std_err.resize(state.numCategories-1, state.numBasis);
    tuple << std_err;

    Matrix t_stats = marginal_effects_trans.cwiseQuotient(std_err);
    tuple << t_stats;

    // Note: p-values will return NULL if numRows <= coef.size
    if (state.numRows > state.numBasis) {
        MutableNativeMatrix p_values(
                this->allocateArray<double>(state.numBasis * (state.numCategories-1)),
                state.numCategories-1,
                state.numBasis);
        for (Index l = 0; l < p_values.rows(); l ++) {
            for (Index m = 0; m < p_values.cols(); m ++) {
                p_values(l,m) = 2. * prob::cdf(prob::normal(), -std::abs(t_stats(l,m)));
            }
        }
        tuple << static_cast<Matrix>(p_values);
    }

    return tuple;
}
// ------------------------ End of Logistic Marginal ---------------------------

} // namespace regress

} // namespace modules

} // namespace madlib


#include "include/matlab_bgl.h"
#include <yasmic/simple_csr_matrix_as_graph.hpp>
#include <boost/property_map.hpp>

template <class EdgeIndexMap, class Graph, class Edge>
void test_edge_index_map_impl(const EdgeIndexMap& m, Graph& g, Edge& e)
{
    using namespace boost;

    int i = 5 + get(m,e);
}

template <class EdgeIndexMap, class Graph>
void test_edge_index_map(EdgeIndexMap m, Graph& g) {
    using namespace boost;
    typedef typename graph_traits<Graph>::edge_descriptor edge_t;
    edge_t e;
    test_edge_index_map_impl(m,g,e);
    function_requires<ReadablePropertyMapConcept<EdgeIndexMap, edge_t> >();
}

template <class EdgeComponentPropertyMap, class Graph>
void test_edge_property_map(EdgeComponentPropertyMap m, Graph& g)
{
    using namespace boost;

    typedef typename graph_traits<Graph>::edge_descriptor edge_t;
    //function_requires<WritablePropertyMapConcept<EdgeComponentPropertyMap, edge_t> >();

    edge_t e;
    mbglIndex ci;
    put(m,e,ci);
}

int main(int argc, char **argv)
{
    using namespace yasmic;
    using namespace boost;

    mbglIndex* ci;

    typedef simple_csr_matrix<mbglIndex,double> crs_graph;
    crs_graph g(0, 0, 0, NULL, NULL, NULL);

    test_edge_index_map(get(edge_index,g),g);
    test_edge_property_map(make_iterator_property_map(ci, get(edge_index,g)), g);

    /*boost::tie(num_bicomps, oi) =
                biconnected_components(g,
                    make_iterator_property_map(ci, get(edge_index,g)),
                    a);*/
}


#include "Integrator2GL.hh"

#include <Eigen/Dense>
#include <map>
#include <iterator>

#include "GSLSamplerGL.hh"
#include "TypesFunctions.hh"

using namespace Eigen;
using namespace std;

Integrator2GL::Integrator2GL(size_t xbins, int xorders, size_t ybins, int  yorders) :
Integrator2GL(xbins, xorders, nullptr, ybins, yorders, nullptr)
{}

Integrator2GL::Integrator2GL(size_t xbins, int* xorders, size_t ybins, int* yorders) :
Integrator2GL(xbins, xorders, nullptr, ybins, yorders, nullptr)
{}

Integrator2GL::Integrator2GL(size_t xbins, int xorders, double* xedges, size_t ybins, int  yorders, double* yedges) :
Integrator2Base(xbins, xorders, xedges, ybins, yorders, yedges)
{
  init_sampler();
}

Integrator2GL::Integrator2GL(size_t xbins, int* xorders, double* xedges, size_t ybins, int* yorders, double* yedges) :
Integrator2Base(xbins, xorders, xedges, ybins, yorders, yedges)
{
  init_sampler();
}

void Integrator2GL::sample(FunctionArgs& fargs){
  auto& rets=fargs.rets;
  GSLSamplerGL sampler;
  auto& x=rets[0];
  auto& y=rets[1];
  sampler.fill_bins(m_xorders.size(), m_xorders.data(), m_xedges.data(), x.buffer, m_xweights.data());
  sampler.fill_bins(m_yorders.size(), m_yorders.data(), m_yedges.data(), y.buffer, m_yweights.data());

  m_weights = m_xweights.matrix() * m_yweights.matrix().transpose();

  rets[2].x = m_xedges.cast<double>();
  rets[3].x = m_yedges.cast<double>();

  rets[4].mat = x.vec.replicate(1, m_yweights.size());
  rets[5].mat = y.vec.transpose().replicate(m_xweights.size(), 1);
  rets[6].x = 0.0;
  rets[7].x = 0.0;
  auto xnpoints=m_xedges.size()-1;
  auto ynpoints=m_yedges.size()-1;
  rets[8].x = 0.5*(m_xedges.tail(xnpoints)+m_xedges.head(xnpoints));
  rets[9].x = 0.5*(m_yedges.tail(ynpoints)+m_yedges.head(ynpoints));
  rets.untaint();
  rets.freeze();
}


#include <iostream>
#include <armadillo>
#include <fstream>
#include <sstream>
#include <iomanip>
#include <chrono>
#include <ctime>
#include "helper_functions.h"
#include "solve_network.h"
#include "solve_dde.h"
#include "backprop.h"
#include "numerical_gradient.h"
#include "katana_get_params.hh"
#include "global_constants.h"

using namespace std;
using namespace arma;
using namespace globalconstants;

int main(int argc, char const *argv[])
{
	// for cpu time measuring
	clock_t start_overall = clock();
	double cumulative_solve_time = 0.0;
	double cumulative_backprop_time = 0.0;

	arma_rng::set_seed_random();

	// ### ### ### --- OPTIONS --- ### ### ###

	// file name of the text file where parameters and results will be saved
	string results_file_name = katana::getCmdOption(argv, argv + argc, "-filename", "results.txt");
	// print message to results file
	string print_msg = "Simulation of the deep learning delay system.";

	// task, possible options are "MNIST", "Fashion-MNIST", "CIFAR-10", "SVHN"
	string task = "Fashion-MNIST-denoising";
	// modify global_constants.h accordingly!

	// number of example images to save
	int save_examples = katana::getCmdOption(argv, argv + argc, "-save_examples", 0);

	// print weights (and diagonals) after each training epoch to text files in weights folder
	bool print_weights_to_file = false;

	// record data for time-signal video (or plot), only for MNIST or Fashion-MNIST
	bool record_data = katana::getCmdOption_bool(argv, argv + argc, "-record_data", false);
	int number_of_examples = katana::getCmdOption(argv, argv + argc, "-record_examples", 10);

	// If the following option is true, the program will not train the machine learning system.
	// Instead it will just choose diag indices n_prime according to the given paramters
	// and save them in the text file diag.txt. This will be done 6 times, i.e. the text file
	// will contain 6 lines of diag indices, which can be used for 6-fold cross validation
	bool make_diags = katana::getCmdOption_bool(argv, argv + argc, "-make_diags", false);
	// If the following option is true, the program will create initial weight files and abort.
	bool save_init_weights = katana::getCmdOption_bool(argv, argv + argc, "-save_init_weights", false);
	// If the following option is true, the program will load the initial weights from files instead of creating random weights.
	// Note that the corresponding diag file must be loaded as well.
	bool init_weights_from_file = katana::getCmdOption_bool(argv, argv + argc, "-init_weights_from_file", false);

	// option for system simulation
	// "dde_ibp":	semi-analytic heun method with modified trapezoidal rule employing integration-by-parts
	// "dde_heun":	semi-analytic heun method with standard trapezoidal rule
	// "network":	solves the equivalent network equations
	// "network_decoupled": solves network ignoring the additional direct linear connections of neighboring nodes and layers
	string system_simu = katana::getCmdOption(argv, argv + argc, "-system_simu", "dde_ibp");

	// option for gradient computation
	// backprop_standard: newly derived backpropagation algorithm for the deep learning delay system (was derived for equivalent network)
	// backprop_classic: classical backpropagation algorithm, ignores the additional direct linear connections of neighboring nodes and layers
	string grad_comp = katana::getCmdOption(argv, argv + argc, "-grad_comp", "backprop_standard");

	// gradient_check: compute numerical gradient (for dde_ibp) for comparision
	bool gradient_check = katana::getCmdOption_bool(argv, argv + argc, "-gradient_check", false);
	double epsilon_gradient_check = 1e-9;
	bool print_gradients_to_file = false;

	// ### ### ### --- PARAMETERS --- ### ### ###

	// M and P are defined in global_constants.h

	// task parameter
	double noise_sigma = 1.0;

	// ... for the system/network architectur:

	int N = katana::getCmdOption(argv, argv + argc, "-N", 100);  // number of nodes per hidden layer
	int L = katana::getCmdOption(argv, argv + argc, "-L", 2);  // number of hidden layers
	int D = katana::getCmdOption(argv, argv + argc, "-D", 50);  // number of delays (NOTE: must be the correct number also if diag_method == "from_file")

	double theta = katana::getCmdOption(argv, argv + argc, "-theta", 0.5);  // node separation
	double alpha = -1.0;  // factor in linear part of delay system

	string diag_method = katana::getCmdOption(argv, argv + argc, "-diag_method", "uniform");  // method to choose diagonals
	// "uniform": uniform distribution with the additional condition that there is at least one n'_d < 0 and one n'_d > 0
	// "equi_dist": diagonals with equal distances in between, if D is an odd number: main diagonal and same number of upper and lower,
	//              if D is an even number and diag_distance is even: centered, e.g. -15, -5, 5, 15,
	//              if D is even and diag_distance is odd: centered around -0.5, e.g. -8, -3, 2, 7.
	// "from_file": diag indices are taken from a given text file
	// For a banded matrix as connection matrix choose equi_dist and diag_distance = 1
	// For a full connection matrix choose D = 2 * N - 1 (and either uniform or equi_dist with diag_distance = 1)
	int diag_distance = katana::getCmdOption(argv, argv + argc, "-diag_distance", 0);  // distance between diagonals if diag_method == "equi_dist",
							// minimum distance between diagonals if diag_method is "uniform", otherwise ignored
							// Do not choose diag_distance too large. If (D - 1) * (2 * diag_distance - 1) >= 2 * (N - diag_margin) - 1 and method is uniform, the program will abort.
	string diag_file_path = katana::getCmdOption(argv, argv + argc, "-diag_file_path", "diag.txt"); // path to text file containing D interger numbers n_prime_d, ignored if method is not "from_file"

	// ... for the training:
	int number_of_epochs = katana::getCmdOption(argv, argv + argc, "-number_of_epochs", 100);
	double eta_0 = katana::getCmdOption(argv, argv + argc, "-eta0", 0.001);
	double eta_1 = katana::getCmdOption(argv, argv + argc, "-eta1", 1000.0);  // learning rate eta = min(eta_0, eta_1 / step)
	bool pixel_shift = katana::getCmdOption_bool(argv, argv + argc, "-pixel_shift", false);  // on-off switch for training input random 1-pixel shift
	int max_pixel_shift = katana::getCmdOption(argv, argv + argc, "-max_pixel_shift", 1);
	bool input_noise = katana::getCmdOption_bool(argv, argv + argc, "-training_noise", false);  // on-off switch for training input gaussian noise
	double training_noise_sigma = katana::getCmdOption(argv, argv + argc, "-sigma", 0.01);  // standard deviation of gaussian noise to disturb training input
	bool rotation = katana::getCmdOption_bool(argv, argv + argc, "-rotation", false);
	double max_rotation_degrees = katana::getCmdOption(argv, argv + argc, "-max_rotation_degrees", 15.0);
	bool horizontal_flip = katana::getCmdOption_bool(argv, argv + argc, "-flip", false);  // only for CIFAR-10
	// dropout
	double dropout_rate = katana::getCmdOption(argv, argv + argc, "-dropout", 0.0);

	// ... for weight initialization:
	double initial_input_weigt_radius = sqrt(6.0 / ((double)D/2.0 + (double)M + 1.0));
	double initial_hidden_weigt_radius =  sqrt(6.0 / ((double)D + 1.0));
	double initial_output_weigt_radius =  sqrt(6.0 / ((double)D/2.0 + (double)P + 1.0));

	// ... for numerics:

	int N_h = max(32 ,(int)(16 * theta));  // computation steps per virtual node for solving DDE

	// computational precision for sums with exp(alpha * theta * n) factor in summands
	// in the functions "get_deltas" and "get_gradient" in "backprop.cpp".
	// exp_precision = -35.0 means that terms, where exponential factor
	// is smaller than exp(-35), are ignored.
	// Since exp(-35) is approximately 6.3e-16, the gradient will still be computed with double precision.
	double exp_precision = -35.0;

	// ### ### ### --- ETC. --- ### ### ###

	// make diag text file and end program if make_diags option is true
	if (make_diags){
		ofstream diag_file;
		diag_file.open("diag.txt");
		for (int i = 0; i < 6; ++i){
			vector<int> diag_indices = get_diag_indices(N, D, diag_method, diag_distance, diag_file_path, 0);
			for (int d = 0; d < D - 1; ++d){
				diag_file << diag_indices[d] << " ";
			}
			diag_file << diag_indices[D - 1] << endl;
		}
		diag_file.close();
		cout << "The option make_diags was true." << endl;
		cout << "Program made (or overrode) diag.txt" << endl;
		return 0;
	}

	// make initial weight files and diag file and end program if make_diags save_init_weights option is true
	if (save_init_weights){
		vector<int> diag_indices;
		ofstream diag_file;
		diag_file.open("diag.txt");
		for (int i = 0; i < 6; ++i){
			diag_indices = get_diag_indices(N, D, diag_method, diag_distance, diag_file_path, 0);
			for (int d = 0; d < D - 1; ++d){
				diag_file << diag_indices[d] << " ";
			}
			diag_file << diag_indices[D - 1] << endl;
		}
		diag_file.close();
		cout << "The option save_init_weights was true." << endl;
		cout << "Program made (or overrode) diag.txt" << endl;
		mat input_weights(N, M + 1);
		mat output_weights(P, N + 1);
		cube hidden_weights(L - 1, N, N + 1);
		initialize_weights(input_weights, hidden_weights, output_weights, D, L, N, diag_indices,
						   initial_input_weigt_radius, initial_hidden_weigt_radius, initial_output_weigt_radius, true);
		cout << "Save initial weights and end program because the option save_init_weights is true." << endl;
		return 0;
	}

	// The following lines are to create a look up table "exp_table"
	// which contains the values exp(alpha * theta * n)
	// which are often needed by the functions "get_deltas" and "get_gradient" in "backprop.cpp".
	vector<double> exp_table;
	int n = 0;
	while (alpha * theta * double(n) >= - 35.0){
		double exp_value = exp(alpha * theta * double(n));
		exp_table.push_back(exp_value);
		++n;
	}

	// make results file and print information about parameters
	print_parameters(results_file_name, print_msg, task,
					system_simu, grad_comp, gradient_check,
					N, L, D, theta, alpha,
					diag_method, diag_distance, diag_file_path,
					number_of_epochs, eta_0, eta_1,
					pixel_shift, input_noise, training_noise_sigma,
					N_h, exp_precision);

	// determine data directory
	string data_dir = "data-Fashion-MNIST";

	// read image data from files to arrays:
	cube train_images(number_of_training_batches, training_batch_size, M);
	mat test_images(test_batch_size, M);
	int train_labels[number_of_training_batches][training_batch_size];
	int test_labels[test_batch_size];
	read_files(train_images, test_images, train_labels, test_labels, data_dir);

	// for video:
	vector<int> step_indices_for_video;
	if (record_data){
		fstream index_file;
		string index_string;
		index_file.open("video/step_indices.txt", ios::in);
		do {
			getline(index_file, index_string);
			if (index_string != ""){
				int i = stoi(index_string);
				step_indices_for_video.push_back(i);
			}
		} while (index_string != "");
		index_file.close();
	}

	// ### ### ### --- INITIALIZATION --- ### ### ###

	vector<int> training_batch_indices;
	cout << "Begin training." << endl;
	for (int i = 0; i < number_of_training_batches; ++i){
		training_batch_indices.push_back(i);
	}

	// initialize arrays which are used below to store the current system states
	mat activations(L, N);
	mat node_states(L, N);
	double output_activations[P];
	double outputs[P];
	vec g_primes(N);

	// initialize arrays to store deltas and gradient
	mat deltas(L, N);
	double output_deltas[P];
	mat input_weight_gradient(N, M + 1, fill::zeros);
	mat output_weight_gradient(P, N + 1, fill::zeros);
	cube weight_gradient(L - 1, N, N + 1, fill::zeros);

	mat deltas_0(L, N);
	double output_deltas_0[P];
	mat input_weight_gradient_0(N, M + 1, fill::zeros);
	mat output_weight_gradient_0(P, N + 1, fill::zeros);
	cube weight_gradient_0(L - 1, N, N + 1, fill::zeros);

	// initialize arrays to eventually store numerical gradient
	mat num_input_weight_gradient(N, M + 1, fill::zeros);
	mat num_output_weight_gradient(P, N + 1, fill::zeros);
	cube num_weight_gradient(L - 1, N, N + 1, fill::zeros);

	// The function "get_diag_indices" returns D intergers n'_d between - N + 1 and N - 1.
	// The delays are then tau_d = N - n'_d.
	vector<int> diag_indices;
	diag_indices = get_diag_indices(N, D, diag_method, diag_distance, diag_file_path, 0);

	// initialize weights.
	mat input_weights(N, M + 1, fill::zeros);
	mat output_weights(P, N + 1, fill::zeros);
	cube hidden_weights(L - 1, N, N + 1, fill::zeros);
	if (init_weights_from_file){
		load_initial_weights(input_weights, hidden_weights, output_weights);
	} else {
		initialize_weights(input_weights, hidden_weights, output_weights, D, L, N, diag_indices,
					   initial_input_weigt_radius, initial_hidden_weigt_radius, initial_output_weigt_radius, false);
	}

	// weights for test_runs.
	mat input_weights_scaled(N, M + 1, fill::zeros);
	mat output_weights_scaled(P, N + 1, fill::zeros);
	cube hidden_weights_scaled(L - 1, N, N + 1, fill::zeros);

	// dropout mask for weights:
	mat input_weights_mask(N, M + 1, fill::ones);
	cube hidden_weights_mask(L - 1, N, N + 1, fill::ones);
	mat output_weights_mask(P, N + 1, fill::ones);

	// ### ### ### --- STOCHASTIC GRADIENT DESCENT TRAINING --- ### ### ###

	// vectors to track training accuracy and validition accuracy (and eventually cosine similarity):
	vector<double> training_accuracy_vector;
	vector<double> accuracy_vector;
	vector<double> similarity_vector;

	// vector and variables to measure and save cpu time needed for each epoch
	vector<double> time_vector;
	clock_t start;
	clock_t ende;
	double epoch_time;

	// loop over training epochs:
	for (int epoch = 0; epoch < number_of_epochs; ++epoch){

		start = clock();

		// make vector with randomly shuffled indices between 0 and 59999 for each epoch of the stochastic gradient descent
		vector<int> index_vector;
		for (int index = 0; index < number_of_training_batches*training_batch_size; ++index){
			index_vector.push_back(index);
		}
		shuffle(begin(index_vector), std::end(index_vector), rng);

		// loop over training steps:
		int step_index = 0;
		for (int index : index_vector){
			++step_index;

			// choose random dropout nodes
			input_weights_mask = mat(N, M + 1, fill::ones);
			hidden_weights_mask = cube(L - 1, N, N + 1, fill::ones);
			output_weights_mask = mat(P, N + 1, fill::ones);
			// input layer
			for (int m = 0; m < M; ++m){
				double random_num = uniform(0.0, 1.0);
				if (random_num < dropout_rate){
					for (int n = 0; n < N; ++n){
						input_weights_mask(n, m) = 0.0;
					}
				}
			}
			// first hidden layer
			if (L > 1){
				for (int n = 0; n < N; ++n){
					double random_num = uniform(0.0, 1.0);
					if (random_num < dropout_rate){
						for (int m = 0; m < M + 1; ++m){
							input_weights_mask(n, m) = 0.0;
						}
						for (int j = 0; j < N; ++j){
							hidden_weights_mask(0, j, n) = 0.0;
						}
					}
				}
			}
			// hidden layers except first and last
			for (int l = 1; l < L - 1; ++l){
				for (int n = 0; n < N; ++n){
					double random_num = uniform(0.0, 1.0);
					if (random_num < dropout_rate){
						for (int i = 0; i < N + 1; ++i){
							hidden_weights_mask(l - 1, n, i) = 0.0;
						}
						for (int j = 0; j < N; ++j){
							hidden_weights_mask(l, j, n) = 0.0;
						}
					}
				}
			}
			// last hidden layer
			if (L > 1){
				for (int n = 0; n < N; ++n){
					double random_num = uniform(0.0, 1.0);
					if (random_num < dropout_rate){
						for (int i = 0; i < N + 1; ++i){
							hidden_weights_mask(L - 2, n, i) = 0.0;
						}
						for (int p = 0; p < P; ++p){
							output_weights_mask(p, n) = 0.0;
						}
					}
				}
			}

			// record data for video: step 0
			if (record_data && epoch == 0 && step_index == 1){
				int rec_step = 0;
				cout << "record data after step: " << rec_step << endl;
				//save weights
				string input_file_name = "video/weights_input_step_" + to_string(rec_step) + ".txt";
				string hidden_file_name = "video/weights_hidden_step_" + to_string(rec_step) + ".txt";
				string output_file_name = "video/weights_output_step_" + to_string(rec_step) + ".txt";
				input_weights.save(input_file_name, csv_ascii);
				hidden_weights.save(hidden_file_name, raw_ascii);
				output_weights.save(output_file_name, csv_ascii);
				//run system with examples from validation batch and save input, output and x_states
				for (int example_index = 0; example_index < number_of_examples; ++example_index){
					// select image as input
					vec input_data = test_images.row(example_index).t();
					int label = test_labels[index];
					// run system and record data for x_states
					solve_dde_ibp(activations, node_states, output_activations, outputs, g_primes,
							  input_data, input_weights, hidden_weights, output_weights, diag_indices, theta, alpha,
							  N, L, N_h, example_index+1, rec_step);
					// save input and output
					string input_file_name = "video/vector_input_step_" + to_string(rec_step) + "_example_" + to_string(example_index + 1) + ".txt";
					input_data.save(input_file_name, csv_ascii);
					string output_file_name = "video/vector_output_step_" + to_string(rec_step) + "_example_" + to_string(example_index + 1) + ".txt";
					ofstream output_file;
					output_file.open(output_file_name);
					for (double y_p : outputs){
						output_file << y_p << endl;
					}
					output_file.close();
				}
			}

			double eta = learning_rate(epoch, step_index, eta_0, eta_1);

			// select image as input
			div_t div_result = div(index, training_batch_size);
			int batch_index = training_batch_indices[div_result.quot];
			int image_index = div_result.rem;
			vec input_data = train_images.tube(batch_index, image_index);
			int label = train_labels[batch_index][image_index];

			// data augmentation or regression
			if (rotation && M == 3072){
				double rotation_degrees = uniform(-max_rotation_degrees, max_rotation_degrees);
				rotation32(input_data, rotation_degrees);
			}

			if (pixel_shift && M == 784){
				pixel_shift28(input_data, max_pixel_shift);
			}
			if (pixel_shift && M == 3072){
				pixel_shift32(input_data, max_pixel_shift);
			}
			if (input_noise){
				vec training_noise = training_noise_sigma * vec(M, fill::randn);
				input_data += training_noise;
			}
			if (horizontal_flip && M == 3072){
				double random_val = uniform(0,1);
				if (random_val > 0.5){
					horizontal_flip32(input_data);
				}
			}

			// get target
			double targets[P];
			for (int p = 0; p < P; ++p){
				targets[p] = input_data(p);
			}

			// add noise to input:
			vec noise = noise_sigma * vec(M, fill::randn);
			input_data += noise;
			for (int m = 0; m < M; ++m){
				if (input_data[m] < 0.0){
					input_data[m] = 0.0;
				}
				if (input_data[m] > 1.0){
					input_data[m] = 1.0;
				}
			}

			// solve the DDE (or network)
			clock_t start_solve = clock();
			if (system_simu == "dde_ibp"){
				solve_dde_ibp(activations, node_states, output_activations, outputs, g_primes,
							  input_data, input_weights%input_weights_mask, hidden_weights%hidden_weights_mask, output_weights%output_weights_mask, diag_indices, theta, alpha,
							  N, L, N_h);
			} else if (system_simu == "dde_heun"){
				solve_dde_heun(activations, node_states, output_activations, outputs, g_primes,
							  input_data, input_weights%input_weights_mask, hidden_weights%hidden_weights_mask, output_weights%output_weights_mask, diag_indices, theta, alpha,
							  N, L, N_h);
			} else if (system_simu == "network"){
				solve_network(activations, node_states, output_activations, outputs, g_primes,
							  input_data, input_weights%input_weights_mask, hidden_weights%hidden_weights_mask, output_weights%output_weights_mask, theta, alpha,
							  N, L);
			} else if (system_simu == "network_decoupled"){
				solve_network_decoupled(activations, node_states, output_activations, outputs, g_primes,
							  input_data, input_weights%input_weights_mask, hidden_weights%hidden_weights_mask, output_weights%output_weights_mask, N, L);
			} else {
				cout << system_simu << " is not a valid value for the system_simu option." << endl;
				abort();
			}
			clock_t end_solve = clock();
			cumulative_solve_time += (end_solve - start_solve) / (double)CLOCKS_PER_SEC;

			// compute deltas and gradient
			clock_t start_backprop = clock();
			mat f_prime_activations = f_prime_matrix(activations);
			if (grad_comp == "backprop_standard"){
				get_gradient_node_by_node(input_weight_gradient, weight_gradient, output_weight_gradient, input_data, node_states, f_prime_activations, g_primes, outputs, targets, hidden_weights%hidden_weights_mask, output_weights%output_weights_mask, diag_indices, N, L, theta, alpha);
			} else if (grad_comp == "backprop_classic"){
				get_gradient_classical_backprop(input_weight_gradient, weight_gradient, output_weight_gradient, input_data, node_states, f_prime_activations, g_primes, outputs, targets, hidden_weights%hidden_weights_mask, output_weights%output_weights_mask, diag_indices, N, L);
			} else {
				cout << grad_comp << " is not a valid value for the grad_comp option." << endl;
				abort();
			}
			clock_t end_backprop = clock();
			cumulative_backprop_time += (end_backprop - start_backprop) / (double)CLOCKS_PER_SEC;
			// eventually compute numerical gradient for comparision and save numerical gradient and backprop gradient in files
			if (gradient_check && step_index == 1){
				if (system_simu != "dde_ibp"){
					cout << "The option to compute the gradient numerically is only available if the method 'dde_ibp' is used to solve the delay system. " << endl;
					cout << "Set the option 'system_simu' to 'dde_ibp' or set the option 'num_gradient' to false." << endl;
					abort();
				}
				get_num_gradient(num_input_weight_gradient, num_weight_gradient, num_output_weight_gradient,
								 activations, node_states, output_activations, outputs, g_primes, label,
								 input_data, input_weights, hidden_weights, output_weights, diag_indices, theta, alpha, N, L, N_h, epsilon_gradient_check);
				if (print_gradients_to_file){
					write_gradients_to_file(input_weight_gradient, weight_gradient, output_weight_gradient, num_input_weight_gradient, num_weight_gradient, num_output_weight_gradient, epoch + 1, step_index, 0);
				}
				double cos_sim = cosine_similarity(input_weight_gradient, weight_gradient, output_weight_gradient, num_input_weight_gradient, num_weight_gradient, num_output_weight_gradient);
				similarity_vector.push_back(cos_sim);
			}

			input_weight_gradient = input_weight_gradient % input_weights_mask;
			weight_gradient = weight_gradient % hidden_weights_mask;
			output_weight_gradient = output_weight_gradient % output_weights_mask;

			// perform weight updates
			input_weights += - eta * input_weight_gradient;
			hidden_weights += - eta * weight_gradient;
			output_weights += - eta * output_weight_gradient;

			// record data for video: step 1 to end
			if (record_data && find(step_indices_for_video.begin(), step_indices_for_video.end(), epoch * 50000 + step_index) != step_indices_for_video.end()){
				int rec_step = epoch * 50000 + step_index;
				cout << "record data after step: " << rec_step << endl;
				//save weights
				string input_file_name = "video/weights_input_step_" + to_string(rec_step) + ".txt";
				string hidden_file_name = "video/weights_hidden_step_" + to_string(rec_step) + ".txt";
				string output_file_name = "video/weights_output_step_" + to_string(rec_step) + ".txt";
				input_weights.save(input_file_name, csv_ascii);
				hidden_weights.save(hidden_file_name, raw_ascii);
				output_weights.save(output_file_name, csv_ascii);
				//run system with examples from validation batch and save input, output and x_states
				for (int example_index = 0; example_index < number_of_examples; ++example_index){
					// select image as input
					vec input_data = test_images.row(example_index).t();
					int label = test_labels[index];
					// run system and record data for x_states
					solve_dde_ibp(activations, node_states, output_activations, outputs, g_primes,
							  input_data, input_weights, hidden_weights, output_weights, diag_indices, theta, alpha,
							  N, L, N_h, example_index+1, rec_step);
					// save input and output
					string input_file_name = "video/vector_input_step_" + to_string(rec_step) + "_example_" + to_string(example_index + 1) + ".txt";
					input_data.save(input_file_name, csv_ascii);
					string output_file_name = "video/vector_output_step_" + to_string(rec_step) + "_example_" + to_string(example_index + 1) + ".txt";
					ofstream output_file;
					output_file.open(output_file_name);
					for (double y_p : outputs){
						output_file << y_p << endl;
					}
					output_file.close();
				}
			}
		}

		//weight scaling
		input_weights_scaled = input_weights / (1.0 - dropout_rate);
		hidden_weights_scaled = hidden_weights / (1.0 - dropout_rate);
		output_weights_scaled = output_weights / (1.0 - dropout_rate);

		// loop to get accuracy on training set:
		double mse_sum = 0.0; ;
		for (int index = 0; index < number_of_training_batches *training_batch_size; ++index){
			//cout << "validation step (on training set)" << index + 1 << endl;

			// select image as input
			div_t div_result = div(index, training_batch_size);
			int batch_index = training_batch_indices[div_result.quot];
			int image_index = div_result.rem;
			vec input_data = train_images.tube(batch_index, image_index);
			int label = train_labels[batch_index][image_index];

			// get target
			double targets[P];
			for (int p = 0; p < P; ++p){
				targets[p] = input_data(p);
			}

			// add noise to input:
			vec noise = noise_sigma * vec(M, fill::randn);
			input_data += noise;
			for (int m = 0; m < M; ++m){
				if (input_data[m] < 0.0){
					input_data[m] = 0.0;
				}
				if (input_data[m] > 1.0){
					input_data[m] = 1.0;
				}
			}

			// solve the DDE (or network)
			clock_t start_solve = clock();
			if (system_simu == "dde_ibp"){
				solve_dde_ibp(activations, node_states, output_activations, outputs, g_primes,
							  input_data, input_weights_scaled, hidden_weights_scaled, output_weights_scaled, diag_indices, theta, alpha,
							  N, L, N_h);
			} else if (system_simu == "dde_heun"){
				solve_dde_heun(activations, node_states, output_activations, outputs, g_primes,
							  input_data, input_weights_scaled, hidden_weights_scaled, output_weights_scaled, diag_indices, theta, alpha,
							  N, L, N_h);
			} else if (system_simu == "network"){
				solve_network(activations, node_states, output_activations, outputs, g_primes,
							  input_data, input_weights_scaled, hidden_weights_scaled, output_weights_scaled, theta, alpha,
							  N, L);
			} else if (system_simu == "network_decoupled"){
				solve_network_decoupled(activations, node_states, output_activations, outputs, g_primes,
							  input_data, input_weights_scaled, hidden_weights_scaled, output_weights_scaled, N, L);
			} else {
				cout << system_simu << " is not a valid value for the system_simu parameter." << endl;
				abort();
			}
			clock_t end_solve = clock();
			cumulative_solve_time += (end_solve - start_solve) / (double)CLOCKS_PER_SEC;

			double mse = 0;
			for (int p = 0; p < P; ++p){
				mse += pow(outputs[p] - targets[p], 2.0);
			}
			mse = mse/(double)P;
			mse_sum += mse;
		}
		training_accuracy_vector.push_back(mse_sum / 50000.0);

		// loop for validation:
		mse_sum = 0.0; ;
		for (int index = 0; index < test_batch_size; ++index){

			vec input_data = test_images.row(index).t();
			int label = test_labels[index];

			// get target
			double targets[P];
			for (int p = 0; p < P; ++p){
				targets[p] = input_data(p);
			}

			// add noise to input:
			vec noise = noise_sigma * vec(M, fill::randn);
			input_data += noise;
			for (int m = 0; m < M; ++m){
				if (input_data[m] < 0.0){
					input_data[m] = 0.0;
				}
				if (input_data[m] > 1.0){
					input_data[m] = 1.0;
				}
			}

			// solve the DDE (or network)
			clock_t start_solve = clock();
			if (system_simu == "dde_ibp"){
				solve_dde_ibp(activations, node_states, output_activations, outputs, g_primes,
							  input_data, input_weights_scaled, hidden_weights_scaled, output_weights_scaled, diag_indices, theta, alpha,
							  N, L, N_h);
			} else if (system_simu == "dde_heun"){
				solve_dde_heun(activations, node_states, output_activations, outputs, g_primes,
							  input_data, input_weights_scaled, hidden_weights_scaled, output_weights_scaled, diag_indices, theta, alpha,
							  N, L, N_h);
			} else if (system_simu == "network"){
				solve_network(activations, node_states, output_activations, outputs, g_primes,
							  input_data, input_weights_scaled, hidden_weights_scaled, output_weights_scaled, theta, alpha,
							  N, L);
			} else if (system_simu == "network_decoupled"){
				solve_network_decoupled(activations, node_states, output_activations, outputs, g_primes,
							  input_data, input_weights_scaled, hidden_weights_scaled, output_weights_scaled, N, L);
			} else {
				cout << system_simu << " is not a valid value for the system_simu parameter." << endl;
				abort();
			}
			clock_t end_solve = clock();
			cumulative_solve_time += (end_solve - start_solve) / (double)CLOCKS_PER_SEC;

			// save example images
			if (index < save_examples){
				string original_file_name = "example_results/epoch_" + to_string(epoch + 1) + "_example_" + to_string(index + 1) + "_original.txt";
				string input_file_name = "example_results/epoch_" + to_string(epoch + 1) + "_example_" + to_string(index + 1) + "_input.txt";
				string output_file_name = "example_results/epoch_" + to_string(epoch + 1) + "_example_" + to_string(index + 1) + "_output.txt";
				ofstream original_file;
				original_file.open(original_file_name);
				for (int p = 0; p < P; ++p){
					original_file << targets[p] << endl;
				}
				original_file.close();
				ofstream input_file;
				input_file.open(input_file_name);
				for (int p = 0; p < P; ++p){
					input_file << input_data(p) << endl;
				}
				input_file.close();
				ofstream output_file;
				output_file.open(output_file_name);
				for (int p = 0; p < P; ++p){
					output_file << outputs[p] << endl;
				}
				output_file.close();
			}

			double mse = 0;
			for (int p = 0; p < P; ++p){
				mse += pow(outputs[p] - targets[p], 2.0);
			}
			mse = mse/(double)P;
			mse_sum += mse;
		}
		accuracy_vector.push_back(mse_sum / 10000.0);

		cout << "epoch " << epoch + 1 << ": validation MSE = " << mse_sum / 10000.0 << endl;
		// eventually print weights to file at end of each epoch
		if (print_weights_to_file){
			print_weights(input_weights_scaled, hidden_weights_scaled, output_weights_scaled, diag_indices, 0, epoch);
		}

		ende = clock();
		epoch_time = ((double) (ende - start)) / (double)CLOCKS_PER_SEC;
		time_vector.push_back(epoch_time);
	}

	// print result to file:
	print_results(results_file_name, 0, diag_indices, training_accuracy_vector, accuracy_vector, similarity_vector);

	// for cpu time measuring
	clock_t end_overall = clock();
	double cpu_time_overall = (end_overall - start_overall) / (double)CLOCKS_PER_SEC;
	double cpu_time_residual = cpu_time_overall - cumulative_solve_time - cumulative_backprop_time;
	double cpu_time_solve_percentage = 100.0 * cumulative_solve_time / cpu_time_overall;
	double cpu_time_backprop_percentage = 100.0 * cumulative_backprop_time / cpu_time_overall;
	double cpu_time_residual_percentage = 100.0 * cpu_time_residual / cpu_time_overall;

	// get current time and date and print to results text file
	auto current_clock = chrono::system_clock::now();
	time_t current_time = chrono::system_clock::to_time_t(current_clock);
	ofstream results_file;
	results_file.open(results_file_name, ios_base::app);
	results_file << endl;
	results_file << endl;
	results_file << "total cpu time (in seconds): " << cpu_time_overall << endl;
	results_file << "cumulative cpu time for solving the DDE or network (in seconds): " << cumulative_solve_time << " (" << cpu_time_solve_percentage << "%)" << endl;
	results_file << "cumulative cpu time for backpropagation (in seconds): " << cumulative_backprop_time << " (" << cpu_time_backprop_percentage << "%)" << endl;
	results_file << "residual cpu time (in seconds): " << cpu_time_residual << " (" << cpu_time_residual_percentage << "%)" << endl;
	results_file << endl;
	results_file << "end of simulation: " << ctime(&current_time);
	results_file.close();

	return 0;
}