/*
 Copyright (C) 2020 Quaternion Risk Management Ltd
 All rights reserved.

 This file is part of ORE, a free-software/open-source library
 for transparent pricing and risk analysis - http://opensourcerisk.org

 ORE is free software: you can redistribute it and/or modify it
 under the terms of the Modified BSD License.  You should have received a
 copy of the license along with this program.
 The license is also available online at <http://opensourcerisk.org>

 This program is distributed on the basis that it will form a useful
 contribution to risk analytics and model standardisation, but WITHOUT
 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 FITNESS FOR A PARTICULAR PURPOSE. See the license for more details.
*/

#include <orea/aggregation/dimregressioncalculator.hpp>
#include <ored/utilities/log.hpp>
#include <ored/utilities/vectorutils.hpp>
#include <ql/errors.hpp>
#include <ql/time/calendars/weekendsonly.hpp>
#include <ql/version.hpp>

#include <ql/math/distributions/normaldistribution.hpp>
#include <ql/math/generallinearleastsquares.hpp>
#include <ql/math/kernelfunctions.hpp>
#include <ql/methods/montecarlo/lsmbasissystem.hpp>
#include <ql/time/daycounters/actualactual.hpp>

#include <qle/math/nadarayawatson.hpp>
#include <qle/math/stabilisedglls.hpp>

#include <boost/accumulators/accumulators.hpp>
#include <boost/accumulators/statistics/error_of_mean.hpp>
#include <boost/accumulators/statistics/mean.hpp>
#include <boost/accumulators/statistics/stats.hpp>

using namespace std;
using namespace QuantLib;

using namespace boost::accumulators;

namespace ore {
namespace analytics {

RegressionDynamicInitialMarginCalculator::RegressionDynamicInitialMarginCalculator(
    const boost::shared_ptr<Portfolio>& portfolio, const boost::shared_ptr<NPVCube>& cube,
    const boost::shared_ptr<CubeInterpretation>& cubeInterpretation,
    const boost::shared_ptr<AggregationScenarioData>& scenarioData, Real quantile, Size horizonCalendarDays,
    Size regressionOrder, std::vector<std::string> regressors, Size localRegressionEvaluations,
    Real localRegressionBandWidth, const std::map<std::string, Real>& currentIM)
    : DynamicInitialMarginCalculator(portfolio, cube, cubeInterpretation, scenarioData, quantile, horizonCalendarDays,
                                     currentIM),
      regressionOrder_(regressionOrder), regressors_(regressors),
      localRegressionEvaluations_(localRegressionEvaluations), localRegressionBandWidth_(localRegressionBandWidth) {
    Size dates = cube_->dates().size();
    Size samples = cube_->samples();
    for (Size i = 0; i < nettingSetIds_.size(); ++i) {
        regressorArray_[nettingSetIds_[i]] = vector<vector<Array>>(dates, vector<Array>(samples));
        nettingSetLocalDIM_[nettingSetIds_[i]] = vector<vector<Real>>(dates, vector<Real>(samples, 0.0));
        nettingSetZeroOrderDIM_[nettingSetIds_[i]] = vector<Real>(dates, 0.0);
        nettingSetSimpleDIMh_[nettingSetIds_[i]] = vector<Real>(dates, 0.0);
        nettingSetSimpleDIMp_[nettingSetIds_[i]] = vector<Real>(dates, 0.0);
    }
}

const vector<vector<Real>>&
RegressionDynamicInitialMarginCalculator::localRegressionResults(const std::string& nettingSet) {
    if (nettingSetLocalDIM_.find(nettingSet) != nettingSetLocalDIM_.end())
        return nettingSetLocalDIM_[nettingSet];
    else
        QL_FAIL("netting set " << nettingSet << " not found in Local DIM results");
}

const vector<Real>& RegressionDynamicInitialMarginCalculator::zeroOrderResults(const std::string& nettingSet) {
    if (nettingSetZeroOrderDIM_.find(nettingSet) != nettingSetZeroOrderDIM_.end())
        return nettingSetZeroOrderDIM_[nettingSet];
    else
        QL_FAIL("netting set " << nettingSet << " not found in Zero Order DIM results");
}

const vector<Real>& RegressionDynamicInitialMarginCalculator::simpleResultsUpper(const std::string& nettingSet) {
    if (nettingSetSimpleDIMp_.find(nettingSet) != nettingSetSimpleDIMp_.end())
        return nettingSetSimpleDIMp_[nettingSet];
    else
        QL_FAIL("netting set " << nettingSet << " not found in Simple DIM (p) results");
}

const vector<Real>& RegressionDynamicInitialMarginCalculator::simpleResultsLower(const std::string& nettingSet) {
    if (nettingSetSimpleDIMh_.find(nettingSet) != nettingSetSimpleDIMh_.end())
        return nettingSetSimpleDIMh_[nettingSet];
    else
        QL_FAIL("netting set " << nettingSet << " not found in Simple DIM (c) results");
}

void RegressionDynamicInitialMarginCalculator::build() {
    LOG("DIM Analysis by polynomial regression");

    map<string, Real> currentDim = unscaledCurrentDIM();

    Size stopDatesLoop = datesLoopSize_;
    Size samples = cube_->samples();

    Size polynomOrder = regressionOrder_;
    LOG("DIM regression polynom order = " << regressionOrder_);
    LsmBasisSystem::PolynomType polynomType = LsmBasisSystem::Monomial;
    Size regressionDimension = regressors_.empty() ? 1 : regressors_.size();
    LOG("DIM regression dimension = " << regressionDimension);
#if QL_HEX_VERSION > 0x01150000
    std::vector<ext::function<Real(Array)>> v(
        LsmBasisSystem::multiPathBasisSystem(regressionDimension, polynomOrder, polynomType));
#else // QL 1.14 and below
    std::vector<boost::function1<Real, Array>> v(
        LsmBasisSystem::multiPathBasisSystem(regressionDimension, polynomOrder, polynomType));
#endif
    Real confidenceLevel = QuantLib::InverseCumulativeNormal()(quantile_);
    LOG("DIM confidence level " << confidenceLevel);

    Size simple_dim_index_h = Size(floor(quantile_ * (samples - 1) + 0.5));
    Size simple_dim_index_p = Size(floor((1.0 - quantile_) * (samples - 1) + 0.5));

    Size nettingSetCount = 0;
    for (auto n : nettingSetIds_) {
        LOG("Process netting set " << n);

        if (currentIM_.find(n) != currentIM_.end()) {
            Real t0im = currentIM_[n];
            QL_REQUIRE(currentDim.find(n) != currentDim.end(), "current DIM not found for netting set " << n);
            Real t0dim = currentDim[n];
            Real t0scaling = t0im / t0dim;
            LOG("t0 scaling for netting set " << n << ": t0im" << t0im << " t0dim=" << t0dim
                                              << " t0scaling=" << t0scaling);
            nettingSetScaling_[n] = t0scaling;
        }

        Real nettingSetDimScaling =
            nettingSetScaling_.find(n) == nettingSetScaling_.end() ? 1.0 : nettingSetScaling_[n];
        LOG("Netting set DIM scaling factor: " << nettingSetDimScaling);

        for (Size j = 0; j < stopDatesLoop; ++j) {
            accumulator_set<double, stats<tag::mean, tag::variance>> accDiff;
            accumulator_set<double, stats<tag::mean>> accOneOverNumeraire;
            for (Size k = 0; k < samples; ++k) {
                Real numDefault = cubeInterpretation_->getDefaultAggrionScenarioData(
                    scenarioData_, AggregationScenarioDataType::Numeraire, j, k);
                Real numCloseOut = cubeInterpretation_->getCloseOutAggrionScenarioData(
                    scenarioData_, AggregationScenarioDataType::Numeraire, j, k);
                Real npvDefault = nettingSetNPV_[n][j][k];
                Real flow = nettingSetFLOW_[n][j][k];
                Real npvCloseOut = nettingSetCloseOutNPV_[n][j][k];
                accDiff((npvCloseOut * numCloseOut) + (flow * numDefault) - (npvDefault * numDefault));
                accOneOverNumeraire(1.0 / numDefault);
            }

            Size mporCalendarDays = cubeInterpretation_->getMporCalendarDays(cube_, j);
            Real horizonScaling = sqrt(1.0 * horizonCalendarDays_ / mporCalendarDays);

            Real stdevDiff = sqrt(variance(accDiff));
            Real E_OneOverNumeraire =
                mean(accOneOverNumeraire); // "re-discount" (the stdev is calculated on non-discounted deltaNPVs)

            nettingSetZeroOrderDIM_[n][j] = stdevDiff * horizonScaling * confidenceLevel;
            nettingSetZeroOrderDIM_[n][j] *= E_OneOverNumeraire;

            vector<Real> rx0(samples, 0.0);
            vector<Array> rx(samples, Array());
            vector<Real> ry1(samples, 0.0);
            vector<Real> ry2(samples, 0.0);
            for (Size k = 0; k < samples; ++k) {
                Real numDefault = cubeInterpretation_->getDefaultAggrionScenarioData(
                    scenarioData_, AggregationScenarioDataType::Numeraire, j, k);
                Real numCloseOut = cubeInterpretation_->getCloseOutAggrionScenarioData(
                    scenarioData_, AggregationScenarioDataType::Numeraire, j, k);
                Real x = nettingSetNPV_[n][j][k] * numDefault;
                Real f = nettingSetFLOW_[n][j][k] * numDefault;
                Real y = nettingSetCloseOutNPV_[n][j][k] * numCloseOut;
                Real z = (y + f - x);
                rx[k] = regressors_.empty() ? Array(1, nettingSetNPV_[n][j][k]) : regressorArray(n, j, k);
                rx0[k] = rx[k][0];
                ry1[k] = z;     // for local regression
                ry2[k] = z * z; // for least squares regression
                nettingSetDeltaNPV_[n][j][k] = z;
                regressorArray_[n][j][k] = rx[k];
            }
            vector<Real> delNpvVec_copy = nettingSetDeltaNPV_[n][j];
            sort(delNpvVec_copy.begin(), delNpvVec_copy.end());
            Real simpleDim_h = delNpvVec_copy[simple_dim_index_h];
            Real simpleDim_p = delNpvVec_copy[simple_dim_index_p];
            simpleDim_h *= horizonScaling;                                  // the usual scaling factors
            simpleDim_p *= horizonScaling;                                  // the usual scaling factors
            nettingSetSimpleDIMh_[n][j] = simpleDim_h * E_OneOverNumeraire; // discounted DIM
            nettingSetSimpleDIMp_[n][j] = simpleDim_p * E_OneOverNumeraire; // discounted DIM

            QL_REQUIRE(rx.size() > v.size(), "not enough points for regression with polynom order " << polynomOrder);
            if (close_enough(stdevDiff, 0.0)) {
                LOG("DIM: Zero std dev estimation at step " << j);
                // Skip IM calculation if all samples have zero NPV (e.g. after latest maturity)
                for (Size k = 0; k < samples; ++k) {
                    nettingSetDIM_[n][j][k] = 0.0;
                    nettingSetLocalDIM_[n][j][k] = 0.0;
                }
            } else {
                // Least squares polynomial regression with specified polynom order
                QuantExt::StabilisedGLLS ls(rx, ry2, v, QuantExt::StabilisedGLLS::MeanStdDev);
                LOG("DIM data normalisation at time step "
                    << j << ": " << scientific << setprecision(6) << " x-shift = " << ls.xShift() << " x-multiplier = "
                    << ls.xMultiplier() << " y-shift = " << ls.yShift() << " y-multiplier = " << ls.yMultiplier());
                LOG("DIM regression coefficients at time step " << j << ": " << fixed << setprecision(6)
                                                                << ls.transformedCoefficients());

                // Local regression versus first regression variable (i.e. we do not perform a
                // multidimensional local regression):
                // We evaluate this at a limited number of samples only for validation purposes.
                // Note that computational effort scales quadratically with number of samples.
                // NadarayaWatson needs a large number of samples for good results.
                QuantExt::NadarayaWatson lr(rx0.begin(), rx0.end(), ry1.begin(),
                                            GaussianKernel(0.0, localRegressionBandWidth_));
                Size localRegressionSamples = samples;
                if (localRegressionEvaluations_ > 0)
                    localRegressionSamples = Size(floor(1.0 * samples / localRegressionEvaluations_ + .5));

                // Evaluate regression function to compute DIM for each scenario
                for (Size k = 0; k < samples; ++k) {
                    // Real num1 = scenarioData_->get(j, k, AggregationScenarioDataType::Numeraire);
                    Real numDefault = cubeInterpretation_->getDefaultAggrionScenarioData(
                        scenarioData_, AggregationScenarioDataType::Numeraire, j, k);
                    Array regressor = regressors_.empty() ? Array(1, nettingSetNPV_[n][j][k]) : regressorArray(n, j, k);
                    Real e = ls.eval(regressor, v);
                    if (e < 0.0)
                        LOG("Negative variance regression for date " << j << ", sample " << k
                                                                     << ", regressor = " << regressor);

                    // Note:
                    // 1) We assume vanishing mean of "z", because the drift over a MPOR is usually small,
                    //    and to avoid a second regression for the conditional mean
                    // 2) In particular the linear regression function can yield negative variance values in
                    //    extreme scenarios where an exact analytical or delta VaR calculation would yield a
                    //    variance aproaching zero. We correct this here by taking the positive part.
                    Real std = sqrt(std::max(e, 0.0));
                    Real scalingFactor = horizonScaling * confidenceLevel * nettingSetDimScaling;
                    // Real dim = std * scalingFactor / num1;
                    Real dim = std * scalingFactor / numDefault;
                    dimCube_->set(dim, nettingSetCount, j, k);
                    nettingSetDIM_[n][j][k] = dim;
                    nettingSetExpectedDIM_[n][j] += dim / samples;

                    // Evaluate the Kernel regression for a subset of the samples only (performance)
                    if (localRegressionEvaluations_ > 0 && (k % localRegressionSamples == 0))
                        // nettingSetLocalDIM_[n][j][k] = lr.standardDeviation(regressor[0]) * scalingFactor / num1;
                        nettingSetLocalDIM_[n][j][k] = lr.standardDeviation(regressor[0]) * scalingFactor / numDefault;
                    else
                        nettingSetLocalDIM_[n][j][k] = 0.0;
                }
            }
        }

        nettingSetCount++;
    }
    LOG("DIM by polynomial regression done");
}

Disposable<Array> RegressionDynamicInitialMarginCalculator::regressorArray(string nettingSet, Size dateIndex,
                                                                           Size sampleIndex) {
    Array a(regressors_.size());
    for (Size i = 0; i < regressors_.size(); ++i) {
        string variable = regressors_[i];
        if (boost::to_upper_copy(variable) ==
            "NPV") // this allows possibility to include NPV as a regressor alongside more fundamental risk factors
            a[i] = nettingSetNPV_[nettingSet][dateIndex][sampleIndex];
        else if (scenarioData_->has(AggregationScenarioDataType::IndexFixing, variable))
            a[i] = cubeInterpretation_->getDefaultAggrionScenarioData(
                scenarioData_, AggregationScenarioDataType::IndexFixing, dateIndex, sampleIndex, variable);
        else if (scenarioData_->has(AggregationScenarioDataType::FXSpot, variable))
            a[i] = cubeInterpretation_->getDefaultAggrionScenarioData(
                scenarioData_, AggregationScenarioDataType::FXSpot, dateIndex, sampleIndex, variable);
        else if (scenarioData_->has(AggregationScenarioDataType::Generic, variable))
            a[i] = cubeInterpretation_->getDefaultAggrionScenarioData(
                scenarioData_, AggregationScenarioDataType::Generic, dateIndex, sampleIndex, variable);
        else
            QL_FAIL("scenario data does not provide data for " << variable);
    }
    return a;
}

map<string, Real> RegressionDynamicInitialMarginCalculator::unscaledCurrentDIM() {
    // In this function we proxy the model-implied T0 IM by looking at the
    // cube grid horizon lying closest to t0+mpor. We measure diffs relative
    // to the mean of the distribution at this same time horizon, thus avoiding
    // any cashflow-specific jumps

    Date today = cube_->asof();
    Size relevantDateIdx = 0;
    Real sqrtTimeScaling = 1.0;
    for (Size i = 0; i < cube_->dates().size(); ++i) {
        Size daysFromT0 = (cube_->dates()[i] - today);
        if (daysFromT0 < horizonCalendarDays_) {
            // iterate until we straddle t0+mpor
            continue;
        } else if (daysFromT0 == horizonCalendarDays_) {
            // this date corresponds to t0+mpor, so use it
            relevantDateIdx = i;
            sqrtTimeScaling = 1.0;
            break;
        } else if (daysFromT0 > horizonCalendarDays_) {
            // the first date greater than t0+MPOR, check if it is closest
            Size lastIdx = (i == 0) ? 0 : (i - 1);
            Size lastDaysFromT0 = (cube_->dates()[lastIdx] - today);
            int daysFromT0CloseOut = daysFromT0 - horizonCalendarDays_;
            int prevDaysFromT0CloseOut = lastDaysFromT0 - horizonCalendarDays_;
            if (std::abs(daysFromT0CloseOut) <= std::abs(prevDaysFromT0CloseOut)) {
                relevantDateIdx = i;
                sqrtTimeScaling = std::sqrt(Real(horizonCalendarDays_) / Real(daysFromT0));
            } else {
                relevantDateIdx = lastIdx;
                sqrtTimeScaling = std::sqrt(Real(horizonCalendarDays_) / Real(lastDaysFromT0));
            }
            break;
        }
    }
    // set some reasonable bounds on the sqrt time scaling, so that we are not looking at a ridiculous time horizon
    if (sqrtTimeScaling < std::sqrt(0.5) || sqrtTimeScaling > std::sqrt(2.0)) {
        WLOG("T0 IM Estimation - The estimation time horizon from grid is not sufficiently close to t0+MPOR - "
             << QuantLib::io::iso_date(cube_->dates()[relevantDateIdx])
             << ", the T0 IM estimate might be inaccurate. Consider inserting a first grid tenor closer to the dim "
                "horizon");
    }

    // TODO: Ensure that the simulation containers read-from below are indeed populated

    Real confidenceLevel = QuantLib::InverseCumulativeNormal()(quantile_);
    Size simple_dim_index_h = Size(floor(quantile_ * (cube_->samples() - 1) + 0.5));
    map<string, Real> t0dimReg, t0dimSimple;
    for (auto it_map = nettingSetNPV_.begin(); it_map != nettingSetNPV_.end(); ++it_map) {
        string key = it_map->first;
        vector<Real> t0_dist = it_map->second[relevantDateIdx];
        Size dist_size = t0_dist.size();
        QL_REQUIRE(dist_size == cube_->samples(),
                   "T0 IM - cube samples size mismatch - " << dist_size << ", " << cube_->samples());
        Real mean_t0_dist = std::accumulate(t0_dist.begin(), t0_dist.end(), 0.0);
        mean_t0_dist /= dist_size;
        vector<Real> t0_delMtM_dist(dist_size, 0.0);
        accumulator_set<double, stats<tag::mean, tag::variance>> acc_delMtm;
        accumulator_set<double, stats<tag::mean>> acc_OneOverNum;
        for (Size i = 0; i < dist_size; ++i) {
            Real numeraire = scenarioData_->get(relevantDateIdx, i, AggregationScenarioDataType::Numeraire);
            Real deltaMtmFromMean = numeraire * (t0_dist[i] - mean_t0_dist) * sqrtTimeScaling;
            t0_delMtM_dist[i] = deltaMtmFromMean;
            acc_delMtm(deltaMtmFromMean);
            acc_OneOverNum(1.0 / numeraire);
        }
        Real E_OneOverNumeraire = mean(acc_OneOverNum);
        Real variance_t0 = variance(acc_delMtm);
        Real sqrt_t0 = sqrt(variance_t0);
        t0dimReg[key] = (sqrt_t0 * confidenceLevel * E_OneOverNumeraire);
        std::sort(t0_delMtM_dist.begin(), t0_delMtM_dist.end());
        t0dimSimple[key] = (t0_delMtM_dist[simple_dim_index_h] * E_OneOverNumeraire);

        LOG("T0 IM (Reg) - {" << key << "} = " << t0dimReg[key]);
        LOG("T0 IM (Simple) - {" << key << "} = " << t0dimSimple[key]);
    }
    LOG("T0 IM Calculations Completed");

    return t0dimReg;
}

void RegressionDynamicInitialMarginCalculator::exportDimEvolution(ore::data::Report& dimEvolutionReport) {

    Size samples = dimCube_->samples();
    Size stopDatesLoop = datesLoopSize_;
    Date asof = cube_->asof();

    dimEvolutionReport.addColumn("TimeStep", Size())
        .addColumn("Date", Date())
        .addColumn("DaysInPeriod", Size())
        .addColumn("ZeroOrderDIM", Real(), 6)
        .addColumn("AverageDIM", Real(), 6)
        .addColumn("AverageFLOW", Real(), 6)
        .addColumn("SimpleDIM", Real(), 6)
        .addColumn("NettingSet", string())
        .addColumn("Time", Real(), 6);

    for (auto nettingSet : dimCube_->ids()) {

        LOG("Export DIM evolution for netting set " << nettingSet);
        for (Size i = 0; i < stopDatesLoop; ++i) {
            Real expectedFlow = 0.0;
            for (Size j = 0; j < samples; ++j) {
                expectedFlow += nettingSetFLOW_[nettingSet][i][j] / samples;
            }

            Date defaultDate = dimCube_->dates()[i];
            Time t = ActualActual().yearFraction(asof, defaultDate);
            Size days = cubeInterpretation_->getMporCalendarDays(dimCube_, i);
            dimEvolutionReport.next()
                .add(i)
                .add(defaultDate)
                .add(days)
                .add(nettingSetZeroOrderDIM_[nettingSet][i])
                .add(nettingSetExpectedDIM_[nettingSet][i])
                .add(expectedFlow)
                .add(nettingSetSimpleDIMh_[nettingSet][i])
                .add(nettingSet)
                .add(t);
        }
    }
    dimEvolutionReport.end();
    LOG("Exporting expected DIM through time done");
}

void RegressionDynamicInitialMarginCalculator::exportDimRegression(
    const std::string& nettingSet, const std::vector<Size>& timeSteps,
    const std::vector<boost::shared_ptr<ore::data::Report>>& dimRegReports) {

    QL_REQUIRE(dimRegReports.size() == timeSteps.size(),
               "number of file names (" << dimRegReports.size() << ") does not match number of time steps ("
                                        << timeSteps.size() << ")");
    for (Size ii = 0; ii < timeSteps.size(); ++ii) {
        Size timeStep = timeSteps[ii];
        LOG("Export DIM by sample for netting set " << nettingSet << " and time step " << timeStep);

        Size dates = dimCube_->dates().size();
        const std::vector<std::string>& ids = dimCube_->ids();

        int index = -1;
        for (Size i = 0; i < ids.size(); ++i) {
            if (ids[i] == nettingSet) {
                index = i;
                break;
            }
        }
        QL_REQUIRE(index >= 0, "netting set " << nettingSet << " not found in DIM cube");

        QL_REQUIRE(timeStep < dates - 1, "selected time step " << timeStep << " out of range [0, " << dates - 1 << "]");

        Size samples = cube_->samples();
        vector<Real> numeraires(samples, 0.0);
        for (Size k = 0; k < samples; ++k)
            // numeraires[k] = scenarioData_->get(timeStep, k, AggregationScenarioDataType::Numeraire);
            numeraires[k] = cubeInterpretation_->getDefaultAggrionScenarioData(
                scenarioData_, AggregationScenarioDataType::Numeraire, timeStep, k);

        auto p = sort_permutation(regressorArray_[nettingSet][timeStep], lessThan);
        vector<Array> reg = apply_permutation(regressorArray_[nettingSet][timeStep], p);
        vector<Real> dim = apply_permutation(nettingSetDIM_[nettingSet][timeStep], p);
        vector<Real> ldim = apply_permutation(nettingSetLocalDIM_[nettingSet][timeStep], p);
        vector<Real> delta = apply_permutation(nettingSetDeltaNPV_[nettingSet][timeStep], p);
        vector<Real> num = apply_permutation(numeraires, p);

        boost::shared_ptr<ore::data::Report> regReport = dimRegReports[ii];
        regReport->addColumn("Sample", Size());
        for (Size k = 0; k < reg[0].size(); ++k) {
            ostringstream o;
            o << "Regressor_" << k << "_";
            o << (regressors_.empty() ? "NPV" : regressors_[k]);
            regReport->addColumn(o.str(), Real(), 6);
        }
        regReport->addColumn("RegressionDIM", Real(), 6)
            .addColumn("LocalDIM", Real(), 6)
            .addColumn("ExpectedDIM", Real(), 6)
            .addColumn("ZeroOrderDIM", Real(), 6)
            .addColumn("DeltaNPV", Real(), 6)
            .addColumn("SimpleDIM", Real(), 6);

        // Note that RegressionDIM, LocalDIM, DeltaNPV are _not_ reduced by the numeraire in this output,
        // but ExpectedDIM, ZeroOrderDIM and SimpleDIM _are_ reduced by the numeraire.
        // This is so that the regression formula can be manually validated

        for (Size j = 0; j < reg.size(); ++j) {
            regReport->next().add(j);
            for (Size k = 0; k < reg[j].size(); ++k)
                regReport->add(reg[j][k]);
            regReport->add(dim[j] * num[j])
                .add(ldim[j] * num[j])
                .add(nettingSetExpectedDIM_[nettingSet][timeStep])
                .add(nettingSetZeroOrderDIM_[nettingSet][timeStep])
                .add(delta[j])
                .add(nettingSetSimpleDIMh_[nettingSet][timeStep]);
        }
        regReport->end();
        LOG("Exporting DIM by Sample done for");
    }
}

} // namespace analytics
} // namespace ore


/*
-------------------------------------------------------------------------
   This file is part of BayesOpt, an efficient C++ library for
   Bayesian optimization.

   Copyright (C) 2011-2015 Ruben Martinez-Cantin <rmcantin@unizar.es>

   BayesOpt is free software: you can redistribute it and/or modify it
   under the terms of the GNU Affero General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   BayesOpt is distributed in the hope that it will be useful, but
   WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with BayesOpt.  If not, see <http://www.gnu.org/licenses/>.
------------------------------------------------------------------------
*/

#include "bayesopt/bayesopt.hpp"

#include <limits>
#include <boost/numeric/ublas/matrix_proxy.hpp>

#include "lhs.hpp"
#include "randgen.hpp"
#include "log.hpp"
#include "boundingbox.hpp"
#include "inneroptimization.hpp"

namespace bayesopt  {

  class CritCallback: public RBOptimizable
  {
  public:
    explicit CritCallback(ContinuousModel* model):mBO(model){};
    double evaluate(const vectord &query)
    {
      return mBO->evaluateCriteria(query);
    }
  private:
    ContinuousModel* mBO;
  };

  ContinuousModel::ContinuousModel(size_t dim, Parameters parameters):
    BayesOptBase(dim,parameters)
  {
    mCallback.reset(new CritCallback(this));
    cOptimizer.reset(new NLOPT_Optimization(mCallback.get(),dim));
    cOptimizer->setAlgorithm(COMBINED);
    cOptimizer->setMaxEvals(parameters.n_inner_iterations);

    vectord lowerBound = zvectord(mDims);
    vectord upperBound = svectord(mDims,1.0);
    mBB.reset(new utils::BoundingBox<vectord>(lowerBound,upperBound));
  } // Constructor

  ContinuousModel::~ContinuousModel()
  {
    //    delete cOptimizer;
  } // Default destructor

  void ContinuousModel::setBoundingBox(const vectord &lowerBound,
				       const vectord &upperBound)
  {
    // We don't change the bounds of the inner optimization because,
    // thanks to this bounding box model, everything is mapped to the
    // unit hypercube, thus the default inner optimization are just
    // right.
    mBB.reset(new utils::BoundingBox<vectord>(lowerBound,upperBound));

    FILE_LOG(logINFO) << "Bounds: ";
    FILE_LOG(logINFO) << lowerBound;
    FILE_LOG(logINFO) << upperBound;
  } //setBoundingBox

  //////////////////////////////////////////////////////////////////////

  vectord ContinuousModel::samplePoint()
  {
    randFloat drawSample(mEngine,realUniformDist(0,1));
    vectord Xnext(mDims);
    for(vectord::iterator x = Xnext.begin(); x != Xnext.end(); ++x)
      {
	*x = drawSample();
      }
    return Xnext;
  };

  void ContinuousModel::findOptimal(vectord &xOpt)
  {
    double minf = cOptimizer->run(xOpt);

    //Let's try some local exploration like spearmint
    randNFloat drawSample(mEngine,normalDist(0,0.001));
    for(size_t ii = 0;ii<5; ++ii)
      {
	vectord pert = getPointAtMinimum();
	for(size_t j=0; j<xOpt.size(); ++j)
	  {
	    pert(j) += drawSample();
	  }
	try
	  {
	    double minf2 = cOptimizer->localTrialAround(pert);
	    if (minf2<minf)
	      {
		minf = minf2;
		FILE_LOG(logDEBUG) << "Local beats Global";
		xOpt = pert;
	      }
	  }
	catch(std::invalid_argument& e)
	  {
	    //We ignore this one
	  }
      }
  };

  vectord ContinuousModel::remapPoint(const vectord& x)
  {
    return mBB->unnormalizeVector(x);
  }

  void ContinuousModel::generateInitialPoints(matrixd& xPoints)
  {
    utils::samplePoints(xPoints,mParameters.init_method,mEngine);
  }
}  //namespace bayesopt


///////////////////////////////////////////////////////////////
//  Copyright 2013 John Maddock. Distributed under the Boost
//  Software License, Version 1.0. (See accompanying file
//  LICENSE_1_0.txt or copy at https://www.boost.org/LICENSE_1_0.txt

//[logged_adaptor

#include <boost/multiprecision/mpfi.hpp>
#include <boost/multiprecision/logged_adaptor.hpp>
#include <iostream>
#include <iomanip>
//
// Begin by overloading log_postfix_event so we can capture each arithmetic event as it happens:
//
namespace boost{ namespace multiprecision{

template <unsigned D>
inline void log_postfix_event(const mpfi_float_backend<D>& val, const char* event_description)
{
   // Print out the (relative) diameter of the interval:
   using namespace boost::multiprecision;
   number<mpfr_float_backend<D> > diam;
   mpfi_diam(diam.backend().data(), val.data());
   std::cout << "Diameter was " << diam << " after operation: " << event_description << std::endl;
}
template <unsigned D, class T>
inline void log_postfix_event(const mpfi_float_backend<D>&, const T&, const char* event_description)
{
   // This version is never called in this example.
}

}}

int main()
{
   using namespace boost::multiprecision;
   typedef number<logged_adaptor<mpfi_float_backend<17> > > logged_type;
   //
   // Test case deliberately introduces cancellation error, relative size of interval
   // gradually gets larger after each operation:
   //
   logged_type a = 1;
   a /= 10;

   for(unsigned i = 0; i < 13; ++i)
   {
      logged_type b = a * 9;
      b /= 10;
      a -= b;
   }
   std::cout << "Final value was: " << a << std::endl;
   return 0;
}

//]

/*
//[logged_adaptor_output

Diameter was nan after operation: Default construct
Diameter was 0 after operation: Assignment from arithmetic type
Diameter was 4.33681e-18 after operation: /=
Diameter was nan after operation: Default construct
Diameter was 7.70988e-18 after operation: *
Diameter was 9.63735e-18 after operation: /=
Diameter was 1.30104e-16 after operation: -=
Diameter was nan after operation: Default construct
Diameter was 1.30104e-16 after operation: *
Diameter was 1.38537e-16 after operation: /=
Diameter was 2.54788e-15 after operation: -=
Diameter was nan after operation: Default construct
Diameter was 2.54788e-15 after operation: *
Diameter was 2.54863e-15 after operation: /=
Diameter was 4.84164e-14 after operation: -=
Diameter was nan after operation: Default construct
Diameter was 4.84164e-14 after operation: *
Diameter was 4.84221e-14 after operation: /=
Diameter was 9.19962e-13 after operation: -=
Diameter was nan after operation: Default construct
Diameter was 9.19962e-13 after operation: *
Diameter was 9.19966e-13 after operation: /=
Diameter was 1.74793e-11 after operation: -=
Diameter was nan after operation: Default construct
Diameter was 1.74793e-11 after operation: *
Diameter was 1.74793e-11 after operation: /=
Diameter was 3.32107e-10 after operation: -=
Diameter was nan after operation: Default construct
Diameter was 3.32107e-10 after operation: *
Diameter was 3.32107e-10 after operation: /=
Diameter was 6.31003e-09 after operation: -=
Diameter was nan after operation: Default construct
Diameter was 6.31003e-09 after operation: *
Diameter was 6.31003e-09 after operation: /=
Diameter was 1.19891e-07 after operation: -=
Diameter was nan after operation: Default construct
Diameter was 1.19891e-07 after operation: *
Diameter was 1.19891e-07 after operation: /=
Diameter was 2.27792e-06 after operation: -=
Diameter was nan after operation: Default construct
Diameter was 2.27792e-06 after operation: *
Diameter was 2.27792e-06 after operation: /=
Diameter was 4.32805e-05 after operation: -=
Diameter was nan after operation: Default construct
Diameter was 4.32805e-05 after operation: *
Diameter was 4.32805e-05 after operation: /=
Diameter was 0.00082233 after operation: -=
Diameter was nan after operation: Default construct
Diameter was 0.00082233 after operation: *
Diameter was 0.00082233 after operation: /=
Diameter was 0.0156243 after operation: -=
Diameter was nan after operation: Default construct
Diameter was 0.0156243 after operation: *
Diameter was 0.0156243 after operation: /=
Diameter was 0.296861 after operation: -=
Final value was: {8.51569e-15,1.14843e-14}

//]
*/


#ifndef UTIL_HH
#define UTIL_HH

#include <algorithm>
#include <vector>

// Visual Studio math.h doesn't have log1p and log1pf functions varjokal 24.3.2010
// This needs to be defined for VS varjokal 24.3.2010
#ifdef _MSC_VER
#include <boost/math/tr1.hpp>
using namespace boost::math::tr1;
#endif

#include <math.h>

/** Common utility functions. */
namespace util {

  /** Function evaluation object for search function(s) */
  class FuncEval {
  public:
    virtual double evaluate_function(double p) = 0;
    virtual ~FuncEval() {}
  };
  /** Binary search for finding the parameter value with which the
   * given function evaluates to max_value within the given accuracy. */
  double bin_search_param_max_value(double lower_bound, double low_value,
                                    double upper_bound, double up_value,
                                    double max_value, double value_acc,
                                    double param_acc, FuncEval &f);

  /** The square of the value. */
  template <typename T>
  T sqr(T a)
  {
    return a * a;
  }

  /** Median of the values in a vector.
   * \note For odd number (2n + 1) of values, the n'th value is returned.
   */
  template <typename T>
  T
  median(std::vector<T> v)
  {
    std::sort(v.begin(), v.end());
    return v[v.size() / 2];
  }

  /** Absolute value. */
  template <typename T>
  T
  abs(const T &value)
  {
    if (value < 0)
      return -value;
    return value;
  }

  /** Maximum of two values. */
  template <typename T>
  T
  max(const T &a, const T &b)
  {
    if (a < b)
      return b;
    return a;
  }

  inline float
  log10addf(float a, float b)
  {
    // M_LN10 is not part of C++ standard and not defined in every compiler.
    const float LN10 = 2.30258509299404568402;
    const float LOG10TOe = LN10;
    const float LOGeTO10 = 1.0 / LN10;

    a = a * LOG10TOe;
    b = b * LOG10TOe;

    float delta = a - b;
    if (delta > 64.0) {
      b += 64;
      delta = -delta;
    }
    return (b + log1pf(exp(delta))) * LOGeTO10;
  }

  inline double
  log10add(double a, double b)
  {
    // M_LN10 is not part of C++ standard and not defined in every compiler.
    const double LN10 = 2.30258509299404568402;
    const double LOG10TOe = LN10;
    const double LOGeTO10 = 1.0 / LN10;

    a = a * LOG10TOe;
    b = b * LOG10TOe;

    double delta = a - b;
    if (delta > 64.0) {
      b += 64;
      delta = -delta;
    }
    return (b + log1p(exp(delta))) * LOGeTO10;
  }

  inline float
  logaddf(float a, float b)
  {
    float delta = a - b;
    if (delta > 0) {
      b = a;
      delta = -delta;
    }
    return b + log1pf(expf(delta));
  }

  inline double
  logadd(double a, double b)
  {
    double delta = a - b;
    if (delta > 0) {
      b = a;
      delta = -delta;
    }
    return b + log1p(exp(delta));
  }

  static const double tiny_for_log = 1e-50;
  inline double safe_log(double x)
  {
    if (x < tiny_for_log)
      return log(tiny_for_log);
    else
      return log(x);
  }

  /** Compute modulo of two values so that negative arguments are
   * handled correctly. */
  inline int modulo(int a, int b)
  {
    int result = a % b;
    if (result < 0)
      result += b;
    return result;
  }

  inline float sinc(float x)
  {
    // M_PI is not part of C++ standard and not defined in every compiler.
    const double PI = 3.14159265358979323846;
    if (fabs(x) < 1e-8)
      return 1;
    double y = PI*x;
    return sin(y)/y;
  }

};

#endif /* UTIL_HH */


#include "ktamura.h"

#include "kprogressbar.h"
#include "common.h"
#include "kutility.h"
#include "kpicinfo.h"
#include "kimagecvt.h" //for image convert

#include <boost/math/special_functions/powm1.hpp>
#include <boost/math/constants/constants.hpp>

#include <vector>
#include <map>
#include <utility>
#include <limits>
#include <cmath>

#include <QRegularExpression>
#include <QCoreApplication>
#include <QDir>

using std::vector;
using std::map;
using std::make_pair;

KTamura::KTamura(QString input)
    :m_iMaxExtend(32)
{
    QRegularExpression re("[\\\\/]");

    GDALDataset * piDataset = (GDALDataset *) GDALOpen(input.toUtf8().constData(), GA_ReadOnly );

    K_OPEN_ASSERT(piDataset,input.toStdString());

    m_sOutputRoot = getDirRoot(input);

    QString tempName=input.right(input.length()-input.lastIndexOf(re)-1);
    m_sFileNoExtName=tempName.left(tempName.lastIndexOf("."));

    GDALDataset *poDataset = NULL;

    if((poDataset = KImageCvt::img2gray(piDataset,poDataset,m_sOutputRoot+"temp"+QDir::separator()+m_sFileNoExtName+"-gray")) == NULL) { std::cout<<"image convert failed\a"<<std::endl; exit(1); }

    m_lXSize = piDataset->GetRasterXSize();
    m_lYSize = piDataset->GetRasterYSize();
    m_fOrgArray = (float *) CPLMalloc(sizeof(float)*m_lXSize*m_lYSize);
    m_fExtArray = (float *) CPLMalloc(sizeof(float)*(m_lXSize+2*m_iMaxExtend)*(m_lYSize+2*m_iMaxExtend));

    // gray only
    GDALRasterBand * poBand = poDataset->GetRasterBand(1);
    poBand->RasterIO( GF_Read, 0, 0, m_lXSize, m_lYSize, m_fOrgArray, m_lXSize, m_lYSize, GDT_Float32, 0, 0 );

    KUtility::reflectExtend<float>(m_fOrgArray,m_fExtArray,m_lXSize,m_lYSize,m_iMaxExtend);

    GDALClose(piDataset);
    GDALClose(poDataset);
}

KTamura::~KTamura()
{
    CPLFree(m_fOrgArray);
    CPLFree(m_fExtArray);
}

void KTamura::build()
{

    m_dCoarseness = calCoarseness();
    m_dContrast = calContrast();
    m_dDirectionality = calDirectionality();
}

double KTamura::calCoarseness()
{
    map<int,std::pair<double,double> > mapAvrGray;
    double coarseness = 0.;
    float * tempOut = new float[m_lXSize*m_lYSize];

    unsigned long int lineLength = m_lXSize+2*m_iMaxExtend;
    KProgressBar progressBar("CalCoarseness",m_lXSize*m_lYSize,80);
    K_PROGRESS_START(progressBar);
    long int orgY = m_iMaxExtend;
    for(unsigned long int desY = 0;orgY<m_lYSize+m_iMaxExtend;++orgY,++desY){
        long int orgX = m_iMaxExtend;
        for(unsigned long int desX = 0;orgX<m_lXSize+m_iMaxExtend;++orgX,++desX){
            mapAvrGray.clear();
            double hDiff = abs(m_fExtArray[orgY*lineLength+orgX-1]-m_fExtArray[orgY*lineLength+orgX+1]);
            double vDiff = abs(m_fExtArray[(orgY-1)*lineLength+orgX]-m_fExtArray[(orgY+1)*lineLength+orgX]);
            mapAvrGray[0] = make_pair(hDiff,vDiff);

            for(int k = 1;k < 6;k++){
                long binSize = boost::math::powm1(2,k)+1;
                hDiff = abs(getAverageGray(m_fExtArray,lineLength,orgX-binSize/2,orgY,binSize)-getAverageGray(m_fExtArray,lineLength,orgX+binSize/2,orgY,binSize));
                vDiff = abs(getAverageGray(m_fExtArray,lineLength,orgX,orgY-binSize/2,binSize)-getAverageGray(m_fExtArray,lineLength,orgX,orgY+binSize/2,binSize));
                mapAvrGray[k] = make_pair(hDiff,vDiff);
            }

            double eMax = (std::numeric_limits<double>::min)();
            int kMax=-1;
            // find the max E
            for(map<int,std::pair<double,double> >::iterator it = mapAvrGray.begin();
                it != mapAvrGray.end(); ++it){
                double tempMax = std::max(it->second.first,it->second.second);
                if(eMax<tempMax){ kMax=it->first; eMax=tempMax; }
            }
            // find optimal E & k
            int tempKMax=kMax;
            for(map<int,std::pair<double,double> >::iterator it = mapAvrGray.begin();
                it != mapAvrGray.end(); ++it){
                if(it->first>kMax){
                    double tempMax = std::max(it->second.first,it->second.second);
                    if(tempMax>0.9*eMax){
                        if(tempKMax<it->first) tempKMax = it->first;
                    }
                }
            }
            kMax=tempKMax;
            tempOut[desY*m_lXSize+desX] = boost::math::powm1(2,kMax)+1;
            progressBar.autoUpdate();
        }
    }
    K_PROGRESS_END(progressBar);

    for(long iY = 0;iY<m_lYSize;++iY){
        for(long iX = 0;iX<m_lXSize;++iX){
            coarseness += tempOut[iY*m_lXSize+iX];
        }
    }

    coarseness = coarseness / (m_lXSize*m_lYSize);

    delete [] tempOut;
    return coarseness;
}

double KTamura::calContrast()
{
    double alpha4 = 0.;
    double miu4 = 0.;
    double delta2 = 0.;
    double delta = 0.;
    double avr = 0.;
    double contrast = 0.;
    /* alpha4 = Sum((Yi - Yavr)^4)/((N-1)*delta^4) */
    // get Average
    for(long int iY = 0;iY<m_lYSize;++iY){
        for(long int iX = 0;iX<m_lXSize;++iX){
            avr+=m_fOrgArray[iY*m_lXSize+iX];
        }
    }
    avr /= (m_lYSize*m_lXSize);
    // get Variance and mean fourth
    for(long int iY = 0;iY<m_lYSize;++iY){
        for(long int iX = 0;iX<m_lXSize;++iX){
            double tempValue = m_fOrgArray[iY*m_lXSize+iX];
            double temp = 0.;
            temp = abs(tempValue-avr);
            delta2 += temp * temp;
            miu4 += delta2 * delta2;
        }
    }
    delta2 /= (m_lYSize*m_lXSize);
    delta = boost::math::powm1(delta2,0.5)+1;
    miu4 /= (m_lYSize*m_lXSize - 1);
    alpha4 = miu4 / (delta2*delta2);
    contrast = delta/(boost::math::powm1(alpha4,0.25)+1);
    return contrast;
}

double KTamura::calDirectionality()
{
    const int quantizationLevel = 16;
    const int thresT = 12;
    double directionality = 0.;
    float * tempDeltaG = new float[m_lXSize*m_lYSize];
    float * tempTheta = new float[m_lXSize*m_lYSize];
    long * histogramD = new long[quantizationLevel];
    long imageLength = m_lXSize*m_lYSize;
    unsigned long int lineLength = m_lXSize+2*m_iMaxExtend;
    //memset(histogramD,0,quantizationLevel);
    for(int index = 0;index < quantizationLevel;++index){
        histogramD[index]=0;
    }
    long int orgY = m_iMaxExtend;
    for(unsigned long int desY = 0;orgY<m_lYSize+m_iMaxExtend;++orgY,++desY){
        long int orgX = m_iMaxExtend;
        for(unsigned long int desX = 0;orgX<m_lXSize+m_iMaxExtend;++orgX,++desX){
            float tempDeltaH=0.;
            float tempDeltaV=0.;
            tempDeltaH = m_fExtArray[(orgY-1)*lineLength+orgX+1]+m_fExtArray[orgY*lineLength+orgX+1]+m_fExtArray[(orgY+1)*lineLength+orgX+1]-
                         m_fExtArray[(orgY-1)*lineLength+orgX-1]-m_fExtArray[orgY*lineLength+orgX-1]-m_fExtArray[(orgY+1)*lineLength+orgX-1];
            tempDeltaV = m_fExtArray[(orgY-1)*lineLength+orgX-1]+m_fExtArray[(orgY-1)*lineLength+orgX]+m_fExtArray[(orgY-1)*lineLength+orgX+1]-
                         m_fExtArray[(orgY+1)*lineLength+orgX-1]-m_fExtArray[(orgY+1)*lineLength+orgX]-m_fExtArray[(orgY+1)*lineLength+orgX+1];
            tempDeltaG[desY*m_lXSize+desX] = (abs(tempDeltaV) + abs(tempDeltaH))/2;
            if(tempDeltaH<0.0001&&tempDeltaH>0.) tempDeltaH=0.0001;
            if(tempDeltaH>-0.0001&&tempDeltaH<0.) tempDeltaH=-0.0001;
            if(tempDeltaH==0.) tempDeltaH+=0.00001;
            tempTheta[desY*m_lXSize+desX] = atan(tempDeltaV/tempDeltaH)+boost::math::constants::pi<float>()/2;
            //if(tempTheta[desY*m_lXSize+desX]<0) tempTheta[desY*m_lXSize+desX]+=boost::math::constants::pi<float>();
            //qDebug()<<"DeltaG"<<tempDeltaG[desY*m_lXSize+desX]<<"DeltaH:"<<tempDeltaH<<"DeltaV:"<<tempDeltaV<<"tempTheta:"<<tempTheta[desY*m_lXSize+desX];
        }
    }
    // get HistogramD
    float sumHistogramD = 0.;
    for(int index = 0;index < quantizationLevel;++index){
        //qDebug()<<"start:"<<histogramD[index];
        float thetaLow = (2*index)*boost::math::constants::pi<float>()/(2*quantizationLevel);
        float thetaHigh = (2*index+2)*boost::math::constants::pi<float>()/(2*quantizationLevel);
        //qDebug()<<"thetaLow"<<thetaLow<<"thetaHigh"<<thetaHigh;

        for(long imageIndex = 0;imageIndex < imageLength;++imageIndex){
            //qDebug()<<tempTheta[imageIndex];
            if(tempDeltaG[imageIndex]>thresT){
                if(tempTheta[imageIndex]>=thetaLow&&tempTheta[imageIndex]<thetaHigh){
                    histogramD[index]+=1;
                }
            }
        }
        //qDebug()<<"a:"<<a;
        //qDebug()<<"histogramD:"<<histogramD[index];
        sumHistogramD += histogramD[index];
    }
//    qDebug()<<sumHistogramD;
//    for(int index = 0;index < quantizationLevel;++index){
//        //qDebug()<<"1:"<<histogramD[index];
//        histogramD[index]=histogramD[index];
//        //qDebug()<<"2:"<<histogramD[index];
//    }
//    QString temp("");
//    for(int index = 0;index < quantizationLevel;++index){
//        temp+=QString("%1:%2 ").arg(index).arg(histogramD[index]);
//    }
//    qDebug()<<temp;
    // get Directionality
    // find all the valley and it's position
    vector<std::pair<int,float> > tempValleyArray;
    vector<std::pair<int,float> > valleyArray;
    vector<std::pair<int,float> > finalValley;
    int firstPeak = getNextValleyPeak(histogramD,0,quantizationLevel,true);
    int lastPos = firstPeak;
    tempValleyArray.push_back(make_pair(firstPeak,histogramD[firstPeak]));
    int tempPeakValley = getNextValleyPeak(histogramD,lastPos,quantizationLevel);
    while((lastPos>=firstPeak && tempPeakValley>lastPos) || tempPeakValley < firstPeak){
        tempValleyArray.push_back(make_pair(tempPeakValley, histogramD[tempPeakValley]));
        lastPos = tempPeakValley;
        tempPeakValley = getNextValleyPeak(histogramD, lastPos, quantizationLevel);
    }
    //qDebug()<<"tempValleyArray"<<tempValleyArray.size();
    //for(unsigned int index = 0;index < tempValleyArray.size();index++){
    //    qDebug()<<tempValleyArray[index].second;
    //}
    float maxPeakRate = (std::numeric_limits<float>::min)();
    int maxRatePos = -1;
    // remove the peak and valley which peak/valley<2.0
    for(unsigned int index = 0;index < tempValleyArray.size();index+=2){
        float tempRate = tempValleyArray[index].second/tempValleyArray[index+1].second;
        if(tempRate>maxPeakRate){ maxPeakRate = tempRate; maxRatePos = index; }
        if(tempRate>2.){
            if(0==index) valleyArray.push_back(tempValleyArray[tempValleyArray.size()-1]);
            else valleyArray.push_back(tempValleyArray[index-1]);
            valleyArray.push_back(tempValleyArray[index]);
            valleyArray.push_back(tempValleyArray[index+1]);
        }
    }
    if(0 == valleyArray.size() && maxPeakRate>1.){
        if(0==maxRatePos) valleyArray.push_back(tempValleyArray[tempValleyArray.size()-1]);
        else valleyArray.push_back(tempValleyArray[maxRatePos-1]);
        valleyArray.push_back(tempValleyArray[maxRatePos]);
        valleyArray.push_back(tempValleyArray[maxRatePos+1]);
    }
    // get the mian peak and the secondmain peak
    float maxPeak = (std::numeric_limits<float>::min)();
    int maxPos = -1;
    float secondMaxPeak = (std::numeric_limits<float>::min)();
    int secondMaxPos = -1;
    //qDebug()<<"valleySize:"<<valleyArray.size();
    for(unsigned int index = 1;index < valleyArray.size();index+=3){
        if(valleyArray[index].second>maxPeak){ maxPeak = valleyArray[index].second; maxPos=index; }
    }
    for(unsigned int index = 1;index < valleyArray.size();index+=3){
        if(static_cast<int>(index)!=maxPos && valleyArray[index].second>secondMaxPeak){ secondMaxPeak = valleyArray[index].second; secondMaxPos=index; }
    }
    //qDebug()<<"max:"<<maxPos;
    if(-1 == maxPos){
        delete [] tempDeltaG;
        delete [] tempTheta;
        delete [] histogramD;
        return 0.;
    }
    //for(unsigned int index = 0;index < tempValleyArray.size();index++)
    //        qDebug()<<index<<":"<<tempValleyArray[index].first;
    //for(unsigned int index = 0;index < valleyArray.size();index++)
    //    qDebug()<<index<<":"<<valleyArray[index].first;
    finalValley.push_back(valleyArray[maxPos-1]);
    finalValley.push_back(valleyArray[maxPos]);
    finalValley.push_back(valleyArray[maxPos+1]);
    if(-1 != secondMaxPos){
        if(maxPeak/secondMaxPeak<5.){
            finalValley.push_back(valleyArray[secondMaxPos-1]);
            finalValley.push_back(valleyArray[secondMaxPos]);
            finalValley.push_back(valleyArray[secondMaxPos+1]);
        }
    }

    int nPeaks = finalValley.size()/3;
//    vector<long> doubleHistogramD;
//    for(int index = 0;index<2*quantizationLevel;++index){
//        doubleHistogramD.push_back(histogramD[index%quantizationLevel]);
//    }
    for(unsigned int index = 1;index < finalValley.size();index+=3){
        vector<int> realOrder;
        int start=0;
        int peak=0;
        int end=0;
        realOrder.push_back(finalValley[index-1].first);
        realOrder.push_back(finalValley[index-1].first+quantizationLevel);
        realOrder.push_back(finalValley[index].first);
        realOrder.push_back(finalValley[index].first+quantizationLevel);
        realOrder.push_back(finalValley[index+1].first);
        realOrder.push_back(finalValley[index+1].first+quantizationLevel);

        std::sort(realOrder.begin(),realOrder.end());
        //for(unsigned int index = 0;index < realOrder.size();index++)
        //    qDebug()<<realOrder[index];
        vector<int>::iterator it=realOrder.begin();
        if((it=std::find(realOrder.begin(),realOrder.end(),finalValley[index].first))!=realOrder.begin()){
            it--;
            start = *it++;
            peak = *it++;
            end = *it++;
        }else{
//            if(realOrder[0]==realOrder[1]){
//                start = realOrder[0];
//                peak = realOrder[1];
//                end = realOrder[2];
//            }else{
            it=std::find(realOrder.begin(),realOrder.end(),finalValley[index].first+quantizationLevel);
            it--;
            start = *it++;
            peak = *it++;
            end = *it++;
//            }
        }
        double tempDir = 0.;
        for(int position = start;position<end+1;++position){
            double temp=abs(peak-position);
            tempDir += temp*temp*histogramD[position%quantizationLevel];
        }

        directionality+=tempDir;
    }
    // calculate sharpness of the peak
//    int nPeaks = finalValley.size()/3;
//    for(unsigned int index = 1;index < finalValley.size();index+=3){
//        double tempDir = 0.;
//        // first valley after peak
//        if(finalValley[index-1].first>finalValley[index].first){
//            for(int position = finalValley[index-1].first;position<quantizationLevel;++position){
//                double temp=quantizationLevel-position+finalValley[index].first;
//                tempDir += temp*temp*histogramD[position];
//            }
//            for(int position = 0;position<finalValley[index+1].first;++position){
//                double temp=position-finalValley[index].first;
//                tempDir += temp*temp*histogramD[position];
//            }
//            directionality+=tempDir;
//            continue;
//        }
//        // second valley before peak
//        if(finalValley[index+1].first<finalValley[index].first){
//            for(int position = finalValley[index-1].first;position<quantizationLevel;++position){
//                double temp=position-finalValley[index].first;
//                tempDir += temp*temp*histogramD[position];
//            }
//            for(int position = 0;position<finalValley[index+1].first;++position){
//                double temp=quantizationLevel+position-finalValley[index].first;
//                tempDir += temp*temp*histogramD[position];
//            }
//            directionality+=tempDir;
//            continue;
//        }
//        for(int position = finalValley[index-1].first;position<finalValley[index+1].first;++position){
//            double temp=position-finalValley[index].first;
//            tempDir += temp*temp*histogramD[position];
//        }
//        directionality+=tempDir;
//    }

    delete [] tempDeltaG;
    delete [] tempTheta;
    delete [] histogramD;
    //qDebug()<<nPeaks*directionality*1./sumHistogramD;
    return nPeaks*directionality*1./sumHistogramD;//*(boost::math::powm1(10,23)+1);
}

int KTamura::getNextValleyPeak(long *histogram, int start, int length,bool bePeak)
{
    static bool bePeakValley = false;
    if(bePeak) bePeakValley = false;
    bePeakValley = !bePeakValley;
    start %= length;
    int tempPos = start;
    for (int index = start; index < start + length; ++index){
        if (bePeakValley){
            if (histogram[index%length]<histogram[(index + 1) % length]){
                for (; index < start + length; ++index){
                    if (histogram[index%length] >= histogram[(index + 1) % length]) break;
                }
                tempPos = index;// %length;
                if (histogram[index%length] == histogram[(index + 1) % length]){
                    for (; index < start + length; ++index){
                        if (histogram[index%length]>histogram[(index + 1) % length]) break;
                    }
                }
                tempPos = ((tempPos + index) / 2) % length;
                break;
            }
        }
        else{
            if (histogram[index%length]>histogram[(index + 1) % length]){
                for (; index < start + length; ++index){
                    if (histogram[index%length] <= histogram[(index + 1) % length]) break;
                }
                tempPos = index;//%length;
                if (histogram[index%length] == histogram[(index + 1) % length]){
                    for (; index < start + length; ++index){
                        if (histogram[index%length]<histogram[(index + 1) % length]) break;
                    }
                }
                tempPos = ((tempPos + index) / 2) % length;
                break;
            }
        }
    }
    return tempPos;
}

double KTamura::getAverageGray(float * image, unsigned long lineLength, unsigned long lX, unsigned long lY, long size)
{
    double sumGray = 0.;
    for(unsigned long nY = lY-size/2; nY < lY+size/2; ++nY){
        for(unsigned long nX = lX-size/2; nX < lX+size/2; ++nX){
            sumGray += image[lineLength*nY+nX];
        }
    }
    sumGray = sumGray/(size*size);
    return sumGray;
}

QString KTamura::getDirRoot(QString filename)
{
    QString tempRet("");
    QRegularExpression re("[\\\\/]+");

    if(filename.contains('\\')||filename.contains('/')){
        QStringList tempList = filename.split(re);
        for(int pos = 0;pos<tempList.length()-1;++pos)
        {
            tempRet+=tempList[pos];
            tempRet+=QDir::separator();
        }
    }else{
        tempRet=QCoreApplication::applicationDirPath()+QDir::separator();
    }
    return tempRet;
}

QString KTamura::getSVMString(int start)
{
    QString temp("");
    for(int index= start;index<start+3;++index){
        temp+=QString("%1:%%2 ").arg(index).arg(index-start+1);
    }
    return QString(temp).arg(m_dCoarseness).arg(m_dContrast).arg(m_dDirectionality);
}