/*
 Copyright (C) 2016 Quaternion Risk Management Ltd
 All rights reserved.

 This file is part of ORE, a free-software/open-source library
 for transparent pricing and risk analysis - http://opensourcerisk.org

 ORE is free software: you can redistribute it and/or modify it
 under the terms of the Modified BSD License.  You should have received a
 copy of the license along with this program.
 The license is also available online at <http://opensourcerisk.org>

 This program is distributed on the basis that it will form a useful
 contribution to risk analytics and model standardisation, but WITHOUT
 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 FITNESS FOR A PARTICULAR PURPOSE. See the license for more details.
*/

/*! \file scenario/scenariosimmarket.cpp
    \brief A Market class that can be updated by Scenarios
    \ingroup
*/

#include <orea/engine/observationmode.hpp>
#include <orea/scenario/scenariosimmarket.hpp>
#include <orea/scenario/simplescenario.hpp>
#include <ql/experimental/credit/basecorrelationstructure.hpp>
#include <ql/math/interpolations/loginterpolation.hpp>
#include <ql/termstructures/credit/interpolatedsurvivalprobabilitycurve.hpp>
#include <ql/termstructures/defaulttermstructure.hpp>
#include <ql/termstructures/volatility/capfloor/capfloortermvolatilitystructure.hpp>
#include <ql/termstructures/volatility/capfloor/capfloortermvolsurface.hpp>
#include <ql/termstructures/volatility/equityfx/blackvariancecurve.hpp>
#include <ql/termstructures/volatility/equityfx/blackvoltermstructure.hpp>
#include <ql/termstructures/volatility/optionlet/strippedoptionlet.hpp>
#include <ql/termstructures/volatility/optionlet/strippedoptionletadapter.hpp>
#include <ql/termstructures/volatility/swaption/swaptionconstantvol.hpp>
#include <ql/termstructures/volatility/swaption/swaptionvolcube.hpp>
#include <ql/termstructures/volatility/swaption/swaptionvolmatrix.hpp>
#include <ql/termstructures/volatility/swaption/swaptionvolstructure.hpp>
#include <ql/termstructures/yield/discountcurve.hpp>
#include <ql/time/calendars/target.hpp>
#include <ql/time/daycounters/actual365fixed.hpp>
#include <ql/time/daycounters/actualactual.hpp>

#include <qle/indexes/inflationindexobserver.hpp>
#include <qle/indexes/inflationindexwrapper.hpp>
#include <qle/termstructures/dynamicblackvoltermstructure.hpp>
#include <qle/termstructures/dynamicswaptionvolmatrix.hpp>
#include <qle/termstructures/flatcorrelation.hpp>
#include <qle/termstructures/interpolatedcorrelationcurve.hpp>
#include <qle/termstructures/pricecurve.hpp>
#include <qle/termstructures/strippedoptionletadapter2.hpp>
#include <qle/termstructures/survivalprobabilitycurve.hpp>
#include <qle/termstructures/swaptionvolatilityconverter.hpp>
#include <qle/termstructures/swaptionvolconstantspread.hpp>
#include <qle/termstructures/swaptionvolcube2.hpp>
#include <qle/termstructures/swaptionvolcubewithatm.hpp>
#include <qle/termstructures/yoyinflationcurveobservermoving.hpp>
#include <qle/termstructures/zeroinflationcurveobservermoving.hpp>

#include <boost/timer.hpp>

#include <ored/utilities/indexparser.hpp>
#include <ored/utilities/log.hpp>

using namespace QuantLib;
using namespace QuantExt;
using namespace ore::data;
using namespace std;

typedef QuantLib::BaseCorrelationTermStructure<QuantLib::BilinearInterpolation> BilinearBaseCorrelationTermStructure;

namespace ore {
namespace analytics {

RiskFactorKey::KeyType yieldCurveRiskFactor(const ore::data::YieldCurveType y) {

    if (y == ore::data::YieldCurveType::Discount) {
        return RiskFactorKey::KeyType::DiscountCurve;
    } else if (y == ore::data::YieldCurveType::Yield) {
        return RiskFactorKey::KeyType::YieldCurve;
    } else if (y == ore::data::YieldCurveType::EquityDividend) {
        return RiskFactorKey::KeyType::DividendYield;
    } else if (y == ore::data::YieldCurveType::EquityForecast) {
        return RiskFactorKey::KeyType::EquityForecastCurve;
    } else {
        QL_FAIL("yieldCurveType not supported");
    }
}

namespace {
ReactionToTimeDecay parseDecayMode(const string& s) {
    static map<string, ReactionToTimeDecay> m = {{"ForwardVariance", ForwardForwardVariance},
                                                 {"ConstantVariance", ConstantVariance}};

    auto it = m.find(s);
    if (it != m.end()) {
        return it->second;
    } else {
        QL_FAIL("Decay mode \"" << s << "\" not recognized");
    }
}

} // namespace

void ScenarioSimMarket::addYieldCurve(const boost::shared_ptr<Market>& initMarket, const std::string& configuration,
                                      const ore::data::YieldCurveType y, const string& key,
                                      const vector<Period>& tenors, const string& dayCounter) {
    Handle<YieldTermStructure> wrapper = initMarket->yieldCurve(y, key, configuration);
    QL_REQUIRE(!wrapper.empty(), "yield curve not provided for " << key);
    QL_REQUIRE(tenors.front() > 0 * Days, "yield curve tenors must not include t=0");
    // include today

    // constructing yield curves
    DayCounter dc = ore::data::parseDayCounter(dayCounter); // used to convert YieldCurve Periods to Times
    vector<Time> yieldCurveTimes(1, 0.0);                   // include today
    vector<Date> yieldCurveDates(1, asof_);
    for (auto& tenor : tenors) {
        yieldCurveTimes.push_back(dc.yearFraction(asof_, asof_ + tenor));
        yieldCurveDates.push_back(asof_ + tenor);
    }

    vector<Handle<Quote>> quotes;
    boost::shared_ptr<SimpleQuote> q(new SimpleQuote(1.0));
    quotes.push_back(Handle<Quote>(q));
    vector<Real> discounts(yieldCurveTimes.size());
    RiskFactorKey::KeyType rf = yieldCurveRiskFactor(y);
    std::map<RiskFactorKey, boost::shared_ptr<SimpleQuote>> simDataTmp;
    for (Size i = 0; i < yieldCurveTimes.size() - 1; i++) {
        boost::shared_ptr<SimpleQuote> q(new SimpleQuote(wrapper->discount(yieldCurveDates[i + 1])));
        Handle<Quote> qh(q);
        quotes.push_back(qh);

        // Check if the risk factor is simulated before adding it
        if (nonSimulatedFactors_.find(rf) == nonSimulatedFactors_.end()) {
            simDataTmp.emplace(std::piecewise_construct, std::forward_as_tuple(rf, key, i), std::forward_as_tuple(q));
            DLOG("ScenarioSimMarket yield curve " << key << " discount[" << i << "]=" << q->value());
        }
    }

    boost::shared_ptr<YieldTermStructure> yieldCurve;

    if (ObservationMode::instance().mode() == ObservationMode::Mode::Unregister) {
        yieldCurve = boost::shared_ptr<YieldTermStructure>(
            new QuantExt::InterpolatedDiscountCurve(yieldCurveTimes, quotes, 0, TARGET(), dc));
    } else {
        yieldCurve = boost::shared_ptr<YieldTermStructure>(
            new QuantExt::InterpolatedDiscountCurve2(yieldCurveTimes, quotes, dc));
    }

    Handle<YieldTermStructure> ych(yieldCurve);
    if (wrapper->allowsExtrapolation())
        ych->enableExtrapolation();
    yieldCurves_.insert(pair<tuple<string, ore::data::YieldCurveType, string>, Handle<YieldTermStructure>>(
        make_tuple(Market::defaultConfiguration, y, key), ych));
    simData_.insert(simDataTmp.begin(), simDataTmp.end());
}

ScenarioSimMarket::ScenarioSimMarket(const boost::shared_ptr<Market>& initMarket,
                                     const boost::shared_ptr<ScenarioSimMarketParameters>& parameters,
                                     const Conventions& conventions, const std::string& configuration,
                                     const bool continueOnError)
    : SimMarket(conventions), parameters_(parameters), filter_(boost::make_shared<ScenarioFilter>()) {

    LOG("building ScenarioSimMarket...");
    asof_ = initMarket->asofDate();
    LOG("AsOf " << QuantLib::io::iso_date(asof_));

    // Set non simulated risk factor key types
    if (!parameters->simulateSwapVols())
        nonSimulatedFactors_.insert(RiskFactorKey::KeyType::SwaptionVolatility);
    if (!parameters->simulateCapFloorVols())
        nonSimulatedFactors_.insert(RiskFactorKey::KeyType::OptionletVolatility);
    if (!parameters->simulateSurvivalProbabilities())
        nonSimulatedFactors_.insert(RiskFactorKey::KeyType::SurvivalProbability);
    if (!parameters->simulateCdsVols())
        nonSimulatedFactors_.insert(RiskFactorKey::KeyType::CDSVolatility);
    if (!parameters->simulateFXVols())
        nonSimulatedFactors_.insert(RiskFactorKey::KeyType::FXVolatility);
    if (!parameters->simulateEquityVols())
        nonSimulatedFactors_.insert(RiskFactorKey::KeyType::EquityVolatility);
    if (!parameters->simulateBaseCorrelations())
        nonSimulatedFactors_.insert(RiskFactorKey::KeyType::BaseCorrelation);
    if (!parameters->simulateEquityForecastCurve())
        nonSimulatedFactors_.insert(RiskFactorKey::KeyType::EquityForecastCurve);
    if (!parameters->simulateDividendYield())
        nonSimulatedFactors_.insert(RiskFactorKey::KeyType::DividendYield);
    if (!parameters->commodityCurveSimulate())
        nonSimulatedFactors_.insert(RiskFactorKey::KeyType::CommodityCurve);
    if (!parameters->commodityVolSimulate())
        nonSimulatedFactors_.insert(RiskFactorKey::KeyType::CommodityVolatility);
    if (!parameters->securitySpreadsSimulate())
        nonSimulatedFactors_.insert(RiskFactorKey::KeyType::SecuritySpread);
    if (!parameters->simulateFxSpots())
        nonSimulatedFactors_.insert(RiskFactorKey::KeyType::FXSpot);
    if (!parameters->simulateCorrelations())
        nonSimulatedFactors_.insert(RiskFactorKey::KeyType::Correlation);

    // Build fixing manager
    fixingManager_ = boost::make_shared<FixingManager>(asof_);

    // constructing fxSpots_
    LOG("building FX triangulation..");
    for (const auto& ccyPair : parameters->fxCcyPairs()) {
        try {
            LOG("adding " << ccyPair << " FX rates");
            boost::shared_ptr<SimpleQuote> q(new SimpleQuote(initMarket->fxSpot(ccyPair, configuration)->value()));
            Handle<Quote> qh(q);
            fxSpots_[Market::defaultConfiguration].addQuote(ccyPair, qh);
            // Check if the risk factor is simulated before adding it
            RiskFactorKey::KeyType rf = RiskFactorKey::KeyType::FXSpot;
            if (nonSimulatedFactors_.find(rf) == nonSimulatedFactors_.end()) {
                simData_.emplace(std::piecewise_construct, std::forward_as_tuple(rf, ccyPair),
                                 std::forward_as_tuple(q));
            }
        } catch (const std::exception& e) {
            if (continueOnError) {
                ALOG("skipping this object: " << e.what());
            } else {
                QL_FAIL(e.what());
            }
        }
    }

    LOG("FX triangulation done");

    // constructing discount yield curves
    LOG("building discount yield curves...");
    for (const auto& ccy : parameters->ccys()) {
        try {
            LOG("building " << ccy << " discount yield curve..");
            vector<Period> tenors = parameters->yieldCurveTenors(ccy);
            addYieldCurve(initMarket, configuration, ore::data::YieldCurveType::Discount, ccy, tenors,
                          parameters->yieldCurveDayCounter(ccy));
            LOG("building " << ccy << " discount yield curve done");
        } catch (const std::exception& e) {
            if (continueOnError) {
                ALOG("skipping this object: " << e.what());
            } else {
                QL_FAIL(e.what());
            }
        }
    }
    LOG("discount yield curves done");

    LOG("building benchmark yield curves...");
    for (const auto& name : parameters->yieldCurveNames()) {
        try {
            LOG("building benchmark yield curve name " << name);
            vector<Period> tenors = parameters->yieldCurveTenors(name);
            addYieldCurve(initMarket, configuration, ore::data::YieldCurveType::Yield, name, tenors,
                          parameters->yieldCurveDayCounter(name));
            LOG("building benchmark yield curve " << name << " done");
        } catch (const std::exception& e) {
            if (continueOnError) {
                ALOG("skipping this object: " << e.what());
            } else {
                QL_FAIL(e.what());
            }
        }
    }
    LOG("benchmark yield curves done");

    // building equity yield curves
    LOG("building equity curves...");
    for (const auto& eqName : parameters->equityNames()) {
        try {
            // building equity spots
            LOG("adding " << eqName << " equity spot...");
            Real spotVal = initMarket->equitySpot(eqName, configuration)->value();
            boost::shared_ptr<SimpleQuote> q(new SimpleQuote(spotVal));
            Handle<Quote> qh(q);
            equitySpots_.insert(
                pair<pair<string, string>, Handle<Quote>>(make_pair(Market::defaultConfiguration, eqName), qh));
            simData_.emplace(std::piecewise_construct,
                             std::forward_as_tuple(RiskFactorKey::KeyType::EquitySpot, eqName),
                             std::forward_as_tuple(q));
            LOG("adding " << eqName << " equity spot done");
        } catch (const std::exception& e) {
            if (continueOnError) {
                ALOG("skipping this object: " << e.what());
            } else {
                QL_FAIL(e.what());
            }
        }
        try {
            LOG("building " << eqName << " equity dividend yield curve..");
            vector<Period> divTenors = parameters->equityDividendTenors(eqName);
            addYieldCurve(initMarket, configuration, ore::data::YieldCurveType::EquityDividend, eqName, divTenors,
                          parameters->yieldCurveDayCounter(eqName));
            LOG("building " << eqName << " equity dividend yield curve done");
        } catch (const std::exception& e) {
            if (continueOnError) {
                ALOG("skipping this object: " << e.what());
            } else {
                QL_FAIL(e.what());
            }
        }
        try {
            LOG("building " << eqName << " equity forecast curve..");
            vector<Period> foreTenors = parameters->equityForecastTenors(eqName);
            addYieldCurve(initMarket, configuration, ore::data::YieldCurveType::EquityForecast, eqName, foreTenors,
                          parameters->yieldCurveDayCounter(eqName));
            LOG("building " << eqName << " forecast curve done");
        } catch (const std::exception& e) {
            if (continueOnError) {
                ALOG("skipping this object: " << e.what());
            } else {
                QL_FAIL(e.what());
            }
        }
        try {
            LOG("building " << eqName << " equity index curve..");
            Handle<EquityIndex> curve = initMarket->equityCurve(eqName, configuration);
            boost::shared_ptr<EquityIndex> ei(curve->clone(
                equitySpot(eqName, configuration), yieldCurve(YieldCurveType::EquityForecast, eqName, configuration),
                yieldCurve(YieldCurveType::EquityDividend, eqName, configuration)));
            Handle<EquityIndex> eh(ei);
            equityCurves_.insert(
                pair<pair<string, string>, Handle<EquityIndex>>(make_pair(Market::defaultConfiguration, eqName), eh));
        } catch (const std::exception& e) {
            if (continueOnError) {
                ALOG("skipping this object: " << e.what());
            } else {
                QL_FAIL(e.what());
            }
        }
    }
    LOG("equity yield curves done");

    // building security spreads
    LOG("building security spreads...");
    for (const auto& name : parameters->securities()) {
        DLOG("Adding security spread " << name << " from configuration " << configuration);
        try {
            boost::shared_ptr<SimpleQuote> spreadQuote(
                new SimpleQuote(initMarket->securitySpread(name, configuration)->value()));
            if (parameters->securitySpreadsSimulate()) {
                simData_.emplace(std::piecewise_construct,
                                 std::forward_as_tuple(RiskFactorKey::KeyType::SecuritySpread, name),
                                 std::forward_as_tuple(spreadQuote));
            }
            securitySpreads_.insert(pair<pair<string, string>, Handle<Quote>>(
                make_pair(Market::defaultConfiguration, name), Handle<Quote>(spreadQuote)));
        } catch (const std::exception& e) {
            if (continueOnError) {
                ALOG("skipping this object: " << e.what());
            } else {
                QL_FAIL(e.what());
            }
        }
    }
    LOG("security spreads done...");

    // building security recovery rates
    LOG("building security recovery rates...");
    for (const auto& name : parameters->securities()) {
        DLOG("Adding security recovery rate " << name << " from configuration " << configuration);
        try {
            boost::shared_ptr<SimpleQuote> recoveryQuote(
                new SimpleQuote(initMarket->recoveryRate(name, configuration)->value()));
            // TODO this comes from the default curves section in the parameters,
            // do we want to specify the simulation of security recovery rates separately?
            if (parameters->simulateRecoveryRates()) {
                simData_.emplace(std::piecewise_construct,
                                 std::forward_as_tuple(RiskFactorKey::KeyType::RecoveryRate, name),
                                 std::forward_as_tuple(recoveryQuote));
            }
            recoveryRates_.insert(pair<pair<string, string>, Handle<Quote>>(
                make_pair(Market::defaultConfiguration, name), Handle<Quote>(recoveryQuote)));
        } catch (const std::exception& e) {
            // security recovery rates are optional, therefore we never throw
            ALOG("skipping this object: " << e.what());
        }
    }
    LOG("security recovery rates done...");

    // building security cpr
    LOG("building security cpr...");
    for (const auto& name : parameters->securities()) {
        DLOG("Adding security cpr " << name << " from configuration " << configuration);
        // security cpr are optional, so we need a try-catch block
        try {
            boost::shared_ptr<SimpleQuote> cprQuote(new SimpleQuote(initMarket->cpr(name, configuration)->value()));
            if (parameters->cprSimulate()) {
                simData_.emplace(std::piecewise_construct, std::forward_as_tuple(RiskFactorKey::KeyType::CPR, name),
                                 std::forward_as_tuple(cprQuote));
            }
            cprs_.insert(pair<pair<string, string>, Handle<Quote>>(make_pair(Market::defaultConfiguration, name),
                                                                   Handle<Quote>(cprQuote)));
        } catch (...) {
        }
    }
    LOG("security cprs done...");

    // constructing index curves
    LOG("building index curves...");
    for (const auto& ind : parameters->indices()) {
        try {
            LOG("building " << ind << " index curve");
            std::vector<string> indexTokens;
            split(indexTokens, ind, boost::is_any_of("-"));
            Handle<IborIndex> index;
            if (indexTokens[1] == "GENERIC") {
                // If we have a generic curve build the index using the index currency's discount curve
                index =
                    Handle<IborIndex>(parseIborIndex(ind, initMarket->discountCurve(indexTokens[0], configuration)));
            } else {
                index = initMarket->iborIndex(ind, configuration);
            }
            QL_REQUIRE(!index.empty(), "index object for " << ind << " not provided");
            Handle<YieldTermStructure> wrapperIndex = index->forwardingTermStructure();
            QL_REQUIRE(!wrapperIndex.empty(), "no termstructure for index " << ind);
            vector<string> keys(parameters->yieldCurveTenors(ind).size());

            DayCounter dc = ore::data::parseDayCounter(
                parameters->yieldCurveDayCounter(ind)); // used to convert YieldCurve Periods to Times
            vector<Time> yieldCurveTimes(1, 0.0);       // include today
            vector<Date> yieldCurveDates(1, asof_);
            QL_REQUIRE(parameters->yieldCurveTenors(ind).front() > 0 * Days, "yield curve tenors must not include t=0");
            for (auto& tenor : parameters->yieldCurveTenors(ind)) {
                yieldCurveTimes.push_back(dc.yearFraction(asof_, asof_ + tenor));
                yieldCurveDates.push_back(asof_ + tenor);
            }

            // include today
            vector<Handle<Quote>> quotes;
            boost::shared_ptr<SimpleQuote> q(new SimpleQuote(1.0));
            quotes.push_back(Handle<Quote>(q));

            std::map<RiskFactorKey, boost::shared_ptr<SimpleQuote>> simDataTmp;
            for (Size i = 0; i < yieldCurveTimes.size() - 1; i++) {
                boost::shared_ptr<SimpleQuote> q(new SimpleQuote(wrapperIndex->discount(yieldCurveDates[i + 1])));
                Handle<Quote> qh(q);
                quotes.push_back(qh);
                simDataTmp.emplace(std::piecewise_construct,
                                   std::forward_as_tuple(RiskFactorKey::KeyType::IndexCurve, ind, i),
                                   std::forward_as_tuple(q));

                DLOG("ScenarioSimMarket index curve " << ind << " discount[" << i << "]=" << q->value());
            }
            // FIXME interpolation fixed to linear, added to xml??
            boost::shared_ptr<YieldTermStructure> indexCurve;
            if (ObservationMode::instance().mode() == ObservationMode::Mode::Unregister) {
                indexCurve = boost::shared_ptr<YieldTermStructure>(
                    new QuantExt::InterpolatedDiscountCurve(yieldCurveTimes, quotes, 0, index->fixingCalendar(), dc));
            } else {
                indexCurve = boost::shared_ptr<YieldTermStructure>(
                    new QuantExt::InterpolatedDiscountCurve2(yieldCurveTimes, quotes, dc));
            }

            // wrapped curve, is slower than a native curve
            // boost::shared_ptr<YieldTermStructure> correctedIndexCurve(
            //     new StaticallyCorrectedYieldTermStructure(
            //         discountCurves_[ccy], initMarket->discountCurve(ccy, configuration),
            //         wrapperIndex));

            Handle<YieldTermStructure> ich(indexCurve);
            // Handle<YieldTermStructure> ich(correctedIndexCurve);
            if (wrapperIndex->allowsExtrapolation())
                ich->enableExtrapolation();

            boost::shared_ptr<IborIndex> i(index->clone(ich));
            Handle<IborIndex> ih(i);
            iborIndices_.insert(
                pair<pair<string, string>, Handle<IborIndex>>(make_pair(Market::defaultConfiguration, ind), ih));
            simData_.insert(simDataTmp.begin(), simDataTmp.end());
            LOG("building " << ind << " index curve done");
        } catch (const std::exception& e) {
            ALOG("skipping this object: " << e.what());
        }
    }
    LOG("index curves done");

    // swap indices
    LOG("building swap indices...");
    for (const auto& it : parameters->swapIndices()) {
        try {
            const string& indexName = it.first;
            const string& discounting = it.second;
            LOG("Adding swap index " << indexName << " with discounting index " << discounting);

            addSwapIndex(indexName, discounting, Market::defaultConfiguration);
            LOG("Adding swap index " << indexName << " done.");
        } catch (const std::exception& e) {
            if (continueOnError) {
                ALOG("skipping this object: " << e.what());
            } else {
                QL_FAIL(e.what());
            }
        }
    }

    // constructing swaption volatility curves
    LOG("building swaption volatility curves...");
    for (const auto& ccy : parameters->swapVolCcys()) {
        try {
            LOG("building " << ccy << " swaption volatility curve...");
            RelinkableHandle<SwaptionVolatilityStructure> wrapper(*initMarket->swaptionVol(ccy, configuration));

            LOG("Initial market " << ccy << " swaption volatility type = " << wrapper->volatilityType());

            string shortSwapIndexBase = initMarket->shortSwapIndexBase(ccy, configuration);
            string swapIndexBase = initMarket->swapIndexBase(ccy, configuration);

            bool isCube = parameters->swapVolIsCube();

            // If swaption volatility type is not Normal, convert to Normal for the simulation
            if (wrapper->volatilityType() != Normal) {
                // FIXME we can not convert constant swaption vol structures yet
                if (boost::dynamic_pointer_cast<ConstantSwaptionVolatility>(*wrapper) != nullptr) {
                    ALOG("Constant swaption volatility found in configuration "
                         << configuration << " for currency " << ccy << " will not be converted to normal");
                } else {
                    // Get swap index associated with this volatility structure
                    string swapIndexName = initMarket->swapIndexBase(ccy, configuration);
                    string shortSwapIndexName = initMarket->shortSwapIndexBase(ccy, configuration);
                    Handle<SwapIndex> swapIndex = initMarket->swapIndex(swapIndexName, configuration);
                    Handle<SwapIndex> shortSwapIndex = initMarket->swapIndex(shortSwapIndexName, configuration);

                    // Set up swaption volatility converter
                    SwaptionVolatilityConverter converter(asof_, *wrapper, *swapIndex, *shortSwapIndex, Normal);
                    wrapper.linkTo(converter.convert());

                    LOG("Converting swaption volatilities in configuration "
                        << configuration << " with currency " << ccy << " to normal swaption volatilities");
                }
            }

            Handle<SwaptionVolatilityStructure> svp;
            std::map<RiskFactorKey, boost::shared_ptr<SimpleQuote>> simDataTmp;
            if (parameters->simulateSwapVols()) {
                LOG("Simulating (" << wrapper->volatilityType() << ") Swaption vols for ccy " << ccy);
                vector<Period> optionTenors = parameters->swapVolExpiries();
                vector<Period> swapTenors = parameters->swapVolTerms();
                vector<Real> strikeSpreads = parameters->swapVolStrikeSpreads();
                bool atmOnly = parameters->simulateSwapVolATMOnly();
                DLOG("Swaption atmOnly : " << (atmOnly ? "True" : "False"));
                DLOG("Swaption isCube : " << (isCube ? "True" : "False"));
                if (atmOnly) {
                    QL_REQUIRE(strikeSpreads.size() == 1 && close_enough(strikeSpreads[0], 0),
                               "for atmOnly strikeSpreads must be {0.0}");
                } else {
                    QL_REQUIRE(isCube, "Only atmOnly simulation supported for swaption vol surfaces");
                }
                boost::shared_ptr<QuantLib::SwaptionVolatilityCube> cube;
                if (isCube) {
                    boost::shared_ptr<SwaptionVolCubeWithATM> tmp =
                        boost::dynamic_pointer_cast<SwaptionVolCubeWithATM>(*wrapper);
                    QL_REQUIRE(tmp, "swaption cube missing")
                    cube = tmp->cube();
                }
                vector<vector<Handle<Quote>>> quotes, atmQuotes;
                quotes.resize(optionTenors.size() * swapTenors.size(),
                              vector<Handle<Quote>>(strikeSpreads.size(), Handle<Quote>()));
                atmQuotes.resize(optionTenors.size(), std::vector<Handle<Quote>>(swapTenors.size(), Handle<Quote>()));
                vector<vector<Real>> shift(optionTenors.size(), vector<Real>(swapTenors.size(), 0.0));
                Size atmSlice = std::find_if(strikeSpreads.begin(), strikeSpreads.end(),
                                             [](const Real s) { return close_enough(s, 0.0); }) -
                                strikeSpreads.begin();
                QL_REQUIRE(atmSlice < strikeSpreads.size(),
                           "could not find atm slice (strikeSpreads do not contain 0.0)");
                for (Size k = 0; k < strikeSpreads.size(); ++k) {
                    for (Size i = 0; i < optionTenors.size(); ++i) {
                        for (Size j = 0; j < swapTenors.size(); ++j) {
                            Real strike = Null<Real>();
                            if (!atmOnly && cube)
                                strike = cube->atmStrike(optionTenors[i], swapTenors[j]) + strikeSpreads[k];
                            Real vol = wrapper->volatility(optionTenors[i], swapTenors[j], strike, true);
                            boost::shared_ptr<SimpleQuote> q(new SimpleQuote(vol));

                            Size index = i * swapTenors.size() * strikeSpreads.size() + j * strikeSpreads.size() + k;

                            simDataTmp.emplace(
                                std::piecewise_construct,
                                std::forward_as_tuple(RiskFactorKey::KeyType::SwaptionVolatility, ccy, index),
                                std::forward_as_tuple(q));
                            auto tmp = Handle<Quote>(q);
                            quotes[i * swapTenors.size() + j][k] = tmp;
                            if (k == atmSlice) {
                                atmQuotes[i][j] = tmp;
                                shift[i][j] = wrapper->volatilityType() == ShiftedLognormal
                                                  ? wrapper->shift(optionTenors[i], swapTenors[j])
                                                  : 0.0;
                            }
                        }
                    }
                }
                bool flatExtrapolation = true; // FIXME: get this from curve configuration
                VolatilityType volType = wrapper->volatilityType();
                DayCounter dc = ore::data::parseDayCounter(parameters->swapVolDayCounter(ccy));
                Handle<SwaptionVolatilityStructure> atm(boost::make_shared<SwaptionVolatilityMatrix>(
                    wrapper->calendar(), wrapper->businessDayConvention(), optionTenors, swapTenors, atmQuotes, dc,
                    flatExtrapolation, volType, shift));
                if (atmOnly) {
                    // floating reference date matrix in sim market
                    // if we have a cube, we keep the vol spreads constant under scenarios
                    // notice that cube is from todaysmarket, so it has a fixed reference date, which means that
                    // we keep the smiles constant in terms of vol spreads when moving forward in time;
                    // notice also that the volatility will be "sticky strike", i.e. it will not react to
                    // changes in the ATM level
                    if (isCube) {
                        svp = Handle<SwaptionVolatilityStructure>(
                            boost::make_shared<SwaptionVolatilityConstantSpread>(atm, wrapper));
                    } else {
                        svp = atm;
                    }
                } else {
                    QL_REQUIRE(isCube, "Only atmOnly simulation supported for swaption vol surfaces");
                    boost::shared_ptr<SwaptionVolatilityCube> tmp(new QuantExt::SwaptionVolCube2(
                        atm, optionTenors, swapTenors, strikeSpreads, quotes,
                        *initMarket->swapIndex(swapIndexBase, configuration),
                        *initMarket->swapIndex(shortSwapIndexBase, configuration), false, flatExtrapolation, false));
                    svp = Handle<SwaptionVolatilityStructure>(boost::make_shared<SwaptionVolCubeWithATM>(tmp));
                }
            } else {
                string decayModeString = parameters->swapVolDecayMode();
                ReactionToTimeDecay decayMode = parseDecayMode(decayModeString);
                LOG("Dynamic (" << wrapper->volatilityType() << ") Swaption vols (" << decayModeString << ") for ccy "
                                << ccy);
                if (isCube)
                    WLOG("Only ATM slice is considered from init market's cube");
                boost::shared_ptr<QuantLib::SwaptionVolatilityStructure> svolp =
                    boost::make_shared<QuantExt::DynamicSwaptionVolatilityMatrix>(*wrapper, 0, NullCalendar(),
                                                                                decayMode);
                svp = Handle<SwaptionVolatilityStructure>(svolp);
            }

            svp->enableExtrapolation(); // FIXME
            swaptionCurves_.insert(pair<pair<string, string>, Handle<SwaptionVolatilityStructure>>(
                make_pair(Market::defaultConfiguration, ccy), svp));

            LOG("Simulaton market " << ccy << " swaption volatility type = " << svp->volatilityType());

            swaptionIndexBases_.insert(pair<pair<string, string>, pair<string, string>>(
                make_pair(Market::defaultConfiguration, ccy), make_pair(shortSwapIndexBase, swapIndexBase)));
            swaptionIndexBases_.insert(pair<pair<string, string>, pair<string, string>>(
                make_pair(Market::defaultConfiguration, ccy), make_pair(swapIndexBase, swapIndexBase)));
            simData_.insert(simDataTmp.begin(), simDataTmp.end());
        } catch (const std::exception& e) {
            if (continueOnError) {
                ALOG("skipping this object: " << e.what());
            } else {
                QL_FAIL(e.what());
            }
        }
    }

    LOG("swaption volatility curves done");

    // Constructing caplet/floorlet volatility surfaces
    LOG("building cap/floor volatility curves...");
    for (const auto& ccy : parameters->capFloorVolCcys()) {
        try {
            LOG("building " << ccy << " cap/floor volatility curve...");
            Handle<OptionletVolatilityStructure> wrapper = initMarket->capFloorVol(ccy, configuration);

            LOG("Initial market cap/floor volatility type = " << wrapper->volatilityType());

            Handle<OptionletVolatilityStructure> hCapletVol;

            std::map<RiskFactorKey, boost::shared_ptr<SimpleQuote>> simDataTmp;
            if (parameters->simulateCapFloorVols()) {
                LOG("Simulating Cap/Floor Optionlet vols for ccy " << ccy);
                vector<Period> optionTenors = parameters->capFloorVolExpiries(ccy);
                vector<Date> optionDates(optionTenors.size());
                vector<Real> strikes = parameters->capFloorVolStrikes();
                vector<vector<Handle<Quote>>> quotes(optionTenors.size(),
                                                     vector<Handle<Quote>>(strikes.size(), Handle<Quote>()));
                for (Size i = 0; i < optionTenors.size(); ++i) {
                    optionDates[i] = wrapper->optionDateFromTenor(optionTenors[i]);
                    for (Size j = 0; j < strikes.size(); ++j) {
                        Real vol = wrapper->volatility(optionTenors[i], strikes[j], wrapper->allowsExtrapolation());
                        boost::shared_ptr<SimpleQuote> q(new SimpleQuote(vol));
                        Size index = i * strikes.size() + j;
                        simDataTmp.emplace(
                            std::piecewise_construct,
                            std::forward_as_tuple(RiskFactorKey::KeyType::OptionletVolatility, ccy, index),
                            std::forward_as_tuple(q));
                        quotes[i][j] = Handle<Quote>(q);
                    }
                }
                DayCounter dc = ore::data::parseDayCounter(parameters->capFloorVolDayCounter(ccy));
                // FIXME: Works as of today only, i.e. for sensitivity/scenario analysis.
                // TODO: Build floating reference date StrippedOptionlet class for MC path generators
                boost::shared_ptr<StrippedOptionlet> optionlet = boost::make_shared<StrippedOptionlet>(
                    0, // FIXME: settlement days
                    wrapper->calendar(), wrapper->businessDayConvention(),
                    boost::shared_ptr<IborIndex>(), // FIXME: required for ATM vol calculation
                    optionDates, strikes, quotes, dc, wrapper->volatilityType(), wrapper->displacement());
                boost::shared_ptr<StrippedOptionletAdapter2> adapter =
                    boost::make_shared<StrippedOptionletAdapter2>(optionlet, true); // FIXME always flat extrapolation
                hCapletVol = Handle<OptionletVolatilityStructure>(adapter);
            } else {
                string decayModeString = parameters->capFloorVolDecayMode();
                ReactionToTimeDecay decayMode = parseDecayMode(decayModeString);
                boost::shared_ptr<OptionletVolatilityStructure> capletVol =
                    boost::make_shared<DynamicOptionletVolatilityStructure>(*wrapper, 0, NullCalendar(), decayMode);
                hCapletVol = Handle<OptionletVolatilityStructure>(capletVol);
            }

            hCapletVol->enableExtrapolation(); // FIXME
            capFloorCurves_.emplace(std::piecewise_construct, std::forward_as_tuple(Market::defaultConfiguration, ccy),
                                    std::forward_as_tuple(hCapletVol));
            simData_.insert(simDataTmp.begin(), simDataTmp.end());

            LOG("Simulaton market cap/floor volatility type = " << hCapletVol->volatilityType());
        } catch (const std::exception& e) {
            if (continueOnError) {
                ALOG("skipping this object: " << e.what());
            } else {
                QL_FAIL(e.what());
            }
        }
    }

    LOG("cap/floor volatility curves done");

    // building default curves
    LOG("building default curves...");
    for (const auto& name : parameters->defaultNames()) {
        try {
            LOG("building " << name << " default curve..");
            Handle<DefaultProbabilityTermStructure> wrapper = initMarket->defaultCurve(name, configuration);
            vector<Handle<Quote>> quotes;

            QL_REQUIRE(parameters->defaultTenors(name).front() > 0 * Days, "default curve tenors must not include t=0");

            vector<Date> dates(1, asof_);

            for (Size i = 0; i < parameters->defaultTenors(name).size(); i++) {
                dates.push_back(asof_ + parameters->defaultTenors(name)[i]);
            }

            boost::shared_ptr<SimpleQuote> q(new SimpleQuote(1.0));
            quotes.push_back(Handle<Quote>(q));
            std::map<RiskFactorKey, boost::shared_ptr<SimpleQuote>> simDataTmp;
            for (Size i = 0; i < dates.size() - 1; i++) {
                Probability prob = wrapper->survivalProbability(dates[i + 1], true);
                boost::shared_ptr<SimpleQuote> q(new SimpleQuote(prob));
                if (parameters->simulateSurvivalProbabilities()) {
                    simDataTmp.emplace(std::piecewise_construct,
                                       std::forward_as_tuple(RiskFactorKey::KeyType::SurvivalProbability, name, i),
                                       std::forward_as_tuple(q));
                    DLOG("ScenarioSimMarket default curve " << name << " survival[" << i << "]=" << prob);
                }
                Handle<Quote> qh(q);
                quotes.push_back(qh);
            }
            DayCounter dc = ore::data::parseDayCounter(parameters->defaultCurveDayCounter(name));
            Calendar cal = ore::data::parseCalendar(parameters->defaultCurveCalendar(name));
            // FIXME riskmarket uses SurvivalProbabilityCurve but this isn't added to ore
            boost::shared_ptr<DefaultProbabilityTermStructure> defaultCurve(
                new QuantExt::SurvivalProbabilityCurve<Linear>(dates, quotes, dc, cal));
            Handle<DefaultProbabilityTermStructure> dch(defaultCurve);

            dch->enableExtrapolation();

            // add recovery rate
            boost::shared_ptr<SimpleQuote> rrQuote(
                new SimpleQuote(initMarket->recoveryRate(name, configuration)->value()));
            if (parameters->simulateRecoveryRates()) {
                simDataTmp.emplace(std::piecewise_construct,
                                   std::forward_as_tuple(RiskFactorKey::KeyType::RecoveryRate, name),
                                   std::forward_as_tuple(rrQuote));
            }
            defaultCurves_.insert(pair<pair<string, string>, Handle<DefaultProbabilityTermStructure>>(
                make_pair(Market::defaultConfiguration, name), dch));
            recoveryRates_.insert(pair<pair<string, string>, Handle<Quote>>(
                make_pair(Market::defaultConfiguration, name), Handle<Quote>(rrQuote)));
            simData_.insert(simDataTmp.begin(), simDataTmp.end());
        } catch (const std::exception& e) {
            if (continueOnError) {
                ALOG("skipping this object: " << e.what());
            } else {
                QL_FAIL(e.what());
            }
        }
    }
    LOG("default curves done");

    // building cds volatilities
    LOG("building cds volatilities...");
    for (const auto& name : parameters->cdsVolNames()) {
        try {
            LOG("building " << name << "  cds vols..");
            Handle<BlackVolTermStructure> wrapper = initMarket->cdsVol(name, configuration);
            Handle<BlackVolTermStructure> cvh;
            std::map<RiskFactorKey, boost::shared_ptr<SimpleQuote>> simDataTmp;
            if (parameters->simulateCdsVols()) {
                LOG("Simulating CDS Vols for " << name);
                vector<Handle<Quote>> quotes;
                vector<Time> times;
                for (Size i = 0; i < parameters->cdsVolExpiries().size(); i++) {
                    Date date = asof_ + parameters->cdsVolExpiries()[i];
                    Volatility vol = wrapper->blackVol(date, Null<Real>(), true);
                    times.push_back(wrapper->timeFromReference(date));
                    boost::shared_ptr<SimpleQuote> q(new SimpleQuote(vol));
                    if (parameters->simulateCdsVols()) {
                        simDataTmp.emplace(std::piecewise_construct,
                                           std::forward_as_tuple(RiskFactorKey::KeyType::CDSVolatility, name, i),
                                           std::forward_as_tuple(q));
                    }
                    quotes.emplace_back(q);
                }

                DayCounter dc = ore::data::parseDayCounter(parameters->cdsVolDayCounter(name));
                boost::shared_ptr<BlackVolTermStructure> cdsVolCurve(
                    new BlackVarianceCurve3(0, NullCalendar(), wrapper->businessDayConvention(), dc, times, quotes));

                cvh = Handle<BlackVolTermStructure>(cdsVolCurve);
            } else {
                string decayModeString = parameters->cdsVolDecayMode();
                LOG("Deterministic CDS Vols with decay mode " << decayModeString << " for " << name);
                ReactionToTimeDecay decayMode = parseDecayMode(decayModeString);

                // currently only curves (i.e. strike indepdendent) CDS volatility structures are
                // supported, so we use a) the more efficient curve tag and b) a hard coded sticky
                // strike stickyness, since then no yield term structures and no fx spot are required
                // that define the ATM level
                cvh = Handle<BlackVolTermStructure>(
                    boost::make_shared<QuantExt::DynamicBlackVolTermStructure<tag::curve>>(wrapper, 0, NullCalendar(),
                                                                                decayMode, StickyStrike));
            }

            if (wrapper->allowsExtrapolation())
                cvh->enableExtrapolation();
            cdsVols_.insert(pair<pair<string, string>, Handle<BlackVolTermStructure>>(
                make_pair(Market::defaultConfiguration, name), cvh));
            simData_.insert(simDataTmp.begin(), simDataTmp.end());
        } catch (const std::exception& e) {
            if (continueOnError) {
                ALOG("skipping this object: " << e.what());
            } else {
                QL_FAIL(e.what());
            }
        }
    }
    LOG("cds volatilities done");

    // building fx volatilities
    LOG("building fx volatilities...");
    for (const auto& ccyPair : parameters->fxVolCcyPairs()) {
        try {
            Handle<BlackVolTermStructure> wrapper = initMarket->fxVol(ccyPair, configuration);
            Handle<Quote> spot = fxSpot(ccyPair);
            QL_REQUIRE(ccyPair.length() == 6, "invalid ccy pair length");
            string forCcy = ccyPair.substr(0, 3);
            string domCcy = ccyPair.substr(3, 3);
            Handle<YieldTermStructure> forTS = discountCurve(forCcy);
            Handle<YieldTermStructure> domTS = discountCurve(domCcy);
            Handle<BlackVolTermStructure> fvh;

            std::map<RiskFactorKey, boost::shared_ptr<SimpleQuote>> simDataTmp;
            if (parameters->simulateFXVols()) {
                LOG("Simulating FX Vols (BlackVarianceCurve3) for " << ccyPair);
                Size n = parameters->fxVolExpiries().size();
                Size m = parameters->fxVolMoneyness().size();
                vector<vector<Handle<Quote>>> quotes(m, vector<Handle<Quote>>(n, Handle<Quote>()));
                Calendar cal = wrapper->calendar();
                // FIXME hardcoded in todaysmarket
                DayCounter dc = ore::data::parseDayCounter(parameters->fxVolDayCounter(ccyPair));
                vector<Time> times;

                for (Size i = 0; i < n; i++) {
                    Date date = asof_ + parameters->fxVolExpiries()[i];

                    times.push_back(wrapper->timeFromReference(date));

                    for (Size j = 0; j < m; j++) {
                        Size idx = j * n + i;
                        Real mon = parameters->fxVolMoneyness()[j]; // 0 if ATM

                        // strike (assuming forward prices)
                        Real k = spot->value() * mon * forTS->discount(date) / domTS->discount(date);
                        Volatility vol = wrapper->blackVol(date, k, true);
                        boost::shared_ptr<SimpleQuote> q(new SimpleQuote(vol));
                        simDataTmp.emplace(std::piecewise_construct,
                                           std::forward_as_tuple(RiskFactorKey::KeyType::FXVolatility, ccyPair, idx),
                                           std::forward_as_tuple(q));
                        quotes[j][i] = Handle<Quote>(q);
                    }
                }

                boost::shared_ptr<BlackVolTermStructure> fxVolCurve;
                if (parameters->fxVolIsSurface()) {
                    bool stickyStrike = true;
                    fxVolCurve = boost::shared_ptr<BlackVolTermStructure>(new BlackVarianceSurfaceMoneynessForward(
                        cal, spot, times, parameters->fxVolMoneyness(), quotes, dc, forTS, domTS, stickyStrike));
                } else {
                    fxVolCurve = boost::shared_ptr<BlackVolTermStructure>(new BlackVarianceCurve3(
                        0, NullCalendar(), wrapper->businessDayConvention(), dc, times, quotes[0]));
                }
                fvh = Handle<BlackVolTermStructure>(fxVolCurve);

            } else {
                string decayModeString = parameters->fxVolDecayMode();
                LOG("Deterministic FX Vols with decay mode " << decayModeString << " for " << ccyPair);
                ReactionToTimeDecay decayMode = parseDecayMode(decayModeString);

                // currently only curves (i.e. strike indepdendent) FX volatility structures are
                // supported, so we use a) the more efficient curve tag and b) a hard coded sticky
                // strike stickyness, since then no yield term structures and no fx spot are required
                // that define the ATM level - to be revisited when FX surfaces are supported
                fvh = Handle<BlackVolTermStructure>(
                    boost::make_shared<QuantExt::DynamicBlackVolTermStructure<tag::curve>>(wrapper, 0, NullCalendar(),
                                                                                decayMode, StickyStrike));
            }

            fvh->enableExtrapolation();
            fxVols_.insert(pair<pair<string, string>, Handle<BlackVolTermStructure>>(
                make_pair(Market::defaultConfiguration, ccyPair), fvh));

            // build inverted surface
            QL_REQUIRE(ccyPair.size() == 6, "Invalid Ccy pair " << ccyPair);
            string reverse = ccyPair.substr(3) + ccyPair.substr(0, 3);
            Handle<QuantLib::BlackVolTermStructure> ifvh(boost::make_shared<BlackInvertedVolTermStructure>(fvh));
            ifvh->enableExtrapolation();
            fxVols_.insert(pair<pair<string, string>, Handle<BlackVolTermStructure>>(
                make_pair(Market::defaultConfiguration, reverse), ifvh));
            simData_.insert(simDataTmp.begin(), simDataTmp.end());
        } catch (const std::exception& e) {
            if (continueOnError) {
                ALOG("skipping this object: " << e.what());
            } else {
                QL_FAIL(e.what());
            }
        }
    }
    LOG("fx volatilities done");

    // building eq volatilities
    LOG("building eq volatilities...");
    for (const auto& equityName : parameters->equityVolNames()) {
        try {
            Handle<BlackVolTermStructure> wrapper = initMarket->equityVol(equityName, configuration);

            Handle<BlackVolTermStructure> evh;

            std::map<RiskFactorKey, boost::shared_ptr<SimpleQuote>> simDataTmp;
            if (parameters->simulateEquityVols()) {
                Handle<Quote> spot = equitySpots_[make_pair(Market::defaultConfiguration, equityName)];
                Size n = parameters->equityVolMoneyness().size();
                Size m = parameters->equityVolExpiries().size();
                vector<vector<Handle<Quote>>> quotes(n, vector<Handle<Quote>>(m, Handle<Quote>()));
                vector<Time> times(m);
                Calendar cal = wrapper->calendar();
                DayCounter dc = ore::data::parseDayCounter(parameters->equityVolDayCounter(equityName));
                bool atmOnly = parameters->simulateEquityVolATMOnly();

                for (Size i = 0; i < n; i++) {
                    Real mon = parameters->equityVolMoneyness()[i];
                    // strike
                    Real k = atmOnly ? Null<Real>() : spot->value() * mon;

                    for (Size j = 0; j < m; j++) {
                        // Index is expires then moneyness. TODO: is this the best?
                        Size idx = i * m + j;
                        times[j] = dc.yearFraction(asof_, asof_ + parameters->equityVolExpiries()[j]);
                        Volatility vol = wrapper->blackVol(asof_ + parameters->equityVolExpiries()[j], k);
                        boost::shared_ptr<SimpleQuote> q(new SimpleQuote(vol));
                        simDataTmp.emplace(
                            std::piecewise_construct,
                            std::forward_as_tuple(RiskFactorKey::KeyType::EquityVolatility, equityName, idx),
                            std::forward_as_tuple(q));
                        quotes[i][j] = Handle<Quote>(q);
                    }
                }
                boost::shared_ptr<BlackVolTermStructure> eqVolCurve;
                if (!parameters->simulateEquityVolATMOnly()) {
                    LOG("Simulating EQ Vols (BlackVarianceSurfaceMoneyness) for " << equityName);
                    // If true, the strikes are fixed, if false they move with the spot handle
                    // Should probably be false, but some people like true for sensi runs.
                    bool stickyStrike = true;

                    eqVolCurve = boost::shared_ptr<BlackVolTermStructure>(new BlackVarianceSurfaceMoneynessSpot(
                        cal, spot, times, parameters->equityVolMoneyness(), quotes, dc, stickyStrike));
                    eqVolCurve->enableExtrapolation();
                } else {
                    LOG("Simulating EQ Vols (BlackVarianceCurve3) for " << equityName);
                    eqVolCurve = boost::shared_ptr<BlackVolTermStructure>(new BlackVarianceCurve3(
                        0, NullCalendar(), wrapper->businessDayConvention(), dc, times, quotes[0]));
                }

                // if we have a surface but are only simulating atm vols we wrap the atm curve and the full t0 surface
                if (parameters->equityVolIsSurface() && parameters->simulateEquityVolATMOnly()) {
                    LOG("Simulating EQ Vols (EquityVolatilityConstantSpread) for " << equityName);
                    evh = Handle<BlackVolTermStructure>(boost::make_shared<EquityVolatilityConstantSpread>(
                        Handle<BlackVolTermStructure>(eqVolCurve), wrapper));
                } else {
                    evh = Handle<BlackVolTermStructure>(eqVolCurve);
                }
            } else {
                string decayModeString = parameters->equityVolDecayMode();
                DLOG("Deterministic EQ Vols with decay mode " << decayModeString << " for " << equityName);
                ReactionToTimeDecay decayMode = parseDecayMode(decayModeString);

                // currently only curves (i.e. strike indepdendent) EQ volatility structures are
                // supported, so we use a) the more efficient curve tag and b) a hard coded sticky
                // strike stickyness, since then no yield term structures and no EQ spot are required
                // that define the ATM level - to be revisited when EQ surfaces are supported
                evh = Handle<BlackVolTermStructure>(
                    boost::make_shared<QuantExt::DynamicBlackVolTermStructure<tag::curve>>(wrapper, 0, NullCalendar(),
                                                                                decayMode, StickyStrike));
            }
            if (wrapper->allowsExtrapolation())
                evh->enableExtrapolation();
            equityVols_.insert(pair<pair<string, string>, Handle<BlackVolTermStructure>>(
                make_pair(Market::defaultConfiguration, equityName), evh));
            simData_.insert(simDataTmp.begin(), simDataTmp.end());
            DLOG("EQ volatility curve built for " << equityName);
        } catch (const std::exception& e) {
            if (continueOnError) {
                ALOG("skipping this object: " << e.what());
            } else {
                QL_FAIL(e.what());
            }
        }
    }
    LOG("equity volatilities done");

    // building base correlation structures
    LOG("building base correlations...");
    for (const auto& bcName : parameters->baseCorrelationNames()) {
        try {
            Handle<BaseCorrelationTermStructure<BilinearInterpolation>> wrapper =
                initMarket->baseCorrelation(bcName, configuration);
            if (!parameters->simulateBaseCorrelations())
                baseCorrelations_.insert(
                    pair<pair<string, string>, Handle<BaseCorrelationTermStructure<BilinearInterpolation>>>(
                        make_pair(Market::defaultConfiguration, bcName), wrapper));
            else {
                Size nd = parameters->baseCorrelationDetachmentPoints().size();
                Size nt = parameters->baseCorrelationTerms().size();
                vector<vector<Handle<Quote>>> quotes(nd, vector<Handle<Quote>>(nt));
                vector<Period> terms(nt);
                std::map<RiskFactorKey, boost::shared_ptr<SimpleQuote>> simDataTmp;
                for (Size i = 0; i < nd; ++i) {
                    Real lossLevel = parameters->baseCorrelationDetachmentPoints()[i];
                    for (Size j = 0; j < nt; ++j) {
                        Period term = parameters->baseCorrelationTerms()[j];
                        if (i == 0)
                            terms[j] = term;
                        Real bc = wrapper->correlation(asof_ + term, lossLevel, true); // extrapolate
                        boost::shared_ptr<SimpleQuote> q(new SimpleQuote(bc));
                        simDataTmp.emplace(
                            std::piecewise_construct,
                            std::forward_as_tuple(RiskFactorKey::KeyType::BaseCorrelation, bcName, i * nt + j),
                            std::forward_as_tuple(q));
                        quotes[i][j] = Handle<Quote>(q);
                    }
                }

                // FIXME: Same change as in ored/market/basecorrelationcurve.cpp
                if (nt == 1) {
                    terms.push_back(terms[0] + 1 * Days); // arbitrary, but larger than the first term
                    for (Size i = 0; i < nd; ++i)
                        quotes[i].push_back(quotes[i][0]);
                }
                DayCounter dc = ore::data::parseDayCounter(parameters->baseCorrelationDayCounter(bcName));
                boost::shared_ptr<BilinearBaseCorrelationTermStructure> bcp =
                    boost::make_shared<BilinearBaseCorrelationTermStructure>(
                        wrapper->settlementDays(), wrapper->calendar(), wrapper->businessDayConvention(), terms,
                        parameters->baseCorrelationDetachmentPoints(), quotes, dc);

                bcp->enableExtrapolation(wrapper->allowsExtrapolation());
                Handle<BilinearBaseCorrelationTermStructure> bch(bcp);
                baseCorrelations_.insert(
                    pair<pair<string, string>, Handle<BaseCorrelationTermStructure<BilinearInterpolation>>>(
                        make_pair(Market::defaultConfiguration, bcName), bch));
                simData_.insert(simDataTmp.begin(), simDataTmp.end());
            }
            DLOG("Base correlations built for " << bcName);
        } catch (const std::exception& e) {
            if (continueOnError) {
                ALOG("skipping this object: " << e.what());
            } else {
                QL_FAIL(e.what());
            }
        }
    }
    LOG("base correlations done");

    LOG("building CPI Indices...");
    for (const auto& ci : parameters->cpiIndices()) {
        try {

            DLOG("adding " << ci << " base CPI price");
            Handle<ZeroInflationIndex> zeroInflationIndex = initMarket->zeroInflationIndex(ci, configuration);
            Period obsLag = zeroInflationIndex->zeroInflationTermStructure()->observationLag();
            Date fixingDate = zeroInflationIndex->zeroInflationTermStructure()->baseDate();
            Real baseCPI = zeroInflationIndex->fixing(fixingDate);

            boost::shared_ptr<SimpleQuote> q(new SimpleQuote(baseCPI));
            Handle<Quote> qh(q);

            boost::shared_ptr<InflationIndex> inflationIndex =
                boost::dynamic_pointer_cast<InflationIndex>(*zeroInflationIndex);
            Handle<InflationIndexObserver> inflObserver(
                boost::make_shared<InflationIndexObserver>(inflationIndex, qh, obsLag));

            baseCpis_.insert(pair<pair<string, string>, Handle<InflationIndexObserver>>(
                make_pair(Market::defaultConfiguration, ci), inflObserver));
            simData_.emplace(std::piecewise_construct, std::forward_as_tuple(RiskFactorKey::KeyType::CPIIndex, ci),
                             std::forward_as_tuple(q));
        } catch (const std::exception& e) {
            if (continueOnError) {
                ALOG("skipping this object: " << e.what());
            } else {
                QL_FAIL(e.what());
            }
        }
    }
    LOG("CPI Indices done");

    LOG("building zero inflation curves...");
    for (const auto& zic : parameters->zeroInflationIndices()) {
        try {
            LOG("building " << zic << " zero inflation curve");

            Handle<ZeroInflationIndex> inflationIndex = initMarket->zeroInflationIndex(zic, configuration);
            Handle<ZeroInflationTermStructure> inflationTs = inflationIndex->zeroInflationTermStructure();
            vector<string> keys(parameters->zeroInflationTenors(zic).size());

            string ccy = inflationIndex->currency().code();
            Handle<YieldTermStructure> yts = discountCurve(ccy, configuration);

            Date date0 = asof_ - inflationTs->observationLag();
            DayCounter dc = ore::data::parseDayCounter(parameters->zeroInflationDayCounter(zic));
            vector<Date> quoteDates;
            vector<Time> zeroCurveTimes(
                1, -dc.yearFraction(inflationPeriod(date0, inflationTs->frequency()).first, asof_));
            vector<Handle<Quote>> quotes;
            QL_REQUIRE(parameters->zeroInflationTenors(zic).front() > 0 * Days,
                       "zero inflation tenors must not include t=0");

            for (auto& tenor : parameters->zeroInflationTenors(zic)) {
                Date inflDate = inflationPeriod(date0 + tenor, inflationTs->frequency()).first;
                zeroCurveTimes.push_back(dc.yearFraction(asof_, inflDate));
                quoteDates.push_back(asof_ + tenor);
            }

            std::map<RiskFactorKey, boost::shared_ptr<SimpleQuote>> simDataTmp;
            for (Size i = 1; i < zeroCurveTimes.size(); i++) {
                boost::shared_ptr<SimpleQuote> q(new SimpleQuote(inflationTs->zeroRate(quoteDates[i - 1])));
                Handle<Quote> qh(q);
                if (i == 1) {
                    // add the zero rate at first tenor to the T0 time, to ensure flat interpolation of T1 rate
                    // for time t T0 < t < T1
                    quotes.push_back(qh);
                }
                quotes.push_back(qh);
                simDataTmp.emplace(std::piecewise_construct,
                                   std::forward_as_tuple(RiskFactorKey::KeyType::ZeroInflationCurve, zic, i - 1),
                                   std::forward_as_tuple(q));
                DLOG("ScenarioSimMarket index curve " << zic << " zeroRate[" << i << "]=" << q->value());
            }

            // FIXME: Settlement days set to zero - needed for floating term structure implementation
            boost::shared_ptr<ZeroInflationTermStructure> zeroCurve;
            dc = ore::data::parseDayCounter(parameters->zeroInflationDayCounter(zic));
            zeroCurve = boost::shared_ptr<ZeroInflationCurveObserverMoving<Linear>>(
                new ZeroInflationCurveObserverMoving<Linear>(
                    0, inflationIndex->fixingCalendar(), dc, inflationTs->observationLag(), inflationTs->frequency(),
                    inflationTs->indexIsInterpolated(), yts, zeroCurveTimes, quotes, inflationTs->seasonality()));

            Handle<ZeroInflationTermStructure> its(zeroCurve);
            its->enableExtrapolation();
            boost::shared_ptr<ZeroInflationIndex> i =
                ore::data::parseZeroInflationIndex(zic, false, Handle<ZeroInflationTermStructure>(its));
            Handle<ZeroInflationIndex> zh(i);
            zeroInflationIndices_.insert(pair<pair<string, string>, Handle<ZeroInflationIndex>>(
                make_pair(Market::defaultConfiguration, zic), zh));
            simData_.insert(simDataTmp.begin(), simDataTmp.end());

            LOG("building " << zic << " zero inflation curve done");
        } catch (const std::exception& e) {
            if (continueOnError) {
                ALOG("skipping this object: " << e.what());
            } else {
                QL_FAIL(e.what());
            }
        }
    }
    LOG("zero inflation curves done");

    LOG("building yoy inflation curves...");
    for (const auto& yic : parameters->yoyInflationIndices()) {
        try {

            Handle<YoYInflationIndex> yoyInflationIndex = initMarket->yoyInflationIndex(yic, configuration);
            Handle<YoYInflationTermStructure> yoyInflationTs = yoyInflationIndex->yoyInflationTermStructure();
            vector<string> keys(parameters->yoyInflationTenors(yic).size());

            string ccy = yoyInflationIndex->currency().code();
            Handle<YieldTermStructure> yts = initMarket->discountCurve(ccy, configuration);

            Date date0 = asof_ - yoyInflationTs->observationLag();
            DayCounter dc = ore::data::parseDayCounter(parameters->yoyInflationDayCounter(yic));
            vector<Date> quoteDates;
            vector<Time> yoyCurveTimes(
                1, -dc.yearFraction(inflationPeriod(date0, yoyInflationTs->frequency()).first, asof_));
            vector<Handle<Quote>> quotes;
            QL_REQUIRE(parameters->yoyInflationTenors(yic).front() > 0 * Days,
                       "zero inflation tenors must not include t=0");

            for (auto& tenor : parameters->yoyInflationTenors(yic)) {
                Date inflDate = inflationPeriod(date0 + tenor, yoyInflationTs->frequency()).first;
                yoyCurveTimes.push_back(dc.yearFraction(asof_, inflDate));
                quoteDates.push_back(asof_ + tenor);
            }

            std::map<RiskFactorKey, boost::shared_ptr<SimpleQuote>> simDataTmp;
            for (Size i = 1; i < yoyCurveTimes.size(); i++) {
                boost::shared_ptr<SimpleQuote> q(new SimpleQuote(yoyInflationTs->yoyRate(quoteDates[i - 1])));
                Handle<Quote> qh(q);
                if (i == 1) {
                    // add the zero rate at first tenor to the T0 time, to ensure flat interpolation of T1 rate
                    // for time t T0 < t < T1
                    quotes.push_back(qh);
                }
                quotes.push_back(qh);
                simDataTmp.emplace(std::piecewise_construct,
                                   std::forward_as_tuple(RiskFactorKey::KeyType::YoYInflationCurve, yic, i - 1),
                                   std::forward_as_tuple(q));
                DLOG("ScenarioSimMarket index curve " << yic << " zeroRate[" << i << "]=" << q->value());
            }

            boost::shared_ptr<YoYInflationTermStructure> yoyCurve;
            // Note this is *not* a floating term structure, it is only suitable for sensi runs
            // TODO: floating
            yoyCurve =
                boost::shared_ptr<YoYInflationCurveObserverMoving<Linear>>(new YoYInflationCurveObserverMoving<Linear>(
                    0, yoyInflationIndex->fixingCalendar(), dc, yoyInflationTs->observationLag(),
                    yoyInflationTs->frequency(), yoyInflationTs->indexIsInterpolated(), yts, yoyCurveTimes, quotes,
                    yoyInflationTs->seasonality()));

            Handle<YoYInflationTermStructure> its(yoyCurve);
            its->enableExtrapolation();
            boost::shared_ptr<YoYInflationIndex> i(yoyInflationIndex->clone(its));
            Handle<YoYInflationIndex> zh(i);
            yoyInflationIndices_.insert(pair<pair<string, string>, Handle<YoYInflationIndex>>(
                make_pair(Market::defaultConfiguration, yic), zh));
            simData_.insert(simDataTmp.begin(), simDataTmp.end());
        } catch (const std::exception& e) {
            if (continueOnError) {
                ALOG("skipping this object: " << e.what());
            } else {
                QL_FAIL(e.what());
            }
        }
    }
    LOG("yoy inflation curves done");

    LOG("building commodity spots");
    for (const auto& name : parameters->commodityNames()) {
        try {
            Real spot = initMarket->commoditySpot(name, configuration)->value();
            DLOG("adding " << name << " commodity spot price");
            boost::shared_ptr<SimpleQuote> q = boost::make_shared<SimpleQuote>(spot);
            commoditySpots_.emplace(piecewise_construct, forward_as_tuple(Market::defaultConfiguration, name),
                                    forward_as_tuple(q));
            simData_.emplace(piecewise_construct, forward_as_tuple(RiskFactorKey::KeyType::CommoditySpot, name),
                             forward_as_tuple(q));
        } catch (const std::exception& e) {
            if (continueOnError) {
                ALOG("skipping this object: " << e.what());
            } else {
                QL_FAIL(e.what());
            }
        }
    }
    LOG("commodity spots done");

    LOG("building commodity curves");
    for (const string& name : parameters->commodityNames()) {
        try {

            LOG("building commodity curve for " << name);

            // Time zero initial market commodity curve
            Handle<PriceTermStructure> initialCommodityCurve = initMarket->commodityPriceCurve(name, configuration);
            bool allowsExtrapolation = initialCommodityCurve->allowsExtrapolation();

            // Get prices at specified simulation tenors from time 0 market curve and place in quotes
            vector<Period> simulationTenors = parameters->commodityCurveTenors(name);
            DayCounter commodityCurveDayCounter = parseDayCounter(parameters->commodityCurveDayCounter(name));
            vector<Time> times(simulationTenors.size());
            vector<Handle<Quote>> quotes(simulationTenors.size());

            std::map<RiskFactorKey, boost::shared_ptr<SimpleQuote>> simDataTmp;
            for (Size i = 0; i < simulationTenors.size(); i++) {
                times[i] = commodityCurveDayCounter.yearFraction(asof_, asof_ + simulationTenors[i]);
                Real price = initialCommodityCurve->price(times[i], allowsExtrapolation);
                boost::shared_ptr<SimpleQuote> quote = boost::make_shared<SimpleQuote>(price);
                quotes[i] = Handle<Quote>(quote);

                // If we are simulating commodities, add the quote to simData_
                if (parameters->commodityCurveSimulate()) {
                    simDataTmp.emplace(piecewise_construct,
                                       forward_as_tuple(RiskFactorKey::KeyType::CommodityCurve, name, i),
                                       forward_as_tuple(quote));
                }
            }

            // Create a commodity price curve with simulation tenors as pillars and store
            // Hard-coded linear interpolation here - may need to make this more dynamic
            Handle<PriceTermStructure> simCommodityCurve(
                boost::make_shared<InterpolatedPriceCurve<Linear>>(times, quotes, commodityCurveDayCounter));
            simCommodityCurve->enableExtrapolation(allowsExtrapolation);

            commodityCurves_.emplace(piecewise_construct, forward_as_tuple(Market::defaultConfiguration, name),
                                     forward_as_tuple(simCommodityCurve));
            simData_.insert(simDataTmp.begin(), simDataTmp.end());
        } catch (const std::exception& e) {
            if (continueOnError) {
                ALOG("skipping this object: " << e.what());
            } else {
                QL_FAIL(e.what());
            }
        }
    }
    LOG("commodity curves done");

    LOG("building commodity volatilities");
    for (const auto& name : parameters->commodityVolNames()) {
        try {
            // Get initial base volatility structure
            Handle<BlackVolTermStructure> baseVol = initMarket->commodityVolatility(name, configuration);

            Handle<BlackVolTermStructure> newVol;
            std::map<RiskFactorKey, boost::shared_ptr<SimpleQuote>> simDataTmp;
            if (parameters->commodityVolSimulate()) {
                Handle<Quote> spot = commoditySpot(name, configuration);
                const vector<Real>& moneyness = parameters->commodityVolMoneyness(name);
                QL_REQUIRE(!moneyness.empty(),
                           "Commodity volatility moneyness for " << name << " should have at least one element");
                const vector<Period>& expiries = parameters->commodityVolExpiries(name);
                QL_REQUIRE(!expiries.empty(),
                           "Commodity volatility expiries for " << name << " should have at least one element");

                // Create surface of quotes
                vector<vector<Handle<Quote>>> quotes(moneyness.size(), vector<Handle<Quote>>(expiries.size()));
                vector<Time> expiryTimes(expiries.size());
                Size index = 0;
                DayCounter dayCounter = baseVol->dayCounter();

                for (Size i = 0; i < quotes.size(); i++) {
                    Real strike = moneyness[i] * spot->value();
                    for (Size j = 0; j < quotes[0].size(); j++) {
                        if (i == 0)
                            expiryTimes[j] = dayCounter.yearFraction(asof_, asof_ + expiries[j]);
                        boost::shared_ptr<SimpleQuote> quote =
                            boost::make_shared<SimpleQuote>(baseVol->blackVol(asof_ + expiries[j], strike));
                        simDataTmp.emplace(piecewise_construct,
                                           forward_as_tuple(RiskFactorKey::KeyType::CommodityVolatility, name, index++),
                                           forward_as_tuple(quote));
                        quotes[i][j] = Handle<Quote>(quote);
                    }
                }

                // Create volatility structure
                if (moneyness.size() == 1) {
                    // We have a term structure of volatilities with no strike dependence
                    LOG("Simulating commodity volatilites for " << name << " using BlackVarianceCurve3.");
                    newVol = Handle<BlackVolTermStructure>(boost::make_shared<BlackVarianceCurve3>(
                        0, NullCalendar(), baseVol->businessDayConvention(), dayCounter, expiryTimes, quotes[0]));
                } else {
                    // We have a volatility surface
                    LOG("Simulating commodity volatilites for " << name << " using BlackVarianceSurfaceMoneynessSpot.");
                    bool stickyStrike = true;
                    newVol = Handle<BlackVolTermStructure>(boost::make_shared<BlackVarianceSurfaceMoneynessSpot>(
                        baseVol->calendar(), spot, expiryTimes, moneyness, quotes, dayCounter, stickyStrike));
                }

            } else {
                string decayModeString = parameters->equityVolDecayMode();
                DLOG("Deterministic commodity volatilities with decay mode " << decayModeString << " for " << name);
                ReactionToTimeDecay decayMode = parseDecayMode(decayModeString);
                // Copy what was done for equity here
                // May need to revisit when looking at commodity RFE
                newVol = Handle<BlackVolTermStructure>(
                    boost::make_shared<QuantExt::DynamicBlackVolTermStructure<tag::curve>>(baseVol, 0, NullCalendar(),
                                                                                decayMode, StickyStrike));
            }

            newVol->enableExtrapolation(baseVol->allowsExtrapolation());

            commodityVols_.emplace(piecewise_construct, forward_as_tuple(Market::defaultConfiguration, name),
                                   forward_as_tuple(newVol));
            simData_.insert(simDataTmp.begin(), simDataTmp.end());

            DLOG("Commodity volatility curve built for " << name);
        } catch (const std::exception& e) {
            if (continueOnError) {
                ALOG("skipping this object: " << e.what());
            } else {
                QL_FAIL(e.what());
            }
        }
    }
    LOG("commodity volatilities done");

    // building correlations
    LOG("building correlations... " << parameters->correlationPairs().size());
    for (const auto& pair : parameters->correlationPairs()) {
        LOG("Adding correlations for " << pair.first << ":" << pair.second << " from configuration " << configuration);
        boost::shared_ptr<QuantExt::CorrelationTermStructure> corr;
        Handle<QuantExt::CorrelationTermStructure> baseCorr =
            initMarket->correlationCurve(pair.first, pair.second, configuration);

        Handle<QuantExt::CorrelationTermStructure> ch;
        if (parameters->simulateCorrelations()) {

            string label = pair.first + ":" + pair.second;

            Size n = parameters->correlationStrikes().size();
            Size m = parameters->correlationExpiries().size();
            vector<vector<Handle<Quote>>> quotes(n, vector<Handle<Quote>>(m, Handle<Quote>()));
            vector<Time> times(m);
            Calendar cal = baseCorr->calendar();
            DayCounter dc = ore::data::parseDayCounter(parameters->correlationDayCounter(pair.first, pair.second));

            for (Size i = 0; i < n; i++) {
                Real strike = parameters->correlationStrikes()[i];

                for (Size j = 0; j < m; j++) {
                    // Index is expiries then strike TODO: is this the best?
                    Size idx = i * m + j;
                    times[j] = dc.yearFraction(asof_, asof_ + parameters->correlationExpiries()[j]);
                    Real correlation = baseCorr->correlation(asof_ + parameters->correlationExpiries()[j], strike);
                    boost::shared_ptr<SimpleQuote> q(new SimpleQuote(correlation));
                    simData_.emplace(std::piecewise_construct,
                                     std::forward_as_tuple(RiskFactorKey::KeyType::Correlation, label, idx),
                                     std::forward_as_tuple(q));
                    quotes[i][j] = Handle<Quote>(q);
                }
            }

            if (n == 1 && m == 1) {
                ch = Handle<QuantExt::CorrelationTermStructure>(
                    boost::make_shared<FlatCorrelation>(baseCorr->settlementDays(), cal, quotes[0][0], dc));
            } else if (n == 1) {
                ch = Handle<QuantExt::CorrelationTermStructure>(
                    boost::make_shared<InterpolatedCorrelationCurve<Linear>>(times, quotes[0], dc, cal));
            } else {
                QL_FAIL("only atm or flat correlation termstructures currently supported");
            }

            ch->enableExtrapolation(baseCorr->allowsExtrapolation());
        } else {
            ch = Handle<QuantExt::CorrelationTermStructure>(*baseCorr);
        }

        correlationCurves_[make_tuple(Market::defaultConfiguration, pair.first, pair.second)] = ch;
    }
    LOG("security recovery rates done...");

    LOG("building base scenario");
    baseScenario_ = boost::make_shared<SimpleScenario>(initMarket->asofDate(), "BASE", 1.0);
    for (auto const& data : simData_) {
        baseScenario_->add(data.first, data.second->value());
    }
    LOG("building base scenario done");
}

void ScenarioSimMarket::applyScenario(const boost::shared_ptr<Scenario>& scenario) {
    const vector<RiskFactorKey>& keys = scenario->keys();

    Size count = 0;
    for (const auto& key : keys) {
        // Loop through the scenario keys and check which keys are present in simData_,
        // adding to the count when a match is identified
        // Then check that the count=simData_.size - this ensures that simData_ is a valid
        // subset of the scenario - fails is a member of simData is not present in the
        // scenario
        auto it = simData_.find(key);
        if (it == simData_.end()) {
            ALOG("simulation data point missing for key " << key);
        } else {
            if (filter_->allow(key))
                it->second->setValue(scenario->get(key));
            count++;
        }
    }

    if (count != simData_.size()) {
        ALOG("mismatch between scenario and sim data size, " << count << " vs " << simData_.size());
        for (auto it : simData_) {
            if (!scenario->has(it.first))
                ALOG("Key " << it.first << " missing in scenario");
        }
        QL_FAIL("mismatch between scenario and sim data size, exit.");
    }

    // update market asof date
    asof_ = scenario->asof();
}

void ScenarioSimMarket::reset() {
    auto filterBackup = filter_;
    // no filter
    filter_ = boost::make_shared<ScenarioFilter>();
    // reset eval date
    Settings::instance().evaluationDate() = baseScenario_->asof();
    // reset numeraire
    numeraire_ = baseScenario_->getNumeraire();
    // reset term structures
    applyScenario(baseScenario_);
    // see the comment in update() for why this is necessary...
    if (ObservationMode::instance().mode() == ObservationMode::Mode::Unregister) {
        boost::shared_ptr<QuantLib::Observable> obs = QuantLib::Settings::instance().evaluationDate();
        obs->notifyObservers();
    }
    // reset fixing manager
    fixingManager_->reset();
    // restore the filter
    filter_ = filterBackup;
}

void ScenarioSimMarket::update(const Date& d) {
    // DLOG("ScenarioSimMarket::update called with Date " << QuantLib::io::iso_date(d));
    QL_REQUIRE(scenarioGenerator_ != nullptr, "ScenarioSimMarket::update: no scenario generator set");

    ObservationMode::Mode om = ObservationMode::instance().mode();
    if (om == ObservationMode::Mode::Disable)
        ObservableSettings::instance().disableUpdates(false);
    else if (om == ObservationMode::Mode::Defer)
        ObservableSettings::instance().disableUpdates(true);

    boost::shared_ptr<Scenario> scenario = scenarioGenerator_->next(d);
    QL_REQUIRE(scenario->asof() == d, "Invalid Scenario date " << scenario->asof() << ", expected " << d);

    numeraire_ = scenario->getNumeraire();

    if (d != Settings::instance().evaluationDate())
        Settings::instance().evaluationDate() = d;
    else if (om == ObservationMode::Mode::Unregister) {
        // Due to some of the notification chains having been unregistered,
        // it is possible that some lazy objects might be missed in the case
        // that the evaluation date has not been updated. Therefore, we
        // manually kick off an observer notification from this level.
        // We have unit regression tests in OREAnalyticsTestSuite to ensure
        // the various ObservationMode settings return the anticipated results.
        boost::shared_ptr<QuantLib::Observable> obs = QuantLib::Settings::instance().evaluationDate();
        obs->notifyObservers();
    }

    applyScenario(scenario);

    // Observation Mode - key to update these before fixings are set
    if (om == ObservationMode::Mode::Disable) {
        refresh();
        ObservableSettings::instance().enableUpdates();
    } else if (om == ObservationMode::Mode::Defer) {
        ObservableSettings::instance().enableUpdates();
    }

    // Apply fixings as historical fixings. Must do this before we populate ASD
    fixingManager_->update(d);

    if (asd_) {
        // add additional scenario data to the given container, if required
        for (auto i : parameters_->additionalScenarioDataIndices()) {
            boost::shared_ptr<QuantLib::Index> index;
            try {
                index = *iborIndex(i);
            } catch (...) {
            }
            try {
                index = *swapIndex(i);
            } catch (...) {
            }
            QL_REQUIRE(index != nullptr, "ScenarioSimMarket::update() index " << i << " not found in sim market");
            asd_->set(index->fixing(d), AggregationScenarioDataType::IndexFixing, i);
        }

        for (auto c : parameters_->additionalScenarioDataCcys()) {
            if (c != parameters_->baseCcy())
                asd_->set(fxSpot(c + parameters_->baseCcy())->value(), AggregationScenarioDataType::FXSpot, c);
        }

        asd_->set(numeraire_, AggregationScenarioDataType::Numeraire);

        asd_->next();
    }

    // DLOG("ScenarioSimMarket::update done");
}

bool ScenarioSimMarket::isSimulated(const RiskFactorKey::KeyType& factor) const {
    return std::find(nonSimulatedFactors_.begin(), nonSimulatedFactors_.end(), factor) == nonSimulatedFactors_.end();
}

} // namespace analytics
} // namespace ore


//          Copyright Gavin Band 2008 - 2012.
// Distributed under the Boost Software License, Version 1.0.
//    (See accompanying file LICENSE_1_0.txt or copy at
//          http://www.boost.org/LICENSE_1_0.txt)

#include <vector>
#include <boost/bind.hpp>
#include <Eigen/Core>
#include "genfile/VariantEntry.hpp"
#include "genfile/vcf/get_set_eigen.hpp"
#include "genfile/CohortIndividualSource.hpp"
#include "genfile/SNPDataSource.hpp"
#include "components/SNPSummaryComponent/SNPSummaryComputation.hpp"
#include "components/SNPSummaryComponent/CrossDataSetHaplotypeComparisonComputation.hpp"

namespace snp_stats {
	CrossDataSetHaplotypeComparisonComputation::CrossDataSetHaplotypeComparisonComputation(
		genfile::CohortIndividualSource const& samples,
		std::string const& main_dataset_sample_id_column
	):
		m_sample_mapper( samples, main_dataset_sample_id_column ),
		m_call_threshhold( 0.9 )
	{}

	void CrossDataSetHaplotypeComparisonComputation::set_comparer( genfile::VariantIdentifyingData::CompareFields const& comparer ) {
		m_comparer = comparer ;
	}

	void CrossDataSetHaplotypeComparisonComputation::set_match_alleles() {
		m_match_alleles = true ;
	}

	void CrossDataSetHaplotypeComparisonComputation::set_alternate_dataset(
		genfile::CohortIndividualSource::UniquePtr samples,
		std::string const& comparison_dataset_sample_id_column,
		genfile::SNPDataSource::UniquePtr snps
	) {
		m_sample_mapper.set_alternate_dataset( samples, comparison_dataset_sample_id_column ) ;
		m_alt_dataset_snps = snps ;
		m_relative_phase.setZero( m_sample_mapper.sample_mapping().size() ) ;
	}

	void CrossDataSetHaplotypeComparisonComputation::operator()(
		VariantIdentifyingData const& snp,
		Genotypes const&,
		Ploidy const&,
		genfile::VariantDataReader& data_reader,
		ResultCallback callback
	) {
		using genfile::string_utils::to_string ;
		VariantIdentifyingData alt_snp ;
		genfile::MissingValue const NA ;
		if( m_alt_dataset_snps->get_next_snp_matching( &alt_snp, snp, m_comparer )) {
			{
				genfile::vcf::PhasedGenotypeSetter< Eigen::MatrixXd > setter( m_haplotypes1, m_nonmissingness1 ) ;
				data_reader.get( ":genotypes:", setter ) ;
			}
			{
				double A_coding = 0 ;
				double B_coding = 1 ;
				bool matching_alleles = true ;
				if( m_match_alleles ) {
					if( alt_snp.get_allele(0) == snp.get_allele(0) && alt_snp.get_allele(1) == snp.get_allele(1) ) {
						A_coding = 0 ;
						B_coding = 1 ;
					}
					else if( alt_snp.get_allele(0) == snp.get_allele(1) && alt_snp.get_allele(1) == snp.get_allele(0) ) {
						A_coding = 1 ;
						B_coding = 0 ;
					} else {
						matching_alleles = false ;
					}
				}
				genfile::vcf::PhasedGenotypeSetter< Eigen::MatrixXd > setter( m_haplotypes2, m_nonmissingness2, 0, A_coding, B_coding ) ;
				m_alt_dataset_snps->read_variant_data()->get( ":genotypes:", setter ) ;

				if( !matching_alleles ) {
					callback( "comment", "Alleles in main and comparison datasets do not match." ) ;
					// alleles don't match, don't  bother doing any computation.
					return ;
				}
			}

			callback( "compared_variant_rsid", alt_snp.get_primary_id() ) ;
			callback( "compared_variant_alleleA", alt_snp.get_allele(0) ) ;
			callback( "compared_variant_alleleB", alt_snp.get_allele(1) ) ;

			callback( "pairwise_non-missing_haplotypes", NA ) ;
			callback( "pairwise_concordant_haplotypes", NA ) ;
			callback( "concordant_heterozygous_haplotypes", NA ) ;
			callback( "concordant_heterozygous_haplotypes_with_switch_error", NA ) ;
			// Threshhold the calls and set to missing any rows not meeting the
			// threshhold.
			m_haplotypes1.array() *= ( m_haplotypes1.array() >= m_call_threshhold ).cast< double >() ;
			m_haplotypes2.array() *= ( m_haplotypes2.array() >= m_call_threshhold ).cast< double >() ;

			// FIX MISSINGNESS HERE

			CrossDataSetSampleMapper::SampleMapping::const_iterator i = m_sample_mapper.sample_mapping().begin() ;
			CrossDataSetSampleMapper::SampleMapping::const_iterator const end_i = m_sample_mapper.sample_mapping().end() ;
			int call_count = 0 ;
			int concordant_call_count = 0 ;
			int het_call_count = 0 ;
			int switch_error_count = 0 ;
			for( std::size_t count = 0; i != end_i; ++i, ++count ) {
				std::string const stub = (
					m_sample_mapper.dataset1_sample_ids()[ i->first ].as< std::string >()
					+ "("
					+ to_string( i->first + 1 )
					+ "~"
					+ to_string( i->second + 1 )
					+ ")"
				) ;

				if( m_nonmissingness1.row( i->first ).sum() == 2 && m_nonmissingness2.row( i->second ).sum() == 2 ) {
					++call_count ;

					Eigen::MatrixXd::RowXpr const& h1 = m_haplotypes1.row( i->first ) ;
					Eigen::MatrixXd::RowXpr const& h2 = m_haplotypes2.row( i->second ) ;
					int const concordant = ( h1.sum() == h2.sum() ) ;
					callback( stub + ":concordance", concordant ) ;
					concordant_call_count += concordant ;

					int const heterozygote = ( h1.sum() == 1 ) ;
					het_call_count += heterozygote ;
					if( concordant && heterozygote ) {

						int const relative_phase = ( h1 == h2 ) ? 1 : -1 ;
						int const switch_error = ( m_relative_phase( count ) != 0 ) && ( m_relative_phase( count ) != relative_phase ) ;
						callback( stub + ":switch_error", switch_error ) ;
						// take account of the switch for the next SNP.
						m_relative_phase( count ) = relative_phase ;
						switch_error_count += switch_error ;
					} else {
						callback( stub + ":switch_error", genfile::MissingValue() ) ;
					}
				} else {
					callback( stub + ":concordance", genfile::MissingValue() ) ;
					callback( stub + ":switch_error", genfile::MissingValue() ) ;
				}
			}
			callback( "pairwise_non-missing_haplotypes", call_count ) ;
			callback( "pairwise_concordant_haplotypes", concordant_call_count ) ;
			callback( "concordant_heterozygous_haplotypes", het_call_count ) ;
			callback( "concordant_heterozygous_haplotypes_with_switch_error", switch_error_count ) ;
		}
	}

	std::string CrossDataSetHaplotypeComparisonComputation::get_summary( std::string const& prefix, std::size_t column_width ) const {
		return prefix + "CrossDataSetHaplotypeComparisonComputation" ;
	}
}


#include <avalon/physics/utils.h>

#include <boost/any.hpp>
#include <avalon/physics/Box2dContainer.h>

namespace {

std::shared_ptr<b2PolygonShape> initRectangleShape(float width, float height, float pixelsInMeter)
{
    auto shape = std::make_shared<b2PolygonShape>();
    shape->SetAsBox((width / pixelsInMeter) * 0.5f, (height / pixelsInMeter) * 0.5f);
    return shape;
}

std::shared_ptr<b2ChainShape> initChainShape(const cocos2d::ValueVector& points, float pixelsInMeter, bool loop = false)
{
    auto convert = [&pixelsInMeter](const cocos2d::Value& value) -> b2Vec2 {
        return {value.asValueMap().at("x").asFloat() / pixelsInMeter, (value.asValueMap().at("y").asFloat() / pixelsInMeter) * -1};
    };

    std::vector<b2Vec2> vecs;
    vecs.reserve(points.size());
    std::transform(points.begin(), points.end(), vecs.begin(), convert);

    auto shape = std::make_shared<b2ChainShape>();
    if (loop) {
        shape->CreateLoop(&vecs[0], points.size());
    } else {
        shape->CreateChain(&vecs[0], points.size());
    }

    return shape;
}

std::shared_ptr<b2EdgeShape> initEdgeShape(cocos2d::Point p1, cocos2d::Point p2, float pixelsInMeter)
{
    p1 = p1 / pixelsInMeter;
    p2 = p2 / pixelsInMeter;
    auto shape = std::make_shared<b2EdgeShape>();
    shape->Set({p1.x, p1.y}, {p2.x, p2.y});
    return shape;
}

std::shared_ptr<b2Shape> initShapeFromPoints(const cocos2d::ValueVector& points, float pixelsInMeter, bool loop = false)
{
    if (points.size() == 2) {
        cocos2d::Point p1 {points.front().asValueMap().at("x").asFloat(), points.front().asValueMap().at("y").asFloat()};
        cocos2d::Point p2 {points.back().asValueMap().at("x").asFloat(), points.back().asValueMap().at("y").asFloat()};
        return initEdgeShape(p1, p2, pixelsInMeter);
    } else {
        return initChainShape(points, pixelsInMeter, loop);
    }
}

} // namespace

namespace avalon {
namespace physics {
namespace utils {

b2BodyType getBodyTypeFromString(const std::string& type)
{
    if      (type == "static")    return b2_staticBody;
    else if (type == "dynamic")   return b2_dynamicBody;
    else if (type == "kinematic") return b2_kinematicBody;
    else                          throw new std::invalid_argument("Unknown box2d type");
}

avalon::io::TiledMapLoader::Callback rectLoader(b2Filter filter, bool isSensor, float rectWidth, float rectHeight)
{
    return [filter, isSensor, rectWidth, rectHeight](const avalon::io::TiledMapLoader::Configuration& config)
    {
        auto rWidth = rectWidth;
        auto rHeight = rectHeight;

        if (rWidth == 0) rWidth = config.map.getTileSize().width;

        if (rHeight == 0) rHeight = config.map.getTileSize().height;

        auto fixtureDef = config.box2dContainer->defaultFixtureDef;

        fixtureDef.isSensor = isSensor;
        fixtureDef.filter = filter;

        float x = config.settings.at("x").asFloat();
        float y = config.settings.at("y").asFloat();
        const float width = rWidth / config.box2dContainer->pixelsInMeter;
        const float height = rHeight / config.box2dContainer->pixelsInMeter;

        y = (config.map.getMapSize().height - (y));

        x = (x + 0.5) * config.map.getTileSize().width;
        y = y * config.map.getTileSize().height - config.map.getTileSize().height / 2.f;

        const auto pos = config.box2dContainer->convertToBox2d({static_cast<float>(x), static_cast<float>(y)});

        std::shared_ptr<b2PolygonShape> polygonShape = std::make_shared<b2PolygonShape>();
        polygonShape->SetAsBox(width / 2.f, height / 2.f);

        b2BodyDef bodyDef;
        bodyDef.type = b2_staticBody;
        bodyDef.position.Set(pos.x, pos.y);

        fixtureDef.shape = polygonShape.get();

        auto body = config.box2dContainer->getWorld().CreateBody(&bodyDef);
        body->CreateFixture(&fixtureDef);
    };
}

avalon::io::TiledMapLoader::Callback shapeLoader(int filterCategory, bool isSensor)
{
    return [filterCategory, isSensor](const avalon::io::TiledMapLoader::Configuration& config)
    {
        const float x = config.settings.at("x").asFloat();
        const float y = config.settings.at("y").asFloat();
        const float width = config.settings.at("width").asFloat();
        const float height = config.settings.at("height").asFloat();
        const float pixelsInMeter = config.box2dContainer->pixelsInMeter;
        const auto pos = config.box2dContainer->convertToBox2d({static_cast<float>(x + width * 0.5), static_cast<float>(y + height * 0.5)});

        auto fixtureDef = config.box2dContainer->defaultFixtureDef;
        fixtureDef.isSensor = isSensor;
        fixtureDef.filter.categoryBits = filterCategory;

        std::string bodytype = "static";
        if (config.settings.count("friction"))          fixtureDef.friction = config.settings.at("friction").asFloat();
        if (config.settings.count("density"))           fixtureDef.density = config.settings.at("density").asFloat();
        if (config.settings.count("restitution"))       fixtureDef.restitution = config.settings.at("restitution").asFloat();
        if (config.settings.count("bodytype"))          bodytype = config.settings.at("bodytype").asString();
        if (config.settings.count("filterCategory"))    fixtureDef.filter.categoryBits = config.settings.at("filterCategory").asInt();

        std::shared_ptr<b2Shape> shape;
        if (config.settings.count("polylinePoints")) {
            auto points = config.settings.at("polylinePoints").asValueVector();
            shape = initShapeFromPoints(points, pixelsInMeter);
        } else if (config.settings.count("points") > 0) {
            auto points = config.settings.at("points").asValueVector();
            shape = initShapeFromPoints(points, pixelsInMeter, true);
        } else {
            cocos2d::log("%f,%f,%f,%f",x,y,width,height);
            shape = initRectangleShape(width, height, pixelsInMeter);
        }
        fixtureDef.shape = shape.get();

        b2BodyDef bodyDef;
        bodyDef.type = getBodyTypeFromString(bodytype);
        bodyDef.position.Set(pos.x, pos.y);

        auto body = config.box2dContainer->getWorld().CreateBody(&bodyDef);
        body->CreateFixture(&fixtureDef);
    };
}

} // namespace utils
} // namespace physics
} // namespace avalon


#include "FWCore/ServiceRegistry/interface/Service.h"
#include "CondCore/DBOutputService/interface/PoolDBOutputService.h"
#include "CondFormats/Calibration/interface/mySiStripNoises.h"

#include <boost/random/linear_congruential.hpp>
#include <boost/random/uniform_real.hpp>
#include <boost/random/variate_generator.hpp>

#include "writeBlob.h"

typedef boost::minstd_rand base_generator_type;
writeBlob::writeBlob(const edm::ParameterSet& iConfig):
  m_StripRecordName("mySiStripNoisesRcd")
{
}

writeBlob::~writeBlob()
{
  std::cout<<"writeBlob::writeBlob"<<std::endl;
}

void
writeBlob::analyze( const edm::Event& evt, const edm::EventSetup& evtSetup)
{
  std::cout<<"writeBlob::analyze "<<std::endl;
  base_generator_type rng(42u);
  boost::uniform_real<> uni_dist(0,1);
  boost::variate_generator<base_generator_type&, boost::uniform_real<> > uni(rng, uni_dist);
  edm::Service<cond::service::PoolDBOutputService> mydbservice;
  if( !mydbservice.isAvailable() ){
    std::cout<<"db service unavailable"<<std::endl;
    return;
  }
  try{
    mySiStripNoises* me = new mySiStripNoises;
    unsigned int detidseed=1234;
    unsigned int bsize=100;
    unsigned int nAPV=2;
    for (uint32_t detid=detidseed;detid<(detidseed+bsize);detid++){
      //Generate Noise for det detid
      std::vector<short> theSiStripVector;
      for(unsigned int strip=0; strip<128*nAPV; ++strip){
	float noise = uni();;
	me->setData(noise,theSiStripVector);
      }
      me->put(detid,theSiStripVector);
    }

    mydbservice->writeOne(me,mydbservice->currentTime(),m_StripRecordName);
  }catch(const cond::Exception& er){
    throw cms::Exception("DBOutputServiceUnitTestFailure","failed writeBlob",er);
    //std::cout<<er.what()<<std::endl;
  }catch(const cms::Exception& er){
    throw cms::Exception("DBOutputServiceUnitTestFailure","failed writeBlob",er);
  }/*catch(const std::exception& er){
    std::cout<<"caught std::exception "<<er.what()<<std::endl;
  }catch(...){
    std::cout<<"Funny error"<<std::endl;
    }*/
}
#include "FWCore/Framework/interface/MakerMacros.h"
DEFINE_FWK_MODULE(writeBlob);