/*
Copyright (C) 2018 Quaternion Risk Management Ltd
All rights reserved.

This file is part of ORE, a free-software/open-source library
for transparent pricing and risk analysis - http://opensourcerisk.org

ORE is free software: you can redistribute it and/or modify it
under the terms of the Modified BSD License.  You should have received a
copy of the license along with this program.
The license is also available online at <http://opensourcerisk.org>

This program is distributed on the basis that it will form a useful
contribution to risk analytics and model standardisation, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the license for more details.
*/

#include <algorithm>
#include <boost/algorithm/string/join.hpp>
#include <boost/algorithm/string/replace.hpp>
#include <boost/range/adaptor/indexed.hpp>
#include <boost/range/adaptor/transformed.hpp>
#include <ored/marketdata/commodityvolcurve.hpp>
#include <ored/utilities/conventionsbasedfutureexpiry.hpp>
#include <ored/utilities/indexparser.hpp>
#include <ored/utilities/log.hpp>
#include <ored/utilities/parsers.hpp>
#include <ored/utilities/to_string.hpp>
#include <ored/utilities/wildcard.hpp>
#include <ql/math/interpolations/bicubicsplineinterpolation.hpp>
#include <ql/math/interpolations/loginterpolation.hpp>
#include <ql/math/matrix.hpp>
#include <ql/termstructures/volatility/equityfx/blackconstantvol.hpp>
#include <ql/termstructures/volatility/equityfx/blackvariancecurve.hpp>
#include <ql/termstructures/volatility/equityfx/blackvariancesurface.hpp>
#include <ql/time/calendars/weekendsonly.hpp>
#include <qle/indexes/commodityindex.hpp>
#include <qle/math/flatextrapolation.hpp>
#include <qle/termstructures/aposurface.hpp>
#include <qle/termstructures/blackvariancesurfacemoneyness.hpp>
#include <qle/termstructures/blackvariancesurfacesparse.hpp>
#include <qle/termstructures/blackvolsurfacedelta.hpp>
#include <qle/termstructures/eqcommoptionsurfacestripper.hpp>
#include <qle/termstructures/pricetermstructureadapter.hpp>

using namespace std;
using namespace QuantLib;
using namespace QuantExt;

namespace {

// Utility struct to represent (expiry, strike) pair when storing volatilities below in map.
struct ExpiryStrike {
    ExpiryStrike(const Date& exp = Date(), Real stk = Null<Real>()) : expiry(exp), strike(stk) {}
    Date expiry;
    Real strike;
};

ostream& operator<<(ostream& os, const ExpiryStrike& es) {
    return os << "(" << io::iso_date(es.expiry) << "," << es.strike << ")";
}

// Comparator so that ExpiryStrike may be used as a map key.
struct ExpiryStrikeComp {
    bool operator() (const ExpiryStrike& lhs, const ExpiryStrike& rhs) const {
        return (lhs.expiry < rhs.expiry) || (!(rhs.expiry < lhs.expiry) &&
            !close(lhs.strike, rhs.strike) && (lhs.strike < rhs.strike));
    }
};

// Struct that stores a call data point (price or vol) and a put data point (price or vol).
// Null<Real>() indicates the absence of the data point.
struct CallPutDatum {
    CallPutDatum() : call(Null<Real>()), put(Null<Real>()) {}

    CallPutDatum(Option::Type optionType, Real value)
        : call(Null<Real>()), put(Null<Real>()) {
        if (optionType == Option::Call) {
            call = value;
        } else {
            put = value;
        }
    }

    Real call;
    Real put;
};

// Container that stores all the relevant points.
struct CallPutData {

    void addDatum(ExpiryStrike node, Option::Type optionType, Real value) {

        auto it = data.find(node);
        if (it != data.end()) {
            // Already have call or put data for the (expiry, strike)
            CallPutDatum& cpd = it->second;
            Real& toUpdate = optionType == Option::Call ? cpd.call : cpd.put;
            if (toUpdate == Null<Real>()) {
                toUpdate = value;
                TLOG("Updated " << optionType << " option data point with value " << fixed <<
                    setprecision(9) << value << " for expiry strike pair " << node << ".");
            } else {
                TLOG("Expiry strike pair " << node << " already has " << optionType << " option data " <<
                    "point with value " << fixed << setprecision(9) << toUpdate <<
                    " so did not update with value " << value << ".");
            }
        } else {
            // Have no call or put data for the (expiry, strike)
            data[node] = CallPutDatum(optionType, value);
            TLOG("Added " << optionType << " option data point with value " << fixed << setprecision(9) <<
                value << " for expiry strike pair " << node << ".");
        }
    }

    map<ExpiryStrike, CallPutDatum, ExpiryStrikeComp> data;
};

// Return the relevant surface using the data
boost::shared_ptr<OptionPriceSurface> optPriceSurface(const CallPutData& cpData,
    const Date& asof, const DayCounter& dc, bool forCall) {

    DLOG("Creating " << (forCall ? "Call" : "Put") << " option price surface.");

    const auto& priceData = cpData.data;
    auto n = priceData.size();

    vector<Date> expiries; expiries.reserve(n);
    vector<Real> strikes; strikes.reserve(n);
    vector<Real> prices; prices.reserve(n);

    for (const auto& kv : cpData.data) {

        if ((forCall && kv.second.call != Null<Real>()) || (!forCall && kv.second.put != Null<Real>())) {

            expiries.push_back(kv.first.expiry);
            strikes.push_back(kv.first.strike);
            prices.push_back(forCall ? kv.second.call : kv.second.put);

            TLOG("Using option datum (" << (forCall ? "Call" : "Put") << "," << io::iso_date(expiries.back()) <<
                "," << fixed << setprecision(9) << strikes.back() << "," << prices.back() << ")");
        }
    }

    // This is not enough but OptionPriceSurface should throw if data is not sufficient.
    QL_REQUIRE(!prices.empty(), "Need at least one point for " << (forCall ? "Call" : "Put") <<
        " commodity option price surface.");

    return boost::make_shared<OptionPriceSurface>(asof, expiries, strikes, prices, dc);
}

}

namespace ore {
namespace data {

CommodityVolCurve::CommodityVolCurve(const Date& asof, const CommodityVolatilityCurveSpec& spec, const Loader& loader,
                                     const CurveConfigurations& curveConfigs, const Conventions& conventions,
                                     const map<string, boost::shared_ptr<YieldCurve>>& yieldCurves,
                                     const map<string, boost::shared_ptr<CommodityCurve>>& commodityCurves,
                                     const map<string, boost::shared_ptr<CommodityVolCurve>>& commodityVolCurves) {

    try {
        LOG("CommodityVolCurve: start building commodity volatility structure with ID " << spec.curveConfigID());

        auto config = *curveConfigs.commodityVolatilityConfig(spec.curveConfigID());

        if (!config.futureConventionsId().empty()) {
            const auto& cId = config.futureConventionsId();
            QL_REQUIRE(conventions.has(cId), "Conventions, " << cId <<
                " for config " << config.curveID() << " not found.");
            convention_ = boost::dynamic_pointer_cast<CommodityFutureConvention>(conventions.get(cId));
            QL_REQUIRE(convention_, "Convention with ID '" << cId << "' should be of type CommodityFutureConvention");
            expCalc_ = boost::make_shared<ConventionsBasedFutureExpiry>(*convention_);
        }

        calendar_ = parseCalendar(config.calendar());
        dayCounter_ = parseDayCounter(config.dayCounter());

        // Do different things depending on the type of volatility configured
        boost::shared_ptr<VolatilityConfig> vc = config.volatilityConfig();
        if (auto cvc = boost::dynamic_pointer_cast<ConstantVolatilityConfig>(vc)) {
            buildVolatility(asof, config, *cvc, loader);
        } else if (auto vcc = boost::dynamic_pointer_cast<VolatilityCurveConfig>(vc)) {
            buildVolatility(asof, config, *vcc, loader);
        } else if (auto vssc = boost::dynamic_pointer_cast<VolatilityStrikeSurfaceConfig>(vc)) {
            // Try to populate the price and yield term structure. Need them in some cases here.
            populateCurves(config, yieldCurves, commodityCurves, true, true);
            buildVolatility(asof, config, *vssc, loader);
        } else if (auto vdsc = boost::dynamic_pointer_cast<VolatilityDeltaSurfaceConfig>(vc)) {
            // Need a yield curve and price curve to create a delta surface.
            populateCurves(config, yieldCurves, commodityCurves, true);
            buildVolatility(asof, config, *vdsc, loader);
        } else if (auto vmsc = boost::dynamic_pointer_cast<VolatilityMoneynessSurfaceConfig>(vc)) {
            // Need a yield curve (if forward moneyness) and price curve to create a moneyness surface.
            MoneynessStrike::Type moneynessType = parseMoneynessType(vmsc->moneynessType());
            bool fwdMoneyness = moneynessType == MoneynessStrike::Type::Forward;
            populateCurves(config, yieldCurves, commodityCurves, fwdMoneyness);
            buildVolatility(asof, config, *vmsc, loader);
        } else if (auto vapo = boost::dynamic_pointer_cast<VolatilityApoFutureSurfaceConfig>(vc)) {

            // Get the base conventions and create the associated expiry calculator.
            QL_REQUIRE(!vapo->baseConventionsId().empty(),
                       "The APO FutureConventions must be populated to build a future APO surface");
            QL_REQUIRE(conventions.has(vapo->baseConventionsId()), "Conventions, " << vapo->baseConventionsId()
                                                                                << " for config " << config.curveID()
                                                                                << " not found.");
            auto convention =
                boost::dynamic_pointer_cast<CommodityFutureConvention>(conventions.get(vapo->baseConventionsId()));
            QL_REQUIRE(convention, "Convention with ID '" << config.futureConventionsId()
                                                          << "' should be of type CommodityFutureConvention");
            auto baseExpCalc = boost::make_shared<ConventionsBasedFutureExpiry>(*convention);

            // Need to get the base commodity volatility structure.
            QL_REQUIRE(!vapo->baseVolatilityId().empty(),
                       "The APO VolatilityId must be populated to build a future APO surface.");
            auto itVs = commodityVolCurves.find(vapo->baseVolatilityId());
            QL_REQUIRE(itVs != commodityVolCurves.end(),
                       "Can't find commodity volatility with id " << vapo->baseVolatilityId());
            auto baseVs = Handle<BlackVolTermStructure>(itVs->second->volatility());

            // Need to get the base price curve
            QL_REQUIRE(!vapo->basePriceCurveId().empty(),
                       "The APO PriceCurveId must be populated to build a future APO surface.");
            auto itPts = commodityCurves.find(vapo->basePriceCurveId());
            QL_REQUIRE(itPts != commodityCurves.end(), "Can't find price curve with id " << vapo->basePriceCurveId());
            auto basePts = Handle<PriceTermStructure>(itPts->second->commodityPriceCurve());

            // Need a yield curve and price curve to create an APO surface.
            populateCurves(config, yieldCurves, commodityCurves, true);

            buildVolatility(asof, config, *vapo, baseVs, basePts, conventions);

        } else {
            QL_FAIL("Unexpected VolatilityConfig in CommodityVolatilityConfig");
        }

        LOG("CommodityVolCurve: finished building commodity volatility structure with ID " << spec.curveConfigID());

    } catch (exception& e) {
        QL_FAIL("Commodity volatility curve building failed : " << e.what());
    } catch (...) {
        QL_FAIL("Commodity volatility curve building failed: unknown error");
    }
}

void CommodityVolCurve::buildVolatility(const Date& asof, const CommodityVolatilityConfig& vc,
                                        const ConstantVolatilityConfig& cvc, const Loader& loader) {

    LOG("CommodityVolCurve: start building constant volatility structure");

    // Loop over all market datums and find the single quote
    // Return error if there are duplicates (this is why we do not use loader.get() method)
    Real quoteValue = Null<Real>();
    for (const boost::shared_ptr<MarketDatum>& md : loader.loadQuotes(asof)) {
        if (md->asofDate() == asof && md->instrumentType() == MarketDatum::InstrumentType::COMMODITY_OPTION) {

            boost::shared_ptr<CommodityOptionQuote> q = boost::dynamic_pointer_cast<CommodityOptionQuote>(md);

            if (q->name() == cvc.quote()) {
                TLOG("Found the constant volatility quote " << q->name());
                QL_REQUIRE(quoteValue == Null<Real>(), "Duplicate quote found for quote with id " << cvc.quote());
                quoteValue = q->quote()->value();
            }
        }
    }
    QL_REQUIRE(quoteValue != Null<Real>(), "Quote not found for id " << cvc.quote());

    DLOG("Creating BlackConstantVol structure");
    volatility_ = boost::make_shared<BlackConstantVol>(asof, calendar_, quoteValue, dayCounter_);

    LOG("CommodityVolCurve: finished building constant volatility structure");
}

void CommodityVolCurve::buildVolatility(const QuantLib::Date& asof, const CommodityVolatilityConfig& vc,
                                        const VolatilityCurveConfig& vcc, const Loader& loader) {

    LOG("CommodityVolCurve: start building 1-D volatility curve");

    // Must have at least one quote
    QL_REQUIRE(vcc.quotes().size() > 0, "No quotes specified in config " << vc.curveID());

    QL_REQUIRE(vcc.quoteType() == MarketDatum::QuoteType::RATE_LNVOL, "CommodityVolCurve: only quote type" <<
        " RATE_LNVOL is currently supported for 1-D commodity volatility curves.");

    // Check if we are using a regular expression to select the quotes for the curve. If we are, the quotes should
    // contain exactly one element.
    auto wildcard = getUniqueWildcard(vcc.quotes());

    // curveData will be populated with the expiry dates and volatility values.
    map<Date, Real> curveData;

    // Different approaches depending on whether we are using a regex or searching for a list of explicit quotes.
    if (wildcard) {

        DLOG("Have single quote with pattern " << (*wildcard).regex());

        // Loop over quotes and process commodity option quotes matching pattern on asof
        for (const boost::shared_ptr<MarketDatum>& md : loader.loadQuotes(asof)) {

            // Go to next quote if the market data point's date does not equal our asof
            if (md->asofDate() != asof)
                continue;

            auto q = boost::dynamic_pointer_cast<CommodityOptionQuote>(md);
            if (q && (*wildcard).matches(q->name())) {

                TLOG("The quote " << q->name() << " matched the pattern");

                Date expiryDate = getExpiry(asof, q->expiry(), vc.futureConventionsId(), vc.optionExpiryRollDays());
                if (expiryDate > asof) {
                    // Add the quote to the curve data
                    auto it = curveData.find(expiryDate);
                    if (it != curveData.end()) {
                        LOG("Already added quote " << fixed << setprecision(9) << it->second << " for the expiry" <<
                            " date " << io::iso_date(expiryDate) << " for commodity curve " << vc.curveID() <<
                            " so not adding " << q->name());
                    } else {
                        curveData[expiryDate] = q->quote()->value();
                        TLOG("Added quote " << q->name() << ": (" << io::iso_date(expiryDate) << "," <<
                            fixed << setprecision(9) << q->quote()->value() << ")");
                    }
                }
            }
        }

        // Check that we have quotes in the end
        QL_REQUIRE(curveData.size() > 0, "No quotes found matching regular expression " << vcc.quotes()[0]);

    } else {

        DLOG("Have " << vcc.quotes().size() << " explicit quotes");

        // Loop over quotes and process commodity option quotes that are explicitly specified in the config
        for (const boost::shared_ptr<MarketDatum>& md : loader.loadQuotes(asof)) {

            // Go to next quote if the market data point's date does not equal our asof
            if (md->asofDate() != asof)
                continue;

            if (auto q = boost::dynamic_pointer_cast<CommodityOptionQuote>(md)) {

                // Find quote name in configured quotes.
                auto it = find(vcc.quotes().begin(), vcc.quotes().end(), q->name());

                if (it != vcc.quotes().end()) {
                    TLOG("Found the configured quote " << q->name());

                    Date expiryDate = getExpiry(asof, q->expiry(), vc.futureConventionsId(), vc.optionExpiryRollDays());
                    QL_REQUIRE(expiryDate > asof, "Commodity volatility quote '" << q->name()
                                                                                << "' has expiry in the past ("
                                                                                << io::iso_date(expiryDate) << ")");
                    QL_REQUIRE(curveData.count(expiryDate) == 0, "Duplicate quote for the date "
                                                                     << io::iso_date(expiryDate)
                                                                     << " provided by commodity volatility config "
                                                                     << vc.curveID());
                    curveData[expiryDate] = q->quote()->value();

                    TLOG("Added quote " << q->name() << ": (" << io::iso_date(expiryDate) << "," << fixed
                                        << setprecision(9) << q->quote()->value() << ")");
                }
            }
        }

        // Check that we have found all of the explicitly configured quotes
        QL_REQUIRE(curveData.size() == vcc.quotes().size(), "Found " << curveData.size() << " quotes, but "
                                                                     << vcc.quotes().size()
                                                                     << " quotes were given in config.");
    }

    // Create the dates and volatility vector
    vector<Date> dates;
    vector<Volatility> volatilities;
    for (const auto& datum : curveData) {
        dates.push_back(datum.first);
        volatilities.push_back(datum.second);
        TLOG("Added data point (" << io::iso_date(dates.back()) << "," << fixed << setprecision(9)
                                  << volatilities.back() << ")");
    }

    DLOG("Creating BlackVarianceCurve object.");
    auto tmp = boost::make_shared<BlackVarianceCurve>(asof, dates, volatilities, dayCounter_);

    // Set the interpolation.
    if (vcc.interpolation() == "Linear") {
        DLOG("Interpolation set to Linear.");
    } else if (vcc.interpolation() == "Cubic") {
        DLOG("Setting interpolation to Cubic.");
        tmp->setInterpolation<Cubic>();
    } else if (vcc.interpolation() == "LogLinear") {
        DLOG("Setting interpolation to LogLinear.");
        tmp->setInterpolation<LogLinear>();
    } else {
        DLOG("Interpolation " << vcc.interpolation() << " not recognised so leaving it Linear.");
    }

    // Set the volatility_ member after we have possibly updated the interpolation.
    volatility_ = tmp;

    // Set the extrapolation
    if (parseExtrapolation(vcc.extrapolation()) == Extrapolation::Flat) {
        DLOG("Enabling BlackVarianceCurve flat volatility extrapolation.");
        volatility_->enableExtrapolation();
    } else if (parseExtrapolation(vcc.extrapolation()) == Extrapolation::None) {
        DLOG("Disabling BlackVarianceCurve extrapolation.");
        volatility_->disableExtrapolation();
    } else if (parseExtrapolation(vcc.extrapolation()) == Extrapolation::UseInterpolator) {
        DLOG("BlackVarianceCurve does not support using interpolator for extrapolation "
             << "so default to flat volatility extrapolation.");
        volatility_->enableExtrapolation();
    } else {
        DLOG("Unexpected extrapolation so default to flat volatility extrapolation.");
        volatility_->enableExtrapolation();
    }

    LOG("CommodityVolCurve: finished building 1-D volatility curve");
}

void CommodityVolCurve::buildVolatility(const Date& asof, CommodityVolatilityConfig& vc,
                                        const VolatilityStrikeSurfaceConfig& vssc, const Loader& loader) {

    LOG("CommodityVolCurve: start building 2-D volatility absolute strike surface");

    // We are building a commodity volatility surface here of the form expiry vs strike where the strikes are absolute
    // numbers. The list of expiries may be explicit or contain a single wildcard character '*'. Similarly, the list
    // of strikes may be explicit or contain a single wildcard character '*'. So, we have four options here which
    // ultimately lead to two different types of surface being built:
    // 1. explicit strikes and explicit expiries => BlackVarianceSurface
    // 2. wildcard strikes and/or wildcard expiries (3 combinations) => BlackVarianceSurfaceSparse

    bool expWc = false;
    if (find(vssc.expiries().begin(), vssc.expiries().end(), "*") != vssc.expiries().end()) {
        expWc = true;
        QL_REQUIRE(vssc.expiries().size() == 1, "Wild card expiry specified but more expiries also specified.");
        DLOG("Have expiry wildcard pattern " << vssc.expiries()[0]);
    }

    bool strkWc = false;
    if (find(vssc.strikes().begin(), vssc.strikes().end(), "*") != vssc.strikes().end()) {
        strkWc = true;
        QL_REQUIRE(vssc.strikes().size() == 1, "Wild card strike specified but more strikes also specified.");
        DLOG("Have strike wildcard pattern " << vssc.strikes()[0]);
    }

    // If we do not have a strike wild card, we expect a list of absolute strike values
    vector<Real> configuredStrikes;
    if (!strkWc) {
        // Parse the list of absolute strikes
        configuredStrikes = parseVectorOfValues<Real>(vssc.strikes(), &parseReal);
        sort(configuredStrikes.begin(), configuredStrikes.end(), [](Real x, Real y) { return !close(x, y) && x < y; });
        QL_REQUIRE(adjacent_find(configuredStrikes.begin(), configuredStrikes.end(),
                                 [](Real x, Real y) { return close(x, y); }) == configuredStrikes.end(),
                   "The configured strikes contain duplicates");
        DLOG("Parsed " << configuredStrikes.size() << " unique configured absolute strikes");
    }

    // If we do not have an expiry wild card, parse the configured expiries.
    vector<boost::shared_ptr<Expiry>> configuredExpiries;
    if (!expWc) {
        // Parse the list of expiry strings.
        for (const string& strExpiry : vssc.expiries()) {
            configuredExpiries.push_back(parseExpiry(strExpiry));
        }
        DLOG("Parsed " << configuredExpiries.size() << " unique configured expiries");
    }

    // If there are no wildcard strikes or wildcard expiries, delegate to buildVolatilityExplicit.
    if (!expWc && !strkWc) {
        buildVolatilityExplicit(asof, vc, vssc, loader, configuredStrikes);
        return;
    }

    DLOG("Expiries and or strikes have been configured via wildcards so building a "
         << "wildcard based absolute strike surface");

    // Store grid points along with call and/or put data point
    CallPutData cpData;

    // If the price term structure is not empty, populate the map with (expiry, forward) pairs
    // that will be used below when choosing between a call and a put if have volatilities.
    map<Date, Real> fwdCurve;

    // Loop over quotes and process any commodity option quote that matches a wildcard
    for (const boost::shared_ptr<MarketDatum>& md : loader.loadQuotes(asof)) {

        // Go to next quote if the market data point's date does not equal our asof.
        if (md->asofDate() != asof)
            continue;

        // Go to next quote if not a commodity option quote.
        auto q = boost::dynamic_pointer_cast<CommodityOptionQuote>(md);
        if (!q)
            continue;

        // Go to next quote if not a commodity name or currency or quote type does not match config.
        if (vc.curveID() != q->commodityName() || vc.currency() != q->quoteCurrency() ||
            vssc.quoteType() != q->quoteType())
            continue;

        // This surface is for absolute strikes only.
        auto strike = boost::dynamic_pointer_cast<AbsoluteStrike>(q->strike());
        if (!strike)
            continue;

        // If we have been given a list of explicit expiries, check that the quote matches one of them.
        // Move to the next quote if it does not.
        if (!expWc) {
            auto expiryIt = find_if(configuredExpiries.begin(), configuredExpiries.end(),
                                    [&q](boost::shared_ptr<Expiry> e) { return *e == *q->expiry(); });
            if (expiryIt == configuredExpiries.end())
                continue;
        }

        // If we have been given a list of explicit strikes, check that the quote matches one of them.
        // Move to the next quote if it does not.
        if (!strkWc) {
            auto strikeIt = find_if(configuredStrikes.begin(), configuredStrikes.end(),
                                    [&strike](Real s) { return close(s, strike->strike()); });
            if (strikeIt == configuredStrikes.end())
                continue;
        }

        // Store in the data container.
        Date expiry = getExpiry(asof, q->expiry(), vc.futureConventionsId(), vc.optionExpiryRollDays());
        ExpiryStrike node(expiry, strike->strike());
        cpData.addDatum(ExpiryStrike(expiry, strike->strike()), q->optionType(), q->quote()->value());

        // If we have a price term structure, add forward for expiry if not there already.
        if (vssc.quoteType() == MarketDatum::QuoteType::RATE_LNVOL && !pts_.empty() && fwdCurve.count(expiry) == 0) {
            fwdCurve[expiry] = pts_->price(expiry);
        }

        TLOG("Added quote " << q->name() << " to intermediate data.");
    }

    // Set the strike extrapolation which only matters if extrapolation is turned on for the whole surface.
    bool flatStrikeExtrap = true;
    bool flatTimeExtrap = true;
    if (vssc.extrapolation()) {

        auto strikeExtrapType = parseExtrapolation(vssc.strikeExtrapolation());
        if (strikeExtrapType == Extrapolation::UseInterpolator) {
            DLOG("Strike extrapolation switched to using interpolator.");
            flatStrikeExtrap = false;
        } else if (strikeExtrapType == Extrapolation::None) {
            DLOG("Strike extrapolation cannot be turned off on its own so defaulting to flat.");
        } else if (strikeExtrapType == Extrapolation::Flat) {
            DLOG("Strike extrapolation has been set to flat.");
        } else {
            DLOG("Strike extrapolation " << strikeExtrapType << " not expected so default to flat.");
        }

        auto timeExtrapType = parseExtrapolation(vssc.timeExtrapolation());
        if (timeExtrapType == Extrapolation::UseInterpolator) {
            DLOG("Time extrapolation switched to using interpolator.");
            flatTimeExtrap = false;
        } else if (timeExtrapType == Extrapolation::None) {
            DLOG("Time extrapolation cannot be turned off on its own so defaulting to flat.");
        } else if (timeExtrapType == Extrapolation::Flat) {
            DLOG("Time extrapolation has been set to flat.");
        } else {
            DLOG("Time extrapolation " << timeExtrapType << " not expected so default to flat.");
        }

    } else {
        DLOG("Extrapolation is turned off for the whole surface so the time and"
             << " strike extrapolation settings are ignored");
    }

    // Determine which option type to pick at each expiry and strike.
    bool preferOutOfTheMoney = !vc.preferOutOfTheMoney() ? true : *vc.preferOutOfTheMoney();
    TLOG("Prefer out of the money set to: " << ore::data::to_string(preferOutOfTheMoney) << ".");

    // Build the surface depending on the quote type.
    if (vssc.quoteType() == MarketDatum::QuoteType::RATE_LNVOL) {

        DLOG("Creating the BlackVarianceSurfaceSparse object");

        // Now pick the quotes that we want from the data container. If the fwdCurve was populated above, we use
        // it in conjunction with preferOutOfTheMoney, to pick the volatilities. If fwdCurve is empty, we just pick
        // a call volatility if there is one else a put volatility (should have at least one).
        vector<Real> strikes;
        vector<Date> expiries;
        vector<Volatility> vols;
        Size quotesAdded = 0;

        for (const auto& kv : cpData.data) {

            const Date& expiry = kv.first.expiry;
            const Real& stk = kv.first.strike;
            const CallPutDatum& cpd = kv.second;
            QL_REQUIRE(cpd.call != Null<Real>() || cpd.put != Null<Real>(),
                "CommodityVolCurve: expected the call or put value to be populated.");

            expiries.push_back(expiry);
            strikes.push_back(stk);
            bool useCall = true;

            if ((cpd.call != Null<Real>() && cpd.put != Null<Real>()) && !fwdCurve.empty()) {
                // We have a call and a put and a forward curve so use it to decide.
                auto it = fwdCurve.find(expiry);
                QL_REQUIRE(it != fwdCurve.end(), "CommodityVolCurve: expected forwards for all expiries.");
                const Real& fwd = it->second;
                useCall = (preferOutOfTheMoney && stk >= fwd) || (!preferOutOfTheMoney && stk <= fwd);
                TLOG("(Forward,Strike,UseCall) = (" << fixed << setprecision(6) <<
                    fwd << "," << stk << "," << to_string(useCall) << ").");
            }

            vols.push_back(useCall ? cpd.call : cpd.put);
            quotesAdded++;

            TLOG("Using option datum (" << (useCall ? "Call" : "Put") << "," << io::iso_date(expiries.back()) <<
                "," << fixed << setprecision(9) << strikes.back() << "," << vols.back() << ")");
        }

        LOG("CommodityVolCurve: added " << quotesAdded << " quotes building wildcard based absolute strike surface.");
        QL_REQUIRE(quotesAdded > 0, "No quotes loaded for " << vc.curveID());

        volatility_ = boost::make_shared<BlackVarianceSurfaceSparse>(asof, calendar_, expiries, strikes,
            vols, dayCounter_, flatStrikeExtrap, flatStrikeExtrap, flatTimeExtrap);

    } else if (vssc.quoteType() == MarketDatum::QuoteType::PRICE) {

        QL_REQUIRE(!pts_.empty(), "CommodityVolCurve: require non-empty price term structure" <<
            " to strip volatilities from prices.");
        QL_REQUIRE(!yts_.empty(), "CommodityVolCurve: require non-empty yield term structure" <<
            " to strip volatilities from prices.");

        // Create the 1D solver options used in the price stripping.
        Solver1DOptions solverOptions = vc.solverConfig();

        // Need to create the call and put option price surfaces.
        auto cSurface = optPriceSurface(cpData, asof, dayCounter_, true);
        auto pSurface = optPriceSurface(cpData, asof, dayCounter_, false);

        DLOG("CommodityVolCurve: stripping volatility surface from the option premium surfaces.");
        auto coss = boost::make_shared<CommodityOptionSurfaceStripper>(pts_, yts_, cSurface, pSurface, calendar_,
            dayCounter_, vssc.exerciseType(), flatStrikeExtrap, flatStrikeExtrap, flatTimeExtrap,
            preferOutOfTheMoney, solverOptions);
        volatility_ = coss->volSurface();

        // If data level logging, output the stripped volatilities.
        if (Log::instance().filter(ORE_DATA)) {
            volatility_->enableExtrapolation(vssc.extrapolation());
            TLOG("CommodityVolCurve: stripped volatilities:");
            TLOG("expiry,strike,forward_price,call_price,put_price,discount,volatility");
            for (const auto& kv : cpData.data) {
                const Date& expiry = kv.first.expiry;
                const Real& strike = kv.first.strike;
                Real fwd = pts_->price(expiry);
                string cPrice;
                if (kv.second.call != Null<Real>()) {
                    stringstream ss;
                    ss << fixed << setprecision(6) << kv.second.call;
                    cPrice = ss.str();
                }
                string pPrice;
                if (kv.second.put != Null<Real>()) {
                    stringstream ss;
                    ss << fixed << setprecision(6) << kv.second.put;
                    pPrice = ss.str();
                }
                Real discount = yts_->discount(expiry);
                Real vol = volatility_->blackVol(expiry, strike);
                TLOG(io::iso_date(expiry) << "," << fixed << setprecision(6) << strike << "," <<
                    fwd << "," << cPrice << "," << pPrice << "," << discount << "," << vol);
            }
        }

    } else {
        QL_FAIL("CommodityVolCurve: invalid quote type " << vssc.quoteType() << " provided.");
    }

    DLOG("Setting BlackVarianceSurfaceSparse extrapolation to " << to_string(vssc.extrapolation()));
    volatility_->enableExtrapolation(vssc.extrapolation());

    LOG("CommodityVolCurve: finished building 2-D volatility absolute strike surface");
}

void CommodityVolCurve::buildVolatilityExplicit(const Date& asof, CommodityVolatilityConfig& vc,
                                                const VolatilityStrikeSurfaceConfig& vssc, const Loader& loader,
                                                const vector<Real>& configuredStrikes) {

    LOG("CommodityVolCurve: start building 2-D volatility absolute strike surface with explicit strikes and expiries");

    QL_REQUIRE(vssc.quoteType() == MarketDatum::QuoteType::RATE_LNVOL, "CommodityVolCurve: only quote type" <<
        " RATE_LNVOL is currently supported for 2-D volatility strike surface with explicit strikes and expiries.");

    // Map to hold the rows of the commodity volatility matrix. The keys are the expiry dates and the values are the
    // vectors of volatilities, one for each configured strike.
    map<Date, vector<Real>> surfaceData;

    // Count the number of quotes added. We check at the end that we have added all configured quotes.
    Size quotesAdded = 0;

    // Loop over quotes and process commodity option quotes that have been requested
    for (const boost::shared_ptr<MarketDatum>& md : loader.loadQuotes(asof)) {

        // Go to next quote if the market data point's date does not equal our asof.
        if (md->asofDate() != asof)
            continue;

        // Go to next quote if not a commodity option quote.
        auto q = boost::dynamic_pointer_cast<CommodityOptionQuote>(md);
        if (!q)
            continue;

        // This surface is for absolute strikes only.
        auto strike = boost::dynamic_pointer_cast<AbsoluteStrike>(q->strike());
        if (!strike)
            continue;

        // If the quote is not in the configured quotes continue
        auto it = find(vc.quotes().begin(), vc.quotes().end(), q->name());
        if (it == vc.quotes().end())
            continue;

        // Process the quote
        Date eDate = getExpiry(asof, q->expiry(), vc.futureConventionsId(), vc.optionExpiryRollDays());

        // Position of quote in vector of strikes
        auto strikeIt = find_if(configuredStrikes.begin(), configuredStrikes.end(),
                                [&strike](Real s) { return close(s, strike->strike()); });
        Size pos = distance(configuredStrikes.begin(), strikeIt);
        QL_REQUIRE(
            pos < configuredStrikes.size(),
            "The quote '" << q->name()
                          << "' is in the list of configured quotes but does not match any of the configured strikes");

        // Add quote to surface
        if (surfaceData.count(eDate) == 0)
            surfaceData[eDate] = vector<Real>(configuredStrikes.size(), Null<Real>());

        QL_REQUIRE(surfaceData[eDate][pos] == Null<Real>(),
                   "Quote " << q->name() << " provides a duplicate quote for the date " << io::iso_date(eDate)
                            << " and the strike " << configuredStrikes[pos]);
        surfaceData[eDate][pos] = q->quote()->value();
        quotesAdded++;

        TLOG("Added quote " << q->name() << ": (" << io::iso_date(eDate) << "," << fixed << setprecision(9)
                            << configuredStrikes[pos] << "," << q->quote()->value() << ")");
    }

    LOG("CommodityVolCurve: added " << quotesAdded << " quotes in building explicit absolute strike surface.");

    QL_REQUIRE(vc.quotes().size() == quotesAdded,
               "Found " << quotesAdded << " quotes, but " << vc.quotes().size() << " quotes required by config.");

    // Set up the BlackVarianceSurface. Note that the rows correspond to strikes and that the columns correspond to
    // the expiry dates in the matrix that is fed to the BlackVarianceSurface ctor.
    vector<Date> expiryDates;
    Matrix volatilities = Matrix(configuredStrikes.size(), surfaceData.size());
    Size expiryIdx = 0;
    for (const auto& expiryRow : surfaceData) {
        expiryDates.push_back(expiryRow.first);
        for (Size i = 0; i < configuredStrikes.size(); i++) {
            volatilities[i][expiryIdx] = expiryRow.second[i];
        }
        expiryIdx++;
    }

    // Trace log the surface
    TLOG("Explicit strike surface grid points:");
    TLOG("expiry,strike,volatility");
    for (Size i = 0; i < volatilities.rows(); i++) {
        for (Size j = 0; j < volatilities.columns(); j++) {
            TLOG(io::iso_date(expiryDates[j])
                 << "," << fixed << setprecision(9) << configuredStrikes[i] << "," << volatilities[i][j]);
        }
    }

    // Set the strike extrapolation which only matters if extrapolation is turned on for the whole surface.
    // BlackVarianceSurface time extrapolation is hard-coded to constant in volatility.
    BlackVarianceSurface::Extrapolation strikeExtrap = BlackVarianceSurface::ConstantExtrapolation;
    if (vssc.extrapolation()) {

        auto strikeExtrapType = parseExtrapolation(vssc.strikeExtrapolation());
        if (strikeExtrapType == Extrapolation::UseInterpolator) {
            DLOG("Strike extrapolation switched to using interpolator.");
            strikeExtrap = BlackVarianceSurface::InterpolatorDefaultExtrapolation;
        } else if (strikeExtrapType == Extrapolation::None) {
            DLOG("Strike extrapolation cannot be turned off on its own so defaulting to flat.");
        } else if (strikeExtrapType == Extrapolation::Flat) {
            DLOG("Strike extrapolation has been set to flat.");
        } else {
            DLOG("Strike extrapolation " << strikeExtrapType << " not expected so default to flat.");
        }

        auto timeExtrapType = parseExtrapolation(vssc.timeExtrapolation());
        if (timeExtrapType != Extrapolation::Flat) {
            DLOG("BlackVarianceSurface only supports flat volatility extrapolation in the time direction");
        }
    } else {
        DLOG("Extrapolation is turned off for the whole surface so the time and"
             << " strike extrapolation settings are ignored");
    }

    DLOG("Creating BlackVarianceSurface object");
    auto tmp = boost::make_shared<BlackVarianceSurface>(asof, calendar_, expiryDates, configuredStrikes, volatilities,
                                                        dayCounter_, strikeExtrap, strikeExtrap);

    // Set the interpolation if configured properly. The default is Bilinear.
    if (!(vssc.timeInterpolation() == "Linear" && vssc.strikeInterpolation() == "Linear")) {
        if (vssc.timeInterpolation() != vssc.strikeInterpolation()) {
            DLOG("Time and strike interpolation must be the same for BlackVarianceSurface but we got strike "
                 << "interpolation " << vssc.strikeInterpolation() << " and time interpolation "
                 << vssc.timeInterpolation());
        } else if (vssc.timeInterpolation() == "Cubic") {
            DLOG("Setting interpolation to BiCubic");
            tmp->setInterpolation<Bicubic>();
        } else {
            DLOG("Interpolation type " << vssc.timeInterpolation() << " not supported in 2 dimensions");
        }
    }

    // Set the volatility_ member after we have possibly updated the interpolation.
    volatility_ = tmp;

    DLOG("Setting BlackVarianceSurface extrapolation to " << to_string(vssc.extrapolation()));
    volatility_->enableExtrapolation(vssc.extrapolation());

    LOG("CommodityVolCurve: finished building 2-D volatility absolute strike "
        << "surface with explicit strikes and expiries");
}

void CommodityVolCurve::buildVolatility(const Date& asof, CommodityVolatilityConfig& vc,
                                        const VolatilityDeltaSurfaceConfig& vdsc, const Loader& loader) {

    using boost::adaptors::transformed;
    using boost::algorithm::join;

    LOG("CommodityVolCurve: start building 2-D volatility delta strike surface");

    QL_REQUIRE(vdsc.quoteType() == MarketDatum::QuoteType::RATE_LNVOL, "CommodityVolCurve: only quote type" <<
        " RATE_LNVOL is currently supported for a 2-D volatility delta strike surface.");

    // Parse, sort and check the vector of configured put deltas
    vector<Real> putDeltas = parseVectorOfValues<Real>(vdsc.putDeltas(), &parseReal);
    sort(putDeltas.begin(), putDeltas.end(), [](Real x, Real y) { return !close(x, y) && x < y; });
    QL_REQUIRE(adjacent_find(putDeltas.begin(), putDeltas.end(), [](Real x, Real y) { return close(x, y); }) ==
                   putDeltas.end(),
               "The configured put deltas contain duplicates");
    DLOG("Parsed " << putDeltas.size() << " unique configured put deltas");
    DLOG("Put deltas are: " << join(putDeltas | transformed([](Real d) { return ore::data::to_string(d); }), ","));

    // Parse, sort descending and check the vector of configured call deltas
    vector<Real> callDeltas = parseVectorOfValues<Real>(vdsc.callDeltas(), &parseReal);
    sort(callDeltas.begin(), callDeltas.end(), [](Real x, Real y) { return !close(x, y) && x > y; });
    QL_REQUIRE(adjacent_find(callDeltas.begin(), callDeltas.end(), [](Real x, Real y) { return close(x, y); }) ==
                   callDeltas.end(),
               "The configured call deltas contain duplicates");
    DLOG("Parsed " << callDeltas.size() << " unique configured call deltas");
    DLOG("Call deltas are: " << join(callDeltas | transformed([](Real d) { return ore::data::to_string(d); }), ","));

    // Expiries may be configured with a wildcard or given explicitly
    bool expWc = false;
    if (find(vdsc.expiries().begin(), vdsc.expiries().end(), "*") != vdsc.expiries().end()) {
        expWc = true;
        QL_REQUIRE(vdsc.expiries().size() == 1, "Wild card expiry specified but more expiries also specified.");
        DLOG("Have expiry wildcard pattern " << vdsc.expiries()[0]);
    }

    // Map to hold the rows of the commodity volatility matrix. The keys are the expiry dates and the values are the
    // vectors of volatilities, one for each configured delta.
    map<Date, vector<Real>> surfaceData;

    // Number of strikes = number of put deltas + ATM + number of call deltas
    Size numStrikes = putDeltas.size() + 1 + callDeltas.size();

    // Count the number of quotes added. We check at the end that we have added all configured quotes.
    Size quotesAdded = 0;

    // Configured delta and Atm types.
    DeltaVolQuote::DeltaType deltaType = parseDeltaType(vdsc.deltaType());
    DeltaVolQuote::AtmType atmType = parseAtmType(vdsc.atmType());
    boost::optional<DeltaVolQuote::DeltaType> atmDeltaType;
    if (!vdsc.atmDeltaType().empty()) {
        atmDeltaType = parseDeltaType(vdsc.atmDeltaType());
    }

    // Populate the configured strikes.
    vector<boost::shared_ptr<BaseStrike>> strikes;
    for (const auto& pd : putDeltas) {
        strikes.push_back(boost::make_shared<DeltaStrike>(deltaType, Option::Put, pd));
    }
    strikes.push_back(boost::make_shared<AtmStrike>(atmType, atmDeltaType));
    for (const auto& cd : callDeltas) {
        strikes.push_back(boost::make_shared<DeltaStrike>(deltaType, Option::Call, cd));
    }

    // Read the quotes to fill the expiry dates and vols matrix.
    for (const boost::shared_ptr<MarketDatum>& md : loader.loadQuotes(asof)) {

        // Go to next quote if the market data point's date does not equal our asof.
        if (md->asofDate() != asof)
            continue;

        // Go to next quote if not a commodity option quote.
        auto q = boost::dynamic_pointer_cast<CommodityOptionQuote>(md);
        if (!q)
            continue;

        // Go to next quote if not a commodity name or currency do not match config.
        if (vc.curveID() != q->commodityName() || vc.currency() != q->quoteCurrency())
            continue;

        // Iterator to one of the configured strikes.
        vector<boost::shared_ptr<BaseStrike>>::iterator strikeIt;

        if (!expWc) {

            // If we have explicitly configured expiries and the quote is not in the configured quotes continue.
            auto it = find(vc.quotes().begin(), vc.quotes().end(), q->name());
            if (it == vc.quotes().end())
                continue;

            // Check if quote's strike is in the configured strikes.
            // It should be as we have selected from the explicitly configured quotes in the last step.
            strikeIt = find_if(strikes.begin(), strikes.end(),
                               [&q](boost::shared_ptr<BaseStrike> s) { return *s == *q->strike(); });
            QL_REQUIRE(strikeIt != strikes.end(),
                       "The quote '"
                           << q->name()
                           << "' is in the list of configured quotes but does not match any of the configured strikes");

        } else {

            // Check if quote's strike is in the configured strikes and continue if it is not.
            strikeIt = find_if(strikes.begin(), strikes.end(),
                               [&q](boost::shared_ptr<BaseStrike> s) { return *s == *q->strike(); });
            if (strikeIt == strikes.end())
                continue;
        }

        // Position of quote in vector of strikes
        Size pos = std::distance(strikes.begin(), strikeIt);

        // Process the quote
        Date eDate = getExpiry(asof, q->expiry(), vc.futureConventionsId(), vc.optionExpiryRollDays());

        // Add quote to surface
        if (surfaceData.count(eDate) == 0)
            surfaceData[eDate] = vector<Real>(numStrikes, Null<Real>());

        QL_REQUIRE(surfaceData[eDate][pos] == Null<Real>(),
                   "Quote " << q->name() << " provides a duplicate quote for the date " << io::iso_date(eDate)
                            << " and strike " << *q->strike());
        surfaceData[eDate][pos] = q->quote()->value();
        quotesAdded++;

        TLOG("Added quote " << q->name() << ": (" << io::iso_date(eDate) << "," << *q->strike() << "," << fixed
                            << setprecision(9) << "," << q->quote()->value() << ")");
    }

    LOG("CommodityVolCurve: added " << quotesAdded << " quotes in building delta strike surface.");

    // Check the data gathered.
    if (!expWc) {
        // If expiries were configured explicitly, the number of configured quotes should equal the
        // number of quotes added.
        QL_REQUIRE(vc.quotes().size() == quotesAdded,
                   "Found " << quotesAdded << " quotes, but " << vc.quotes().size() << " quotes required by config.");
    } else {
        // If the expiries were configured via a wildcard, check that no surfaceData element has a Null<Real>().
        for (const auto& kv : surfaceData) {
            for (Size j = 0; j < numStrikes; j++) {
                QL_REQUIRE(kv.second[j] != Null<Real>(), "Volatility for expiry date "
                                                             << io::iso_date(kv.first) << " and strike " << *strikes[j]
                                                             << " not found. Cannot proceed with a sparse matrix.");
            }
        }
    }

    // Populate the matrix of volatilities and the expiry dates.
    vector<Date> expiryDates;
    Matrix vols(surfaceData.size(), numStrikes);
    for (const auto row : surfaceData | boost::adaptors::indexed(0)) {
        expiryDates.push_back(row.value().first);
        copy(row.value().second.begin(), row.value().second.end(), vols.row_begin(row.index()));
    }

    // Need to multiply each put delta value by -1 before passing it to the BlackVolatilitySurfaceDelta ctor
    // i.e. a put delta of 0.25 that is passed in to the config must be -0.25 when passed to the ctor.
    transform(putDeltas.begin(), putDeltas.end(), putDeltas.begin(), [](Real pd) { return -1.0 * pd; });
    DLOG("Multiply put deltas by -1.0 before creating BlackVolatilitySurfaceDelta object.");
    DLOG("Put deltas are: " << join(putDeltas | transformed([](Real d) { return ore::data::to_string(d); }), ","));

    // Set the strike extrapolation which only matters if extrapolation is turned on for the whole surface.
    // BlackVolatilitySurfaceDelta time extrapolation is hard-coded to constant in volatility.
    bool flatExtrapolation = true;
    if (vdsc.extrapolation()) {

        auto strikeExtrapType = parseExtrapolation(vdsc.strikeExtrapolation());
        if (strikeExtrapType == Extrapolation::UseInterpolator) {
            DLOG("Strike extrapolation switched to using interpolator.");
            flatExtrapolation = false;
        } else if (strikeExtrapType == Extrapolation::None) {
            DLOG("Strike extrapolation cannot be turned off on its own so defaulting to flat.");
        } else if (strikeExtrapType == Extrapolation::Flat) {
            DLOG("Strike extrapolation has been set to flat.");
        } else {
            DLOG("Strike extrapolation " << strikeExtrapType << " not expected so default to flat.");
        }

        auto timeExtrapType = parseExtrapolation(vdsc.timeExtrapolation());
        if (timeExtrapType != Extrapolation::Flat) {
            DLOG("BlackVolatilitySurfaceDelta only supports flat volatility extrapolation in the time direction");
        }
    } else {
        DLOG("Extrapolation is turned off for the whole surface so the time and"
             << " strike extrapolation settings are ignored");
    }

    // Time interpolation
    if (vdsc.timeInterpolation() != "Linear") {
        DLOG("BlackVolatilitySurfaceDelta only supports linear time interpolation.");
    }

    // Strike interpolation
    InterpolatedSmileSection::InterpolationMethod im;
    if (vdsc.strikeInterpolation() == "Linear") {
        im = InterpolatedSmileSection::InterpolationMethod::Linear;
    } else if (vdsc.strikeInterpolation() == "NaturalCubic") {
        im = InterpolatedSmileSection::InterpolationMethod::NaturalCubic;
    } else if (vdsc.strikeInterpolation() == "FinancialCubic") {
        im = InterpolatedSmileSection::InterpolationMethod::FinancialCubic;
    } else {
        im = InterpolatedSmileSection::InterpolationMethod::Linear;
        DLOG("BlackVolatilitySurfaceDelta does not support strike interpolation '" << vdsc.strikeInterpolation()
                                                                                << "' so setting it to linear.");
    }

    // Apply correction to future price term structure if requested and we have a valid expiry calculator
    QL_REQUIRE(!pts_.empty(), "Expected the price term structure to be populated for a delta surface.");
    Handle<PriceTermStructure> cpts = pts_;
    if (vdsc.futurePriceCorrection() && expCalc_)
        cpts = correctFuturePriceCurve(asof, vc.futureConventionsId(), *pts_, expiryDates);

    // Need to construct a dummy spot and foreign yts such that fwd = spot * DF_for / DF
    QL_REQUIRE(!yts_.empty(), "Expected the yield term structure to be populated for a delta surface.");
    Handle<Quote> spot(boost::make_shared<DerivedPriceQuote>(cpts));
    Handle<YieldTermStructure> pyts =
        Handle<YieldTermStructure>(boost::make_shared<PriceTermStructureAdapter>(*cpts, *yts_));
    pyts->enableExtrapolation();

    DLOG("Creating BlackVolatilitySurfaceDelta object");
    bool hasAtm = true;
    volatility_ = boost::make_shared<BlackVolatilitySurfaceDelta>(
        asof, expiryDates, putDeltas, callDeltas, hasAtm, vols, dayCounter_, calendar_, spot, yts_, pyts, deltaType,
        atmType, atmDeltaType, 0 * Days, deltaType, atmType, atmDeltaType, im, flatExtrapolation);

    DLOG("Setting BlackVolatilitySurfaceDelta extrapolation to " << to_string(vdsc.extrapolation()));
    volatility_->enableExtrapolation(vdsc.extrapolation());

    LOG("CommodityVolCurve: finished building 2-D volatility delta strike surface");
}

void CommodityVolCurve::buildVolatility(const Date& asof, CommodityVolatilityConfig& vc,
                                        const VolatilityMoneynessSurfaceConfig& vmsc, const Loader& loader) {

    using boost::adaptors::transformed;
    using boost::algorithm::join;

    LOG("CommodityVolCurve: start building 2-D volatility moneyness strike surface");

    QL_REQUIRE(vmsc.quoteType() == MarketDatum::QuoteType::RATE_LNVOL, "CommodityVolCurve: only quote type" <<
        " RATE_LNVOL is currently supported for a 2-D volatility moneyness strike surface.");

    // Parse, sort and check the vector of configured moneyness levels
    vector<Real> moneynessLevels = checkMoneyness(vmsc.moneynessLevels());

    // Expiries may be configured with a wildcard or given explicitly
    bool expWc = false;
    if (find(vmsc.expiries().begin(), vmsc.expiries().end(), "*") != vmsc.expiries().end()) {
        expWc = true;
        QL_REQUIRE(vmsc.expiries().size() == 1, "Wild card expiry specified but more expiries also specified.");
        DLOG("Have expiry wildcard pattern " << vmsc.expiries()[0]);
    }

    // Map to hold the rows of the commodity volatility matrix. The keys are the expiry dates and the values are the
    // vectors of volatilities, one for each configured moneyness.
    map<Date, vector<Real>> surfaceData;

    // Count the number of quotes added. We check at the end that we have added all configured quotes.
    Size quotesAdded = 0;

    // Configured moneyness type.
    MoneynessStrike::Type moneynessType = parseMoneynessType(vmsc.moneynessType());

    // Populate the configured strikes.
    vector<boost::shared_ptr<BaseStrike>> strikes;
    for (const auto& moneynessLevel : moneynessLevels) {
        strikes.push_back(boost::make_shared<MoneynessStrike>(moneynessType, moneynessLevel));
    }

    // Read the quotes to fill the expiry dates and vols matrix.
    for (const boost::shared_ptr<MarketDatum>& md : loader.loadQuotes(asof)) {

        // Go to next quote if the market data point's date does not equal our asof.
        if (md->asofDate() != asof)
            continue;

        // Go to next quote if not a commodity option quote.
        auto q = boost::dynamic_pointer_cast<CommodityOptionQuote>(md);
        if (!q)
            continue;

        // Go to next quote if not a commodity name or currency do not match config.
        if (vc.curveID() != q->commodityName() || vc.currency() != q->quoteCurrency())
            continue;

        // Iterator to one of the configured strikes.
        vector<boost::shared_ptr<BaseStrike>>::iterator strikeIt;

        if (!expWc) {

            // If we have explicitly configured expiries and the quote is not in the configured quotes continue.
            auto it = find(vc.quotes().begin(), vc.quotes().end(), q->name());
            if (it == vc.quotes().end())
                continue;

            // Check if quote's strike is in the configured strikes.
            // It should be as we have selected from the explicitly configured quotes in the last step.
            strikeIt = find_if(strikes.begin(), strikes.end(),
                               [&q](boost::shared_ptr<BaseStrike> s) { return *s == *q->strike(); });
            QL_REQUIRE(strikeIt != strikes.end(),
                       "The quote '"
                           << q->name()
                           << "' is in the list of configured quotes but does not match any of the configured strikes");

        } else {

            // Check if quote's strike is in the configured strikes and continue if it is not.
            strikeIt = find_if(strikes.begin(), strikes.end(),
                               [&q](boost::shared_ptr<BaseStrike> s) { return *s == *q->strike(); });
            if (strikeIt == strikes.end())
                continue;
        }

        // Position of quote in vector of strikes
        Size pos = std::distance(strikes.begin(), strikeIt);

        // Process the quote
        Date eDate = getExpiry(asof, q->expiry(), vc.futureConventionsId(), vc.optionExpiryRollDays());

        // Add quote to surface
        if (surfaceData.count(eDate) == 0)
            surfaceData[eDate] = vector<Real>(moneynessLevels.size(), Null<Real>());

        QL_REQUIRE(surfaceData[eDate][pos] == Null<Real>(),
                   "Quote " << q->name() << " provides a duplicate quote for the date " << io::iso_date(eDate)
                            << " and strike " << *q->strike());
        surfaceData[eDate][pos] = q->quote()->value();
        quotesAdded++;

        TLOG("Added quote " << q->name() << ": (" << io::iso_date(eDate) << "," << *q->strike() << "," << fixed
                            << setprecision(9) << "," << q->quote()->value() << ")");
    }

    LOG("CommodityVolCurve: added " << quotesAdded << " quotes in building moneyness strike surface.");

    // Check the data gathered.
    if (!expWc) {
        // If expiries were configured explicitly, the number of configured quotes should equal the
        // number of quotes added.
        QL_REQUIRE(vc.quotes().size() == quotesAdded,
                   "Found " << quotesAdded << " quotes, but " << vc.quotes().size() << " quotes required by config.");
    } else {
        // If the expiries were configured via a wildcard, check that no surfaceData element has a Null<Real>().
        for (const auto& kv : surfaceData) {
            for (Size j = 0; j < moneynessLevels.size(); j++) {
                QL_REQUIRE(kv.second[j] != Null<Real>(), "Volatility for expiry date "
                                                             << io::iso_date(kv.first) << " and strike " << *strikes[j]
                                                             << " not found. Cannot proceed with a sparse matrix.");
            }
        }
    }

    // Populate the volatility quotes and the expiry times.
    // Rows are moneyness levels and columns are expiry times - this is what the ctor needs below.
    vector<Date> expiryDates(surfaceData.size());
    vector<Time> expiryTimes(surfaceData.size());
    vector<vector<Handle<Quote>>> vols(moneynessLevels.size());
    for (const auto row : surfaceData | boost::adaptors::indexed(0)) {
        expiryDates[row.index()] = row.value().first;
        expiryTimes[row.index()] = dayCounter_.yearFraction(asof, row.value().first);
        for (Size i = 0; i < row.value().second.size(); i++) {
            vols[i].push_back(Handle<Quote>(boost::make_shared<SimpleQuote>(row.value().second[i])));
        }
    }

    // Set the strike extrapolation which only matters if extrapolation is turned on for the whole surface.
    // BlackVarianceSurfaceMoneyness time extrapolation is hard-coded to constant in volatility.
    bool flatExtrapolation = true;
    if (vmsc.extrapolation()) {

        auto strikeExtrapType = parseExtrapolation(vmsc.strikeExtrapolation());
        if (strikeExtrapType == Extrapolation::UseInterpolator) {
            DLOG("Strike extrapolation switched to using interpolator.");
            flatExtrapolation = false;
        } else if (strikeExtrapType == Extrapolation::None) {
            DLOG("Strike extrapolation cannot be turned off on its own so defaulting to flat.");
        } else if (strikeExtrapType == Extrapolation::Flat) {
            DLOG("Strike extrapolation has been set to flat.");
        } else {
            DLOG("Strike extrapolation " << strikeExtrapType << " not expected so default to flat.");
        }

        auto timeExtrapType = parseExtrapolation(vmsc.timeExtrapolation());
        if (timeExtrapType != Extrapolation::Flat) {
            DLOG("BlackVarianceSurfaceMoneyness only supports flat volatility extrapolation in the time direction");
        }
    } else {
        DLOG("Extrapolation is turned off for the whole surface so the time and"
             << " strike extrapolation settings are ignored");
    }

    // Time interpolation
    if (vmsc.timeInterpolation() != "Linear") {
        DLOG("BlackVarianceSurfaceMoneyness only supports linear time interpolation in variance.");
    }

    // Strike interpolation
    if (vmsc.strikeInterpolation() != "Linear") {
        DLOG("BlackVarianceSurfaceMoneyness only supports linear strike interpolation in variance.");
    }

    // Apply correction to future price term structure if requested and we have a valid expiry calculator
    QL_REQUIRE(!pts_.empty(), "Expected the price term structure to be populated for a moneyness surface.");
    Handle<PriceTermStructure> cpts = pts_;
    if (vmsc.futurePriceCorrection() && expCalc_)
        cpts = correctFuturePriceCurve(asof, vc.futureConventionsId(), *pts_, expiryDates);

    // Both moneyness surfaces need a spot quote.
    Handle<Quote> spot(boost::make_shared<DerivedPriceQuote>(cpts));

    // The choice of false here is important for forward moneyness. It means that we use the cpts and yts in the
    // BlackVarianceSurfaceMoneynessForward to get the forward value at all times and in particular at times that
    // are after the last expiry time. If we set it to true, BlackVarianceSurfaceMoneynessForward uses a linear
    // interpolated forward curve on the expiry times internally which is poor.
    bool stickyStrike = false;

    if (moneynessType == MoneynessStrike::Type::Forward) {

        QL_REQUIRE(!yts_.empty(), "Expected yield term structure to be populated for a forward moneyness surface.");
        Handle<YieldTermStructure> pyts =
            Handle<YieldTermStructure>(boost::make_shared<PriceTermStructureAdapter>(*cpts, *yts_));
        pyts->enableExtrapolation();

        DLOG("Creating BlackVarianceSurfaceMoneynessForward object");
        volatility_ = boost::make_shared<BlackVarianceSurfaceMoneynessForward>(calendar_, spot, expiryTimes,
                                                                               moneynessLevels, vols, dayCounter_, pyts,
                                                                               yts_, stickyStrike, flatExtrapolation);

    } else {

        DLOG("Creating BlackVarianceSurfaceMoneynessSpot object");
        volatility_ = boost::make_shared<BlackVarianceSurfaceMoneynessSpot>(
            calendar_, spot, expiryTimes, moneynessLevels, vols, dayCounter_, stickyStrike, flatExtrapolation);
    }

    DLOG("Setting BlackVarianceSurfaceMoneyness extrapolation to " << to_string(vmsc.extrapolation()));
    volatility_->enableExtrapolation(vmsc.extrapolation());

    LOG("CommodityVolCurve: finished building 2-D volatility moneyness strike surface");
}

void CommodityVolCurve::buildVolatility(const Date& asof, CommodityVolatilityConfig& vc,
                                        const VolatilityApoFutureSurfaceConfig& vapo,
                                        const Handle<BlackVolTermStructure>& baseVts,
                                        const Handle<PriceTermStructure>& basePts, const Conventions& conventions) {

    LOG("CommodityVolCurve: start building the APO surface");

    QL_REQUIRE(vapo.quoteType() == MarketDatum::QuoteType::RATE_LNVOL, "CommodityVolCurve: only quote type" <<
        " RATE_LNVOL is currently supported for an APO surface.");

    // Get the base conventions and create the associated expiry calculator.
    QL_REQUIRE(!vapo.baseConventionsId().empty(),
               "The APO FutureConventions must be populated to build a future APO surface");
    QL_REQUIRE(conventions.has(vapo.baseConventionsId()),
               "Conventions, " << vapo.baseConventionsId() << " for config " << vc.curveID() << " not found.");
    auto baseConvention =
        boost::dynamic_pointer_cast<CommodityFutureConvention>(conventions.get(vapo.baseConventionsId()));
    QL_REQUIRE(baseConvention,
               "Convention with ID '" << vapo.baseConventionsId() << "' should be of type CommodityFutureConvention");

    auto baseExpCalc = boost::make_shared<ConventionsBasedFutureExpiry>(*baseConvention);

    // Get the max tenor from the config if provided.
    boost::optional<QuantLib::Period> maxTenor;
    if (!vapo.maxTenor().empty())
        maxTenor = parsePeriod(vapo.maxTenor());

    // Get the moneyness levels
    vector<Real> moneynessLevels = checkMoneyness(vapo.moneynessLevels());

    // Get the beta parameter to use for valuing the APOs in the surface
    Real beta = vapo.beta();

    // Construct the commodity index.
    auto index = parseCommodityIndex(baseConvention->id(), conventions, false, basePts);

    // Set the strike extrapolation which only matters if extrapolation is turned on for the whole surface.
    // BlackVarianceSurfaceMoneyness, which underlies the ApoFutureSurface, has time extrapolation hard-coded to
    // constant in volatility.
    bool flatExtrapolation = true;
    if (vapo.extrapolation()) {

        auto strikeExtrapType = parseExtrapolation(vapo.strikeExtrapolation());
        if (strikeExtrapType == Extrapolation::UseInterpolator) {
            DLOG("Strike extrapolation switched to using interpolator.");
            flatExtrapolation = false;
        } else if (strikeExtrapType == Extrapolation::None) {
            DLOG("Strike extrapolation cannot be turned off on its own so defaulting to flat.");
        } else if (strikeExtrapType == Extrapolation::Flat) {
            DLOG("Strike extrapolation has been set to flat.");
        } else {
            DLOG("Strike extrapolation " << strikeExtrapType << " not expected so default to flat.");
        }

        auto timeExtrapType = parseExtrapolation(vapo.timeExtrapolation());
        if (timeExtrapType != Extrapolation::Flat) {
            DLOG("ApoFutureSurface only supports flat volatility extrapolation in the time direction");
        }
    } else {
        DLOG("Extrapolation is turned off for the whole surface so the time and"
             << " strike extrapolation settings are ignored");
    }

    // Time interpolation
    if (vapo.timeInterpolation() != "Linear") {
        DLOG("ApoFutureSurface only supports linear time interpolation in variance.");
    }

    // Strike interpolation
    if (vapo.strikeInterpolation() != "Linear") {
        DLOG("ApoFutureSurface only supports linear strike interpolation in variance.");
    }

    DLOG("Creating ApoFutureSurface object");
    volatility_ = boost::make_shared<ApoFutureSurface>(asof, moneynessLevels, index, pts_, yts_, expCalc_, baseVts,
                                                       baseExpCalc, beta, flatExtrapolation, maxTenor);

    DLOG("Setting ApoFutureSurface extrapolation to " << to_string(vapo.extrapolation()));
    volatility_->enableExtrapolation(vapo.extrapolation());

    LOG("CommodityVolCurve: finished building the APO surface");
}

Handle<PriceTermStructure> CommodityVolCurve::correctFuturePriceCurve(const Date& asof, const string& contractName,
                                                                      const boost::shared_ptr<PriceTermStructure>& pts,
                                                                      const vector<Date>& optionExpiries) const {

    LOG("CommodityVolCurve: start adding future price correction at option expiry.");

    // Gather curve dates and prices
    map<Date, Real> curveData;

    // Get existing curve dates and prices. Messy but can't think of another way.
    vector<Date> ptsDates;
    if (auto ipc = boost::dynamic_pointer_cast<InterpolatedPriceCurve<Linear>>(pts)) {
        ptsDates = ipc->pillarDates();
    } else if (auto ipc = boost::dynamic_pointer_cast<InterpolatedPriceCurve<LogLinear>>(pts)) {
        ptsDates = ipc->pillarDates();
    } else if (auto ipc = boost::dynamic_pointer_cast<InterpolatedPriceCurve<Cubic>>(pts)) {
        ptsDates = ipc->pillarDates();
    } else if (auto ipc = boost::dynamic_pointer_cast<InterpolatedPriceCurve<LinearFlat>>(pts)) {
        ptsDates = ipc->pillarDates();
    } else if (auto ipc = boost::dynamic_pointer_cast<InterpolatedPriceCurve<LogLinearFlat>>(pts)) {
        ptsDates = ipc->pillarDates();
    } else if (auto ipc = boost::dynamic_pointer_cast<InterpolatedPriceCurve<CubicFlat>>(pts)) {
        ptsDates = ipc->pillarDates();
    } else {
        DLOG("Could not cast the price term structure so do not have its pillar dates.");
    }

    // Add existing dates and prices to data.
    DLOG("Got " << ptsDates.size() << " pillar dates from the price curve.");
    for (const Date& d : ptsDates) {
        curveData[d] = pts->price(d);
        TLOG("Added (" << io::iso_date(d) << "," << fixed << setprecision(9) << curveData.at(d) << ").");
    }

    // Add future price at option expiry to the curve i.e. override any interpolation between future option
    // expiry (oed) and future expiry (fed).
    for (const Date& oed : optionExpiries) {
        Date fed = expCalc_->nextExpiry(true, oed, 0, false);
        // In general, expect that the future expiry date will already be in curveData but maybe not.
        auto it = curveData.find(fed);
        if (it != curveData.end()) {
            curveData[oed] = it->second;
            TLOG("Found future expiry in existing data. (Option Expiry,Future Expiry,Future Price) is ("
                 << io::iso_date(oed) << "," << io::iso_date(fed) << "," << fixed << setprecision(9)
                 << curveData.at(oed) << ").");
        } else {
            curveData[oed] = pts->price(fed);
            curveData[fed] = pts->price(fed);
            TLOG("Future expiry not found in existing data. (Option Expiry,Future Expiry,Future Price) is ("
                 << io::iso_date(oed) << "," << io::iso_date(fed) << "," << fixed << setprecision(9)
                 << curveData.at(oed) << ").");
        }
    }

    // Gather data for building the "corrected" curve.
    vector<Date> curveDates;
    vector<Real> curvePrices;
    for (const auto& kv : curveData) {
        curveDates.push_back(kv.first);
        curvePrices.push_back(kv.second);
    }
    DayCounter dc = pts->dayCounter();

    // Create the "corrected" curve. Again messy but can't think of another way.
    boost::shared_ptr<PriceTermStructure> cpts;
    if (auto ipc = boost::dynamic_pointer_cast<InterpolatedPriceCurve<Linear>>(pts)) {
        cpts = boost::make_shared<InterpolatedPriceCurve<Linear>>(asof, curveDates, curvePrices, dc, pts->currency());
    } else if (auto ipc = boost::dynamic_pointer_cast<InterpolatedPriceCurve<LogLinear>>(pts)) {
        cpts =
            boost::make_shared<InterpolatedPriceCurve<LogLinear>>(asof, curveDates, curvePrices, dc, pts->currency());
    } else if (auto ipc = boost::dynamic_pointer_cast<InterpolatedPriceCurve<Cubic>>(pts)) {
        cpts = boost::make_shared<InterpolatedPriceCurve<Cubic>>(asof, curveDates, curvePrices, dc, pts->currency());
    } else if (auto ipc = boost::dynamic_pointer_cast<InterpolatedPriceCurve<LinearFlat>>(pts)) {
        cpts =
            boost::make_shared<InterpolatedPriceCurve<LinearFlat>>(asof, curveDates, curvePrices, dc, pts->currency());
    } else if (auto ipc = boost::dynamic_pointer_cast<InterpolatedPriceCurve<LogLinearFlat>>(pts)) {
        cpts = boost::make_shared<InterpolatedPriceCurve<LogLinearFlat>>(asof, curveDates, curvePrices, dc,
                                                                         pts->currency());
    } else if (auto ipc = boost::dynamic_pointer_cast<InterpolatedPriceCurve<CubicFlat>>(pts)) {
        cpts =
            boost::make_shared<InterpolatedPriceCurve<CubicFlat>>(asof, curveDates, curvePrices, dc, pts->currency());
    } else {
        DLOG("Could not cast the price term structure so corrected curve is a linear InterpolatedPriceCurve.");
        cpts = boost::make_shared<InterpolatedPriceCurve<Linear>>(asof, curveDates, curvePrices, dc, pts->currency());
    }
    cpts->enableExtrapolation(pts->allowsExtrapolation());

    LOG("CommodityVolCurve: finished adding future price correction at option expiry.");

    return Handle<PriceTermStructure>(cpts);
}

Date CommodityVolCurve::getExpiry(const Date& asof, const boost::shared_ptr<Expiry>& expiry, const string& name,
                                  Natural rollDays) const {

    Date result;

    if (auto expiryDate = boost::dynamic_pointer_cast<ExpiryDate>(expiry)) {
        result = expiryDate->expiryDate();
    } else if (auto expiryPeriod = boost::dynamic_pointer_cast<ExpiryPeriod>(expiry)) {
        // We may need more conventions here eventually.
        result = calendar_.adjust(asof + expiryPeriod->expiryPeriod());
    } else if (auto fcExpiry = boost::dynamic_pointer_cast<FutureContinuationExpiry>(expiry)) {

        QL_REQUIRE(expCalc_, "CommodityVolCurve::getExpiry: need a future expiry calculator for continuation quotes.");
        QL_REQUIRE(convention_, "CommodityVolCurve::getExpiry: need a future convention for continuation quotes.");
        DLOG("Future option continuation expiry is " << *fcExpiry);

        // Firstly, get the next option expiry on or after the asof date
        result = expCalc_->nextExpiry(true, asof, 0, true);
        TLOG("CommodityVolCurve::getExpiry: next option expiry relative to " << io::iso_date(asof) << " is "
                                                                             << io::iso_date(result) << ".");

        // Market quotes may be delivered with a given number of roll days.
        if (rollDays > 0) {
            Date roll;
            roll = calendar_.advance(result, -static_cast<Integer>(rollDays), Days);
            TLOG("CommodityVolCurve::getExpiry: roll days is " << rollDays << " giving a roll date "
                                                               << io::iso_date(roll) << ".");
            // Take the next option expiry if the roll days means the roll date is before asof.
            if (roll < asof) {
                result = expCalc_->nextExpiry(true, asof, 1, true);
                roll = calendar_.advance(result, -static_cast<Integer>(rollDays), Days);
                QL_REQUIRE(roll > asof, "CommodityVolCurve::getExpiry: expected roll to be greater than asof.");
                TLOG("CommodityVolCurve::getExpiry: roll date " << io::iso_date(roll) << " is less than asof "
                                                                << io::iso_date(asof) << " so take next option expiry "
                                                                << io::iso_date(result));
            }
        }

        // At this stage, 'result' should hold the next option expiry on or after the asof date accounting for roll
        // days.
        TLOG("CommodityVolCurve::getExpiry: first option expiry is " << io::iso_date(result) << ".");

        // If the continuation index is greater than 1 get the corresponding expiry.
        Natural fcIndex = fcExpiry->expiryIndex();

        // The option continuation expiry may be mapped to another one.
        const auto& ocm = convention_->optionContinuationMappings();
        auto it = ocm.find(fcIndex);
        if (it != ocm.end())
            fcIndex = it->second;

        if (fcIndex > 1) {
            result += 1 * Days;
            result = expCalc_->nextExpiry(true, result, fcIndex - 2, true);
        }

        DLOG("Expiry date corresponding to continuation expiry, " << *fcExpiry <<
            ", is " << io::iso_date(result) << ".");

    } else {
        QL_FAIL("CommodityVolCurve::getExpiry: cannot determine expiry type.");
    }

    return result;
}

void CommodityVolCurve::populateCurves(const CommodityVolatilityConfig& config,
                                       const map<string, boost::shared_ptr<YieldCurve>>& yieldCurves,
                                       const map<string, boost::shared_ptr<CommodityCurve>>& commodityCurves,
                                       bool searchYield, bool dontThrow) {

    if (searchYield) {
        const string& ytsId = config.yieldCurveId();
        if (!ytsId.empty()) {
            auto itYts = yieldCurves.find(ytsId);
            if (itYts != yieldCurves.end()) {
                yts_ = itYts->second->handle();
            } else if (!dontThrow) {
                QL_FAIL("CommodityVolCurve: can't find yield curve with id " << ytsId);
            }
        } else if (!dontThrow) {
            QL_FAIL("CommodityVolCurve: YieldCurveId was not populated for " << config.curveID());
        }
    }

    const string& ptsId = config.priceCurveId();
    if (!ptsId.empty()) {
        auto itPts = commodityCurves.find(ptsId);
        if (itPts != commodityCurves.end()) {
            pts_ = Handle<PriceTermStructure>(itPts->second->commodityPriceCurve());
        } else if (!dontThrow) {
            QL_FAIL("CommodityVolCurve: can't find price curve with id " << ptsId);
        }
    } else if (!dontThrow) {
        QL_FAIL("CommodityVolCurve: PriceCurveId was not populated for " << config.curveID());
    }
}

vector<Real> CommodityVolCurve::checkMoneyness(const vector<string>& strMoneynessLevels) const {

    using boost::adaptors::transformed;
    using boost::algorithm::join;

    vector<Real> moneynessLevels = parseVectorOfValues<Real>(strMoneynessLevels, &parseReal);
    sort(moneynessLevels.begin(), moneynessLevels.end(), [](Real x, Real y) { return !close(x, y) && x < y; });
    QL_REQUIRE(adjacent_find(moneynessLevels.begin(), moneynessLevels.end(),
                             [](Real x, Real y) { return close(x, y); }) == moneynessLevels.end(),
               "The configured moneyness levels contain duplicates");
    DLOG("Parsed " << moneynessLevels.size() << " unique configured moneyness levels.");
    DLOG("The moneyness levels are: " << join(
             moneynessLevels | transformed([](Real d) { return ore::data::to_string(d); }), ","));

    return moneynessLevels;
}

} // namespace data
} // namespace ore


#include "hasher/MD5Hasher.hpp"
#include "types/BufferedChunk.hpp"

#include <boost/uuid/detail/md5.hpp>

#include <iterator>

namespace file_hasher {

MD5Hasher::MD5Hasher() : size_(sizeof(boost::uuids::detail::md5::digest_type)) {}

std::shared_ptr<IChunk> MD5Hasher::hash(const std::shared_ptr<IChunk>& input) {
    using boost::uuids::detail::md5;

    md5 hash;
    hash.process_bytes(input->data(), input->size());
    md5::digest_type digest;
    hash.get_digest(digest);
    std::vector<char> result;
    result.reserve(size_);
    const char* begin = reinterpret_cast<const char*>(&digest);
    std::copy(begin, begin + size_, std::back_inserter(result));
    return std::make_shared<BufferedChunk>(input->id(), std::move(result));
}

size_t MD5Hasher::hashSizeBytes() const {
    return size_;
}

} // namespace file_hasher


// Copyright (c) 2012-2017 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include <consensus/tx_verify.h>
#include <core_io.h>
#include <key.h>
#include <keystore.h>
#include <validation.h>
#include <policy/policy.h>
#include <script/script.h>
#include <script/script_error.h>
#include <script/sign.h>
#include <script/ismine.h>
#include <test/test_bitcoin.h>

#include <vector>

#include <boost/test/unit_test.hpp>
#include <err.h>
#include "../script/standard.h"
#include "../script/script.h"
#include "test_bitcoin.h"
#include "../utiltime.h"
#include "../policy/policy.h"
#include "../amount.h"
#include "../primitives/transaction.h"
#include "../pubkey.h"
#include "../key.h"
#include "../keystore.h"
#include "../script/interpreter.h"
#include "../sync.h"
#include "../validation.h"
#include "../script/ismine.h"
#include "../script/sign.h"
#include "../base58.h"
#include "../uint256.h"
#include "../script/script_error.h"

// Helpers:
static std::vector<unsigned char>
Serialize(const CScript& s)
{
    std::vector<unsigned char> sSerialized(s.begin(), s.end());
    return sSerialized;
}

static bool
Verify(const CScript& scriptSig, const CScript& scriptPubKey, bool fStrict, ScriptError& err)
{
    // Create dummy to/from transactions:
    CMutableTransaction txFrom;
    txFrom.vout.resize(1);
    txFrom.vout[0].scriptPubKey = scriptPubKey;

    CMutableTransaction txTo;
    txTo.vin.resize(1);
    txTo.vout.resize(1);
    txTo.vin[0].prevout.n = 0;
    txTo.vin[0].prevout.hash = txFrom.GetHash();
    txTo.vin[0].scriptSig = scriptSig;
    txTo.vout[0].nValue = 1;

    return VerifyScript(scriptSig, scriptPubKey, nullptr, fStrict ? SCRIPT_VERIFY_P2SH : SCRIPT_VERIFY_NONE, MutableTransactionSignatureChecker(&txTo, 0, txFrom.vout[0].nValue), &err);
}

BOOST_FIXTURE_TEST_SUITE(script_P2SH_tests, BasicTestingSetup)

BOOST_AUTO_TEST_CASE(checklocktimeverify)
{
    //Make two keys and add them to a keystore
    CBasicKeyStore keystore;
    CKey key1;
    key1.MakeNewKey(true);
    keystore.AddKey(key1);
    CPubKey pubkey1 = key1.GetPubKey();
    CKey key2;
    key2.MakeNewKey(true);
    keystore.AddKey(key2);
    CPubKey pubkey2 = key2.GetPubKey();

    unsigned int nUnlockTime = GetTime() + (60*60); // the time that key2 will be able to create a valid spend
    std::cout << "unlock time: " << nUnlockTime << "\n";
    CScript scriptRedeem;
    scriptRedeem << OP_IF << ToByteVector(pubkey1) << OP_CHECKSIGVERIFY << OP_ELSE << CScriptNum(nUnlockTime) << OP_CHECKLOCKTIMEVERIFY << OP_DROP << OP_ENDIF << ToByteVector(pubkey2) << OP_CHECKSIG;

    CScript scriptHash;
    scriptHash << OP_HASH160 << ToByteVector(CScriptID(scriptRedeem)) << OP_EQUAL;

    std::vector<std::vector<unsigned char> > solutions;
    txnouttype whichType;
    BOOST_CHECK(Solver(scriptHash, whichType, solutions));
    BOOST_CHECK_EQUAL(whichType, TX_SCRIPTHASH);
    BOOST_CHECK_EQUAL(solutions.size(), 1);
    BOOST_CHECK(solutions[0] == ToByteVector(CScriptID(scriptRedeem)));
    BOOST_CHECK_MESSAGE(scriptHash.IsPayToScriptHash(), "script is not p2sh");

    //Create a funding tx with a CLTV UTXO
    CMutableTransaction txFrom;
    CAmount amount = 1000;
    CTxOut out(amount, scriptHash);
    txFrom.vout.emplace_back(out);

    std::string reason;
    BOOST_CHECK_MESSAGE(IsStandardTx(txFrom, reason), "funding transaction is not standard");

    //Make a tx that spends the UTXO from key1 (valid to spend at all times)
    CMutableTransaction txTo;
    txTo.vin.resize(1);
    txTo.vout.resize(1);
    txTo.vin[0].prevout.n = 0;
    txTo.vin[0].prevout.hash = txFrom.GetHash();
    txTo.vout[0].nValue = amount;
    txTo.nLockTime = GetTime();
    unsigned int nSequence = 0;
    nSequence = ~nSequence;
    nSequence -= 1;
    txTo.vin[0].nSequence = nSequence;
    std::cout << "nseq: " << txTo.vin[0].nSequence << "\n";

    //Sign the tx with key1
    PrecomputedTransactionData txdata(txTo);
    uint256 hash = SignatureHash(scriptRedeem, txTo, 0, SIGHASH_ALL, amount, SIGVERSION_BASE, &txdata);
    std::vector<unsigned char> vchSig;
    BOOST_CHECK(key1.Sign(hash, vchSig));
    vchSig.push_back((unsigned char)SIGHASH_ALL);
    std::cout << "pubkey1: " << pubkey1.GetHash().GetHex() << " hash=" << hash.GetHex() << "\n";
    BOOST_CHECK_MESSAGE(pubkey1.Verify(hash, vchSig), "failed to verify sig");

    //Create scriptSig which adds the signature + the original redeem script
    CScript scriptSig;
    scriptSig << vchSig << OP_1 << Serialize(scriptRedeem);

    //Check that the scriptSig actually unlocks the locked UTXO
    ScriptError serror;
    unsigned int flags = SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY | SCRIPT_VERIFY_P2SH;// | SCRIPT_VERIFY_STRICTENC;
    std::cout << "before verifyscript\n";
    bool fVerifyScript = VerifyScript(scriptSig, scriptRedeem, nullptr, SCRIPT_VERIFY_P2SH, MutableTransactionSignatureChecker(&txTo, 0, amount), &serror);
    BOOST_CHECK_MESSAGE(fVerifyScript, ScriptErrorString(serror));
    std::cout << "after verifyscript\n";

    bool sigOK = CScriptCheck(txFrom.vout[0], txTo, 0, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_STRICTENC, false, &txdata)();
    BOOST_CHECK_MESSAGE(sigOK, "sig not ok");

    //Try spending using key2 which should not work yet
    uint256 hash2 = SignatureHash(scriptHash, txTo, 0, SIGHASH_ALL, amount, SIGVERSION_BASE);
    std::vector<unsigned char> vchSig2;
    BOOST_CHECK_MESSAGE(key2.Sign(hash2, vchSig2), "failed to sign with key2");

    CScript scriptSig2;
    scriptSig2 << ToByteVector(vchSig2) << OP_1 << Serialize(scriptRedeem);
    fVerifyScript = VerifyScript(scriptSig2, scriptHash, nullptr, flags, MutableTransactionSignatureChecker(&txTo, 0, amount), &serror);
    BOOST_CHECK_MESSAGE(!fVerifyScript, "scriptSig was valid even though key2 is not within the time range");
}

BOOST_AUTO_TEST_CASE(sign)
{
    LOCK(cs_main);
    // Pay-to-script-hash looks like this:
    // scriptSig:    <sig> <sig...> <serialized_script>
    // scriptPubKey: HASH160 <hash> EQUAL

    // Test SignSignature() (and therefore the version of Solver() that signs transactions)
    CBasicKeyStore keystore;
    CKey key[4];
    for (int i = 0; i < 4; i++)
    {
        key[i].MakeNewKey(true);
        keystore.AddKey(key[i]);
    }

    // 8 Scripts: checking all combinations of
    // different keys, straight/P2SH, pubkey/pubkeyhash
    CScript standardScripts[4];
    standardScripts[0] << ToByteVector(key[0].GetPubKey()) << OP_CHECKSIG;
    standardScripts[1] = GetScriptForDestination(key[1].GetPubKey().GetID());
    standardScripts[2] << ToByteVector(key[1].GetPubKey()) << OP_CHECKSIG;
    standardScripts[3] = GetScriptForDestination(key[2].GetPubKey().GetID());
    CScript evalScripts[4];
    for (int i = 0; i < 4; i++)
    {
        keystore.AddCScript(standardScripts[i]);
        evalScripts[i] = GetScriptForDestination(CScriptID(standardScripts[i]));
    }

    CMutableTransaction txFrom;  // Funding transaction:
    std::string reason;
    txFrom.vout.resize(8);
    for (int i = 0; i < 4; i++)
    {
        txFrom.vout[i].scriptPubKey = evalScripts[i];
        txFrom.vout[i].nValue = COIN;
        txFrom.vout[i+4].scriptPubKey = standardScripts[i];
        txFrom.vout[i+4].nValue = COIN;
    }
    BOOST_CHECK(IsStandardTx(txFrom, reason));

    CMutableTransaction txTo[8]; // Spending transactions
    for (int i = 0; i < 8; i++)
    {
        txTo[i].vin.resize(1);
        txTo[i].vout.resize(1);
        txTo[i].vin[0].prevout.n = i;
        txTo[i].vin[0].prevout.hash = txFrom.GetHash();
        txTo[i].vout[0].nValue = 1;
        BOOST_CHECK_MESSAGE(IsMine(keystore, txFrom.vout[i].scriptPubKey), strprintf("IsMine %d", i));
    }
    for (int i = 0; i < 8; i++)
    {
        BOOST_CHECK_MESSAGE(SignSignature(keystore, txFrom, txTo[i], 0, SIGHASH_ALL), strprintf("SignSignature %d", i));
    }
    // All of the above should be OK, and the txTos have valid signatures
    // Check to make sure signature verification fails if we use the wrong ScriptSig:
    for (int i = 0; i < 8; i++) {
        PrecomputedTransactionData txdata(txTo[i]);
        for (int j = 0; j < 8; j++)
        {
            CScript sigSave = txTo[i].vin[0].scriptSig;
            txTo[i].vin[0].scriptSig = txTo[j].vin[0].scriptSig;
            bool sigOK = CScriptCheck(txFrom.vout[txTo[i].vin[0].prevout.n], txTo[i], 0, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_STRICTENC, false, &txdata)();
            if (i == j)
                BOOST_CHECK_MESSAGE(sigOK, strprintf("VerifySignature %d %d", i, j));
            else
                BOOST_CHECK_MESSAGE(!sigOK, strprintf("VerifySignature %d %d", i, j));
            txTo[i].vin[0].scriptSig = sigSave;
        }
    }
}

BOOST_AUTO_TEST_CASE(norecurse)
{
    ScriptError err;
    // Make sure only the outer pay-to-script-hash does the
    // extra-validation thing:
    CScript invalidAsScript;
    invalidAsScript << OP_INVALIDOPCODE << OP_INVALIDOPCODE;

    CScript p2sh = GetScriptForDestination(CScriptID(invalidAsScript));

    CScript scriptSig;
    scriptSig << Serialize(invalidAsScript);

    // Should not verify, because it will try to execute OP_INVALIDOPCODE
    BOOST_CHECK(!Verify(scriptSig, p2sh, true, err));
    BOOST_CHECK_MESSAGE(err == SCRIPT_ERR_BAD_OPCODE, ScriptErrorString(err));

    // Try to recur, and verification should succeed because
    // the inner HASH160 <> EQUAL should only check the hash:
    CScript p2sh2 = GetScriptForDestination(CScriptID(p2sh));
    CScript scriptSig2;
    scriptSig2 << Serialize(invalidAsScript) << Serialize(p2sh);

    BOOST_CHECK(Verify(scriptSig2, p2sh2, true, err));
    BOOST_CHECK_MESSAGE(err == SCRIPT_ERR_OK, ScriptErrorString(err));
}

BOOST_AUTO_TEST_CASE(set)
{
    LOCK(cs_main);
    // Test the CScript::Set* methods
    CBasicKeyStore keystore;
    CKey key[4];
    std::vector<CPubKey> keys;
    for (int i = 0; i < 4; i++)
    {
        key[i].MakeNewKey(true);
        keystore.AddKey(key[i]);
        keys.push_back(key[i].GetPubKey());
    }

    CScript inner[4];
    inner[0] = GetScriptForDestination(key[0].GetPubKey().GetID());
    inner[1] = GetScriptForMultisig(2, std::vector<CPubKey>(keys.begin(), keys.begin()+2));
    inner[2] = GetScriptForMultisig(1, std::vector<CPubKey>(keys.begin(), keys.begin()+2));
    inner[3] = GetScriptForMultisig(2, std::vector<CPubKey>(keys.begin(), keys.begin()+3));

    CScript outer[4];
    for (int i = 0; i < 4; i++)
    {
        outer[i] = GetScriptForDestination(CScriptID(inner[i]));
        keystore.AddCScript(inner[i]);
    }

    CMutableTransaction txFrom;  // Funding transaction:
    std::string reason;
    txFrom.vout.resize(4);
    for (int i = 0; i < 4; i++)
    {
        txFrom.vout[i].scriptPubKey = outer[i];
        txFrom.vout[i].nValue = CENT;
    }
    BOOST_CHECK(IsStandardTx(txFrom, reason));

    CMutableTransaction txTo[4]; // Spending transactions
    for (int i = 0; i < 4; i++)
    {
        txTo[i].vin.resize(1);
        txTo[i].vout.resize(1);
        txTo[i].vin[0].prevout.n = i;
        txTo[i].vin[0].prevout.hash = txFrom.GetHash();
        txTo[i].vout[0].nValue = 1*CENT;
        txTo[i].vout[0].scriptPubKey = inner[i];
        BOOST_CHECK_MESSAGE(IsMine(keystore, txFrom.vout[i].scriptPubKey), strprintf("IsMine %d", i));
    }
    for (int i = 0; i < 4; i++)
    {
        BOOST_CHECK_MESSAGE(SignSignature(keystore, txFrom, txTo[i], 0, SIGHASH_ALL), strprintf("SignSignature %d", i));
        BOOST_CHECK_MESSAGE(IsStandardTx(txTo[i], reason), strprintf("txTo[%d].IsStandard", i));
    }
}

BOOST_AUTO_TEST_CASE(is)
{
    // Test CScript::IsPayToScriptHash()
    uint160 dummy;
    CScript p2sh;
    p2sh << OP_HASH160 << ToByteVector(dummy) << OP_EQUAL;
    BOOST_CHECK(p2sh.IsPayToScriptHash());

    // Not considered pay-to-script-hash if using one of the OP_PUSHDATA opcodes:
    static const unsigned char direct[] =    { OP_HASH160, 20, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, OP_EQUAL };
    BOOST_CHECK(CScript(direct, direct+sizeof(direct)).IsPayToScriptHash());
    static const unsigned char pushdata1[] = { OP_HASH160, OP_PUSHDATA1, 20, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, OP_EQUAL };
    BOOST_CHECK(!CScript(pushdata1, pushdata1+sizeof(pushdata1)).IsPayToScriptHash());
    static const unsigned char pushdata2[] = { OP_HASH160, OP_PUSHDATA2, 20,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, OP_EQUAL };
    BOOST_CHECK(!CScript(pushdata2, pushdata2+sizeof(pushdata2)).IsPayToScriptHash());
    static const unsigned char pushdata4[] = { OP_HASH160, OP_PUSHDATA4, 20,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, OP_EQUAL };
    BOOST_CHECK(!CScript(pushdata4, pushdata4+sizeof(pushdata4)).IsPayToScriptHash());

    CScript not_p2sh;
    BOOST_CHECK(!not_p2sh.IsPayToScriptHash());

    not_p2sh.clear(); not_p2sh << OP_HASH160 << ToByteVector(dummy) << ToByteVector(dummy) << OP_EQUAL;
    BOOST_CHECK(!not_p2sh.IsPayToScriptHash());

    not_p2sh.clear(); not_p2sh << OP_NOP << ToByteVector(dummy) << OP_EQUAL;
    BOOST_CHECK(!not_p2sh.IsPayToScriptHash());

    not_p2sh.clear(); not_p2sh << OP_HASH160 << ToByteVector(dummy) << OP_CHECKSIG;
    BOOST_CHECK(!not_p2sh.IsPayToScriptHash());
}

BOOST_AUTO_TEST_CASE(switchover)
{
    // Test switch over code
    CScript notValid;
    ScriptError err;
    notValid << OP_11 << OP_12 << OP_EQUALVERIFY;
    CScript scriptSig;
    scriptSig << Serialize(notValid);

    CScript fund = GetScriptForDestination(CScriptID(notValid));

    // Validation should succeed under old rules (hash is correct):
    BOOST_CHECK(Verify(scriptSig, fund, false, err));
    BOOST_CHECK_MESSAGE(err == SCRIPT_ERR_OK, ScriptErrorString(err));
    // Fail under new:
    BOOST_CHECK(!Verify(scriptSig, fund, true, err));
    BOOST_CHECK_MESSAGE(err == SCRIPT_ERR_EQUALVERIFY, ScriptErrorString(err));
}

BOOST_AUTO_TEST_CASE(AreInputsStandard)
{
    LOCK(cs_main);
    CCoinsView coinsDummy;
    CCoinsViewCache coins(&coinsDummy);
    CBasicKeyStore keystore;
    CKey key[6];
    std::vector<CPubKey> keys;
    for (int i = 0; i < 6; i++)
    {
        key[i].MakeNewKey(true);
        keystore.AddKey(key[i]);
    }
    for (int i = 0; i < 3; i++)
        keys.push_back(key[i].GetPubKey());

    CMutableTransaction txFrom;
    txFrom.vout.resize(7);

    // First three are standard:
    CScript pay1 = GetScriptForDestination(key[0].GetPubKey().GetID());
    keystore.AddCScript(pay1);
    CScript pay1of3 = GetScriptForMultisig(1, keys);

    txFrom.vout[0].scriptPubKey = GetScriptForDestination(CScriptID(pay1)); // P2SH (OP_CHECKSIG)
    txFrom.vout[0].nValue = 1000;
    txFrom.vout[1].scriptPubKey = pay1; // ordinary OP_CHECKSIG
    txFrom.vout[1].nValue = 2000;
    txFrom.vout[2].scriptPubKey = pay1of3; // ordinary OP_CHECKMULTISIG
    txFrom.vout[2].nValue = 3000;

    // vout[3] is complicated 1-of-3 AND 2-of-3
    // ... that is OK if wrapped in P2SH:
    CScript oneAndTwo;
    oneAndTwo << OP_1 << ToByteVector(key[0].GetPubKey()) << ToByteVector(key[1].GetPubKey()) << ToByteVector(key[2].GetPubKey());
    oneAndTwo << OP_3 << OP_CHECKMULTISIGVERIFY;
    oneAndTwo << OP_2 << ToByteVector(key[3].GetPubKey()) << ToByteVector(key[4].GetPubKey()) << ToByteVector(key[5].GetPubKey());
    oneAndTwo << OP_3 << OP_CHECKMULTISIG;
    keystore.AddCScript(oneAndTwo);
    txFrom.vout[3].scriptPubKey = GetScriptForDestination(CScriptID(oneAndTwo));
    txFrom.vout[3].nValue = 4000;

    // vout[4] is max sigops:
    CScript fifteenSigops; fifteenSigops << OP_1;
    for (unsigned i = 0; i < MAX_P2SH_SIGOPS; i++)
        fifteenSigops << ToByteVector(key[i%3].GetPubKey());
    fifteenSigops << OP_15 << OP_CHECKMULTISIG;
    keystore.AddCScript(fifteenSigops);
    txFrom.vout[4].scriptPubKey = GetScriptForDestination(CScriptID(fifteenSigops));
    txFrom.vout[4].nValue = 5000;

    // vout[5/6] are non-standard because they exceed MAX_P2SH_SIGOPS
    CScript sixteenSigops; sixteenSigops << OP_16 << OP_CHECKMULTISIG;
    keystore.AddCScript(sixteenSigops);
    txFrom.vout[5].scriptPubKey = GetScriptForDestination(CScriptID(fifteenSigops));
    txFrom.vout[5].nValue = 5000;
    CScript twentySigops; twentySigops << OP_CHECKMULTISIG;
    keystore.AddCScript(twentySigops);
    txFrom.vout[6].scriptPubKey = GetScriptForDestination(CScriptID(twentySigops));
    txFrom.vout[6].nValue = 6000;

    AddCoins(coins, txFrom, 0);

    CMutableTransaction txTo;
    txTo.vout.resize(1);
    txTo.vout[0].scriptPubKey = GetScriptForDestination(key[1].GetPubKey().GetID());

    txTo.vin.resize(5);
    for (int i = 0; i < 5; i++)
    {
        txTo.vin[i].prevout.n = i;
        txTo.vin[i].prevout.hash = txFrom.GetHash();
    }
    BOOST_CHECK(SignSignature(keystore, txFrom, txTo, 0, SIGHASH_ALL));
    BOOST_CHECK(SignSignature(keystore, txFrom, txTo, 1, SIGHASH_ALL));
    BOOST_CHECK(SignSignature(keystore, txFrom, txTo, 2, SIGHASH_ALL));
    // SignSignature doesn't know how to sign these. We're
    // not testing validating signatures, so just create
    // dummy signatures that DO include the correct P2SH scripts:
    txTo.vin[3].scriptSig << OP_11 << OP_11 << std::vector<unsigned char>(oneAndTwo.begin(), oneAndTwo.end());
    txTo.vin[4].scriptSig << std::vector<unsigned char>(fifteenSigops.begin(), fifteenSigops.end());

    BOOST_CHECK(::AreInputsStandard(txTo, coins));
    // 22 P2SH sigops for all inputs (1 for vin[0], 6 for vin[3], 15 for vin[4]
    BOOST_CHECK_EQUAL(GetP2SHSigOpCount(txTo, coins), 22U);

    CMutableTransaction txToNonStd1;
    txToNonStd1.vout.resize(1);
    txToNonStd1.vout[0].scriptPubKey = GetScriptForDestination(key[1].GetPubKey().GetID());
    txToNonStd1.vout[0].nValue = 1000;
    txToNonStd1.vin.resize(1);
    txToNonStd1.vin[0].prevout.n = 5;
    txToNonStd1.vin[0].prevout.hash = txFrom.GetHash();
    txToNonStd1.vin[0].scriptSig << std::vector<unsigned char>(sixteenSigops.begin(), sixteenSigops.end());

    BOOST_CHECK(!::AreInputsStandard(txToNonStd1, coins));
    BOOST_CHECK_EQUAL(GetP2SHSigOpCount(txToNonStd1, coins), 16U);

    CMutableTransaction txToNonStd2;
    txToNonStd2.vout.resize(1);
    txToNonStd2.vout[0].scriptPubKey = GetScriptForDestination(key[1].GetPubKey().GetID());
    txToNonStd2.vout[0].nValue = 1000;
    txToNonStd2.vin.resize(1);
    txToNonStd2.vin[0].prevout.n = 6;
    txToNonStd2.vin[0].prevout.hash = txFrom.GetHash();
    txToNonStd2.vin[0].scriptSig << std::vector<unsigned char>(twentySigops.begin(), twentySigops.end());

    BOOST_CHECK(!::AreInputsStandard(txToNonStd2, coins));
    BOOST_CHECK_EQUAL(GetP2SHSigOpCount(txToNonStd2, coins), 20U);
}

BOOST_AUTO_TEST_SUITE_END()