/*

CandyPoker
https://github.com/sweeterthancandy/CandyPoker

MIT License

Copyright (c) 2019 Gerry Candy

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

*/
#include "ps/base/algorithm.h"
#include "ps/base/rank_hasher.h"

#include <boost/range/algorithm.hpp>
#include <map>

namespace ps{

/*

   The idea of this is to 2-tuple of permutations (C,S),
   so for a vector of n-players, we apply the permutation
                        C : c -> c',
   ie
                (p1,p2,p3) -> (p1', p2', p3'),
   and then apply the suit permutation S : s -> s'
                (p1', p2', p3') -> (p1'', p2'', p3''),
   so that given and set of players, we can can an injective
   mapping to a small subset. Ei, AA vs KK == KK vs AA
   with the player orientation swapped around.
        Noting that we can complete ignore any permutation
   of suits for all practical purposes, as always just want
   to find a suit permutation so that it's injective

                    eval(AA,KK vs AA,KK)

        =           eval(AA vs AA ) * w0 +
                    eval(AA vs KK ) * w1 +
                    eval(KK vs AA ) * w2 +
                    eval(KK vs KK ) * w3

        =           eval(AA vs AA ) * w0 +
                    eval(AA vs KK ) * w1 +
            Inverse(eval(AA vs KK )) * w2 +
                    eval(KK vs KK ) * w3

 */
struct rank_info{
        size_t index;
        holdem_id hid;
        rank_hasher::rank_hash_t rank_hash;
        bool is_suited;
        friend bool operator==(rank_info const& l, rank_info const& r)noexcept{
                return ( ! ( l < r ) ) && ( ! ( r < l ) );
        }
        friend bool operator!=(rank_info const& l, rank_info const& r)noexcept{
                return ( ! (l == r) );
        }
        friend bool operator<(rank_info const& l, rank_info const& r)noexcept{
                if( l.rank_hash != r.rank_hash ){
                        return l.rank_hash <  r.rank_hash;
                }
                return l.is_suited <  r.is_suited;
        }
};
struct suit_perm_info{
        std::array<suit_id, 4> suit_perm;
        std::vector<int> player_perm;
        holdem_hand_vector hv;
        size_t card_mask;
        friend bool operator<(suit_perm_info const& l, suit_perm_info const& r)noexcept{
                #if 0
                if( l.card_mask != r.card_mask )
                        return l.card_mask < r.card_mask;
                #endif
                return l.hv < r.hv;
        }
};

std::tuple<
        std::vector<int>,
        std::vector<holdem_id>
> permutate_for_the_better( std::vector<holdem_id> const& players )
{
        enum{ Debug = false };

        std::vector< rank_info > ri;
        for(size_t idx=0;idx!=players.size();++idx){
                auto h =  holdem_hand_decl::get( players[idx] ) ;
                auto rank_hash = rank_hasher::create_from_cards(h.first(), h.second());
                ri.emplace_back(rank_info{idx, players[idx], rank_hash, h.is_suited()});
        }
        boost::sort( ri );

        std::vector< std::vector<rank_info> > level_sets;

        level_sets.emplace_back();
        level_sets.back().push_back( ri[0] );
        for(size_t idx=1; idx < ri.size();++idx){
                if( level_sets.back().front() == ri[idx]){
                        // same set
                        level_sets.back().push_back(ri[idx]);
                } else {
                        // start new set
                        level_sets.emplace_back();
                        level_sets.back().push_back(ri[idx]);
                }
        }

        if( Debug ){
                PS_LOG(trace) << "begin level_sets";
                for(auto const& ls : level_sets ){
                        holdem_hand_vector hv;
                        for(auto const& _ : ls){
                                hv.push_back(_.hid);
                        }
                        PS_LOG(trace) << "    -" << hv;
                }
                PS_LOG(trace) << "end   level_sets";
        }

        std::vector< std::vector< rank_info > > rank_permutations;
        rank_permutations.emplace_back();

        for(auto& ls : level_sets){
                if( ls.size() == 1 ){
                        for(auto& _ : rank_permutations ){
                                _.push_back(ls.back());
                        }
                } else {

                        decltype(rank_permutations) next;

                        // should be sorted anyway
                        boost::sort(ls);
                        do{
                                for(auto rp : rank_permutations ){
                                        for(auto const& item : ls ){
                                                rp.push_back(item);
                                        }
                                        next.push_back(rp);
                                }
                        }while(boost::next_permutation(ls, [](auto const& l, auto const& r){ return l.index < r.index; }));
                        rank_permutations = std::move(next);
                }
        }

        if( Debug ){
                PS_LOG(trace) << "rank_permutations.size() => " << rank_permutations.size();
                PS_LOG(trace) << "begin rank_permutations";
                for(auto const& rp : rank_permutations ){
                        holdem_hand_vector hv;
                        for(auto const& _ : rp){
                                hv.push_back(_.hid);
                        }
                        PS_LOG(trace) << "    -" << hv;
                }
                PS_LOG(trace) << "end   rank_permutations";
        }

        std::vector<suit_perm_info> suit_perm_vec;

        std::array<suit_id, 4> suits = { 0, 1, 2, 3};
        for(auto const& rp : rank_permutations ){
                boost::sort(suits);
                do{
                        std::vector<int> player_perm;
                        holdem_hand_vector hv;
                        size_t mask = 0;

                        for(auto const& _ : rp){

                                auto h =  holdem_hand_decl::get( _.hid ) ;

                                rank_id r0 = h.first().rank();
                                suit_id s0 = h.first().suit();
                                rank_id r1 = h.second().rank();
                                suit_id s1 = h.second().suit();

                                suit_id m0 = suits[s0];
                                suit_id m1 = suits[s1];

                                holdem_id mhid = holdem_hand_decl::make_id(r0, m0, r1, m1);

                                mask |= static_cast<size_t>(1) << mhid;

                                player_perm.push_back(_.index);
                                hv.push_back(mhid);
                        }

                        suit_perm_vec.push_back(suit_perm_info{suits, std::move(player_perm),
                                                               hv, mask});

                }while(boost::next_permutation(suits));
        }

        boost::sort(suit_perm_vec);
        if( Debug ){
                PS_LOG(trace) << "begin rank_permutations";
                for(auto const& sp : suit_perm_vec ){
                        PS_LOG(trace) << "    -" << sp.hv;
                }
                PS_LOG(trace) << "end   rank_permutations";
                PS_LOG(trace) << "hv = " << suit_perm_vec.front().hv;
        }

        return { suit_perm_vec.front().player_perm, suit_perm_vec.front().hv };

}

#if 0
std::tuple<
        std::vector<int>,
        std::vector<holdem_id>
> permutate_for_the_better( std::vector<holdem_id> const& players ){
        // first create vector of n, and token_n = hh_n
        //      (0,hh_0), (1,hh_1), ... (n,hh_n),
        // where first h is greater handk the second h
        std::vector< std::tuple< size_t, std::string> > player_perm;
        for(size_t i=0;i!=players.size();++i){
                auto h =  holdem_hand_decl::get( players[i] ) ;
                player_perm.emplace_back(i, h.first().rank().to_string() +
                                            h.second().rank().to_string() );
        }
        // sort it by the token
        boost::sort(player_perm, [](auto const& left, auto const& right){
                return std::get<1>(left) < std::get<1>(right);
        });

        // new work out the perm used to create it
        std::vector<int> perm;
        for(size_t i=0;i!=players.size();++i){
                perm.emplace_back( std::get<0>(player_perm[i]) );
        }

        std::vector<std::vector<holdem_hand_decl> > decls;

        // now we allocate suits, starting with 0 etc
        std::array< int, 4> rev_suit_map{-1,-1,-1,-1};
        int suit_iter = 0; // using the fact we know suits \in {0,1,2,3}
        // allocate pocket pairs common types first

        // AA KK -> AaAb KcKa
        //       -> AaAb KcKb -> AbAa KcKb

        std::map<int, int> pp_count;
        for(size_t i=0;i!=players.size();++i){
                auto h =  holdem_hand_decl::get( players[perm[i]] ) ;
                if( h.first().rank() != h.second().rank()){
                        continue;
                }
                ++pp_count[h.first().suit()];
                ++pp_count[h.second().suit()];
        }

        for(size_t i=0;i!=players.size();++i){
                auto h =  holdem_hand_decl::get( players[perm[i]] ) ;

                auto a = &h.first();
                auto b = &h.second();
                // TODO pocket pair
                #if 1
                if( a->rank() == b->rank()){
                        if( pp_count[a->suit()] > pp_count[b->suit()] ){
                                std::swap(a,b);
                        }
                }
                #endif

                if(     rev_suit_map[a->suit()] == -1 )
                        rev_suit_map[a->suit()] = suit_iter++;
                if(     rev_suit_map[b->suit()] == -1 )
                        rev_suit_map[b->suit()] = suit_iter++;
        }

        // TODO remove this, unneeded
        for(size_t i=0;i != 4;++i){
                if(     rev_suit_map[i] == -1 )
                        rev_suit_map[i] = suit_iter++;
        }

        // crate map
        std::vector< int> suit_perms;
        for(size_t i=0;i != 4;++i){
                suit_perms.emplace_back(rev_suit_map[i]);
        }

        std::vector<holdem_id> perm_hands;
        for(size_t i=0;i != players.size();++i){
                auto h =  holdem_hand_decl::get( players[perm[i]] ) ;

                #if 0
                if( h.first().rank() == h.second().rank() ){
                        if( suit_perms[h.first().suit()] > suit_perms[h.second().suit()] ){
                                perm_hands.emplace_back(
                                        holdem_hand_decl::make_id(
                                                h.first().rank(),
                                                suit_perms[h.second().suit()],
                                                h.second().rank(),
                                                suit_perms[h.first().suit()]));
                        } else {
                                perm_hands.emplace_back(
                                        holdem_hand_decl::make_id(
                                                h.first().rank(),
                                                suit_perms[h.first().suit()],
                                                h.second().rank(),
                                                suit_perms[h.second().suit()]));
                        }
                } else{
                #endif
                        perm_hands.emplace_back(
                                holdem_hand_decl::make_id(
                                        h.first().rank(),
                                        suit_perms[h.first().suit()],
                                        h.second().rank(),
                                        suit_perms[h.second().suit()]));
                #if 0
                }
                #endif

        }
        return std::make_tuple( perm, perm_hands);
}
#endif

} // ps


#include "ModulusSolver.h"
#include "EigenSparseResponse.h"

#include <sofa/core/ObjectFactory.h>

#include "../utils/anderson.h"
#include "../utils/nlnscg.h"

#include "../utils/scoped.h"

#include "../constraint/CoulombConstraint.h"
#include "../constraint/UnilateralConstraint.h"

#include <Eigen/SparseCholesky>
#include "../utils/sub_kkt.inl"

namespace utils {

template<class Matrix>
struct sub_kkt::traits< Eigen::SimplicialLDLT<Matrix, Eigen::Upper> > {

    typedef Eigen::SimplicialLDLT<Matrix, Eigen::Upper> solver_type;

    static void factor(solver_type& solver, const rmat& matrix) {
        // ldlt expects a cmat so we transpose to ease the conversion
        solver.compute(matrix.triangularView<Eigen::Lower>().transpose());
        if( solver.info() != Eigen::Success ) {
            std::cerr << "error during LDLT factorization !" << std::endl;

            std::cerr << matrix << std::endl;
        }
    }

    static void solve(const solver_type& solver, vec& res, const vec& rhs) {
        res = solver.solve(rhs);
    }

};

}

namespace sofa {
namespace component {
namespace linearsolver {

SOFA_DECL_CLASS(ModulusSolver)
int ModulusSolverClass = core::RegisterObject("Modulus solver").add< ModulusSolver >();

ModulusSolver::ModulusSolver()
    : omega(initData(&omega,
                     (real)1.0,
                     "omega",
                     "magic stuff")),
      anderson(initData(&anderson, unsigned(0),
                        "anderson",
                        "anderson acceleration history size, 0 if none")),
      nlnscg(initData(&nlnscg, false,
                        "nlnscg",
                        "nlnscg acceleration"))

{

}

ModulusSolver::~ModulusSolver() {

}

void ModulusSolver::factor(const system_type& sys) {

    // build unilateral mask
    unilateral = vec::Zero(sys.n);

    // build system TODO hardcoded regularization
    sub.projected_kkt(sys, 1e-14, true);

    const vec Hdiag_inv = sys.P * sys.H.diagonal().cwiseInverse();

    diagonal = vec::Zero(sys.n);

    // change diagonal compliance
    const real omega = this->omega.getValue();

    // compute diagonal + homogenize tangent values
    {
        unsigned off = 0;

        for(unsigned i = 0, n = sys.constraints.size(); i < n; ++i) {
            const unsigned dim = sys.compliant[i]->getMatrixSize();

            for(unsigned j = off, je = off + dim; j < je; ++j) {
                diagonal(j) = sys.J.row(j).dot( Hdiag_inv.asDiagonal() * sys.J.row(j).transpose());
            }

            typedef linearsolver::CoulombConstraintBase proj_type;
            Constraint* c = sys.constraints[i].projector.get();
            if( c && proj_type::checkConstraintType(c) ) {
                assert( dim % 3 == 0 && "non vec3 dofs not handled");
                for(unsigned j = 0, je = dim / 3; j < je; ++j) {

                    // set both tangent values to max tangent value to
                    // avoid friction anisotropy

                    const unsigned t1 = off + 3 * j + 1;
                    const unsigned t2 = off + 3 * j + 2;

                    const SReal value = std::max(diagonal(t1),
                                                 diagonal(t2));

                    diagonal(t1) = value;
                    diagonal(t2) = value;
                }

            }

            off += dim;
        }
        assert( off == sys.n );
    }

    // update system matrix (2, 2 block) with diagonal
    for(unsigned i = 0; i < sys.n; ++i) {

        sub.matrix.coeffRef(sub.primal.cols() + i,
                            sub.primal.cols() + i) += -omega * diagonal(i);
    }

    // factor the modified kkt
    sub.factor( solver );
}

void ModulusSolver::project(vec::SegmentReturnType view, const system_type& sys, bool correct) const {
    unsigned off = 0;

    for(unsigned i = 0, n = sys.constraints.size(); i < n; ++i) {
        const unsigned dim = sys.compliant[i]->getMatrixSize();

        const linearsolver::Constraint* proj = sys.constraints[i].projector.get();
        if( proj ) {
            proj->project(&view[off], dim, 0, correct);
        }

        off += dim;
    }
    assert( off == view.size() );
}

// good luck with that :p
void ModulusSolver::solve(vec& res,
                          const system_type& sys,
                          const vec& rhs) const {

    vec constant = rhs;
    if( sys.n ) constant.tail(sys.n) = - constant.tail(sys.n);

    vec tmp;
    sub.solve(solver, tmp, constant);

    if(!sys.n) {
        res = tmp;
        return;
    }

    // y = (x, z)
    vec y = vec::Zero( sys.size() );

    // TODO warm start ?
    vec zabs;
    vec old;
    real error;

    // TODO
    const bool correct = false;

    const real omega = this->omega.getValue();
    const real precision = this->precision.getValue();

    // utils::anderson accel(sys.n, anderson.getValue(), diagonal);
    utils::nlnscg accel(sys.n, diagonal);

    unsigned k;
    const unsigned kmax = iterations.getValue();

    const bool use_accel = anderson.getValue();

    for(k = 0; k < kmax; ++k) {

        // tmp = (x, |z|)
        tmp = y;

        // |z| = 2 * P(z) - z
        project( tmp.tail(sys.n), sys, correct );
        tmp.tail(sys.n) = 2 * tmp.tail(sys.n) - y.tail(sys.n);

        zabs = tmp.tail(sys.n);

        // TODO fix this mess
        // after this, prod is wrong, we need to add back 2 diag * omega |z|
        {
            scoped::timer step("subkkt prod");
            sub.prod(tmp, tmp, true);
        }

        // tmp += 2 * omega.getValue() * zabs;
        tmp.tail(sys.n).array() += (2 * omega) * diagonal.array() * zabs.array();

        //
        tmp -= constant;

        // solve
        sub.solve(solver, tmp, tmp);

        // backup old dual
        old = y.tail(sys.n);

        y.head(sys.m).array() -= tmp.head(sys.m).array();
        y.tail(sys.n).array() -= tmp.tail(sys.n).array() + y.tail(sys.n).array();

        if( use_accel ) {

            // reuse zabs
            zabs = y.tail(sys.n);
            accel(zabs);
            y.tail(sys.n) = zabs;
        }

        error = (old - y.tail(sys.n)).norm();
        if( error <= precision ) break;
    }

    sout << "fixed point error: " << error << "\titerations: " << k << sendl;

    // and that should be it ?
    res = y;

    // we need to project 2 z to get lambdas
    res.tail(sys.n) *= 2.0;
    project( res.tail(sys.n), sys, correct );

    // TODO we should recompute x based on actual, non sticky lambdas

}

}
}
}


/*
 Copyright (C) 2016 Quaternion Risk Management Ltd
 All rights reserved.

 This file is part of ORE, a free-software/open-source library
 for transparent pricing and risk analysis - http://opensourcerisk.org

 ORE is free software: you can redistribute it and/or modify it
 under the terms of the Modified BSD License.  You should have received a
 copy of the license along with this program.
 The license is also available online at <http://opensourcerisk.org>

 This program is distributed on the basis that it will form a useful
 contribution to risk analytics and model standardisation, but WITHOUT
 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 FITNESS FOR A PARTICULAR PURPOSE. See the license for more details.
*/

#include <boost/make_shared.hpp>
#include <ql/indexes/inflation/euhicp.hpp>
#include <ql/indexes/inflation/ukrpi.hpp>
#include <ql/termstructures/inflation/inflationhelpers.hpp>
#include <ql/termstructures/inflation/piecewiseyoyinflationcurve.hpp>
#include <ql/termstructures/inflation/piecewisezeroinflationcurve.hpp>
#include <ql/time/calendars/nullcalendar.hpp>
#include <ql/time/calendars/target.hpp>
#include <ql/time/calendars/unitedstates.hpp>
#include <qle/indexes/equityindex.hpp>
#include <qle/indexes/inflationindexwrapper.hpp>
#include <qle/termstructures/pricecurve.hpp>

#include <test/testmarket.hpp>

using QuantExt::InterpolatedPriceCurve;
using QuantExt::PriceTermStructure;
using QuantExt::EquityIndex;

namespace testsuite {

TestMarket::TestMarket(Date asof) {
    asof_ = asof;

    // add conventions
    boost::shared_ptr<ore::data::Convention> swapIndexEURConv(
        new ore::data::SwapIndexConvention("EUR-CMS-2Y", "EUR-6M-SWAP-CONVENTIONS"));
    boost::shared_ptr<ore::data::Convention> swapIndexEURLongConv(
        new ore::data::SwapIndexConvention("EUR-CMS-30Y", "EUR-6M-SWAP-CONVENTIONS"));
    boost::shared_ptr<ore::data::Convention> swapIndexUSDConv(
        new ore::data::SwapIndexConvention("USD-CMS-2Y", "USD-3M-SWAP-CONVENTIONS"));
    boost::shared_ptr<ore::data::Convention> swapIndexUSDLongConv(
        new ore::data::SwapIndexConvention("USD-CMS-30Y", "USD-3M-SWAP-CONVENTIONS"));
    boost::shared_ptr<ore::data::Convention> swapIndexGBPConv(
        new ore::data::SwapIndexConvention("GBP-CMS-2Y", "GBP-3M-SWAP-CONVENTIONS"));
    boost::shared_ptr<ore::data::Convention> swapIndexGBPLongConv(
        new ore::data::SwapIndexConvention("GBP-CMS-30Y", "GBP-6M-SWAP-CONVENTIONS"));
    boost::shared_ptr<ore::data::Convention> swapIndexCHFConv(
        new ore::data::SwapIndexConvention("CHF-CMS-2Y", "CHF-3M-SWAP-CONVENTIONS"));
    boost::shared_ptr<ore::data::Convention> swapIndexCHFLongConv(
        new ore::data::SwapIndexConvention("CHF-CMS-30Y", "CHF-6M-SWAP-CONVENTIONS"));
    boost::shared_ptr<ore::data::Convention> swapIndexJPYConv(
        new ore::data::SwapIndexConvention("JPY-CMS-2Y", "JPY-LIBOR-6M-SWAP-CONVENTIONS"));
    boost::shared_ptr<ore::data::Convention> swapIndexJPYLongConv(
        new ore::data::SwapIndexConvention("JPY-CMS-30Y", "JPY-LIBOR-6M-SWAP-CONVENTIONS"));

    conventions_.add(swapIndexEURConv);
    conventions_.add(swapIndexEURLongConv);
    conventions_.add(swapIndexUSDConv);
    conventions_.add(swapIndexUSDLongConv);
    conventions_.add(swapIndexGBPConv);
    conventions_.add(swapIndexGBPLongConv);
    conventions_.add(swapIndexCHFConv);
    conventions_.add(swapIndexCHFLongConv);
    conventions_.add(swapIndexJPYConv);
    conventions_.add(swapIndexJPYLongConv);

    boost::shared_ptr<ore::data::Convention> swapEURConv(new ore::data::IRSwapConvention(
        "EUR-6M-SWAP-CONVENTIONS", "TARGET", "Annual", "MF", "30/360", "EUR-EURIBOR-6M"));
    boost::shared_ptr<ore::data::Convention> swapUSDConv(
        new ore::data::IRSwapConvention("USD-3M-SWAP-CONVENTIONS", "US", "Semiannual", "MF", "30/360", "USD-LIBOR-3M"));
    boost::shared_ptr<ore::data::Convention> swapGBPConv(
        new ore::data::IRSwapConvention("GBP-3M-SWAP-CONVENTIONS", "UK", "Semiannual", "MF", "A365", "GBP-LIBOR-3M"));
    boost::shared_ptr<ore::data::Convention> swapGBPLongConv(
        new ore::data::IRSwapConvention("GBP-6M-SWAP-CONVENTIONS", "UK", "Semiannual", "MF", "A365", "GBP-LIBOR-6M"));
    boost::shared_ptr<ore::data::Convention> swapCHFConv(
        new ore::data::IRSwapConvention("CHF-3M-SWAP-CONVENTIONS", "ZUB", "Annual", "MF", "30/360", "CHF-LIBOR-3M"));
    boost::shared_ptr<ore::data::Convention> swapCHFLongConv(
        new ore::data::IRSwapConvention("CHF-6M-SWAP-CONVENTIONS", "ZUB", "Annual", "MF", "30/360", "CHF-LIBOR-6M"));
    boost::shared_ptr<ore::data::Convention> swapJPYConv(new ore::data::IRSwapConvention(
        "JPY-LIBOR-6M-SWAP-CONVENTIONS", "JP", "Semiannual", "MF", "A365", "JPY-LIBOR-6M"));

    conventions_.add(swapEURConv);
    conventions_.add(swapUSDConv);
    conventions_.add(swapGBPConv);
    conventions_.add(swapGBPLongConv);
    conventions_.add(swapCHFConv);
    conventions_.add(swapCHFLongConv);
    conventions_.add(swapJPYConv);

    // build discount
    yieldCurves_[make_tuple(Market::defaultConfiguration, YieldCurveType::Discount, "EUR")] = flatRateYts(0.02);
    yieldCurves_[make_tuple(Market::defaultConfiguration, YieldCurveType::Discount, "USD")] = flatRateYts(0.03);
    yieldCurves_[make_tuple(Market::defaultConfiguration, YieldCurveType::Discount, "GBP")] = flatRateYts(0.04);
    yieldCurves_[make_tuple(Market::defaultConfiguration, YieldCurveType::Discount, "CHF")] = flatRateYts(0.01);
    yieldCurves_[make_tuple(Market::defaultConfiguration, YieldCurveType::Discount, "JPY")] = flatRateYts(0.005);

    // build ibor indices
    vector<pair<string, Real>> indexData = {{"EUR-EONIA", 0.01},    {"EUR-EURIBOR-6M", 0.02}, {"USD-FedFunds", 0.01},
                                            {"USD-LIBOR-3M", 0.03}, {"USD-LIBOR-6M", 0.05},   {"GBP-SONIA", 0.01},
                                            {"GBP-LIBOR-3M", 0.03}, {"GBP-LIBOR-6M", 0.04},   {"CHF-LIBOR-3M", 0.01},
                                            {"CHF-LIBOR-6M", 0.02}, {"JPY-LIBOR-6M", 0.01}};
    for (auto id : indexData) {
        Handle<IborIndex> h(parseIborIndex(id.first, flatRateYts(id.second)));
        iborIndices_[make_pair(Market::defaultConfiguration, id.first)] = h;

        // set up dummy fixings for the past 400 days
        for (Date d = asof - 400; d < asof; d++) {
            if (h->isValidFixingDate(d))
                h->addFixing(d, 0.01);
        }
    }

    // swap index
    addSwapIndex("EUR-CMS-2Y", "EUR-EONIA", Market::defaultConfiguration);
    addSwapIndex("EUR-CMS-30Y", "EUR-EONIA", Market::defaultConfiguration);
    addSwapIndex("USD-CMS-2Y", "USD-FedFunds", Market::defaultConfiguration);
    addSwapIndex("USD-CMS-30Y", "USD-FedFunds", Market::defaultConfiguration);
    addSwapIndex("GBP-CMS-2Y", "GBP-SONIA", Market::defaultConfiguration);
    addSwapIndex("GBP-CMS-30Y", "GBP-SONIA", Market::defaultConfiguration);
    addSwapIndex("CHF-CMS-2Y", "CHF-LIBOR-6M", Market::defaultConfiguration);
    addSwapIndex("CHF-CMS-30Y", "CHF-LIBOR-6M", Market::defaultConfiguration);
    addSwapIndex("JPY-CMS-2Y", "JPY-LIBOR-6M", Market::defaultConfiguration);
    addSwapIndex("JPY-CMS-30Y", "JPY-LIBOR-6M", Market::defaultConfiguration);

    // add fx rates
    fxSpots_[Market::defaultConfiguration].addQuote("EURUSD", Handle<Quote>(boost::make_shared<SimpleQuote>(1.2)));
    fxSpots_[Market::defaultConfiguration].addQuote("EURGBP", Handle<Quote>(boost::make_shared<SimpleQuote>(0.8)));
    fxSpots_[Market::defaultConfiguration].addQuote("EURCHF", Handle<Quote>(boost::make_shared<SimpleQuote>(1.0)));
    fxSpots_[Market::defaultConfiguration].addQuote("EURJPY", Handle<Quote>(boost::make_shared<SimpleQuote>(128.0)));

    // build fx vols
    fxVols_[make_pair(Market::defaultConfiguration, "EURUSD")] = flatRateFxv(0.12);
    fxVols_[make_pair(Market::defaultConfiguration, "EURGBP")] = flatRateFxv(0.15);
    fxVols_[make_pair(Market::defaultConfiguration, "EURCHF")] = flatRateFxv(0.15);
    fxVols_[make_pair(Market::defaultConfiguration, "EURJPY")] = flatRateFxv(0.15);
    fxVols_[make_pair(Market::defaultConfiguration, "GBPCHF")] = flatRateFxv(0.15);

    // Add Equity Spots
    equitySpots_[make_pair(Market::defaultConfiguration, "SP5")] =
        Handle<Quote>(boost::make_shared<SimpleQuote>(2147.56));
    equitySpots_[make_pair(Market::defaultConfiguration, "Lufthansa")] =
        Handle<Quote>(boost::make_shared<SimpleQuote>(12.75));

    equityVols_[make_pair(Market::defaultConfiguration, "SP5")] = flatRateFxv(0.2514);
    equityVols_[make_pair(Market::defaultConfiguration, "Lufthansa")] = flatRateFxv(0.30);

    yieldCurves_[make_tuple(Market::defaultConfiguration, YieldCurveType::EquityDividend, "SP5")] = flatRateDiv(0.01);
    yieldCurves_[make_tuple(Market::defaultConfiguration, YieldCurveType::EquityDividend, "Lufthansa")] =
        flatRateDiv(0.0);

    equityCurves_[make_pair(Market::defaultConfiguration, "SP5")] = Handle<EquityIndex>(boost::make_shared<EquityIndex>(
        "SP5", UnitedStates(), parseCurrency("USD"), equitySpot("SP5"), yieldCurve(YieldCurveType::Discount, "USD"),
        yieldCurve(YieldCurveType::EquityDividend, "SP5")));
    equityCurves_[make_pair(Market::defaultConfiguration, "Lufthansa")] =
        Handle<EquityIndex>(boost::make_shared<EquityIndex>("Lufthansa", TARGET(), parseCurrency("EUR"), equitySpot("Lufthansa"),
                                                            yieldCurve(YieldCurveType::Discount, "EUR"),
                                                            yieldCurve(YieldCurveType::EquityDividend, "Lufthansa")));

    // build swaption vols
    swaptionCurves_[make_pair(Market::defaultConfiguration, "EUR")] = flatRateSvs(0.20);
    swaptionCurves_[make_pair(Market::defaultConfiguration, "USD")] = flatRateSvs(0.30);
    swaptionCurves_[make_pair(Market::defaultConfiguration, "GBP")] = flatRateSvs(0.25);
    swaptionCurves_[make_pair(Market::defaultConfiguration, "CHF")] = flatRateSvs(0.25);
    swaptionCurves_[make_pair(Market::defaultConfiguration, "JPY")] = flatRateSvs(0.25);
    swaptionIndexBases_[make_pair(Market::defaultConfiguration, "EUR")] = std::make_pair("EUR-CMS-2Y", "EUR-CMS-30Y");
    swaptionIndexBases_[make_pair(Market::defaultConfiguration, "USD")] = std::make_pair("USD-CMS-2Y", "USD-CMS-30Y");
    swaptionIndexBases_[make_pair(Market::defaultConfiguration, "GBP")] = std::make_pair("GBP-CMS-2Y", "GBP-CMS-30Y");
    swaptionIndexBases_[make_pair(Market::defaultConfiguration, "CHF")] = std::make_pair("CHF-CMS-2Y", "CHF-CMS-30Y");
    swaptionIndexBases_[make_pair(Market::defaultConfiguration, "JPY")] = std::make_pair("JPY-CMS-2Y", "JPY-CMS-30Y");

    // build cap/floor vol structures
    capFloorCurves_[make_pair(Market::defaultConfiguration, "EUR")] = flatRateCvs(0.0050, Normal);
    capFloorCurves_[make_pair(Market::defaultConfiguration, "USD")] = flatRateCvs(0.0060, Normal);
    capFloorCurves_[make_pair(Market::defaultConfiguration, "GBP")] = flatRateCvs(0.0055, Normal);
    capFloorCurves_[make_pair(Market::defaultConfiguration, "CHF")] = flatRateCvs(0.0045, Normal);
    capFloorCurves_[make_pair(Market::defaultConfiguration, "JPY")] = flatRateCvs(0.0040, Normal);

    // build default curves
    defaultCurves_[make_pair(Market::defaultConfiguration, "dc")] = flatRateDcs(0.1);
    defaultCurves_[make_pair(Market::defaultConfiguration, "dc2")] = flatRateDcs(0.2);
    defaultCurves_[make_pair(Market::defaultConfiguration, "BondIssuer1")] = flatRateDcs(0.0);

    recoveryRates_[make_pair(Market::defaultConfiguration, "dc")] = Handle<Quote>(boost::make_shared<SimpleQuote>(0.4));
    recoveryRates_[make_pair(Market::defaultConfiguration, "dc2")] =
        Handle<Quote>(boost::make_shared<SimpleQuote>(0.4));
    recoveryRates_[make_pair(Market::defaultConfiguration, "BondIssuer1")] =
        Handle<Quote>(boost::make_shared<SimpleQuote>(0.0));

    yieldCurves_[make_tuple(Market::defaultConfiguration, YieldCurveType::Yield, "BondCurve1")] = flatRateYts(0.05);

    securitySpreads_[make_pair(Market::defaultConfiguration, "Bond1")] =
        Handle<Quote>(boost::make_shared<SimpleQuote>(0.0));

    Handle<IborIndex> hGBP(ore::data::parseIborIndex(
        "GBP-LIBOR-6M", yieldCurves_[make_tuple(Market::defaultConfiguration, YieldCurveType::Discount, "GBP")]));
    iborIndices_[make_pair(Market::defaultConfiguration, "GBP-LIBOR-6M")] = hGBP;

    // build UKRPI fixing history
    Date cpiFixingEnd(1, asof_.month(), asof_.year());
    Date cpiFixingStart = cpiFixingEnd - Period(14, Months);
    Schedule fixingDatesUKRPI = MakeSchedule().from(cpiFixingStart).to(cpiFixingEnd).withTenor(1 * Months);
    Real fixingRatesUKRPI[] = {258.5, 258.9, 258.6, 259.8, 259.6, 259.5, 259.8, 260.6,
                               258.8, 260.0, 261.1, 261.4, 262.1, 264.3, 265.2};

    // build UKRPI index
    boost::shared_ptr<ZeroInflationIndex> ii = parseZeroInflationIndex("UKRPI");
    boost::shared_ptr<YoYInflationIndex> yi = boost::make_shared<QuantExt::YoYInflationIndexWrapper>(ii, false);

    RelinkableHandle<ZeroInflationTermStructure> hcpi;
    bool interp = false;
    ii = boost::shared_ptr<UKRPI>(new UKRPI(interp, hcpi));
    for (Size i = 0; i < fixingDatesUKRPI.size(); i++) {
        // std::cout << i << ", " << fixingDatesUKRPI[i] << ", " << fixingRatesUKRPI[i] << std::endl;
        ii->addFixing(fixingDatesUKRPI[i], fixingRatesUKRPI[i], true);
    };

    // build EUHICPXT fixing history
    Schedule fixingDatesEUHICPXT = MakeSchedule().from(cpiFixingStart).to(cpiFixingEnd).withTenor(1 * Months);
    Real fixingRatesEUHICPXT[] = {258.5, 258.9, 258.6, 259.8, 259.6, 259.5, 259.8, 260.6,
                                  258.8, 260.0, 261.1, 261.4, 262.1, 264.3, 265.2};

    // build EUHICPXT index
    boost::shared_ptr<ZeroInflationIndex> euii = parseZeroInflationIndex("EUHICPXT");
    boost::shared_ptr<YoYInflationIndex> euyi = boost::make_shared<QuantExt::YoYInflationIndexWrapper>(euii, false);

    RelinkableHandle<ZeroInflationTermStructure> euhcpi;
    interp = false;
    euii = boost::shared_ptr<EUHICPXT>(new EUHICPXT(interp, euhcpi));
    for (Size i = 0; i < fixingDatesEUHICPXT.size(); i++) {
        // std::cout << i << ", " << fixingDatesUKRPI[i] << ", " << fixingRatesUKRPI[i] << std::endl;
        euii->addFixing(fixingDatesEUHICPXT[i], fixingRatesEUHICPXT[i], true);
    };

    vector<Date> datesZCII = {asof_,
                              asof_ + 1 * Years,
                              asof_ + 2 * Years,
                              asof_ + 3 * Years,
                              asof_ + 4 * Years,
                              asof_ + 5 * Years,
                              asof_ + 6 * Years,
                              asof_ + 7 * Years,
                              asof_ + 8 * Years,
                              asof_ + 9 * Years,
                              asof_ + 10 * Years,
                              asof_ + 12 * Years,
                              asof_ + 15 * Years,
                              asof_ + 20 * Years};

    vector<Rate> ratesZCII = {2.825, 2.9425, 2.975,  2.983, 3.0,  3.01,  3.008,
                              3.009, 3.013,  3.0445, 3.044, 3.09, 3.109, 3.108};

    zeroInflationIndices_[make_pair(Market::defaultConfiguration, "EUHICPXT")] =
        makeZeroInflationIndex("EUHICPXT", datesZCII, ratesZCII, euii,
                               yieldCurves_[make_tuple(Market::defaultConfiguration, YieldCurveType::Discount, "EUR")]);
    zeroInflationIndices_[make_pair(Market::defaultConfiguration, "UKRPI")] =
        makeZeroInflationIndex("UKRPI", datesZCII, ratesZCII, ii,
                               yieldCurves_[make_tuple(Market::defaultConfiguration, YieldCurveType::Discount, "GBP")]);
    yoyInflationIndices_[make_pair(Market::defaultConfiguration, "UKRPI")] =
        makeYoYInflationIndex("UKRPI", datesZCII, ratesZCII, yi,
                              yieldCurves_[make_tuple(Market::defaultConfiguration, YieldCurveType::Discount, "GBP")]);

    cpiInflationCapFloorPriceSurfaces_[make_pair(Market::defaultConfiguration, "EUHICPXT")] = flatRateCps(
        zeroInflationIndices_[make_pair(Market::defaultConfiguration, "EUHICPXT")], vector<Real>{0.0, 0.01, 0.02},
        vector<Real>{-1.0, -0.99}, vector<Period>{5 * Years, 10 * Years}, Matrix(3, 2, 0.15), Matrix(2, 2, 0.15));
    cpiInflationCapFloorPriceSurfaces_[make_pair(Market::defaultConfiguration, "UKRPI")] = flatRateCps(
        zeroInflationIndices_[make_pair(Market::defaultConfiguration, "UKRPI")], vector<Real>{0.0, 0.01, 0.02},
        vector<Real>{-1.0, -0.99}, vector<Period>{5 * Years, 10 * Years}, Matrix(3, 2, 0.15), Matrix(2, 2, 0.15));

    // Commodity price curves and spots
    Actual365Fixed ccDayCounter;
    vector<Time> times = {0.0, 1.0, 2.0, 5.0};

    // Gold curve
    vector<Real> prices = {1155.593, 1160.9, 1168.1, 1210};
    commodityCurves_[make_pair(Market::defaultConfiguration, "COMDTY_GOLD_USD")] =
        Handle<PriceTermStructure>(boost::make_shared<InterpolatedPriceCurve<Linear>>(times, prices, ccDayCounter));
    commodityCurves_[make_pair(Market::defaultConfiguration, "COMDTY_GOLD_USD")]->enableExtrapolation();
    commoditySpots_[make_pair(Market::defaultConfiguration, "COMDTY_GOLD_USD")] =
        Handle<Quote>(boost::make_shared<SimpleQuote>(1155.593));

    // WTI Oil curve
    prices = {30.89, 41.23, 44.44, 49.18};
    commodityCurves_[make_pair(Market::defaultConfiguration, "COMDTY_WTI_USD")] =
        Handle<PriceTermStructure>(boost::make_shared<InterpolatedPriceCurve<Linear>>(times, prices, ccDayCounter));
    commodityCurves_[make_pair(Market::defaultConfiguration, "COMDTY_WTI_USD")]->enableExtrapolation();
    commoditySpots_[make_pair(Market::defaultConfiguration, "COMDTY_WTI_USD")] =
        Handle<Quote>(boost::make_shared<SimpleQuote>(30.89));

    // Commodity volatilities
    commodityVols_[make_pair(Market::defaultConfiguration, "COMDTY_GOLD_USD")] = flatRateFxv(0.15);
    commodityVols_[make_pair(Market::defaultConfiguration, "COMDTY_WTI_USD")] = flatRateFxv(0.20);

    // Correlations
    correlationCurves_[make_tuple(Market::defaultConfiguration, "EUR-CMS-10Y", "EUR-CMS-1Y")] = flatCorrelation(0.15);
    correlationCurves_[make_tuple(Market::defaultConfiguration, "USD-CMS-10Y", "USD-CMS-1Y")] = flatCorrelation(0.2);
}

Handle<ZeroInflationIndex> TestMarket::makeZeroInflationIndex(string index, vector<Date> dates, vector<Rate> rates,
                                                              boost::shared_ptr<ZeroInflationIndex> ii,
                                                              Handle<YieldTermStructure> yts) {
    // build UKRPI index

    boost::shared_ptr<ZeroInflationTermStructure> cpiTS;
    // now build the helpers ...
    vector<boost::shared_ptr<BootstrapHelper<ZeroInflationTermStructure>>> instruments;
    for (Size i = 0; i < dates.size(); i++) {
        Handle<Quote> quote(boost::shared_ptr<Quote>(new SimpleQuote(rates[i] / 100.0)));
        boost::shared_ptr<BootstrapHelper<ZeroInflationTermStructure>> anInstrument(new ZeroCouponInflationSwapHelper(
            quote, Period(2, Months), dates[i], TARGET(), ModifiedFollowing, ActualActual(), ii));
        anInstrument->unregisterWith(Settings::instance().evaluationDate());
        instruments.push_back(anInstrument);
    };
    // we can use historical or first ZCIIS for this
    // we know historical is WAY off market-implied, so use market implied flat.
    Rate baseZeroRate = rates[0] / 100.0;
    boost::shared_ptr<PiecewiseZeroInflationCurve<Linear>> pCPIts(
        new PiecewiseZeroInflationCurve<Linear>(asof_, TARGET(), ActualActual(), Period(2, Months), ii->frequency(),
                                                ii->interpolated(), baseZeroRate, yts, instruments));
    pCPIts->recalculate();
    cpiTS = boost::dynamic_pointer_cast<ZeroInflationTermStructure>(pCPIts);
    cpiTS->enableExtrapolation(true);
    cpiTS->unregisterWith(Settings::instance().evaluationDate());
    return Handle<ZeroInflationIndex>(parseZeroInflationIndex(index, false, Handle<ZeroInflationTermStructure>(cpiTS)));
}

Handle<YoYInflationIndex> TestMarket::makeYoYInflationIndex(string index, vector<Date> dates, vector<Rate> rates,
                                                            boost::shared_ptr<YoYInflationIndex> ii,
                                                            Handle<YieldTermStructure> yts) {
    // build UKRPI index

    boost::shared_ptr<YoYInflationTermStructure> yoyTS;
    // now build the helpers ...
    vector<boost::shared_ptr<BootstrapHelper<YoYInflationTermStructure>>> instruments;
    for (Size i = 0; i < dates.size(); i++) {
        Handle<Quote> quote(boost::shared_ptr<Quote>(new SimpleQuote(rates[i] / 100.0)));
        boost::shared_ptr<BootstrapHelper<YoYInflationTermStructure>> anInstrument(new YearOnYearInflationSwapHelper(
            quote, Period(2, Months), dates[i], TARGET(), ModifiedFollowing, ActualActual(), ii));
        instruments.push_back(anInstrument);
    };
    // we can use historical or first ZCIIS for this
    // we know historical is WAY off market-implied, so use market implied flat.
    Rate baseZeroRate = rates[0] / 100.0;
    boost::shared_ptr<PiecewiseYoYInflationCurve<Linear>> pYoYts(
        new PiecewiseYoYInflationCurve<Linear>(asof_, TARGET(), ActualActual(), Period(2, Months), ii->frequency(),
                                               ii->interpolated(), baseZeroRate, yts, instruments));
    pYoYts->recalculate();
    yoyTS = boost::dynamic_pointer_cast<YoYInflationTermStructure>(pYoYts);
    return Handle<YoYInflationIndex>(boost::make_shared<QuantExt::YoYInflationIndexWrapper>(
        parseZeroInflationIndex(index, false), false, Handle<YoYInflationTermStructure>(pYoYts)));
}

boost::shared_ptr<ore::data::Conventions> TestConfigurationObjects::conv() {
    boost::shared_ptr<ore::data::Conventions> conventions(new ore::data::Conventions());

    conventions->add(boost::make_shared<ore::data::SwapIndexConvention>("EUR-CMS-2Y", "EUR-6M-SWAP-CONVENTIONS"));
    conventions->add(boost::make_shared<ore::data::SwapIndexConvention>("EUR-CMS-30Y", "EUR-6M-SWAP-CONVENTIONS"));

    // boost::shared_ptr<data::Convention> swapConv(
    //     new data::IRSwapConvention("EUR-6M-SWAP-CONVENTIONS", "TARGET", "Annual", "MF", "30/360", "EUR-EURIBOR-6M"));
    conventions->add(boost::make_shared<ore::data::IRSwapConvention>("EUR-6M-SWAP-CONVENTIONS", "TARGET", "A", "MF",
                                                                     "30/360", "EUR-EURIBOR-6M"));
    conventions->add(boost::make_shared<ore::data::IRSwapConvention>("USD-3M-SWAP-CONVENTIONS", "TARGET", "Q", "MF",
                                                                     "30/360", "USD-LIBOR-3M"));
    conventions->add(boost::make_shared<ore::data::IRSwapConvention>("USD-6M-SWAP-CONVENTIONS", "TARGET", "Q", "MF",
                                                                     "30/360", "USD-LIBOR-6M"));
    conventions->add(boost::make_shared<ore::data::IRSwapConvention>("GBP-6M-SWAP-CONVENTIONS", "TARGET", "A", "MF",
                                                                     "30/360", "GBP-LIBOR-6M"));
    conventions->add(boost::make_shared<ore::data::IRSwapConvention>("JPY-6M-SWAP-CONVENTIONS", "TARGET", "A", "MF",
                                                                     "30/360", "JPY-LIBOR-6M"));
    conventions->add(boost::make_shared<ore::data::IRSwapConvention>("CHF-6M-SWAP-CONVENTIONS", "TARGET", "A", "MF",
                                                                     "30/360", "CHF-LIBOR-6M"));

    conventions->add(boost::make_shared<ore::data::DepositConvention>("EUR-DEP-CONVENTIONS", "EUR-EURIBOR"));
    conventions->add(boost::make_shared<ore::data::DepositConvention>("USD-DEP-CONVENTIONS", "USD-LIBOR"));
    conventions->add(boost::make_shared<ore::data::DepositConvention>("GBP-DEP-CONVENTIONS", "GBP-LIBOR"));
    conventions->add(boost::make_shared<ore::data::DepositConvention>("JPY-DEP-CONVENTIONS", "JPY-LIBOR"));
    conventions->add(boost::make_shared<ore::data::DepositConvention>("CHF-DEP-CONVENTIONS", "CHF-LIBOR"));

    return conventions;
}

boost::shared_ptr<ore::analytics::ScenarioSimMarketParameters> TestConfigurationObjects::setupSimMarketData2() {
    boost::shared_ptr<ore::analytics::ScenarioSimMarketParameters> simMarketData(
        new ore::analytics::ScenarioSimMarketParameters());
    simMarketData->baseCcy() = "EUR";
    simMarketData->setDiscountCurveNames({"EUR", "GBP"});
    simMarketData->setYieldCurveNames({"BondCurve1"});
    simMarketData->setYieldCurveTenors("", {1 * Months, 6 * Months, 1 * Years, 2 * Years, 3 * Years, 4 * Years,
                                            5 * Years, 6 * Years, 7 * Years, 8 * Years, 9 * Years, 10 * Years,
                                            12 * Years, 15 * Years, 20 * Years, 25 * Years, 30 * Years});
    simMarketData->setYieldCurveDayCounters("", "ACT/ACT");
    simMarketData->setIndices({"EUR-EURIBOR-6M", "GBP-LIBOR-6M"});
    simMarketData->setDefaultNames({"BondIssuer1"});
    simMarketData->setDefaultTenors(
        "", {6 * Months, 1 * Years, 2 * Years, 3 * Years, 5 * Years, 7 * Years, 10 * Years, 15 * Years, 20 * Years});
    simMarketData->setSecurities({"Bond1"});
    simMarketData->setSimulateSurvivalProbabilities(true);
    simMarketData->setDefaultCurveDayCounters("", "ACT/ACT");
    simMarketData->setDefaultCurveCalendars("", "TARGET");
    simMarketData->interpolation() = "LogLinear";
    simMarketData->extrapolate() = true;

    simMarketData->swapVolTerms() = {1 * Years, 2 * Years, 3 * Years,  4 * Years,
                                     5 * Years, 7 * Years, 10 * Years, 20 * Years};
    simMarketData->swapVolExpiries() = {6 * Months, 1 * Years, 2 * Years,  3 * Years,
                                        5 * Years,  7 * Years, 10 * Years, 20 * Years};
    simMarketData->setSwapVolCcys({"EUR", "GBP"});
    simMarketData->swapVolDecayMode() = "ForwardVariance";
    simMarketData->setSimulateSwapVols(true);
    simMarketData->setSwapVolDayCounters("", "ACT/ACT");

    simMarketData->fxVolExpiries() = {1 * Months, 3 * Months, 6 * Months, 2 * Years, 3 * Years, 4 * Years, 5 * Years};
    simMarketData->fxVolDecayMode() = "ConstantVariance";
    simMarketData->setSimulateFXVols(true);
    simMarketData->setFxVolCcyPairs({"EURGBP"});
    simMarketData->fxVolIsSurface() = true;
    simMarketData->fxVolMoneyness() = {0.1, 0.2, 0.3, 0.5, 1, 2, 3};
    simMarketData->setFxVolDayCounters("", "ACT/ACT");

    simMarketData->setFxCcyPairs({"EURGBP"});

    simMarketData->setSimulateCapFloorVols(false);

    return simMarketData;
}

boost::shared_ptr<ore::analytics::ScenarioSimMarketParameters> TestConfigurationObjects::setupSimMarketData5() {
    boost::shared_ptr<ore::analytics::ScenarioSimMarketParameters> simMarketData(
        new ore::analytics::ScenarioSimMarketParameters());

    simMarketData->baseCcy() = "EUR";
    simMarketData->setDiscountCurveNames({"EUR", "GBP", "USD", "CHF", "JPY"});
    simMarketData->setYieldCurveTenors("", {1 * Months, 6 * Months, 1 * Years, 2 * Years, 3 * Years, 4 * Years,
                                            5 * Years, 7 * Years, 10 * Years, 15 * Years, 20 * Years, 30 * Years});
    simMarketData->setYieldCurveDayCounters("", "ACT/ACT");
    simMarketData->setIndices(
        {"EUR-EURIBOR-6M", "USD-LIBOR-3M", "USD-LIBOR-6M", "GBP-LIBOR-6M", "CHF-LIBOR-6M", "JPY-LIBOR-6M"});
    simMarketData->swapIndices()["EUR-CMS-2Y"] = "EUR-EURIBOR-6M";
    simMarketData->swapIndices()["EUR-CMS-30Y"] = "EUR-EURIBOR-6M";

    simMarketData->setYieldCurveNames({"BondCurve1"});
    simMarketData->interpolation() = "LogLinear";
    simMarketData->extrapolate() = true;

    simMarketData->swapVolTerms() = {1 * Years, 2 * Years, 3 * Years, 5 * Years, 7 * Years, 10 * Years, 20 * Years};
    simMarketData->swapVolExpiries() = {6 * Months, 1 * Years, 2 * Years,  3 * Years,
                                        5 * Years,  7 * Years, 10 * Years, 20 * Years};
    simMarketData->setSwapVolCcys({"EUR", "GBP", "USD", "CHF", "JPY"});
    simMarketData->swapVolDecayMode() = "ForwardVariance";
    simMarketData->setSimulateSwapVols(true); // false;
    simMarketData->setSwapVolDayCounters("", "ACT/ACT");

    simMarketData->fxVolExpiries() = {1 * Months, 3 * Months, 6 * Months, 2 * Years, 3 * Years, 4 * Years, 5 * Years};
    simMarketData->fxVolDecayMode() = "ConstantVariance";
    simMarketData->setSimulateFXVols(true); // false;
    simMarketData->setFxVolCcyPairs({"EURUSD", "EURGBP", "EURCHF", "EURJPY", "GBPCHF"});
    simMarketData->fxVolIsSurface() = true;
    simMarketData->fxVolMoneyness() = {0.1, 0.2, 0.3, 0.5, 1, 2, 3};
    simMarketData->setFxVolDayCounters("", "ACT/ACT");

    simMarketData->setFxCcyPairs({"EURUSD", "EURGBP", "EURCHF", "EURJPY"});

    simMarketData->setSimulateCapFloorVols(true);
    simMarketData->capFloorVolDecayMode() = "ForwardVariance";
    simMarketData->setCapFloorVolCcys({"EUR", "USD"});
    simMarketData->setCapFloorVolDayCounters("", "ACT/ACT");
    simMarketData->setCapFloorVolExpiries(
        "", {6 * Months, 1 * Years, 2 * Years, 3 * Years, 5 * Years, 7 * Years, 10 * Years, 15 * Years, 20 * Years});
    simMarketData->capFloorVolStrikes() = {0.00, 0.01, 0.02, 0.03, 0.04, 0.05, 0.06};

    simMarketData->setDefaultNames({"BondIssuer1"});
    simMarketData->setDefaultTenors(
        "", {6 * Months, 1 * Years, 2 * Years, 3 * Years, 5 * Years, 7 * Years, 10 * Years, 15 * Years, 20 * Years});
    simMarketData->setSimulateSurvivalProbabilities(true);
    simMarketData->setSecurities({"Bond1"});
    simMarketData->setDefaultCurveDayCounters("", "ACT/ACT");
    simMarketData->setDefaultCurveCalendars("", "TARGET");

    simMarketData->setEquityNames({"SP5", "Lufthansa"});
    simMarketData->setEquityDividendTenors("SP5", {6 * Months, 1 * Years, 2 * Years});
    simMarketData->setEquityDividendTenors("Lufthansa", {6 * Months, 1 * Years, 2 * Years});

    simMarketData->setSimulateEquityVols(true);
    simMarketData->equityVolDecayMode() = "ForwardVariance";
    simMarketData->setEquityVolNames({"SP5", "Lufthansa"});
    simMarketData->equityVolExpiries() = {6 * Months, 1 * Years, 2 * Years,  3 * Years,
                                          5 * Years,  7 * Years, 10 * Years, 20 * Years};
    simMarketData->equityVolIsSurface() = false;
    simMarketData->simulateEquityVolATMOnly() = true;
    simMarketData->equityVolMoneyness() = {1};
    simMarketData->setEquityVolDayCounters("", "ACT/ACT");

    simMarketData->setZeroInflationIndices({"UKRPI"});
    simMarketData->setZeroInflationTenors(
        "UKRPI", {1 * Years, 2 * Years, 3 * Years, 5 * Years, 7 * Years, 10 * Years, 15 * Years, 20 * Years});
    simMarketData->setZeroInflationDayCounters("", "ACT/ACT");
    simMarketData->setYoyInflationIndices({"UKRPI"});
    simMarketData->setYoyInflationTenors(
        "UKRPI", {1 * Years, 2 * Years, 3 * Years, 5 * Years, 7 * Years, 10 * Years, 15 * Years, 20 * Years});
    simMarketData->setYoyInflationDayCounters("", "ACT/ACT");

    simMarketData->setCommodityCurveSimulate(true);
    simMarketData->setCommodityNames({"COMDTY_GOLD_USD", "COMDTY_WTI_USD"});
    simMarketData->setCommodityCurveDayCounter("", "A365");
    simMarketData->setCommodityCurveTenors("", {0 * Days, 1 * Years, 2 * Years, 5 * Years});

    simMarketData->setCommodityVolSimulate(true);
    simMarketData->commodityVolDecayMode() = "ForwardVariance";
    simMarketData->setCommodityVolNames({"COMDTY_GOLD_USD", "COMDTY_WTI_USD"});
    simMarketData->commodityVolExpiries("COMDTY_GOLD_USD") = {1 * Years, 2 * Years, 5 * Years};
    simMarketData->commodityVolMoneyness("COMDTY_GOLD_USD") = {1.0};
    simMarketData->commodityVolExpiries("COMDTY_WTI_USD") = {1 * Years, 5 * Years};
    simMarketData->commodityVolMoneyness("COMDTY_WTI_USD") = {0.9, 0.95, 1.0, 1.05, 1.1};

    return simMarketData;
}

boost::shared_ptr<ore::analytics::SensitivityScenarioData> TestConfigurationObjects::setupSensitivityScenarioData2() {
    boost::shared_ptr<ore::analytics::SensitivityScenarioData> sensiData =
        boost::make_shared<ore::analytics::SensitivityScenarioData>();

    ore::analytics::SensitivityScenarioData::CurveShiftData cvsData;
    cvsData.shiftTenors = {1 * Years, 2 * Years,  3 * Years,  5 * Years,
                           7 * Years, 10 * Years, 15 * Years, 20 * Years}; // multiple tenors: triangular shifts
    cvsData.shiftType = "Absolute";
    cvsData.shiftSize = 0.0001;

    ore::analytics::SensitivityScenarioData::SpotShiftData fxsData;
    fxsData.shiftType = "Relative";
    fxsData.shiftSize = 0.01;

    ore::analytics::SensitivityScenarioData::VolShiftData fxvsData;
    fxvsData.shiftType = "Relative";
    fxvsData.shiftSize = 1.0;
    fxvsData.shiftExpiries = {2 * Years, 5 * Years};

    ore::analytics::SensitivityScenarioData::CapFloorVolShiftData cfvsData;
    cfvsData.shiftType = "Absolute";
    cfvsData.shiftSize = 0.0001;
    cfvsData.shiftExpiries = {1 * Years, 2 * Years, 3 * Years, 5 * Years, 10 * Years};
    cfvsData.shiftStrikes = {0.05};

    ore::analytics::SensitivityScenarioData::GenericYieldVolShiftData swvsData;
    swvsData.shiftType = "Relative";
    swvsData.shiftSize = 0.01;
    swvsData.shiftExpiries = {3 * Years, 5 * Years, 10 * Years};
    swvsData.shiftTerms = {2 * Years, 5 * Years, 10 * Years};

    sensiData->discountCurveShiftData()["EUR"] =
        boost::make_shared<ore::analytics::SensitivityScenarioData::CurveShiftData>(cvsData);

    sensiData->discountCurveShiftData()["GBP"] =
        boost::make_shared<ore::analytics::SensitivityScenarioData::CurveShiftData>(cvsData);

    sensiData->indexCurveShiftData()["EUR-EURIBOR-6M"] =
        boost::make_shared<ore::analytics::SensitivityScenarioData::CurveShiftData>(cvsData);
    sensiData->indexCurveShiftData()["GBP-LIBOR-6M"] =
        boost::make_shared<ore::analytics::SensitivityScenarioData::CurveShiftData>(cvsData);

    sensiData->yieldCurveShiftData()["BondCurve1"] =
        boost::make_shared<ore::analytics::SensitivityScenarioData::CurveShiftData>(cvsData);

    sensiData->fxShiftData()["EURGBP"] = fxsData;

    sensiData->fxVolShiftData()["EURGBP"] = fxvsData;

    sensiData->swaptionVolShiftData()["EUR"] = swvsData;
    sensiData->swaptionVolShiftData()["GBP"] = swvsData;

    sensiData->creditCurveShiftData()["BondIssuer1"] =
        boost::make_shared<ore::analytics::SensitivityScenarioData::CurveShiftData>(cvsData);

    // sensiData->capFloorVolLabel() = "VOL_CAPFLOOR";
    // sensiData->capFloorVolShiftData()["EUR"] = cfvsData;
    // sensiData->capFloorVolShiftData()["EUR"].indexName = "EUR-EURIBOR-6M";
    // sensiData->capFloorVolShiftData()["GBP"] = cfvsData;
    // sensiData->capFloorVolShiftData()["GBP"].indexName = "GBP-LIBOR-6M";

    return sensiData;
}

boost::shared_ptr<ore::analytics::SensitivityScenarioData> TestConfigurationObjects::setupSensitivityScenarioData2b() {
    boost::shared_ptr<ore::analytics::SensitivityScenarioData> sensiData =
        boost::make_shared<ore::analytics::SensitivityScenarioData>();

    // shift curve has more points than underlying curve, has tenor points where the underlying curve hasn't, skips some
    // tenor points that occur in the underlying curve (e.g. 2Y, 3Y, 4Y)
    ore::analytics::SensitivityScenarioData::CurveShiftData cvsData;
    cvsData.shiftTenors = {1 * Years,   15 * Months, 18 * Months, 21 * Months, 27 * Months, 30 * Months, 33 * Months,
                           42 * Months, 54 * Months, 5 * Years,   6 * Years,   7 * Years,   8 * Years,   9 * Years,
                           10 * Years,  11 * Years,  12 * Years,  13 * Years,  14 * Years,  15 * Years,  16 * Years,
                           17 * Years,  18 * Years,  19 * Years,  20 * Years};
    cvsData.shiftType = "Absolute";
    cvsData.shiftSize = 0.0001;

    ore::analytics::SensitivityScenarioData::SpotShiftData fxsData;
    fxsData.shiftType = "Relative";
    fxsData.shiftSize = 0.01;

    ore::analytics::SensitivityScenarioData::VolShiftData fxvsData;
    fxvsData.shiftType = "Relative";
    fxvsData.shiftSize = 1.0;
    fxvsData.shiftExpiries = {2 * Years, 5 * Years};

    ore::analytics::SensitivityScenarioData::CapFloorVolShiftData cfvsData;
    cfvsData.shiftType = "Absolute";
    cfvsData.shiftSize = 0.0001;
    cfvsData.shiftExpiries = {1 * Years, 2 * Years, 3 * Years, 5 * Years, 10 * Years};
    cfvsData.shiftStrikes = {0.05};

    ore::analytics::SensitivityScenarioData::GenericYieldVolShiftData swvsData;
    swvsData.shiftType = "Relative";
    swvsData.shiftSize = 0.01;
    swvsData.shiftExpiries = {3 * Years, 5 * Years, 10 * Years};
    swvsData.shiftTerms = {2 * Years, 5 * Years, 10 * Years};

    sensiData->discountCurveShiftData()["EUR"] =
        boost::make_shared<ore::analytics::SensitivityScenarioData::CurveShiftData>(cvsData);

    sensiData->discountCurveShiftData()["GBP"] =
        boost::make_shared<ore::analytics::SensitivityScenarioData::CurveShiftData>(cvsData);

    sensiData->indexCurveShiftData()["EUR-EURIBOR-6M"] =
        boost::make_shared<ore::analytics::SensitivityScenarioData::CurveShiftData>(cvsData);
    sensiData->indexCurveShiftData()["GBP-LIBOR-6M"] =
        boost::make_shared<ore::analytics::SensitivityScenarioData::CurveShiftData>(cvsData);

    sensiData->yieldCurveShiftData()["BondCurve1"] =
        boost::make_shared<ore::analytics::SensitivityScenarioData::CurveShiftData>(cvsData);

    sensiData->fxShiftData()["EURGBP"] = fxsData;

    sensiData->fxVolShiftData()["EURGBP"] = fxvsData;

    sensiData->swaptionVolShiftData()["EUR"] = swvsData;
    sensiData->swaptionVolShiftData()["GBP"] = swvsData;

    sensiData->creditCurveShiftData()["BondIssuer1"] =
        boost::make_shared<ore::analytics::SensitivityScenarioData::CurveShiftData>(cvsData);

    // sensiData->capFloorVolLabel() = "VOL_CAPFLOOR";
    // sensiData->capFloorVolShiftData()["EUR"] = cfvsData;
    // sensiData->capFloorVolShiftData()["EUR"].indexName = "EUR-EURIBOR-6M";
    // sensiData->capFloorVolShiftData()["GBP"] = cfvsData;
    // sensiData->capFloorVolShiftData()["GBP"].indexName = "GBP-LIBOR-6M";

    return sensiData;
}

boost::shared_ptr<ore::analytics::SensitivityScenarioData> TestConfigurationObjects::setupSensitivityScenarioData5() {
    boost::shared_ptr<ore::analytics::SensitivityScenarioData> sensiData =
        boost::make_shared<ore::analytics::SensitivityScenarioData>();

    ore::analytics::SensitivityScenarioData::CurveShiftData cvsData;
    cvsData.shiftTenors = {6 * Months, 1 * Years,  2 * Years,  3 * Years, 5 * Years,
                           7 * Years,  10 * Years, 15 * Years, 20 * Years}; // multiple tenors: triangular shifts
    cvsData.shiftType = "Absolute";
    cvsData.shiftSize = 0.0001;

    ore::analytics::SensitivityScenarioData::SpotShiftData fxsData;
    fxsData.shiftType = "Relative";
    fxsData.shiftSize = 0.01;

    ore::analytics::SensitivityScenarioData::VolShiftData fxvsData;
    fxvsData.shiftType = "Relative";
    fxvsData.shiftSize = 1.0;
    fxvsData.shiftExpiries = {5 * Years};

    ore::analytics::SensitivityScenarioData::CapFloorVolShiftData cfvsData;
    cfvsData.shiftType = "Absolute";
    cfvsData.shiftSize = 0.0001;
    cfvsData.shiftExpiries = {1 * Years, 2 * Years, 3 * Years, 5 * Years, 10 * Years};
    cfvsData.shiftStrikes = {0.01, 0.02, 0.03, 0.04, 0.05};

    ore::analytics::SensitivityScenarioData::GenericYieldVolShiftData swvsData;
    swvsData.shiftType = "Relative";
    swvsData.shiftSize = 0.01;
    swvsData.shiftExpiries = {2 * Years, 5 * Years, 10 * Years};
    swvsData.shiftTerms = {5 * Years, 10 * Years};

    ore::analytics::SensitivityScenarioData::SpotShiftData eqsData;
    eqsData.shiftType = "Relative";
    eqsData.shiftSize = 0.01;

    ore::analytics::SensitivityScenarioData::VolShiftData eqvsData;
    eqvsData.shiftType = "Relative";
    eqvsData.shiftSize = 0.01;
    eqvsData.shiftExpiries = {5 * Years};

    ore::analytics::SensitivityScenarioData::CurveShiftData zinfData;
    zinfData.shiftType = "Absolute";
    zinfData.shiftSize = 0.0001;
    zinfData.shiftTenors = {1 * Years, 2 * Years, 3 * Years, 5 * Years, 7 * Years, 10 * Years, 15 * Years, 20 * Years};

    ore::analytics::SensitivityScenarioData::SpotShiftData commoditySpotShiftData;
    commoditySpotShiftData.shiftType = "Relative";
    commoditySpotShiftData.shiftSize = 0.01;

    auto commodityShiftData = boost::make_shared<ore::analytics::SensitivityScenarioData::CurveShiftData>();
    commodityShiftData->shiftType = "Relative";
    commodityShiftData->shiftSize = 0.01;
    commodityShiftData->shiftTenors = {0 * Days, 1 * Years, 2 * Years, 5 * Years};

    ore::analytics::SensitivityScenarioData::VolShiftData commodityVolShiftData;
    commodityVolShiftData.shiftType = "Relative";
    commodityVolShiftData.shiftSize = 0.01;
    commodityVolShiftData.shiftExpiries = {1 * Years, 2 * Years, 5 * Years};
    commodityVolShiftData.shiftStrikes = {0.9, 0.95, 1.0, 1.05, 1.1};

    sensiData->discountCurveShiftData()["EUR"] =
        boost::make_shared<ore::analytics::SensitivityScenarioData::CurveShiftData>(cvsData);

    sensiData->discountCurveShiftData()["USD"] =
        boost::make_shared<ore::analytics::SensitivityScenarioData::CurveShiftData>(cvsData);

    sensiData->discountCurveShiftData()["GBP"] =
        boost::make_shared<ore::analytics::SensitivityScenarioData::CurveShiftData>(cvsData);

    sensiData->discountCurveShiftData()["JPY"] =
        boost::make_shared<ore::analytics::SensitivityScenarioData::CurveShiftData>(cvsData);

    sensiData->discountCurveShiftData()["CHF"] =
        boost::make_shared<ore::analytics::SensitivityScenarioData::CurveShiftData>(cvsData);

    sensiData->indexCurveShiftData()["EUR-EURIBOR-6M"] =
        boost::make_shared<ore::analytics::SensitivityScenarioData::CurveShiftData>(cvsData);

    sensiData->indexCurveShiftData()["USD-LIBOR-3M"] =
        boost::make_shared<ore::analytics::SensitivityScenarioData::CurveShiftData>(cvsData);

    sensiData->indexCurveShiftData()["GBP-LIBOR-6M"] =
        boost::make_shared<ore::analytics::SensitivityScenarioData::CurveShiftData>(cvsData);

    sensiData->indexCurveShiftData()["JPY-LIBOR-6M"] =
        boost::make_shared<ore::analytics::SensitivityScenarioData::CurveShiftData>(cvsData);

    sensiData->indexCurveShiftData()["CHF-LIBOR-6M"] =
        boost::make_shared<ore::analytics::SensitivityScenarioData::CurveShiftData>(cvsData);

    sensiData->yieldCurveShiftData()["BondCurve1"] =
        boost::make_shared<ore::analytics::SensitivityScenarioData::CurveShiftData>(cvsData);

    sensiData->creditCurveShiftData()["BondIssuer1"] =
        boost::make_shared<ore::analytics::SensitivityScenarioData::CurveShiftData>(cvsData);

    sensiData->fxShiftData()["EURUSD"] = fxsData;
    sensiData->fxShiftData()["EURGBP"] = fxsData;
    sensiData->fxShiftData()["EURJPY"] = fxsData;
    sensiData->fxShiftData()["EURCHF"] = fxsData;

    sensiData->fxVolShiftData()["EURUSD"] = fxvsData;
    sensiData->fxVolShiftData()["EURGBP"] = fxvsData;
    sensiData->fxVolShiftData()["EURJPY"] = fxvsData;
    sensiData->fxVolShiftData()["EURCHF"] = fxvsData;
    sensiData->fxVolShiftData()["GBPCHF"] = fxvsData;

    sensiData->swaptionVolShiftData()["EUR"] = swvsData;
    sensiData->swaptionVolShiftData()["GBP"] = swvsData;
    sensiData->swaptionVolShiftData()["USD"] = swvsData;
    sensiData->swaptionVolShiftData()["JPY"] = swvsData;
    sensiData->swaptionVolShiftData()["CHF"] = swvsData;

    sensiData->capFloorVolShiftData()["EUR"] = cfvsData;
    sensiData->capFloorVolShiftData()["EUR"].indexName = "EUR-EURIBOR-6M";
    sensiData->capFloorVolShiftData()["USD"] = cfvsData;
    sensiData->capFloorVolShiftData()["USD"].indexName = "USD-LIBOR-3M";

    sensiData->equityShiftData()["SP5"] = eqsData;
    sensiData->equityShiftData()["Lufthansa"] = eqsData;

    sensiData->equityVolShiftData()["SP5"] = eqvsData;
    sensiData->equityVolShiftData()["Lufthansa"] = eqvsData;

    sensiData->zeroInflationCurveShiftData()["UKRPI"] =
        boost::make_shared<ore::analytics::SensitivityScenarioData::CurveShiftData>(zinfData);

    sensiData->yoyInflationCurveShiftData()["UKRPI"] =
        boost::make_shared<ore::analytics::SensitivityScenarioData::CurveShiftData>(zinfData);

    sensiData->commodityShiftData()["COMDTY_GOLD_USD"] = commoditySpotShiftData;
    sensiData->commodityShiftData()["COMDTY_WTI_USD"] = commoditySpotShiftData;
    sensiData->commodityCurveShiftData()["COMDTY_GOLD_USD"] = commodityShiftData;
    sensiData->commodityCurrencies()["COMDTY_GOLD_USD"] = "USD";
    sensiData->commodityCurveShiftData()["COMDTY_WTI_USD"] = commodityShiftData;
    sensiData->commodityCurrencies()["COMDTY_WTI_USD"] = "USD";

    sensiData->commodityVolShiftData()["COMDTY_GOLD_USD"] = commodityVolShiftData;
    sensiData->commodityVolShiftData()["COMDTY_WTI_USD"] = commodityVolShiftData;

    return sensiData;
}

} // namespace testsuite


#include "teca_tc_trajectory.h"

#include "teca_database.h"
#include "teca_table.h"

#include "teca_variant_array.h"
#include "teca_metadata.h"

#include "teca_coordinate_util.h"
#include "teca_distance_function.h"

#include <algorithm>
#include <iostream>
#include <string>
#include <set>
#include <cmath>
#include <chrono>

#if defined(TECA_HAS_BOOST)
#include <boost/program_options.hpp>
#endif

#if defined(TECA_HAS_MPI)
#include <mpi.h>
#endif

using std::cerr;
using std::endl;
using seconds_t = std::chrono::duration<double, std::chrono::seconds::period>;

//#define TECA_DEBUG

namespace internal
{
template<typename coord_t, typename var_t>
int teca_tc_trajectory(var_t r_crit, var_t wind_crit, double n_wind_crit,
    unsigned long step_interval, const long *time_step, const double *time,
    const int *storm_uid, const coord_t *d_lon, const coord_t *d_lat,
    const var_t *wind_max, const var_t *vort_max, const var_t *psl,
    const int *have_twc, const int *have_thick, const var_t *twc_max,
    const var_t *thick_max, unsigned long n_rows, p_teca_table track_table)
{
    const coord_t DEG_TO_RAD = M_PI/180.0;
    unsigned long track_id = 0;

    // convert from dsegrees to radians
    unsigned long nbytes = n_rows*sizeof(coord_t);
    coord_t *r_lon = static_cast<coord_t*>(malloc(nbytes));
    for (unsigned long i = 0; i < n_rows; ++i)
        r_lon[i] = DEG_TO_RAD*d_lon[i];

    coord_t *r_lat = static_cast<coord_t*>(malloc(nbytes));
    for (unsigned long i = 0; i < n_rows; ++i)
        r_lat[i] = DEG_TO_RAD*d_lat[i];

    bool *available = static_cast<bool*>(malloc(n_rows*sizeof(bool)));
    for (unsigned long i = 0; i < n_rows; ++i)
        available[i] = true;

    // build offset to time step table
    unsigned long n_steps;
    std::vector<unsigned long> step_counts;
    std::vector<unsigned long> step_offsets;
    std::vector<unsigned long> step_ids;

    teca_coordinate_util::get_table_offsets(time_step,
        n_rows, n_steps, step_counts, step_offsets, step_ids);

    // build the track start queue.
    // consider all tracks eminating from all storms.
    // as tracks form these are marked as used
    unsigned long n_m1 = n_rows - 1;
    std::vector<unsigned long> track_starts(n_rows);
    for (unsigned long i = 0; i < n_rows; ++i)
        track_starts[n_m1-i] = i;

    while (track_starts.size())
    {
        // for each potential track start
        unsigned long track_start = track_starts.back();
        track_starts.pop_back();

        if (available[track_start])
        {
            // this storm is not part of another track is now considered
            // whether or not it works out it is no longer available for
            // use in other tracks
            available[track_start] = false;

            // start the new track
            unsigned long max_track_len = n_steps - step_ids[track_start];

            std::vector<unsigned long> new_track;
            new_track.reserve(max_track_len);
            new_track.push_back(track_start);

            std::vector<coord_t> storm_speed;
            storm_speed.reserve(max_track_len);
            storm_speed.push_back(coord_t());

            double duration = 0.0;
            double wind_duration = 0.0;

            // now walk forward in time examining each storm in the
            // next time step.active step is next one forward in time
            unsigned long active_step = step_ids[track_start] + 1;
            for (unsigned long j = active_step; j < n_steps; ++j)
            {
                // get position of the end of the track
                unsigned long track_tip = new_track.back();
                unsigned long jj = step_offsets[j];

                coord_t lon_0 = r_lon[track_tip];
                coord_t lat_0 = r_lat[track_tip];

                double t_0 = time[track_tip];
                double t_i = time[jj];
                double dt = t_i - t_0;

                // record duration.
                duration += dt;

                // limit search to adjacent time steps. non-adjacent candidates
                // can occur when a corrupted file is fed into the candidates stage.
                unsigned long ds = time_step[jj] - time_step[track_tip];
                if (ds != step_interval)
                {
                    TECA_WARNING("At index " << j << " missing " << ds << " steps("
                        << dt << " days) of candidate data between steps "
                        << time_step[track_tip] << " and " << time_step[jj])
                    break;
                }

                // apply storm duration criteria
                if ((wind_max[track_tip] >= wind_crit) &&
                  have_twc[track_tip] && have_thick[track_tip])
                  wind_duration += dt;

                // find the closest storm
                // note that r_crit is specified in km per day
                unsigned long closest_storm_id = 0;
                coord_t closest_storm_dist = r_crit*static_cast<coord_t>(dt);
                bool success = false;

                unsigned long n_storms = step_counts[j];
                for (unsigned int i = 0; i < n_storms; ++i)
                {
                    // check the storms distance. check them all since
                    // we need the closest
                    unsigned long storm_id = step_offsets[j] + i;

                    // skip storms that are already part of other tracks
                    if (!available[storm_id])
                      continue;

                    coord_t lon_i = r_lon[storm_id];
                    coord_t lat_i = r_lat[storm_id];

                    // compute the distance to the track tip
                    coord_t storm_dist =
                      teca_distance(lon_0, lat_0, lon_i, lat_i);

                    if (storm_dist <= closest_storm_dist)
                    {
                        // found one that's at least as close as
                        closest_storm_id = storm_id;
                        closest_storm_dist = storm_dist;
                        success = true;
                    }
                }

                if (success)
                {
                    // we were able to extend this track
                    new_track.push_back(closest_storm_id);
                    available[closest_storm_id] = false;

                    coord_t v1 = closest_storm_dist/dt;
                    storm_speed.push_back(v1);
                }
                else
                {
                  // track ends here
                  break;
                }
            }

            // one track has been completed
            unsigned long track_len = new_track.size();

            if (wind_duration >= n_wind_crit)
            {
                // output trajectory info
                for (unsigned long i = 0; i < track_len; ++i)
                {
                    // output trajectory info
                    unsigned long storm_id = new_track[i];

                    track_table << track_id << storm_uid[storm_id]
                        << time_step[storm_id] << time[storm_id] << d_lon[storm_id]
                        << d_lat[storm_id] << duration << wind_duration << wind_max[storm_id]
                        << vort_max[storm_id] << psl[storm_id] << have_twc[storm_id]
                        << have_thick[storm_id] << twc_max[storm_id] << thick_max[storm_id]
                        << storm_speed[i];
                }

                ++track_id;
            }

            // free up memory for this track
            new_track.clear();
        }
    }

    // free up memory
    track_starts.clear();
    free(r_lon);
    free(r_lat);
    free(available);

    return 0;
}
};

// --------------------------------------------------------------------------
teca_tc_trajectory::teca_tc_trajectory() :
    max_daily_distance(1600.0),
    min_wind_speed(17.0),
    min_wind_duration(2.0),
    step_interval(1)
{
    this->set_number_of_input_connections(1);
    this->set_number_of_output_ports(1);
}

// --------------------------------------------------------------------------
teca_tc_trajectory::~teca_tc_trajectory()
{}

#if defined(TECA_HAS_BOOST)
// --------------------------------------------------------------------------
void teca_tc_trajectory::get_properties_description(
    const std::string &prefix, options_description &global_opts)
{
    options_description opts("Options for "
        + (prefix.empty()?"teca_tc_trajectory":prefix));

    opts.add_options()
        TECA_POPTS_GET(double, prefix, max_daily_distance,
            "max distance a storm can move on the same track in single day (1600 km)")
        TECA_POPTS_GET(double, prefix, min_wind_speed,
            "minimum wind speed to be worthy of tracking (17.0 ms^-1)")
        TECA_POPTS_GET(double, prefix, min_wind_duration,
            "minimum number of, not necessarily consecutive, days thickness, "
            "core temp, and wind speed criteria must be satisfied (2.0 days)")
        TECA_POPTS_GET(unsigned long, prefix, step_interval,
            "number of time steps between valid candidate data. (1 step)")
        ;

    global_opts.add(opts);
}

// --------------------------------------------------------------------------
void teca_tc_trajectory::set_properties(
    const std::string &prefix, variables_map &opts)
{
    TECA_POPTS_SET(opts, double, prefix, max_daily_distance)
    TECA_POPTS_SET(opts, double, prefix, min_wind_speed)
    TECA_POPTS_SET(opts, double, prefix, min_wind_duration)
    TECA_POPTS_SET(opts, unsigned long, prefix, step_interval)
}
#endif

// --------------------------------------------------------------------------
teca_metadata teca_tc_trajectory::get_output_metadata(
    unsigned int port, const std::vector<teca_metadata> &input_md)
{
#ifdef TECA_DEBUG
    cerr << teca_parallel_id()
        << "teca_tc_trajectory::get_output_metadata" << endl;
#endif
    (void)port;
    teca_metadata out_md(input_md[0]);
    return out_md;
}

// --------------------------------------------------------------------------
std::vector<teca_metadata> teca_tc_trajectory::get_upstream_request(
    unsigned int port, const std::vector<teca_metadata> &input_md,
    const teca_metadata &request)
{
#ifdef TECA_DEBUG
    cerr << teca_parallel_id()
        << "teca_tc_trajectory::get_upstream_request" << endl;
#endif
    (void)port;
    (void)input_md;

    std::vector<teca_metadata> up_reqs;

    teca_metadata req(request);
    up_reqs.push_back(req);

    return up_reqs;
}

// --------------------------------------------------------------------------
const_p_teca_dataset teca_tc_trajectory::execute(
    unsigned int port, const std::vector<const_p_teca_dataset> &input_data,
    const teca_metadata &request)
{
#ifdef TECA_DEBUG
    cerr << teca_parallel_id() << "teca_tc_trajectory::execute" << endl;
#endif
    (void)port;
    (void)request;

    // get the input mesh
    const_p_teca_table candidates =
        std::dynamic_pointer_cast<const teca_table>(input_data[0]);

    // in parallel only rank 0 is required to have data
    int rank = 0;
#if defined(TECA_HAS_MPI)
    int init = 0;
    MPI_Initialized(&init);
    if (init)
        MPI_Comm_rank(this->get_communicator(), &rank);
#endif
    if (!candidates)
    {
        if (rank == 0)
        {
            TECA_ERROR("empty input or not a table")
        }
        return nullptr;
    }

    // validate the input table
    const char *req_cols[] = {
        "step", "time", "storm_id", "lon", "lat", "surface_wind",
        "850mb_vorticity", "sea_level_pressure", "have_core_temp",
        "have_thickness", "core_temp", "thickness"};

    size_t n_req_cols = sizeof(req_cols)/sizeof(char*);
    for (size_t i = 0; i < n_req_cols; ++i)
    {
        if (!candidates->has_column(req_cols[i]))
        {
            TECA_ERROR("Candidate table missing \"" << req_cols[i] << "\"")
            return nullptr;
        }
    }

    // get the candidate storm properties
    const long *p_step =
        dynamic_cast<const teca_variant_array_impl<long>*>(
        candidates->get_column("step").get())->get();

    const double *p_time =
        dynamic_cast<const teca_variant_array_impl<double>*>(
        candidates->get_column("time").get())->get();

    const int *p_storm_id =
        dynamic_cast<const teca_variant_array_impl<int>*>(
        candidates->get_column("storm_id").get())->get();

    const_p_teca_variant_array lon = candidates->get_column("lon");
    const_p_teca_variant_array lat = candidates->get_column("lat");

    const_p_teca_variant_array wind_max =
        candidates->get_column("surface_wind");

    const_p_teca_variant_array vort_max =
        candidates->get_column("850mb_vorticity");

    const_p_teca_variant_array psl_min =
        candidates->get_column("sea_level_pressure");

    const int *p_have_twc =
        dynamic_cast<const teca_variant_array_impl<int>*>(
        candidates->get_column("have_core_temp").get())->get();

    const int *p_have_thick =
        dynamic_cast<const teca_variant_array_impl<int>*>(
        candidates->get_column("have_thickness").get())->get();

    const_p_teca_variant_array twc_max =
        candidates->get_column("core_temp");

    const_p_teca_variant_array thick_max =
        candidates->get_column("thickness");

    // create the table to hold storm tracks
    p_teca_table storm_tracks = teca_table::New();
    storm_tracks->copy_metadata(candidates);

    std::string time_units;
    storm_tracks->get_time_units(time_units);
    if (time_units.find("days since") == std::string::npos)
    {
        TECA_ERROR("Conversion for \"" << time_units << "\" not implemented")
        return nullptr;
    }

    unsigned long n_rows = candidates->get_number_of_rows();
    std::chrono::high_resolution_clock::time_point t0, t1;

    NESTED_TEMPLATE_DISPATCH_FP(const teca_variant_array_impl,
        lon.get(), _COORD,

        const NT_COORD *p_lon =
            static_cast<const TT_COORD*>(lon.get())->get();

        const NT_COORD *p_lat =
            static_cast<const TT_COORD*>(lat.get())->get();

        NESTED_TEMPLATE_DISPATCH_FP(const teca_variant_array_impl,
            wind_max.get(), _VAR,

            // configure the output
            storm_tracks->declare_columns("track_id", int(),
                "storm_id", int(), "step", long(), "time", double(),
                "lon", NT_COORD(), "lat", NT_COORD(), "duration", double(),
                "wind_duration", double(), "surface_wind", NT_VAR(),
                "850mb_vorticity", NT_VAR(), "sea_level_pressure", NT_VAR(),
                "have_core_temp", int(), "have_thickness", int(),
                "core_temp", NT_VAR(), "thickness", NT_VAR(),
                "storm_speed", NT_COORD());

            const NT_VAR *p_wind_max =
                dynamic_cast<const TT_VAR*>(wind_max.get())->get();

            const NT_VAR *p_vort_max =
                dynamic_cast<const TT_VAR*>(vort_max.get())->get();

            const NT_VAR *p_psl_min =
                dynamic_cast<const TT_VAR*>(psl_min.get())->get();

            const NT_VAR *p_twc_max =
                dynamic_cast<const TT_VAR*>(twc_max.get())->get();

            const NT_VAR *p_thick_max =
                dynamic_cast<const TT_VAR*>(thick_max.get())->get();

            // invoke the track finder
            t0 = std::chrono::high_resolution_clock::now();
            if (internal::teca_tc_trajectory(
                static_cast<NT_VAR>(this->max_daily_distance),
                static_cast<NT_VAR>(this->min_wind_speed), this->min_wind_duration,
                this->step_interval, p_step, p_time, p_storm_id, p_lon, p_lat,
                p_wind_max, p_vort_max, p_psl_min, p_have_twc, p_have_thick,
                p_twc_max, p_thick_max, n_rows, storm_tracks))
            {
                TECA_ERROR("GFDL TC trajectory analysis encountered an error")
                return nullptr;
            }
            t1 = std::chrono::high_resolution_clock::now();
            )
        )

    seconds_t dt(t1 - t0);
    TECA_STATUS("teca_tc_trajectory n_candidates=" << n_rows << ", n_tracks="
        << storm_tracks->get_number_of_rows() << ", dt=" << dt.count() << " sec")

    return storm_tracks;
}


#include <frovedis.hpp>
#include <frovedis/matrix/sliced_vector.hpp>
#include <frovedis/matrix/blas_wrapper.hpp>

#define BOOST_TEST_MODULE FrovedisTest
#include <boost/test/unit_test.hpp>

using namespace frovedis;
using namespace std;

BOOST_AUTO_TEST_CASE( frovedis_test )
{
    int argc = 1;
    char** argv = NULL;
    use_frovedis use(argc, argv);

    std::vector<float> v_in = {1,2,3,4};
    std::vector<float> v_out = {2,4,6,8};

    // creating slice colmajor vector from entire std::vector
    sliced_colmajor_vector_local<float> sv (v_in);

    // scaling entire slice with 2
    scal<float> (sv, 2);

    // if slice was correctly created, this check should pass
    BOOST_CHECK (v_in == v_out);
}


#include "VillageGraph.h"

#include <AdventOfCodeCommon/DisableLibraryWarningsMacros.h>

__BEGIN_LIBRARIES_DISABLE_WARNINGS
#include <boost/algorithm/string.hpp>
#include <boost/lexical_cast.hpp>
__END_LIBRARIES_DISABLE_WARNINGS

namespace AdventOfCode
{
namespace Year2017
{
namespace Day12
{

VillageGraph::VillageGraph(NeighborSetToNodeID neighborSets) noexcept
    : m_neighborSets(std::move(neighborSets))
{

}

unsigned VillageGraph::numNodesInGroupContainingNode(NodeIDType nodeID) const
{
    NodeIDSet visitedNodeIDs{};
    dfsRecordVisitedNodes(nodeID, visitedNodeIDs);

    return visitedNodeIDs.size();
}

unsigned VillageGraph::numTotalGroups() const
{
    unsigned numGroups = 0;
    NodeIDSet visitedNodeIDs{};
    for (NodeIDType nodeID = 0; nodeID < m_neighborSets.size(); ++nodeID)
    {
        if (visitedNodeIDs.find(nodeID) == visitedNodeIDs.end())
        {
            ++numGroups;
            dfsRecordVisitedNodes(nodeID, visitedNodeIDs);
        }
    }

    return numGroups;
}

VillageGraph VillageGraph::fromNeighborsLines(const std::vector<std::string>& neighborsLines)
{
    NodeIDType expectedNodeID = 0;
    NeighborSetToNodeID neighborSets;
    for (const auto& neighborsLine : neighborsLines)
    {
        std::vector<std::string> tokens;
        boost::split(tokens, neighborsLine, boost::is_any_of(", "), boost::algorithm::token_compress_on);

        if (tokens.size() < 2)
        {
            throw std::runtime_error("Each line needs to have at least 2 tokens.");
        }

        NodeIDType nodeID = boost::lexical_cast<NodeIDType>(tokens[0]);
        if (nodeID != expectedNodeID++)
        {
            throw std::runtime_error("Neighbor lines are expected to be in order.");
        }

        std::vector<NodeIDType> neighbors;
        std::transform(tokens.cbegin() + 2, tokens.cend(), std::back_inserter(neighbors), [](const std::string& nodeIDString)
                       {
                           return boost::lexical_cast<NodeIDType>(nodeIDString);
                       });

        NodeIDSet neighborSet{std::make_move_iterator(neighbors.begin()), std::make_move_iterator(neighbors.end())};
        neighborSets.push_back(std::move(neighborSet));
    }

    return VillageGraph{std::move(neighborSets)};
}

void VillageGraph::dfsRecordVisitedNodes(NodeIDType startNodeID, NodeIDSet& visitedNodeIDs) const
{
    visitedNodeIDs.insert(startNodeID);

    for (const auto& neighborNodeID : m_neighborSets[startNodeID])
    {
        if (visitedNodeIDs.find(neighborNodeID) == visitedNodeIDs.end())
        {
            dfsRecordVisitedNodes(neighborNodeID, visitedNodeIDs);
        }
    }
}

}
}
}


//==================================================================================================
/*!
  @file

  @copyright 2016 NumScale SAS
  @copyright 2016 J.T. Lapreste

  Distributed under the Boost Software License, Version 1.0.
  (See accompanying file LICENSE.md or copy at http://boost.org/LICENSE_1_0.txt)
*/
//==================================================================================================
#ifndef BOOST_SIMD_FUNCTION_PROD_HPP_INCLUDED
#define BOOST_SIMD_FUNCTION_PROD_HPP_INCLUDED

#if defined(DOXYGEN_ONLY)
namespace boost { namespace simd
{

 /*!

    @ingroup group-reduction
    Function object implementing prod capabilities

    Returns the product of the elements of the SIMD vector

    @par Semantic:

    For every parameter of type T

    @code
    scalar<T> r = prod(x);
    @endcode

    is similar to:

    @code
    scalar<T> r = One;
    for(result_type i = 0; i != cardinal_of<T>; ++i)
      r *= x[i];
    @endcode

  **/
  const boost::dispatch::functor<tag::prod_> prod = {};
} }
#endif

#include <boost/simd/function/scalar/prod.hpp>
#include <boost/simd/function/simd/prod.hpp>

#endif


#ifndef NETWORKS_HPP
#define NETWORKS_HPP

// For External Library
#include <torch/torch.h>
#include <boost/program_options.hpp>

// Define Namespace
namespace nn = torch::nn;
namespace po = boost::program_options;

// Function Prototype
void weights_init(nn::Module &m);

// -------------------------------------------------
// struct{MC_ResNetImpl}(nn::Module)
// -------------------------------------------------
struct MC_ResNetImpl : nn::Module{
private:
    size_t inplanes;
    nn::Sequential first;
    nn::Sequential layer1, layer2, layer3, layer4;
    nn::Sequential avgpool, classifier;
public:
    MC_ResNetImpl(){}
    MC_ResNetImpl(po::variables_map &vm);
    void init();
    template <typename T> nn::Sequential make_layers(T &block, const size_t planes, const size_t num_blocks, const size_t stride);
    torch::Tensor forward(torch::Tensor x);
};

// -------------------------------------------------
// struct{BasicBlockImpl}(nn::Module)
// -------------------------------------------------
struct BasicBlockImpl : nn::Module{
private:
    bool down;
    nn::Sequential layerA, layerB;
    nn::Sequential downsample;
    nn::Sequential last;
public:
    static const size_t expansion = 1;
    BasicBlockImpl(){}
    BasicBlockImpl(const size_t inplanes, const size_t planes, const size_t stride);
    torch::Tensor forward(torch::Tensor x);
};

// -------------------------------------------------
// struct{BottleneckImpl}(nn::Module)
// -------------------------------------------------
struct BottleneckImpl : nn::Module{
private:
    bool down;
    nn::Sequential layerA, layerB, layerC;
    nn::Sequential downsample;
    nn::Sequential last;
public:
    static const size_t expansion = 4;
    BottleneckImpl(){}
    BottleneckImpl(const size_t inplanes, const size_t planes, const size_t stride);
    torch::Tensor forward(torch::Tensor x);
};

TORCH_MODULE(MC_ResNet);
TORCH_MODULE(BasicBlock);
TORCH_MODULE(Bottleneck);

#endif


#include <vector>
#include <iostream>
#include <Eigen/Core>
#include "pcl/point_types.h"
#include "pcl/point_cloud.h"
#include "pcl/io/pcd_io.h"
#include <pcl/search/kdtree.h>
#include <pcl/features/normal_3d_omp.h>
#include <pcl/filters/voxel_grid.h>
#include <pcl/features/vfh.h>
#include <pcl/visualization/pcl_plotter.h>
#include <pcl/console/parse.h>
#include <pcl/ml/svm_wrapper.h>

int number_ (142);
pcl::SVMParam param_;
int nr_fold_(0);

void
showHelp (char *filename)
{
  std::cout << std::endl;
  std::cout << "***************************************************************************" << std::endl;
  std::cout << "*                                                                         *" << std::endl;
  std::cout << "*                creation of features dataset - Usage Guide               *" << std::endl;
  std::cout << "*                                                                         *" << std::endl;
  std::cout << "***************************************************************************" << std::endl << std::endl;
  std::cout << "Usage: " << filename << " [output_model_file(*.model)] [Options]" << std::endl << std::endl;
  std::cout << "Options:" << std::endl;
  std::cout << "-number val:     the number of models from dataset to calulate the VFH feature from 0 to 142 (default :"<< number_ << ")"  << std::endl;
  printf(
  "-s svm_type : set type of SVM (default 0)\n"
  "	0 -- C-SVC		(multi-class classification)\n"
  "	1 -- nu-SVC		(multi-class classification)\n"
  "	2 -- one-class SVM\n"
  "	3 -- epsilon-SVR	(regression)\n"
  "	4 -- nu-SVR		(regression)\n"
  "-t kernel_type : set type of kernel function (default 2)\n"
  "	0 -- linear: u'*v\n"
  "	1 -- polynomial: (gamma*u'*v + coef0)^degree\n"
  "	2 -- radial basis function: exp(-gamma*|u-v|^2)\n"
  "	3 -- sigmoid: tanh(gamma*u'*v + coef0)\n"
  "	4 -- precomputed kernel (kernel values in training_set_file)\n"
  "-d degree : set degree in kernel function (default 3)\n"
  "-g gamma : set gamma in kernel function (default 1/num_features)\n"
  "-r coef0 : set coef0 in kernel function (default 0)\n"
  "-c cost : set the parameter C of C-SVC, epsilon-SVR, and nu-SVR (default 1)\n"
  "-n nu : set the parameter nu of nu-SVC, one-class SVM, and nu-SVR (default 0.5)\n"
  "-p epsilon : set the epsilon in loss function of epsilon-SVR (default 0.1)\n"
  "-m cachesize : set cache memory size in MB (default 100)\n"
  "-e epsilon : set tolerance of termination criterion (default 0.001)\n"
  "-h shrinking : whether to use the shrinking heuristics, 0 or 1 (default 1)\n"
  "-b probability_estimates : whether to train a SVC or SVR model for probability estimates, 0 or 1 (default 0)\n"
  "-v n: n-fold cross validation mode\n"
  );
}

void
downsample (pcl::PointCloud<pcl::PointXYZ>::Ptr &points, float leaf_size,
            pcl::PointCloud<pcl::PointXYZ>::Ptr &downsampled_out)
{
  pcl::VoxelGrid<pcl::PointXYZ> vox_grid;
  vox_grid.setLeafSize (leaf_size, leaf_size, leaf_size);
  vox_grid.setInputCloud (points);
  vox_grid.filter (*downsampled_out);
}

void
compute_surface_normals (pcl::PointCloud<pcl::PointXYZ>::Ptr &points, float normal_radius,
                         pcl::PointCloud<pcl::Normal>::Ptr &normals_out)
{
  pcl::NormalEstimation<pcl::PointXYZ, pcl::Normal> norm_est;

  // Use a KdTree to perform neighborhood searches
  norm_est.setSearchMethod (pcl::search::KdTree<pcl::PointXYZ>::Ptr (new pcl::search::KdTree<pcl::PointXYZ>));

  // Specify the size of the local neighborhood to use when computing the surface normals
  norm_est.setRadiusSearch (normal_radius);

  // Set the input points
  norm_est.setInputCloud (points);

  // Estimate the surface normals and store the result in "normals_out"
  norm_est.compute (*normals_out);
}

void
compute_VFH_features (pcl::PointCloud<pcl::PointXYZ>::Ptr &points,
                      pcl::PointCloud<pcl::Normal>::Ptr &normals,
                      pcl::PointCloud<pcl::VFHSignature308>::Ptr &descriptors_out)
{
  // Create a PFHEstimation object
  pcl::VFHEstimation<pcl::PointXYZ, pcl::Normal, pcl::VFHSignature308> vfh_est;

  // Set the input points and surface normals
  vfh_est.setInputCloud (points);
  vfh_est.setInputNormals(normals);
  vfh_est.setSearchMethod(pcl::search::KdTree<pcl::PointXYZ>::Ptr (new pcl::search::KdTree<pcl::PointXYZ>));
  // Optionally, we can normalize the bins of the resulting histogram,
  // using the total number of points.
  vfh_est.setNormalizeBins(true);
  // Also, we can normalize the SDC with the maximum size found between
  // the centroid and any of the cluster's points.
  vfh_est.setNormalizeDistance(false);

//  std::cout << "start compute VFH features descriptors" << std::endl;
  // Compute the features
  vfh_est.compute (*descriptors_out);
//  std::cout << "Compute VFH descriptors successfully" << std::endl;

}

/* Function : output the *.txt file for Libsvm use*/
void
write2file (pcl::PointCloud<pcl::VFHSignature308>::Ptr &descriptors_vfh, const int count)
{
  if(count ==0)
  {
    ofstream outFile( "VFH_svm_features.txt" );
    if ( !outFile )
    {
      printf("Error opening output file: %s!\n", "VFH_svm_features.txt");
      exit( 1 );
    }
    for (size_t j = 0; j < descriptors_vfh->size (); j++)
    {
      outFile << "0 ";
      for(int q=0; q< 308; q++)
      {
        outFile << (q+1) << ":"<<descriptors_vfh->at(j).histogram[q] << " ";
      }
    }
    outFile << std::endl;
  }
  else
  {
    ofstream ofresult( "VFH_svm_features.txt", ios::app);
    for (size_t k = 0; k < descriptors_vfh->size (); k++)
    {
      ofresult << count << " ";
      for(int p=0; p< 308; p++)
      {
        ofresult << (p+1) << ":"<<descriptors_vfh->at(k).histogram[p] << " ";
      }
    }
    ofresult << std::endl;
    if(count == number_)   ofresult.close();
  }
}

void
write2svmdata(pcl::PointCloud<pcl::VFHSignature308>::Ptr &descriptors_vfh,
              const int count, std::vector<pcl::SVMData> &trainning_set)
{
  std::vector<pcl::SVMDataPoint> svm_set;
  for (size_t j = 0; j < descriptors_vfh->size (); j++)
  {
    for(int q=0; q< 308; q++)
    {
      pcl::SVMDataPoint svm_point;
      svm_point.idx = int (q+1);
      svm_point.value = descriptors_vfh->at(j).histogram[q];
      svm_set.push_back(svm_point);
    }
  }
  pcl::SVMData svm_data;
  svm_data.label = count;
  svm_data.SV = svm_set;
  trainning_set.push_back(svm_data);
}

void
parseCommandLine (int argc, char *argv[])
{
  //Show help
  if (( argc < 2 ) || pcl::console::find_switch (argc, argv, "-h"))
  {
    showHelp (argv[0]);
    exit (0);
  }

  //General parameters
  pcl::console::parse_argument (argc, argv, "-number", number_);
  pcl::console::parse_argument (argc, argv, "-s", param_.svm_type);
  pcl::console::parse_argument (argc, argv, "-t", param_.kernel_type);
  pcl::console::parse_argument (argc, argv, "-g", param_.gamma);
  pcl::console::parse_argument (argc, argv, "-r", param_.coef0);
  pcl::console::parse_argument (argc, argv, "-n", param_.nu);
  pcl::console::parse_argument (argc, argv, "-m", param_.cache_size);
  pcl::console::parse_argument (argc, argv, "-c", param_.C);
  pcl::console::parse_argument (argc, argv, "-e", param_.eps);
  pcl::console::parse_argument (argc, argv, "-p", param_.p);
  pcl::console::parse_argument (argc, argv, "-h", param_.shrinking);
  pcl::console::parse_argument (argc, argv, "-b", param_.probability);
  pcl::console::parse_argument (argc, argv, "-r", param_.coef0);
  pcl::console::parse_argument (argc, argv, "-n", param_.nu);
  pcl::console::parse_argument (argc, argv, "-v", nr_fold_);
  if(nr_fold_ == 1 && nr_fold_ == 2)
  {
    fprintf(stderr,"n-fold cross validation: n must >= 2\n");
    showHelp (argv[0]);
    exit (0);
  }
}

int main (int argc, char *argv[])
{
  const char *modelFileName = argv[1];
  parseCommandLine (argc, argv);

  std::vector<pcl::SVMData> tranning_dataset;
  for(int i=0; i< (number_+ 1) ; i++)
  {
    pcl::PointCloud<pcl::PointXYZ>::Ptr points1 (new pcl::PointCloud<pcl::PointXYZ>);
    pcl::PointCloud<pcl::PointXYZ>::Ptr downsampled1 (new pcl::PointCloud<pcl::PointXYZ>);
    pcl::PointCloud<pcl::Normal>::Ptr normals1 (new pcl::PointCloud<pcl::Normal>);
    pcl::PointCloud<pcl::VFHSignature308>::Ptr descriptors_vfh1 (new pcl::PointCloud<pcl::VFHSignature308>);

    std::stringstream ss1, ss2;
    ss1 << "training_model_"<<i<<".pcd";
    pcl::io::loadPCDFile (ss1.str(), *points1);
    const float voxel_grid_leaf_size1 = 4.0;
    downsample (points1, voxel_grid_leaf_size1, downsampled1);
    const float normal_radius1 = 4.5;
    compute_surface_normals (downsampled1, normal_radius1, normals1);

    compute_VFH_features (downsampled1, normals1, descriptors_vfh1);

    std::cout<<"write features of model No."<<i<<std::endl;
    // Store the VF histogram into the txt for libsvm input
    write2svmdata(descriptors_vfh1, i, tranning_dataset);
  }
  std::cout<<"All the SVM datas have been created"<<std::endl;

  // SVM trainning
  pcl::SVMTrain Mytrainner;
  Mytrainner.setParameters(param_);
  Mytrainner.resetTrainingSet ();
  Mytrainner.setInputTrainingSet (tranning_dataset);
  Mytrainner.trainClassifier ();
  Mytrainner.saveClassifierModel(modelFileName);
  std::cout<<"Output trainning model : "<< modelFileName <<std::endl;
  return (0);
}


///////////////////////////////////////////////////////////////////////////////
// ars::function::signature.hpp                                               //
//                                                                           //
//  Copyright 2009 Erwann Rogard. Distributed under the Boost                //
//  Software License, Version 1.0. (See accompanying file                    //
//  LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)         //
///////////////////////////////////////////////////////////////////////////////
#ifndef BOOST_STATISTICS_DETAIL_ARS_FUNCTION_SIGNATURE_HPP_ER_2009
#define BOOST_STATISTICS_DETAIL_ARS_FUNCTION_SIGNATURE_HPP_ER_2009
#include <boost/mpl/identity.hpp>

namespace boost{
namespace statistics{
namespace detail{
namespace ars{
namespace function{

    // This is the function signature required by the ars sampler.
    //
    // Given input x, fun(x, y, dy) writes the log unnormalized pdf and its
    // derivative at x to y and dy, respectively.
    template<typename T>
    struct signature : mpl::identity<
        void(const T&,T&,T&)
    >{};

}// function
}// ars
}// detail
}// statistics
}// boost

#endif