/**
 ** Copyright (c) 2014 Illumina, Inc.
 **
 ** This file is part of Illumina's Enhanced Artificial Genome Engine (EAGLE),
 ** covered by the "BSD 2-Clause License" (see accompanying LICENSE file)
 **
 ** \author Lilian Janin
 **/

#include <boost/algorithm/string/predicate.hpp>
#include <boost/bind.hpp>
#include <boost/foreach.hpp>
#include <boost/format.hpp>
#include <boost/lambda/lambda.hpp>
#include <boost/lambda/bind.hpp>
#include <boost/lexical_cast.hpp>
#include "common/Exceptions.hh"
#include "io/Text.hh"
#include "model/Phred.hh"
#include "genome/QualityModel.hh"
#include "libzoo/util/AutoGrowVector.hh"

using namespace std;

//#define DEBUG_QUALITIES

namespace eagle
{
namespace genome
{

class MyDiscreteDist
{
public:
    MyDiscreteDist() {}

    MyDiscreteDist( const vector<double> w )
    {
        setWeights( w );
    }

    void setWeights( const vector<double> w )
    {
        weights_.reserve( w.size() );
        double sum = std::accumulate( w.begin(), w.end(), 0.0 );
        double cur = 0;
        for (unsigned int i=0; i<w.size(); ++i)
        {
            cur += w[i];
            weights_.push_back( cur?cur/sum:-1 );
        }
    }

    unsigned int operator()( const double val ) {
        vector<double>::iterator it = lower_bound( weights_.begin(), weights_.end(), val );
        if ( it != weights_.end() )
            return (&*it - &weights_[0]);
        else
            return weights_.size() - 1;
    }

private:
    vector<double> weights_;
};

QualityModel::QualityModel( const vector<boost::filesystem::path>& qualityTableFiles )
{
    useNewStuff_ = false;
    if (useNewStuff_)
    {
        parseBigQualityTableFile( qualityTableFiles.front() );
        return;
    }
    unsigned int lastCycle = 0;
    BOOST_FOREACH( const boost::filesystem::path& file, qualityTableFiles)
    {
        lastCycle = parseQualityTableFile( file, lastCycle );
    }
}

unsigned int QualityModel::parseBigQualityTableFile( const boost::filesystem::path& filename )
{
    assert (useNewStuff_);
    unsigned long totalSize = boost::filesystem::file_size( filename );
    bigTable_.resize( totalSize / sizeof(unsigned int) );
    ifstream is( filename.string().c_str(), ios_base::binary );
    is.read( reinterpret_cast<char*>(&bigTable_[0]), totalSize );
    return 0;
}

unsigned int QualityModel::parseQualityTableFile( const boost::filesystem::path& filename, const int cycleOffset )
// Returns last cycle processed + 1 (= first cycle of next file to process)
{
    assert (!useNewStuff_);

    io::DsvReader tsvReader( filename );
    vector<string> tokens;
    unsigned int cycle = cycleOffset;
/*
    double overallExpectedErrors = 0.0;
    double overallOutOf = 0.0;
    double overallExpectedErrorsForCurrentRead = 0.0;
    double overallOutOfForCurrentRead = 0.0;
*/

    if ( boost::algorithm::ends_with( filename.string(), ".qtable2" ) )
    {
        // Parsing of new file format

        AutoGrowVector< AutoGrowVector< AutoGrowVector< double > > > qualityTable;
        while( tsvReader.getNextLineFields<'\t'>(tokens) )
        {
            assert( tokens.size() >= 3 && "Quality table should have at least 3 entries per line (profileId, cycle, and 1 or more quality:count)" );

            unsigned int profileId = boost::lexical_cast<unsigned int>( tokens[0] );
            cycle = boost::lexical_cast<unsigned int>( tokens[1] );
            if (cycle == 0 && profileId == 0)
            {
                cycle = 1; // just a convention that profileId 0 and cycle 1 determines the proportion of profileIds
            }
            assert( cycle > 0 ); // cycles are 1-based by convention
            cycle += cycleOffset;

            for (unsigned int i=2; i<tokens.size(); ++i)
            {
                if ( tokens[i].empty() )
                    continue;

                unsigned int quality;
                double count = 0;
                istringstream ss( tokens[i] );
                char colon;
                ss >> quality >> colon >> count;
                assert( colon == ':' && count > 0 && "quality:count pairs must be separated by ':' in quality table files" );
                qualityTable[profileId][cycle][quality] = count;
            }

            // Extend quality vector as appropriate
            if (qualityDistPerCyclePerLastQuality.size() <= cycle)
            {
                qualityDistPerCyclePerLastQuality.resize( cycle+1 );
            }
            if (qualityDistPerCyclePerLastQuality[cycle].size() <= profileId)
            {
                qualityDistPerCyclePerLastQuality[cycle].resize( profileId+1 );
            }
            qualityDistPerCyclePerLastQuality[cycle][profileId] = boost::random::discrete_distribution<>(qualityTable[profileId][cycle]);
        }
        return cycle;
    }

    // Parsing of old file format
    while( tsvReader.getNextLineFields<'\t'>(tokens) )
    {
        assert( (tokens.size() == 43 || tokens.size() == 53) && "Quality table should have 43 or 53 entries per line" );

        cycle = boost::lexical_cast<unsigned int>( tokens[0] );
        assert( cycle > 0 ); // cycles are 1-based by convention
        cycle += cycleOffset;
        unsigned int lastQ = boost::lexical_cast<unsigned int>( tokens[1] );

        vector<double> values;
        try
        {
            std::transform( tokens.begin()+2, tokens.end(), std::back_inserter(values), boost::bind( &boost::lexical_cast<double,std::string>, _1) );
        }
        catch (const boost::bad_lexical_cast &e)
        {
            EAGLE_ERROR("Error while reading quality table: a numerical field seems to contain non-numerical characters");
        }
        assert( values.size() == 41 || values.size() == 51 );

        // Ensure that the distribution to determine the quality level cannot generate a value of zero
        if (lastQ==0 && values[0]!=0)
        {
            EAGLE_ERROR("Error while reading quality table: Column 3 must be zero if column 2 is zero (the distribution to determine the quality level cannot generate a value of zero)");
        }

        // Extend quality vector as appropriate
        if (qualityDistPerCyclePerLastQuality.size() <= cycle)
        {
            qualityDistPerCyclePerLastQuality.resize( cycle+1 );
        }
        if (qualityDistPerCyclePerLastQuality[cycle].size() <= lastQ)
        {
            qualityDistPerCyclePerLastQuality[cycle].resize( lastQ+1 );
        }
        qualityDistPerCyclePerLastQuality[cycle][lastQ] = boost::random::discrete_distribution<>(values);
/*
        // Expected errors statistics
        double expectedErrors = 0.0;
        double outOf = 0.0;

        if (cycle==102 && lastQ==0)
        {
            cout << (boost::format("Quality stats for read 1: expected mismatch rate = %f%%") % (100.0*overallExpectedErrorsForCurrentRead/overallOutOfForCurrentRead) ).str() << endl;
            overallExpectedErrorsForCurrentRead = 0.0;
            overallOutOfForCurrentRead = 0.0;
        }

        for (unsigned int qValue=0; qValue<=40; ++qValue)
        {
            double qCount = values[qValue];
            expectedErrors += qValue?(model::Phred::qualToProb(qValue)*qCount):0; // don't include Q0 in error count
            outOf += qCount;
#ifdef DEBUG_QUALITIES
            cout << (boost::format("Q=%d count=%d => error-rate=%f => %f expected mismatches out of %f bases = %f%%") % qValue % qCount % qualToProb(qValue) % expectedErrors % outOf % (100.0*expectedErrors/outOf) ).str() << endl;
#endif // DEBUG_QUALITIES
        }

        if (outOf > 0)
        {
            overallExpectedErrors += expectedErrors;
            overallOutOf += outOf;
            overallExpectedErrorsForCurrentRead += expectedErrors;
            overallOutOfForCurrentRead += outOf;
        }
#ifdef DEBUG_QUALITIES
        cout << (boost::format("Quality stats for cycle %d: %f expected mismatches out of %f bases = %f%%") % cycle % expectedErrors % outOf % (100.0*expectedErrors/outOf) ).str() << endl;
        cout << (boost::format("Quality stats over all cycles : %f expected mismatches out of %f bases = %f%%") % overallExpectedErrors % overallOutOf % (100.0*overallExpectedErrors/overallOutOf) ).str() << endl;
#endif // DEBUG_QUALITIES
*/
    }
/*
    cout << (boost::format("Quality stats for read 2: expected mismatch rate = %f%%") % (100.0*overallExpectedErrorsForCurrentRead/overallOutOfForCurrentRead) ).str() << endl;
    cout << (boost::format("Quality stats overall   : expected mismatch rate = %f%%") % (100.0*overallExpectedErrors/overallOutOf) ).str() << endl;
*/
    return cycle;
}

/*
static int Qbins[41] = {
        0, 0, 0, 0, 1, 1, 1, 1, 1, 1,
        2, 2, 2, 2, 2, 2, 2, 2, 2, 3,
        3, 3, 3, 3, 3, 4, 4, 4, 4, 4,
        4, 5, 5, 5, 5, 5, 6, 6, 6, 7, 7 };
static int bin2Q[8] = { 0, 6, 15, 22, 27, 33, 37, 40 };
*/

/*
unsigned int QualityModel::getQuality( boost::mt19937& randomGen, const unsigned int cycle, const char bclBase, ClusterErrorModelContext& clusterErrorModelContext )
{
    unsigned int result = bin2Q[ 5 ];
    assert (useNewStuff_);

    unsigned int cycleNum = cycle;
    unsigned int precedingKmer = clusterErrorModelContext.qualityModelContext.kmer & 0x3FF;
    unsigned int newBaseNum = bclBase;
    unsigned int prevQualityBin = Qbins[ clusterErrorModelContext.qualityModelContext.profileNumber ];
    vector<unsigned int> counts(8);
    unsigned int sum = 0;
    for (unsigned int newQualityBin = 0; newQualityBin<8; ++newQualityBin)
    {
//        assert( cycleNum >= 1 && cycleNum <= cycleCount );
        assert( precedingKmer < 1024 );
        assert( newBaseNum < 4 );
        assert( prevQualityBin < 8 );
        assert( newQualityBin < 8 );
        unsigned int entryNum =
            (cycleNum-1) * (1024*4*8*8) +
            precedingKmer * (4*8*8) +
            newBaseNum * (8*8) +
            prevQualityBin * (8) +
            newQualityBin;
        assert( entryNum < bigTable_.size() );
        unsigned int val = bigTable_[entryNum];
        counts[newQualityBin] = val;
        sum += val;
    }

    if (sum != 0)
    {
        unsigned int rnd = (unsigned int)( (double)randomGen() / randomGen.max() * sum );
        for (unsigned int newQualityBin = 0; newQualityBin<8; ++newQualityBin)
        {
            if (rnd >= counts[newQualityBin])
            {
                rnd -= counts[newQualityBin];
            }
            else
            {
                result = bin2Q[ newQualityBin ];
                break;
            }
        }
    }
    else
    {
        result = bin2Q[ 5 ]; //clusterErrorModelContext.qualityModelContext.profileNumber;
    }

    clusterErrorModelContext.qualityModelContext.kmer = (clusterErrorModelContext.qualityModelContext.kmer << 2 ) | (bclBase & 3);
    clusterErrorModelContext.qualityModelContext.profileNumber = result;

    return result;
}
*/

unsigned int QualityModel::getQuality( boost::mt19937& randomGen, const unsigned int cycle, ClusterErrorModelContext& clusterErrorModelContext )
{
    assert (!useNewStuff_);

    if (cycle >= qualityDistPerCyclePerLastQuality.size())
    {
        EAGLE_ERROR( "The quality table doesn't model as many cycles as necessary for this simulation" );
    }

    if (clusterErrorModelContext.qualityModelContext.profileNumber == 0)
    {
        // No profile number assigned to this read => find the last cycle containing a profile spec and use it
        unsigned int cycleForProfileNumber = cycle;
        while (qualityDistPerCyclePerLastQuality[cycleForProfileNumber][0].max() == 0)
        {
            if (cycleForProfileNumber == 0)
            {
                EAGLE_ERROR("Cannot find quality level distribution in quality tables (there should at least be an entry for {cycle=0, profile=0})");
            }
            --cycleForProfileNumber;
        }
        clusterErrorModelContext.qualityModelContext.profileNumber = qualityDistPerCyclePerLastQuality[cycleForProfileNumber][0](randomGen);
        assert( clusterErrorModelContext.qualityModelContext.profileNumber > 0 );
    }
    unsigned int profileNumber = clusterErrorModelContext.qualityModelContext.profileNumber;
    assert( profileNumber > 0 );
    if (profileNumber >= qualityDistPerCyclePerLastQuality[cycle].size())
    {
        EAGLE_ERROR( (boost::format("The quality table doesn't contain the required entry for {cycle=%d, profileNumber=%d}") % cycle % profileNumber).str() );
    }
    unsigned int quality = qualityDistPerCyclePerLastQuality[cycle][profileNumber](randomGen);

#ifdef DEBUG_QUALITIES
    cout << (boost::format("Q=%d => error-rate=%f") % quality % model::Phred::qualToProb(quality)).str() << endl;
#endif // DEBUG_QUALITIES
    return quality;
}

SequencingMismatchModel::SequencingMismatchModel( const boost::filesystem::path& mismatchTableFilename )
{
    // Mismatch model for base 'x': x->A, x->C, x->G, x->T, del, x->insertedA, x->insertedC, x->insertedG, x->insertedT (dup=='x->insertedx')
    // Parse mismatch table file: each line based on the model, for x={A,C,G,T}
    if (mismatchTableFilename == "")
    { // Use default values
        vector< double > errorA = boost::assign::list_of(0.0)(1.0)(1.0)(1.0)(0.0)(0.0)(0.0)(0.0)(0.0);
        vector< double > errorC = boost::assign::list_of(1.0)(0.0)(1.0)(1.0)(0.0)(0.0)(0.0)(0.0)(0.0);
        vector< double > errorG = boost::assign::list_of(1.0)(1.0)(0.0)(1.0)(0.0)(0.0)(0.0)(0.0)(0.0);
        vector< double > errorT = boost::assign::list_of(1.0)(1.0)(1.0)(0.0)(0.0)(0.0)(0.0)(0.0)(0.0);

        errorDistPerBase.push_back( boost::random::discrete_distribution<>(errorA) );
        errorDistPerBase.push_back( boost::random::discrete_distribution<>(errorC) );
        errorDistPerBase.push_back( boost::random::discrete_distribution<>(errorG) );
        errorDistPerBase.push_back( boost::random::discrete_distribution<>(errorT) );
    }
    else
    { // Parse mismatch table file
        io::DsvReader tsvReader( mismatchTableFilename );
        vector<string> tokens;
        const vector<string> expectedRowHeaders = boost::assign::list_of("A")("C")("G")("T");
        BOOST_FOREACH( const string& expectedRowHeader, expectedRowHeaders)
        {
            (void)expectedRowHeader; // prevents "unused variable" warning when asserts are not compiled
            tsvReader.getNextLineFields<'\t'>(tokens);
            assert( tokens.size() == 10 && "There should be 10 entries per line" );
            assert( tokens[0] == expectedRowHeader && "Unexpected value in first column (The first 4 entries - ignoring the lines starting with '#' - should be A,C,G,T)" );
            vector<double> values;
            try
            {
                std::transform( tokens.begin()+1, tokens.end(), std::back_inserter(values), boost::bind( &boost::lexical_cast<double,std::string>, _1) );
            }
            catch (const boost::bad_lexical_cast &e)
            {
                EAGLE_ERROR("Error while reading mismatch table: a numerical field seems to contain non-numerical characters");
            }
            assert( values.size() == 9 );
            errorDistPerBase.push_back( boost::random::discrete_distribution<>(values) );
        }
    }
}

void SequencingMismatchModel::apply( boost::mt19937& randomGen, const double errorRate, unsigned int& randomErrorType, char& bclBase, ClusterErrorModelContext& clusterErrorModelContext )
{
    if (randomGen() > errorRate * randomGen.max())
    {
        randomErrorType = ErrorModel::NoError;
    }
    else
    {
        unsigned char errorType = errorDistPerBase[bclBase](randomGen);
        switch (errorType)
        {
        case 0: // x->A
        case 1: // x->C
        case 2: // x->G
        case 3: // x->T
            randomErrorType = ErrorModel::BaseSubstitution;
            bclBase = errorType;
            break;

        case 4: // del
            randomErrorType = ErrorModel::BaseDeletion;
            break;

        case 5: // x->insertedA
        case 6: // x->insertedC
        case 7: // x->insertedG
        case 8: // x->insertedT
            randomErrorType = ErrorModel::BaseInsertion;
            bclBase = errorType-5;
            break;

        default:
            assert( false );
        }
    }

#ifdef DEBUG_QUALITIES
    // Debugging stats
    static unsigned long mismatches = 0;
    static unsigned long noMismatches = 0;
    if (randomErrorType == ErrorModel::NoError)
    {
        noMismatches++;
    }
    else
    {
        mismatches++;
    }
    cout << (boost::format("mismatch rate = %d mismatches and %d noMismatch out of %d => %f%%") % mismatches % noMismatches % (mismatches + noMismatches) % (100.0*(float)mismatches/(float)(mismatches + noMismatches))).str() << endl;
#endif // DEBUG_QUALITIES
}

HomopolymerIndelModel::HomopolymerIndelModel( const boost::filesystem::path& homopolymerIndelTableFilename )
{
    if (homopolymerIndelTableFilename == "")
    { // Use default values: no indels
        homoDeletionTable_.push_back(0.0);
        homoInsertionTable_.push_back(0.0); // homopolymer of size 0 or more
    }
    else
    { // Parse homopolymer indel table file
        io::DsvReader tsvReader( homopolymerIndelTableFilename );
        vector<string> tokens;
        while ( tsvReader.getNextLineFields<'\t'>(tokens) )
        {
            if ( tokens.size() == 0 ) { continue; }
            assert ( tokens.size() == 3 && "There should be 3 entries per line" );
            vector<double> values;
            try
            {
                std::transform( tokens.begin(), tokens.end(), std::back_inserter(values), boost::bind( &boost::lexical_cast<double,std::string>, _1) );
            }
            catch (const boost::bad_lexical_cast &e)
            {
                EAGLE_ERROR("Error while reading homopolymer indel table: a numerical field seems to contain non-numerical characters");
            }
            assert( values.size() == 3 );
            assert( values[0] == homoDeletionTable_.size() && values[0] == homoInsertionTable_.size() && "First column should contain consecutive numbers starting from 0" );
            homoDeletionTable_.push_back ( values[1] );
            homoInsertionTable_.push_back( values[2] );
        }
    }
}

void HomopolymerIndelModel::apply( boost::mt19937& randomGen, const double errorRate, unsigned int& randomErrorType, char& bclBase, ClusterErrorModelContext& clusterErrorModelContext )
{
    if (bclBase != clusterErrorModelContext.homopolymerModelContext.lastBase)
    {
        clusterErrorModelContext.homopolymerModelContext.lastBase = bclBase;
        clusterErrorModelContext.homopolymerModelContext.homopolymerLength = 1;
    }
    else
    {
        ++(clusterErrorModelContext.homopolymerModelContext.homopolymerLength);
        unsigned int length = clusterErrorModelContext.homopolymerModelContext.homopolymerLength;
        assert( length >= 1 );
        unsigned int tableEntry = std::min<unsigned int>( length, homoDeletionTable_.size()-1 );
        double delErrorRate = homoDeletionTable_ [tableEntry];
        double insErrorRate = homoInsertionTable_[tableEntry];
        double randomValue = (double)randomGen() / (double)randomGen.max();
        switch (clusterErrorModelContext.homopolymerModelContext.errorDirection)
        {
        case 0: // This homopolymer didn't get any insertion or deletion yet
            if (randomValue < delErrorRate )
            {
                randomErrorType = ErrorModel::BaseDeletion;
                clusterErrorModelContext.homopolymerModelContext.errorDirection = -1;
            }
            else if (randomValue < delErrorRate + insErrorRate )
            {
                randomErrorType = ErrorModel::BaseInsertion;
                clusterErrorModelContext.homopolymerModelContext.errorDirection = +1;
            }
            break;
        case +1: // This homopolymer already got one or more insertions
            if (randomValue < insErrorRate )
            {
                randomErrorType = ErrorModel::BaseInsertion;
            }
            break;
        case -1: // This homopolymer already got one or more deletions
            if (randomValue < delErrorRate )
            {
                randomErrorType = ErrorModel::BaseDeletion;
            }
            break;
        }
    }
}

class MotifRepeatQualityDropInfo
{
public:
   MotifRepeatQualityDropInfo()
       : meanQualityDrop( 0 )
        {}

    float meanQualityDrop;
    MyDiscreteDist distribution;
};

eagle::model::IUPAC baseConverter_;

uint64_t kmerStringToInt64( const string &s )
{
    uint64_t result = 0;
    for (unsigned int i=0; i<s.size(); ++i)
    {
        unsigned int binValue;
        if (s[i]>='0' && s[i] <='3')
            binValue = s[i] - '0';
        else
            binValue = baseConverter_.bin(s[i]);
        result = (result << 2) | binValue;
    }
    return result;
}

#define MAX_MOTIF_KMER_LENGTH 10
#define AVERAGE_QUALITY 34
MotifQualityDropModel::MotifQualityDropModel( const boost::filesystem::path& tableFilename )
    : active_( false )
{
    if (tableFilename != "")
    {
        tableData_.resize( MAX_MOTIF_KMER_LENGTH+1 );

        // Parse motif quality drop table file
        io::DsvReader tsvReader( tableFilename );
        vector<string> tokens;
        while ( tsvReader.getNextLineFields<'\t'>(tokens) )
        {
            if ( tokens.size() == 0 ) { continue; }
            assert ( tokens.size() > 3 && "There should be more than 4 entries per line: {kmer, repeatCount, coveredBaseCount, meanQ, (quality:count)+}" );
            uint64_t kmer = kmerStringToInt64( tokens[0] );
            unsigned int repeatCount = boost::lexical_cast<unsigned int, string>( tokens[1] );
            float meanQualityDrop = boost::lexical_cast<double, string>( tokens[2] );
            unsigned int kmerLength = tokens[0].size();
            unsigned int kmerLengthInBits = 2 * kmerLength;
            uint64_t kmerMask = (1ull << kmerLengthInBits) - 1;

            AutoGrowVector<double> distribution;
            for (unsigned int i=3; i<tokens.size(); ++i)
            {
                unsigned int quality;
                double count = 0;
                istringstream ss( tokens[i] );
                char colon;
                ss >> quality >> colon >> count;
                assert( colon == ':' && count > 0 && "quality:count pairs must be separated by ':' in motif Qdrop table" );
                distribution[quality] = count;
            }

            active_ = true;
            boost::shared_ptr< std::vector< MotifRepeatQualityDropInfo > > &mapData = tableData_[kmerLength][ kmer ];
            if (mapData.get() == NULL)
            {
                mapData.reset( new std::vector< MotifRepeatQualityDropInfo > );

                uint64_t kmerPermutation = kmer;
                for (unsigned int permutation=1; permutation < kmerLength; ++permutation)
                {
                    // Permute kmer
                    uint64_t leftMostBase = kmerPermutation >> (kmerLengthInBits - 2);
                    kmerPermutation = ((kmerPermutation << 2) & kmerMask) | leftMostBase;

                    //            cout << "MotifQualityDropModel constructor: Adding " << kmer << "\t" << repeatCount << "\t" << roundf(37 - meanQualityDrop) << endl;
                    boost::shared_ptr< std::vector< MotifRepeatQualityDropInfo > > &mapDataPermutation = tableData_[kmerLength][ kmerPermutation ];
                    assert (mapDataPermutation.get() == NULL);
                    mapDataPermutation = mapData;
                }
            }

            if (repeatCount >= mapData->size())
            {
                mapData->resize( repeatCount+1 );
            }
            assert( (*mapData)[repeatCount].meanQualityDrop == 0 && "Same motif repeat is described in 2 lines of motif Qdrop table" );
            (*mapData)[repeatCount].meanQualityDrop = AVERAGE_QUALITY - meanQualityDrop;
            (*mapData)[repeatCount].distribution.setWeights( distribution );
        }
    }
}

MotifRepeatQualityDropInfo* MotifQualityDropModel::getMotifRepeatQualityDrop( const uint64_t kmer1, const unsigned int repeatKmerLength, const unsigned int repeatCount )
{
//    cout << "getMotifRepeatQualityDrop:" << endl;
//    cout << "  kmer1=" << kmer1 << endl;
//    cout << "  repeatKmerLength=" << repeatKmerLength << endl;
//    cout << "  repeatCount=" << repeatCount << endl;

//    int result = ( repeatCount - 1 ) * repeatKmerLength;
//    cout << " => result=" << result << endl;

    std::map< uint64_t, boost::shared_ptr< std::vector< MotifRepeatQualityDropInfo > > >::iterator it = tableData_[repeatKmerLength].find( kmer1 );
    if (it != tableData_[repeatKmerLength].end())
    {
//        cout << "kmer found up to repeat " << it->second->size() << endl;
        unsigned int repeatCount2 = min<unsigned int>( repeatCount, it->second->size() );
        if (repeatCount2 > 0)
        {
            if ( (*it->second).size() <= repeatCount2 )
                repeatCount2 = (*it->second).size() - 1;
            MotifRepeatQualityDropInfo *info = &((*it->second)[repeatCount2]);
//            int result2 = (int)(info.meanQualityDrop);
//            cout << " => result2=" << result2 << endl;
            return info;
        }
    }

    return 0;
}

void MotifQualityDropModel::applyQualityDrop( unsigned int& quality, const char bclBase, ClusterErrorModelContext& clusterErrorModelContext, const unsigned int cycle, boost::mt19937& randomGen )
{
    if (!active_) { return; }

    uint64_t &kmer = clusterErrorModelContext.motifQualityDropModelContext.kmer;
    unsigned int &kmerLength = clusterErrorModelContext.motifQualityDropModelContext.kmerLength;
    double &qualityDropLevel = clusterErrorModelContext.motifQualityDropModelContext.qualityDropLevel;
    MotifRepeatQualityDropInfo* &shortTermEffect = clusterErrorModelContext.motifQualityDropModelContext.shortTermEffect;
    float &shortTermQualityDrop = clusterErrorModelContext.motifQualityDropModelContext.shortTermQualityDrop;
    int &qualityDropDueToPhasing = clusterErrorModelContext.phasingContext.qualityDrop;

    bool repeatDetected = false;
    MotifRepeatQualityDropInfo *strongestRepeat_effect = 0;
    unsigned int strongestRepeat_repeatLengthExcludingFirst = 0;

    if (kmerLength >= 5) // No point looking for short repeats
    {
        // detect repeated kmer
        const unsigned int maxKmerLength = min<unsigned int>( MAX_MOTIF_KMER_LENGTH, kmerLength );

        for ( unsigned int repeatKmerLength = 1; repeatKmerLength <= maxKmerLength; ++repeatKmerLength )
        {
            unsigned int repeatKmerLengthInBits = 2 * repeatKmerLength;
            uint64_t kmerMask = (1ull << repeatKmerLengthInBits) - 1;
            uint64_t kmer0 = kmer;
            uint64_t kmer1 = kmer & kmerMask;
            uint64_t kmer2;
            unsigned int repeatCount = 0;
            do
            {
                kmer0 >>= repeatKmerLengthInBits;
                ++repeatCount;
                kmer2 = kmer0 & kmerMask;
            } while (kmer1 == kmer2 &&
                     repeatKmerLength * (repeatCount + 1) <= kmerLength &&
                     repeatKmerLengthInBits * (repeatCount + 1) <= 64
                );

            unsigned int repeatLengthExcludingFirst = ( repeatCount - 1 ) * repeatKmerLength;
            const unsigned int repeatLengthThreshold = 4;
            if ( repeatLengthExcludingFirst >= repeatLengthThreshold
                 && repeatLengthExcludingFirst > strongestRepeat_repeatLengthExcludingFirst )
            {
                MotifRepeatQualityDropInfo *effect = getMotifRepeatQualityDrop( kmer1, repeatKmerLength, repeatCount );
                if ( ( effect && strongestRepeat_effect && (int)effect->meanQualityDrop > (int)strongestRepeat_effect->meanQualityDrop )
                     || ( effect && !strongestRepeat_effect && (int)effect->meanQualityDrop > 0 ) )
                {
                    repeatDetected = true;
                    strongestRepeat_effect = effect;
                    strongestRepeat_repeatLengthExcludingFirst = repeatLengthExcludingFirst;
                }
            }
        }

        if ( repeatDetected
             && strongestRepeat_effect->meanQualityDrop > (shortTermEffect?shortTermEffect->meanQualityDrop:0) )
        {
            if (qualityDropLevel == 0)
            {
                qualityDropLevel = (double)randomGen() / randomGen.max();
            }
            int newQuality = strongestRepeat_effect->distribution( qualityDropLevel );
            assert( newQuality != 0 );
            int newQualityDrop = max<int>(AVERAGE_QUALITY - newQuality, 0);

            if (shortTermQualityDrop < newQualityDrop)
            {
                qualityDropDueToPhasing -= (int)shortTermQualityDrop;
                qualityDropDueToPhasing += newQualityDrop;
                shortTermQualityDrop = newQualityDrop;
            }
            shortTermEffect = strongestRepeat_effect;
        }
        else
        {
            // Pulls down the "quality drop effect to cancel"
            float reducedMeanQualityDrop = strongestRepeat_effect?strongestRepeat_effect->meanQualityDrop:0;
            double newShortTermQualityDrop;
            if (shortTermEffect && shortTermEffect->meanQualityDrop)
            {
                assert( reducedMeanQualityDrop <= shortTermEffect->meanQualityDrop );
                newShortTermQualityDrop = shortTermQualityDrop * reducedMeanQualityDrop / shortTermEffect->meanQualityDrop;
            }
            else
            {
                assert( reducedMeanQualityDrop <= 0 );
                newShortTermQualityDrop = reducedMeanQualityDrop;
            }
            const double attenuation = 1.0;
            shortTermQualityDrop = (shortTermQualityDrop * (1.0 - attenuation)) + (newShortTermQualityDrop * attenuation);
            shortTermEffect = strongestRepeat_effect;
        }
    }

    kmer = (kmer << 2) | (bclBase & 3);
    ++kmerLength;
}

RandomQualityDropModel::RandomQualityDropModel( /*const boost::filesystem::path& tableFilename*/ )
{
}

void RandomQualityDropModel::applyQualityDrop( unsigned int& quality, const char bclBase, ClusterErrorModelContext& clusterErrorModelContext )
{
}

QualityGlitchModel::QualityGlitchModel( /*const boost::filesystem::path& tableFilename*/ )
{
}

void QualityGlitchModel::applyQualityDrop( unsigned int& quality, const char bclBase, ClusterErrorModelContext& clusterErrorModelContext )
{
}

HappyPhasingModel::HappyPhasingModel( /*const boost::filesystem::path& tableFilename*/ )
{
}

void HappyPhasingModel::applyQualityDrop( unsigned int& quality, const char bclBase, ClusterErrorModelContext& clusterErrorModelContext )
{
    // Use the values from the MotifQualityDrop plugin, if available
    const int qualityDropDueToPhasing = clusterErrorModelContext.phasingContext.qualityDrop;
    if (qualityDropDueToPhasing < 5)
        return; // Such a small quality drop is not worth correcting

    const unsigned int kmerLength = clusterErrorModelContext.motifQualityDropModelContext.kmerLength;
    if (kmerLength > 3)
    {
        uint64_t kmer = clusterErrorModelContext.motifQualityDropModelContext.kmer;
        float concordance = 0.0;
        float maxConcordance = 0.0;
        float concordanceWeight = 1.0;
        float decay = 0.7;
        for (int i=0; i<3; ++i)
        {
            kmer >>= 2;
            if ((unsigned char)bclBase == (kmer & 3))
                concordance += concordanceWeight;
            maxConcordance += concordanceWeight;
            concordanceWeight *= decay;
        }
        concordance /= maxConcordance;
        concordance *= concordance; // squared seems nice

        // restore quality in happy phasing areas
        quality += (int)(qualityDropDueToPhasing * concordance);
    }
}

QQTable::QQTable( const boost::filesystem::path& qqTableFilename )
{
    if (qqTableFilename == "")
    { // Use default Phred values
        for (unsigned int q=0; q<model::Phred::QUALITY_MAX; ++q)
        {
            qualityToProbability_.push_back( model::Phred::qualToProb( q ) );
        }
        qualityToProbability_.push_back( 0.0 ); // QUALITY_MAX has this special property
    }
    else
    { // Parse QQ table file
        io::DsvReader tsvReader( qqTableFilename );
        vector<string> tokens;
        while ( tsvReader.getNextLineFields<'\t'>(tokens) )
        {
            if ( tokens.size() == 0 ) { continue; }
            assert ( tokens.size() == 2 && "There should be 2 entries per line" );
            vector<double> values;
            try
            {
                std::transform( tokens.begin(), tokens.end(), std::back_inserter(values), boost::bind( &boost::lexical_cast<double,std::string>, _1) );
            }
            catch (const boost::bad_lexical_cast &e)
            {
                EAGLE_ERROR("Error while reading QQ table: a numerical field seems to contain non-numerical characters");
            }
            assert( values.size() == 2 );
            assert( values[0] == qualityToProbability_.size() && "First column should contain consecutive numbers starting from 0" );
            qualityToProbability_.push_back ( values[1] );
        }
    }

    if (qualityToProbability_.size() < model::Phred::QUALITY_MAX+1)
    {
        EAGLE_ERROR("QQ table doesn't contain enough values");
    }
}

double QQTable::qualToErrorRate( const unsigned int qual )
{
    if (qual >= qualityToProbability_.size())
    {
        BOOST_THROW_EXCEPTION( eagle::common::EagleException( 0, "Requested quality is higher than allowed max") );
    }
    double prob = qualityToProbability_[qual];
    return prob;
}

ErrorModel::ErrorModel( const vector<boost::filesystem::path>& qualityTableFilenames, const boost::filesystem::path& mismatchTableFilename, const boost::filesystem::path& homopolymerIndelTableFilename, const boost::filesystem::path& motifQualityDropTableFilename, const boost::filesystem::path& qqTableFilename, const std::vector< std::string >& errorModelOptions )
    : qualityModel_           ( qualityTableFilenames )
    , sequencingMismatchModel_( mismatchTableFilename )
    , homopolymerIndelModel_  ( homopolymerIndelTableFilename )
    , motifQualityDropModel_  ( motifQualityDropTableFilename )
    , randomQualityDropModel_ ()
    , qualityGlitchModel_     ()
    , happyPhasingModel_      ()
    , longreadBaseDuplicationModel_( errorModelOptions )
    , longreadDeletionModel_       ( errorModelOptions )
    , qqTable_                ( qqTableFilename )
{
}

void ErrorModel::getQualityAndRandomError( boost::mt19937& randomGen, const unsigned int cycle, const char base, unsigned int& quality, unsigned int& randomErrorType, char& bclBase, ClusterErrorModelContext& clusterErrorModelContext )
{
    bclBase = baseConverter_.normalizedBcl( base );
    if (bclBase==4)
    {
        // base is N
        randomErrorType = NoError;
        bclBase = 0;
        quality = 0;
        return;
    }
    assert( bclBase >= 0 && bclBase < 4 );

//    quality = qualityModel_.getQuality( randomGen, cycle, bclBase, clusterErrorModelContext );
    quality = qualityModel_.getQuality( randomGen, cycle, clusterErrorModelContext );
    motifQualityDropModel_.applyQualityDrop( quality, bclBase, clusterErrorModelContext, cycle, randomGen );
    randomQualityDropModel_.applyQualityDrop( quality, bclBase, clusterErrorModelContext );
    qualityGlitchModel_.applyQualityDrop( quality, bclBase, clusterErrorModelContext );
    happyPhasingModel_.applyQualityDrop( quality, bclBase, clusterErrorModelContext );

    // Apply quality drop due to phasing, using an additive strategy
    // This quality drop was calculated as part of the previous "applyQualityDrop" methods
    if ((int)quality > clusterErrorModelContext.phasingContext.qualityDrop)
    {
      quality -= clusterErrorModelContext.phasingContext.qualityDrop;
    }
    else
    {
      quality = 0;
    }

    // Make sure quality scores stay above 2
    if ( quality < 2 )
        quality = 2;

    double errorRate = qqTable_.qualToErrorRate( quality );

//#define REPORT_ERROR_RATE
#ifdef REPORT_ERROR_RATE
    static double totalErrors = 0;
    static double totalBases  = 0;
    static double threshold   = 1;
    totalErrors += errorRate;
    totalBases++;
    if (totalBases >= threshold)
    {
        clog << (boost::format("%f @ %f => %f") % totalErrors % totalBases % (totalErrors/totalBases)).str() << endl;
        threshold *= 2;
    }
#endif //ifdef REPORT_ERROR_RATE

    sequencingMismatchModel_.apply( randomGen, errorRate, randomErrorType, bclBase, clusterErrorModelContext );
    homopolymerIndelModel_.apply( randomGen, errorRate, randomErrorType, bclBase, clusterErrorModelContext );
    longreadBaseDuplicationModel_.apply( randomGen, errorRate, randomErrorType, bclBase, clusterErrorModelContext );
    longreadDeletionModel_.apply( randomGen, errorRate, randomErrorType, bclBase, clusterErrorModelContext );
}

} // namespace genome
} // namespace eagle


#include <Python.h>  // NOLINT(build/include_alpha)

// Produce deprecation warnings (needs to come before arrayobject.h inclusion).
#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION

#include <boost/make_shared.hpp>
#include <boost/python.hpp>
#include <boost/python/raw_function.hpp>
#include <boost/python/suite/indexing/vector_indexing_suite.hpp>
#include <numpy/arrayobject.h>
#include "boost/algorithm/string.hpp"
#include "boost/pointer_cast.hpp"
#include "google/protobuf/text_format.h"

// these need to be included after boost on OS X
#include <string>  // NOLINT(build/include_order)
#include <vector>  // NOLINT(build/include_order)
#include <map>
#include <algorithm>  // NOLINT(build/include_order)

#include "caffe/blob.hpp"
#include "caffe/common.hpp"
#include "caffe/net.hpp"
#include "caffe/layer.hpp"
#include "caffe/layers/image_data_layer.hpp"
#include "caffe/proto/caffe.pb.h"
#include "caffe/util/db.hpp"
#include "caffe/util/format.hpp"
#include "caffe/util/io.hpp"
#include "caffe/util/math_functions.hpp"

// Temporary solution for numpy < 1.7 versions: old macro, no promises.
// You're strongly advised to upgrade to >= 1.7.
#ifndef NPY_ARRAY_C_CONTIGUOUS
#define NPY_ARRAY_C_CONTIGUOUS NPY_C_CONTIGUOUS
#define PyArray_SetBaseObject(arr, x) (PyArray_BASE(arr) = (x))
#endif

// for python, we'll just use float as the type
typedef float Dtype;

using caffe::Blob;
using caffe::Caffe;
using caffe::Datum;
using caffe::Net;
using caffe::ImageDataLayer;
using std::string;

namespace db = caffe::db;
namespace bp = boost::python;

template<typename Dtype>
bp::list feature_extraction_pipeline_raw(bp::tuple args, bp::dict kwargs) {

  if (bp::len(kwargs) > 0) {
    throw std::runtime_error("main_raw takes no kwargs");
  }

  const int num_required_args = 7;
  if (bp::len(args) < num_required_args) {
    LOG(ERROR)<<
    "This program takes in a trained network and an input data layer, and then"
    " extract features of the input data produced by the net.\n"
    "Usage: extract_features  pretrained_net_param"
    "  feature_extraction_proto_file  extract_feature_blob_name1[,name2,...]"
    "  save_feature_dataset_name1[,name2,...]  num_mini_batches  db_type"
    "  [CPU/GPU] [DEVICE_ID=0]\n"
    "Note: you can extract multiple features in one pass by specifying"
    " multiple feature blob names and dataset names separated by ','."
    " The names cannot contain white space characters and the number of blobs"
    " and datasets must be equal.";
    return bp::list();
  }

  // force to use CPU
  Caffe::set_mode(Caffe::CPU);

  std::string pretrained_binary_proto( (bp::extract<std::string>(args[1])) );

  // Expected prototxt contains at least one data layer such as
  //  the layer data_layer_name and one feature blob such as the
  //  fc7 top blob to extract features.
  /*
   layers {
     name: "data_layer_name"
     type: DATA
     data_param {
       source: "/path/to/your/images/to/extract/feature/images_leveldb"
       mean_file: "/path/to/your/image_mean.binaryproto"
       batch_size: 128
       crop_size: 227
       mirror: false
     }
     top: "data_blob_name"
     top: "label_blob_name"
   }
   layers {
     name: "drop7"
     type: DROPOUT
     dropout_param {
       dropout_ratio: 0.5
     }
     bottom: "fc7"
     top: "fc7"
   }
   */

  std::string feature_extraction_proto( (bp::extract<std::string>(args[2])) );
  boost::shared_ptr<Net<Dtype> > feature_extraction_net(
      new Net<Dtype>(feature_extraction_proto, caffe::TEST));
  feature_extraction_net->CopyTrainedLayersFrom(pretrained_binary_proto);

  std::string extract_feature_blob_names( (bp::extract<std::string>(args[3])) );
  std::vector<std::string> blob_names;
  boost::split(blob_names, extract_feature_blob_names, boost::is_any_of(","));

  std::string save_feature_dataset_names( (bp::extract<std::string>(args[4])) ) ;
  std::vector<std::string> dataset_names;
  boost::split(dataset_names, save_feature_dataset_names,
               boost::is_any_of(","));
  size_t num_features = blob_names.size();

  int num_mini_batches = bp::extract<int>(args[5]);

  std::vector<boost::shared_ptr<db::DB> > feature_dbs;
  std::vector<boost::shared_ptr<db::Transaction> > txns;
  const char* db_type = bp::extract<const char*>(args[6]);
  for (size_t i = 0; i < dataset_names.size(); ++i) {
    //LOG(INFO)<< "Opening dataset " << dataset_names[i];
    boost::shared_ptr<db::DB> db(db::GetDB(db_type));
    db->Open(dataset_names.at(i), db::WRITE);
    feature_dbs.push_back(db);
    boost::shared_ptr<db::Transaction> txn(db->NewTransaction());
    txns.push_back(txn);
  }

  // storing "fc7" features (in C++)
  Datum datum;
  // sorting "prob" features (in Python)
  bp::list prob_list;

  // the name of the image as the 'key' for feature db
  const boost::shared_ptr<ImageDataLayer<Dtype> > layer_ptr = boost::static_pointer_cast<ImageDataLayer<Dtype> >(feature_extraction_net->layer_by_name("data"));
  const std::vector<std::pair<std::string, int> >& data_names_labels = layer_ptr->GetDataNameLabel();

  std::vector<int> image_indices(num_features, 0);
  for (int batch_index = 0; batch_index < num_mini_batches; ++batch_index) {
    feature_extraction_net->Forward();
    for (int i = 0; i < num_features; ++i) {
      const boost::shared_ptr<Blob<Dtype> > feature_blob =
        feature_extraction_net->blob_by_name(blob_names[i]);
      int batch_size = feature_blob->num();
      int dim_features = feature_blob->count() / batch_size;
      const Dtype* feature_blob_data;
      // blob "fc7"
      if(i == 0){
        for (int n = 0; n < batch_size; ++n) {
          datum.set_height(feature_blob->height());
          datum.set_width(feature_blob->width());
          datum.set_channels(feature_blob->channels());
          datum.clear_data();
          datum.clear_float_data();
          feature_blob_data = feature_blob->cpu_data() +
              feature_blob->offset(n);
          for (int d = 0; d < dim_features; ++d) {
            datum.add_float_data(feature_blob_data[d]);
          }
          string key_str = data_names_labels[batch_index].first;

          // const string ready to be stored
          string out;
          datum.SerializeToString(&out);

          txns.at(i)->Put(key_str, out);
          ++image_indices[i];
          if (image_indices[i] % 1000 == 0) {
            txns.at(i)->Commit();
            txns.at(i).reset(feature_dbs.at(i)->NewTransaction());
          }
        } // for (int n = 0; n < batch_size; ++n)
      } // if(i == 0)
      // blob "prob"
      else{
        for (int n = 0; n < batch_size; ++n) {
          feature_blob_data = feature_blob->cpu_data() +
              feature_blob->offset(n);
          for (int d = 0; d < dim_features; ++d) {
            prob_list.append(feature_blob_data[d]);
            //datum.add_float_data(feature_blob_data[d]);
          }
        } // for (int n = 0; n < batch_size; ++n)
      }
    }  // for (int i = 0; i < num_features; ++i)
  }  // for (int batch_index = 0; batch_index < num_mini_batches; ++batch_index)
  // write the last batch
  for (int i = 0; i < dataset_names.size(); ++i) {
    if (image_indices[i] % 1000 != 0) {
      txns.at(i)->Commit();
    }
    feature_dbs.at(i)->Close();
  }

  printf("%s\n", "Successfully extracted the features!");
  return prob_list;
}

// costum compare object
template <typename K, typename V>
bool cmp(const std::pair<K, V>& lhs, const std::pair<K,V>& rhs)
{
  return lhs.second > rhs.second;
}

// using STL pair for argSort
template <typename Dtype>
std::vector<size_t> argsort_using_stlMap(std::vector<Dtype> const& values) {
  std::vector<size_t> indices(values.size());
  std::vector<std::pair<size_t, Dtype> > pairs;
  for(size_t i=0; i<values.size(); ++i){
    pairs.push_back(std::make_pair(i, values[i]));
  }

  std::sort(pairs.begin(), pairs.end(), cmp<size_t, Dtype> );

  for(size_t i=0; i<pairs.size(); ++i){
    indices[i] = pairs[i].first;
  }
  return indices;
}

// Computing the similarity of two features (e.g. *Cosine similarity or L2-norm)
template <typename Dtype>
float similarity(Datum& datum_1, Datum& datum_2){

  int feature_dim = datum_1.channels();
  Dtype* ptr_1 = datum_1.mutable_float_data()->mutable_data();
  Dtype* ptr_2 = datum_2.mutable_float_data()->mutable_data();

  Dtype dot_val, norm_1, norm_2;

  // L2-norm
  norm_1 = sqrt( caffe::caffe_cpu_dot(feature_dim, ptr_1, ptr_1) );
  norm_2 = sqrt( caffe::caffe_cpu_dot(feature_dim, ptr_2, ptr_2) );
  // dot product
  dot_val = caffe::caffe_cpu_dot(feature_dim, ptr_1, ptr_2);

  return dot_val / (norm_1 * norm_2);
}

/* DEBUG -> dislay Datum instance @Wei-Lin */
template <typename Dtype>
void listDatum(const Datum& datum){
  if(datum.has_height())  std::cout << "H:\t" << datum.height() << std::endl;
  if(datum.has_width())  std::cout << "W:\t" << datum.width() << std::endl;
  if(datum.has_channels())  std::cout << "C:\t" << datum.channels() << std::endl;
  std::cout << "size of float data:\t" << datum.float_data_size() << std::endl;

  // display fc7 floating features
  google::protobuf::RepeatedField<Dtype> data = datum.float_data();
  for( google::protobuf::RepeatedField<float>::iterator iter=data.begin(); iter<data.end(); ++iter ){
    std::cout << *iter << " ";
  }
  std::cout << "\n" << std::endl;
}

template <typename Dtype>
bp::object matching_pipeline_raw(bp::tuple args, bp::dict kwargs) {

  if (bp::len(kwargs) > 0) {
    throw std::runtime_error("main_raw takes no kwargs");
  }

  const int num_required_args = 3;
  if (bp::len(args) < num_required_args) {
    LOG(ERROR)<<
    "This program takes in an image feature database (stored in leveldb/lmdb format)"
    "and a query image feature, and then"
    " compare database features with the query feature to give a short list of relavence.\n"

    "@TODO: define the usage of retrieval interface"

    "Usage: retrieval path_to_feature_db db_type\n"

    "@TODO: modify the notes for retrieval process"

    "Note: you can extract multiple features in one pass by specifying"
    " multiple feature blob names and dataset names separated by ','."
    " The names cannot contain white space characters and the number of blobs"
    " and datasets must be equal.";
    return bp::object();
  }

  // 1st argument: the path to feature db
  std::string feature_db_name( (bp::extract<std::string>(args[1])) );
  std::cout << "feature_db_name: " << feature_db_name << std::endl;
  // 2nd argument: the type of the feature db
  const char* db_type = bp::extract<const char*>(args[2]);

  // Open the feature db
  boost::shared_ptr<db::DB> feature_db(db::GetDB(db_type));
  feature_db->Open(feature_db_name, db::READ);

  // Visit all the features
  std::vector<boost::shared_ptr<Datum> > Datum_vec;
  db::Cursor* db_cursor(feature_db->NewCursor());
  for(db_cursor->SeekToFirst(); db_cursor->valid(); db_cursor->Next()){
    // get the current string value
    std::string s = db_cursor->value();

    // convert to the feature (floating)
    boost::shared_ptr<Datum> datum(new Datum);
    datum->ParseFromString(s);

    Datum_vec.push_back(datum);
  }
  printf("Total feat. is #%zd\n", Datum_vec.size());

  // similarity measure of two features
  Dtype dot_val = similarity<Dtype>(*Datum_vec[0], *Datum_vec[1]);
  std::cout << "The similarity of vector[0] and vector[1] is " << dot_val << std::endl;

  // close the feature db
  feature_db->Close();

  return bp::object();
}

template<typename Dtype>
bp::list retrieval_pipeline_raw(bp::tuple args, bp::dict kwargs) {

  if (bp::len(kwargs) > 0) {
    throw std::runtime_error("main_raw takes no kwargs");
  }

  const int num_required_args = 7;
  if (bp::len(args) < num_required_args) {
    LOG(ERROR)<<
    "This program takes in a trained network and an input data layer, and then"
    " extract features of the input data produced by the net.\n"
    "Usage: extract_features  pretrained_net_param"
    "  feature_extraction_proto_file  extract_feature_blob_name1[,name2,...]"
    "  save_feature_dataset_name1[,name2,...]  num_mini_batches  db_type"
    "  [CPU/GPU] [DEVICE_ID=0]\n"
    "Note: you can extract multiple features in one pass by specifying"
    " multiple feature blob names and dataset names separated by ','."
    " The names cannot contain white space characters and the number of blobs"
    " and datasets must be equal.";
    return bp::list();
  }

  // Expected prototxt contains at least one data layer such as
  //  the layer data_layer_name and one feature blob such as the
  //  fc7 top blob to extract features.
  /*
   layers {
     name: "data_layer_name"
     type: DATA
     data_param {
       source: "/path/to/your/images/to/extract/feature/images_leveldb"
       mean_file: "/path/to/your/image_mean.binaryproto"
       batch_size: 128
       crop_size: 227
       mirror: false
     }
     top: "data_blob_name"
     top: "label_blob_name"
   }
   layers {
     name: "drop7"
     type: DROPOUT
     dropout_param {
       dropout_ratio: 0.5
     }
     bottom: "fc7"
     top: "fc7"
   }
   */

  // force caffe to use CPU
  Caffe::set_mode(Caffe::CPU);

  // 1st argument: excutable binary name (ingored)

  // 2nd argument: model binary
  std::string pretrained_binary_proto( (bp::extract<std::string>(args[1])) );

  // 3rd argument: model proto txt
  std::string feature_extraction_proto( (bp::extract<std::string>(args[2])) );
  boost::shared_ptr<Net<Dtype> > feature_extraction_net(
      new Net<Dtype>(feature_extraction_proto, caffe::TEST));
  feature_extraction_net->CopyTrainedLayersFrom(pretrained_binary_proto);

  // 4nd argument: blob names (features to be extracted)
  std::string extract_feature_blob_names( (bp::extract<std::string>(args[3])) );
  std::vector<std::string> blob_names;
  boost::split(blob_names, extract_feature_blob_names, boost::is_any_of(","));
  size_t num_features = blob_names.size();

  // 5nd argument: size of mini patch
  int num_mini_batches = bp::extract<int>(args[4]);

  // 6nd argument: feature db path
  std::string feature_db_name( (bp::extract<std::string>(args[5])) );

  // 7nd argument: feature db type
  const char* db_type = bp::extract<const char*>(args[6]);

  // storing "fc7" features (in C++)
  Datum query;
  // storing sorted "filenames" (in Python)
  bp::list result_list;
  // storing "prob" features (in Python)
  bp::list prob_list;

  // the name of the image as the 'key' for feature db
  const boost::shared_ptr<ImageDataLayer<Dtype> > layer_ptr = boost::static_pointer_cast<ImageDataLayer<Dtype> >(feature_extraction_net->layer_by_name("data"));
  const std::vector<std::pair<std::string, int> >& data_names_labels = layer_ptr->GetDataNameLabel();

  /* ==== Feature Extraction ==== */
  std::vector<int> image_indices(num_features, 0);
  for (int batch_index = 0; batch_index < num_mini_batches; ++batch_index) {
    feature_extraction_net->Forward();
    for (int i = 0; i < num_features; ++i) {
      const boost::shared_ptr<Blob<Dtype> > feature_blob = feature_extraction_net->blob_by_name(blob_names[i]);
      int batch_size = feature_blob->num();
      int dim_features = feature_blob->count() / batch_size;
      const Dtype* feature_blob_data;
      // blob[fc7]
      if(i == 0){
        for (int n = 0; n < batch_size; ++n) {
          query.set_height(feature_blob->height());
          query.set_width(feature_blob->width());
          query.set_channels(feature_blob->channels());
          query.clear_data();
          query.clear_float_data();
          feature_blob_data = feature_blob->cpu_data() + feature_blob->offset(n);
          for (int d = 0; d < dim_features; ++d) {
            query.add_float_data(feature_blob_data[d]);
          }

          // const string ready to be stored
          string out;
          query.SerializeToString(&out);
        }
      }
      // blob[prob]
      else{
        for (int n = 0; n < batch_size; ++n) {
          feature_blob_data = feature_blob->cpu_data() +
              feature_blob->offset(n);
          for (int d = 0; d < dim_features; ++d) {
            prob_list.append(feature_blob_data[d]);
            //datum.add_float_data(feature_blob_data[d]);
          }
        } // for (int n = 0; n < batch_size; ++n)
      }
    }  // for (int i = 0; i < num_features; ++i)
  }  // for (int batch_index = 0; batch_index < num_mini_batches; ++batch_index)

  /* ==== Retrieval ==== */
  // Open the feature db
  boost::shared_ptr<db::DB> feature_db(db::GetDB(db_type));
  feature_db->Open(feature_db_name, db::READ);
  db::Cursor* db_cursor(feature_db->NewCursor());

  // Visit all the features
  std::vector<std::string> name_vec;
  std::vector<boost::shared_ptr<Datum> > Datum_vec;
  for(db_cursor->SeekToFirst(); db_cursor->valid(); db_cursor->Next()){
    std::string name = db_cursor->key();
    // get the current string value
    std::string feat = db_cursor->value();

    // convert to the feature (floating)
    boost::shared_ptr<Datum> datum(new Datum);
    datum->ParseFromString(feat);

    name_vec.push_back(name);
    Datum_vec.push_back(datum);
  }
  printf("#feature in DB = #%zd\n", Datum_vec.size());

  // similarity measure of features against query
  std::vector<Dtype> scores;
  for(std::vector< boost::shared_ptr<Datum> >::iterator iter=Datum_vec.begin(); iter<Datum_vec.end(); ++iter){
    Dtype dot_val = similarity<Dtype>(query, **iter);
    scores.push_back(dot_val);
    std::cout << "The similarity of two vectors is " << dot_val << std::endl;
  }

  // sort by similarity measures
  std::vector<size_t> indices = argsort_using_stlMap(scores);
  for(std::vector<size_t>::iterator iter=indices.begin(); iter<indices.end(); ++iter){
    std::cout << "Sorted Indices: " << *iter << std::endl;
  }

  // map sorted filename into python list
  for (size_t i=0; i<indices.size(); ++i) {
    result_list.append(name_vec[ indices[i] ]);
  }

  // close the feature db
  feature_db->Close();

  printf("%s\n", "Successfully retrieved similar images!");
  return result_list;
}

BOOST_PYTHON_MODULE(_django_CV){
  bp::def("feature_extraction_pipeline", bp::raw_function(&feature_extraction_pipeline_raw<Dtype>));
  bp::def("matching_pipeline", bp::raw_function(&matching_pipeline_raw<Dtype>));
  bp::def("retrieval_pipeline", bp::raw_function(&retrieval_pipeline_raw<Dtype>));
  // boost python expects a void (missing) return value, while import_array
  // returns NULL for python3. import_array1() forces a void return value.
  import_array1();
}


// Distributed under the MIT License.
// See LICENSE.txt for details.

#include "Evolution/Systems/GrMhd/ValenciaDivClean/PrimitiveFromConservative.hpp"

#include <boost/none.hpp>
#include <boost/optional/optional.hpp>
#include <iomanip>
#include <limits>
#include <ostream>

#include "DataStructures/DataVector.hpp"
#include "DataStructures/Tensor/EagerMath/DotProduct.hpp"
#include "DataStructures/Tensor/Tensor.hpp"
#include "ErrorHandling/Error.hpp"
#include "Evolution/Systems/GrMhd/ValenciaDivClean/NewmanHamlin.hpp"  // IWYU pragma: keep
#include "Evolution/Systems/GrMhd/ValenciaDivClean/PalenzuelaEtAl.hpp"  // IWYU pragma: keep
#include "Evolution/Systems/GrMhd/ValenciaDivClean/PrimitiveRecoveryData.hpp"
#include "Evolution/Systems/GrMhd/ValenciaDivClean/Tags.hpp"  // IWYU pragma: keep
#include "PointwiseFunctions/GeneralRelativity/IndexManipulation.hpp"
#include "PointwiseFunctions/GeneralRelativity/Tags.hpp"  // IWYU pragma: keep
#include "PointwiseFunctions/Hydro/SpecificEnthalpy.hpp"
#include "PointwiseFunctions/Hydro/Tags.hpp"  // IWYU pragma: keep
#include "Utilities/ConstantExpressions.hpp"
#include "Utilities/GenerateInstantiations.hpp"
#include "Utilities/Gsl.hpp"
#include "Utilities/Overloader.hpp"
#include "Utilities/TMPL.hpp"

// IWYU pragma: no_forward_declare EquationsOfState::EquationOfState
// IWYU pragma: no_forward_declare Tensor
// IWYU pragma: no_include <array>

/// \cond
namespace grmhd {
namespace ValenciaDivClean {

template <typename OrderedListOfPrimitiveRecoverySchemes,
          size_t ThermodynamicDim>
void PrimitiveFromConservative<OrderedListOfPrimitiveRecoverySchemes,
                               ThermodynamicDim>::
    apply(
        gsl::not_null<Scalar<DataVector>*> rest_mass_density,
        gsl::not_null<Scalar<DataVector>*> specific_internal_energy,
        gsl::not_null<tnsr::I<DataVector, 3, Frame::Inertial>*>
            spatial_velocity,
        gsl::not_null<tnsr::I<DataVector, 3, Frame::Inertial>*> magnetic_field,
        gsl::not_null<Scalar<DataVector>*> divergence_cleaning_field,
        gsl::not_null<Scalar<DataVector>*> lorentz_factor,
        gsl::not_null<Scalar<DataVector>*> pressure,
        gsl::not_null<Scalar<DataVector>*> specific_enthalpy,
        const Scalar<DataVector>& tilde_d, const Scalar<DataVector>& tilde_tau,
        const tnsr::i<DataVector, 3, Frame::Inertial>& tilde_s,
        const tnsr::I<DataVector, 3, Frame::Inertial>& tilde_b,
        const Scalar<DataVector>& tilde_phi,
        const tnsr::ii<DataVector, 3, Frame::Inertial>& spatial_metric,
        const tnsr::II<DataVector, 3, Frame::Inertial>& inv_spatial_metric,
        const Scalar<DataVector>& sqrt_det_spatial_metric,
        const EquationsOfState::EquationOfState<true, ThermodynamicDim>&
            equation_of_state) noexcept {
  get(*divergence_cleaning_field) =
      get(tilde_phi) / get(sqrt_det_spatial_metric);
  for (size_t i = 0; i < 3; ++i) {
    magnetic_field->get(i) = tilde_b.get(i) / get(sqrt_det_spatial_metric);
  }
  const DataVector total_energy_density =
      (get(tilde_tau) + get(tilde_d)) / get(sqrt_det_spatial_metric);
  const auto tilde_s_upper = raise_or_lower_index(tilde_s, inv_spatial_metric);
  const DataVector momentum_density_squared =
      get(dot_product(tilde_s, tilde_s_upper)) /
      square(get(sqrt_det_spatial_metric));
  const DataVector momentum_density_dot_magnetic_field =
      get(dot_product(tilde_s, *magnetic_field)) / get(sqrt_det_spatial_metric);
  const DataVector magnetic_field_squared =
      get(dot_product(*magnetic_field, *magnetic_field, spatial_metric));
  const DataVector rest_mass_density_times_lorentz_factor =
      get(tilde_d) / get(sqrt_det_spatial_metric);

  for (size_t s = 0; s < total_energy_density.size(); ++s) {
    boost::optional<PrimitiveRecoverySchemes::PrimitiveRecoveryData>
        primitive_data = boost::none;
    tmpl::for_each<OrderedListOfPrimitiveRecoverySchemes>(
        [&primitive_data, &total_energy_density, &momentum_density_squared,
         &momentum_density_dot_magnetic_field, &magnetic_field_squared,
         &rest_mass_density_times_lorentz_factor,
         &equation_of_state, &s](auto scheme) noexcept {
          using primitive_recovery_scheme = tmpl::type_from<decltype(scheme)>;
          if (not primitive_data) {
            primitive_data =
                primitive_recovery_scheme::template apply<ThermodynamicDim>(
                    total_energy_density[s], momentum_density_squared[s],
                    momentum_density_dot_magnetic_field[s],
                    magnetic_field_squared[s],
                    rest_mass_density_times_lorentz_factor[s],
                    equation_of_state);
          }
        });

    if (primitive_data) {
      get(*rest_mass_density)[s] = primitive_data.get().rest_mass_density;
      const double coefficient_of_b =
          momentum_density_dot_magnetic_field[s] /
          (primitive_data.get().rho_h_w_squared *
           (primitive_data.get().rho_h_w_squared + magnetic_field_squared[s]));
      const double coefficient_of_s =
          1.0 /
          (get(sqrt_det_spatial_metric)[s] *
           (primitive_data.get().rho_h_w_squared + magnetic_field_squared[s]));
      for (size_t i = 0; i < 3; ++i) {
        spatial_velocity->get(i)[s] =
            coefficient_of_b * magnetic_field->get(i)[s] +
            coefficient_of_s * tilde_s_upper.get(i)[s];
      }
      get(*lorentz_factor)[s] = primitive_data.get().lorentz_factor;
      get(*pressure)[s] = primitive_data.get().pressure;
    } else {
      ERROR("All primitive inversion schemes failed at s = "
            << s << ".\n"
            << std::setprecision(std::numeric_limits<double>::digits10 + 1)
            << "total_energy_density = " << total_energy_density[s] << "\n"
            << "momentum_density_squared = " << momentum_density_squared[s]
            << "\n"
            << "momentum_density_dot_magnetic_field = "
            << momentum_density_dot_magnetic_field[s] << "\n"
            << "magnetic_field_squared = " << magnetic_field_squared[s] << "\n"
            << "rest_mass_density_times_lorentz_factor = "
            << rest_mass_density_times_lorentz_factor[s] << "\n"
            << "previous_rest_mass_density = " << get(*rest_mass_density)[s]
            << "\n"
            << "previous_pressure = " << get(*pressure)[s] << "\n"
            << "previous_lorentz_factor = " << get(*lorentz_factor)[s] << "\n");
    }
  }
  *specific_internal_energy = make_overloader(
      [&rest_mass_density](const EquationsOfState::EquationOfState<true, 1>&
                               the_equation_of_state) noexcept {
        return the_equation_of_state.specific_internal_energy_from_density(
            *rest_mass_density);
      },
      [&rest_mass_density,
       &pressure ](const EquationsOfState::EquationOfState<true, 2>&
                       the_equation_of_state) noexcept {
        return the_equation_of_state
            .specific_internal_energy_from_density_and_pressure(
                *rest_mass_density, *pressure);
      })(equation_of_state);
  *specific_enthalpy = hydro::specific_enthalpy(
      *rest_mass_density, *specific_internal_energy, *pressure);
}
}  // namespace ValenciaDivClean
}  // namespace grmhd

#define RECOVERY(data) BOOST_PP_TUPLE_ELEM(0, data)
#define THERMODIM(data) BOOST_PP_TUPLE_ELEM(1, data)

#define INSTANTIATION(_, data)                                        \
  template struct grmhd::ValenciaDivClean::PrimitiveFromConservative< \
      RECOVERY(data), THERMODIM(data)>;

using NewmanHamlinThenPalenzuelaEtAl = tmpl::list<
    grmhd::ValenciaDivClean::PrimitiveRecoverySchemes::NewmanHamlin,
    grmhd::ValenciaDivClean::PrimitiveRecoverySchemes::PalenzuelaEtAl>;

GENERATE_INSTANTIATIONS(
    INSTANTIATION,
    (tmpl::list<
         grmhd::ValenciaDivClean::PrimitiveRecoverySchemes::NewmanHamlin>,
     tmpl::list<
         grmhd::ValenciaDivClean::PrimitiveRecoverySchemes::PalenzuelaEtAl>,
     NewmanHamlinThenPalenzuelaEtAl),
    (1, 2))

#undef INSTANTIATION
#undef THERMODIM
#undef RECOVERY
/// \endcond


//  (C) Copyright Edward Diener 2011-2015
//  Use, modification and distribution are subject to the Boost Software License,
//  Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
//  http://www.boost.org/LICENSE_1_0.txt).

#include <boost/vmd/is_number.hpp>
#include <boost/detail/lightweight_test.hpp>

int main()
  {

#if BOOST_PP_VARIADICS

 BOOST_TEST(BOOST_VMD_IS_NUMBER(0));
 BOOST_TEST(BOOST_VMD_IS_NUMBER(44));
 BOOST_TEST(!BOOST_VMD_IS_NUMBER(SQUARE));
 BOOST_TEST(!BOOST_VMD_IS_NUMBER(44 DATA));
 BOOST_TEST(!BOOST_VMD_IS_NUMBER(044));
 BOOST_TEST(BOOST_VMD_IS_NUMBER(256));
 BOOST_TEST(!BOOST_VMD_IS_NUMBER(257));
 BOOST_TEST(!BOOST_VMD_IS_NUMBER(245e2));
 BOOST_TEST(!BOOST_VMD_IS_NUMBER((44)));

#else

BOOST_ERROR("No variadic macro support");

#endif

  return boost::report_errors();

  }


// Boost.Geometry (aka GGL, Generic Geometry Library)

// Copyright (c) 2007-2015 Barend Gehrels, Amsterdam, the Netherlands.
// Copyright (c) 2008-2015 Bruno Lalande, Paris, France.
// Copyright (c) 2009-2015 Mateusz Loskot, London, UK.

// Parts of Boost.Geometry are redesigned from Geodan's Geographic Library
// (geolib/GGL), copyright (c) 1995-2010 Geodan, Amsterdam, the Netherlands.

// Use, modification and distribution is subject to the Boost Software License,
// Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)

#ifndef BOOST_GEOMETRY_ALGORITHMS_SIMPLIFY_HPP
#define BOOST_GEOMETRY_ALGORITHMS_SIMPLIFY_HPP

#include <cstddef>

#include <boost/core/ignore_unused.hpp>
#include <boost/range.hpp>

#include <boost/variant/apply_visitor.hpp>
#include <boost/variant/static_visitor.hpp>
#include <boost/variant/variant_fwd.hpp>

#include <boost/geometry/core/cs.hpp>
#include <boost/geometry/core/closure.hpp>
#include <boost/geometry/core/exterior_ring.hpp>
#include <boost/geometry/core/interior_rings.hpp>
#include <boost/geometry/core/mutable_range.hpp>
#include <boost/geometry/core/tags.hpp>

#include <boost/geometry/geometries/concepts/check.hpp>
#include <boost/geometry/strategies/agnostic/simplify_douglas_peucker.hpp>
#include <boost/geometry/strategies/concepts/simplify_concept.hpp>
#include <boost/geometry/strategies/default_strategy.hpp>
#include <boost/geometry/strategies/distance.hpp>

#include <boost/geometry/algorithms/area.hpp>
#include <boost/geometry/algorithms/clear.hpp>
#include <boost/geometry/algorithms/convert.hpp>
#include <boost/geometry/algorithms/detail/equals/point_point.hpp>
#include <boost/geometry/algorithms/not_implemented.hpp>
#include <boost/geometry/algorithms/is_empty.hpp>
#include <boost/geometry/algorithms/perimeter.hpp>

#include <boost/geometry/algorithms/detail/distance/default_strategies.hpp>

namespace boost { namespace geometry
{

#ifndef DOXYGEN_NO_DETAIL
namespace detail { namespace simplify
{

template <typename Range>
inline bool is_degenerate(Range const& range)
{
    return boost::size(range) == 2
        && detail::equals::equals_point_point(geometry::range::front(range),
                                              geometry::range::back(range));
}

struct simplify_range_insert
{
    template<typename Range, typename Strategy, typename OutputIterator, typename Distance>
    static inline void apply(Range const& range, OutputIterator out,
                             Distance const& max_distance, Strategy const& strategy)
    {
        boost::ignore_unused(strategy);

        if (is_degenerate(range))
        {
            std::copy(boost::begin(range), boost::begin(range) + 1, out);
        }
        else if (boost::size(range) <= 2 || max_distance < 0)
        {
            std::copy(boost::begin(range), boost::end(range), out);
        }
        else
        {
            strategy.apply(range, out, max_distance);
        }
    }
};

struct simplify_copy
{
    template <typename RangeIn, typename RangeOut, typename Strategy, typename Distance>
    static inline void apply(RangeIn const& range, RangeOut& out,
                             Distance const& , Strategy const& )
    {
        std::copy
            (
                boost::begin(range), boost::end(range),
                    geometry::range::back_inserter(out)
            );
    }
};

template <std::size_t MinimumToUseStrategy>
struct simplify_range
{
    template <typename RangeIn, typename RangeOut, typename Strategy, typename Distance>
    static inline void apply(RangeIn const& range, RangeOut& out,
                    Distance const& max_distance, Strategy const& strategy)
    {
        // For a RING:
        // Note that, especially if max_distance is too large,
        // the output ring might be self intersecting while the input ring is
        // not, although chances are low in normal polygons

        if (boost::size(range) <= MinimumToUseStrategy || max_distance < 0)
        {
            simplify_copy::apply(range, out, max_distance, strategy);
        }
        else
        {
            simplify_range_insert::apply
                (
                    range, geometry::range::back_inserter(out), max_distance, strategy
                );
        }

        // Verify the two remaining points are equal. If so, remove one of them.
        // This can cause the output being under the minimum size
        if (is_degenerate(out))
        {
            range::resize(out, 1);
        }
    }
};

struct simplify_ring
{
private :
    template <typename Area>
    static inline int area_sign(Area const& area)
    {
        return area > 0 ? 1 : area < 0 ? -1 : 0;
    }

    template <typename Strategy, typename Ring>
    static std::size_t get_opposite(std::size_t index, Ring const& ring)
    {
        typename Strategy::distance_strategy_type distance_strategy;

        // Verify if it is NOT the case that all points are less than the
        // simplifying distance. If so, output is empty.
        typename Strategy::distance_type max_distance(-1);

        typename geometry::point_type<Ring>::type point = range::at(ring, index);
        std::size_t i = 0;
        for (typename boost::range_iterator<Ring const>::type
                it = boost::begin(ring); it != boost::end(ring); ++it, ++i)
        {
            // This actually is point-segment distance but will result
            // in point-point distance
            typename Strategy::distance_type dist = distance_strategy.apply(*it, point, point);
            if (dist > max_distance)
            {
                max_distance = dist;
                index = i;
            }
        }
        return index;
    }

public :
    template <typename Ring, typename Strategy, typename Distance>
    static inline void apply(Ring const& ring, Ring& out,
                    Distance const& max_distance, Strategy const& strategy)
    {
        std::size_t const size = boost::size(ring);
        if (size == 0)
        {
            return;
        }

        int const input_sign = area_sign(geometry::area(ring));

        std::set<std::size_t> visited_indexes;

        // Rotate it into a copied vector
        // (vector, because source type might not support rotation)
        // (duplicate end point will be simplified away)
        typedef typename geometry::point_type<Ring>::type point_type;

        std::vector<point_type> rotated(size);

        // Closing point (but it will not start here)
        std::size_t index = 0;

        // Iterate (usually one iteration is enough)
        for (std::size_t iteration = 0; iteration < 4u; iteration++)
        {
            // Always take the opposite. Opposite guarantees that no point
            // "halfway" is chosen, creating an artefact (very narrow triangle)
            // Iteration 0: opposite to closing point (1/2, = on convex hull)
            //              (this will start simplification with that point
            //               and its opposite ~0)
            // Iteration 1: move a quarter on that ring, then opposite to 1/4
            //              (with its opposite 3/4)
            // Iteration 2: move an eight on that ring, then opposite (1/8)
            // Iteration 3: again move a quarter, then opposite (7/8)
            // So finally 8 "sides" of the ring have been examined (if it were
            // a semi-circle). Most probably, there are only 0 or 1 iterations.
            switch (iteration)
            {
                case 1 : index = (index + size / 4) % size; break;
                case 2 : index = (index + size / 8) % size; break;
                case 3 : index = (index + size / 4) % size; break;
            }
            index = get_opposite<Strategy>(index, ring);

            if (visited_indexes.count(index) > 0)
            {
                // Avoid trying the same starting point more than once
                continue;
            }

            std::rotate_copy(boost::begin(ring), range::pos(ring, index),
                             boost::end(ring), rotated.begin());

            // Close the rotated copy
            rotated.push_back(range::at(ring, index));

            simplify_range<0>::apply(rotated, out, max_distance, strategy);

            // Verify that what was positive, stays positive (or goes to 0)
            // and what was negative stays negative (or goes to 0)
            int const output_sign = area_sign(geometry::area(out));
            if (output_sign == input_sign)
            {
                // Result is considered as satisfactory (usually this is the
                // first iteration - only for small rings, having a scale
                // similar to simplify_distance, next iterations are tried
                return;
            }

            // Original is simplified away. Possibly there is a solution
            // when another starting point is used
            geometry::clear(out);

            if (iteration == 0
                && geometry::perimeter(ring) < 3 * max_distance)
            {
                // Check if it is useful to iterate. A minimal triangle has a
                // perimeter of a bit more than 3 times the simplify distance
                return;
            }

            // Prepare next try
            visited_indexes.insert(index);
            rotated.resize(size);
        }
    }
};

struct simplify_polygon
{
private:

    template
    <
        typename IteratorIn,
        typename InteriorRingsOut,
        typename Distance,
        typename Strategy
    >
    static inline void iterate(IteratorIn begin, IteratorIn end,
                    InteriorRingsOut& interior_rings_out,
                    Distance const& max_distance, Strategy const& strategy)
    {
        typedef typename boost::range_value<InteriorRingsOut>::type single_type;
        for (IteratorIn it = begin; it != end; ++it)
        {
            single_type out;
            simplify_ring::apply(*it, out, max_distance, strategy);
            if (! geometry::is_empty(out))
            {
                range::push_back(interior_rings_out, out);
            }
        }
    }

    template
    <
        typename InteriorRingsIn,
        typename InteriorRingsOut,
        typename Distance,
        typename Strategy
    >
    static inline void apply_interior_rings(
                    InteriorRingsIn const& interior_rings_in,
                    InteriorRingsOut& interior_rings_out,
                    Distance const& max_distance, Strategy const& strategy)
    {
        range::clear(interior_rings_out);

        iterate(
            boost::begin(interior_rings_in), boost::end(interior_rings_in),
            interior_rings_out,
            max_distance, strategy);
    }

public:
    template <typename Polygon, typename Strategy, typename Distance>
    static inline void apply(Polygon const& poly_in, Polygon& poly_out,
                    Distance const& max_distance, Strategy const& strategy)
    {
        // Note that if there are inner rings, and distance is too large,
        // they might intersect with the outer ring in the output,
        // while it didn't in the input.
        simplify_ring::apply(exterior_ring(poly_in), exterior_ring(poly_out),
            max_distance, strategy);

        apply_interior_rings(interior_rings(poly_in),
            interior_rings(poly_out), max_distance, strategy);
    }
};

template<typename Policy>
struct simplify_multi
{
    template <typename MultiGeometry, typename Strategy, typename Distance>
    static inline void apply(MultiGeometry const& multi, MultiGeometry& out,
                    Distance const& max_distance, Strategy const& strategy)
    {
        range::clear(out);

        typedef typename boost::range_value<MultiGeometry>::type single_type;

        for (typename boost::range_iterator<MultiGeometry const>::type
                it = boost::begin(multi); it != boost::end(multi); ++it)
        {
            single_type single_out;
            Policy::apply(*it, single_out, max_distance, strategy);
            if (! geometry::is_empty(single_out))
            {
                range::push_back(out, single_out);
            }
        }
    }
};

}} // namespace detail::simplify
#endif // DOXYGEN_NO_DETAIL

#ifndef DOXYGEN_NO_DISPATCH
namespace dispatch
{

template
<
    typename Geometry,
    typename Tag = typename tag<Geometry>::type
>
struct simplify: not_implemented<Tag>
{};

template <typename Point>
struct simplify<Point, point_tag>
{
    template <typename Distance, typename Strategy>
    static inline void apply(Point const& point, Point& out,
                    Distance const& , Strategy const& )
    {
        geometry::convert(point, out);
    }
};

// Linestring, keep 2 points (unless those points are the same)
template <typename Linestring>
struct simplify<Linestring, linestring_tag>
    : detail::simplify::simplify_range<2>
{};

template <typename Ring>
struct simplify<Ring, ring_tag>
    : detail::simplify::simplify_ring
{};

template <typename Polygon>
struct simplify<Polygon, polygon_tag>
    : detail::simplify::simplify_polygon
{};

template
<
    typename Geometry,
    typename Tag = typename tag<Geometry>::type
>
struct simplify_insert: not_implemented<Tag>
{};

template <typename Linestring>
struct simplify_insert<Linestring, linestring_tag>
    : detail::simplify::simplify_range_insert
{};

template <typename Ring>
struct simplify_insert<Ring, ring_tag>
    : detail::simplify::simplify_range_insert
{};

template <typename MultiPoint>
struct simplify<MultiPoint, multi_point_tag>
    : detail::simplify::simplify_copy
{};

template <typename MultiLinestring>
struct simplify<MultiLinestring, multi_linestring_tag>
    : detail::simplify::simplify_multi<detail::simplify::simplify_range<2> >
{};

template <typename MultiPolygon>
struct simplify<MultiPolygon, multi_polygon_tag>
    : detail::simplify::simplify_multi<detail::simplify::simplify_polygon>
{};

} // namespace dispatch
#endif // DOXYGEN_NO_DISPATCH

namespace resolve_strategy
{

struct simplify
{
    template <typename Geometry, typename Distance, typename Strategy>
    static inline void apply(Geometry const& geometry,
                             Geometry& out,
                             Distance const& max_distance,
                             Strategy const& strategy)
    {
        dispatch::simplify<Geometry>::apply(geometry, out, max_distance, strategy);
    }

    template <typename Geometry, typename Distance>
    static inline void apply(Geometry const& geometry,
                             Geometry& out,
                             Distance const& max_distance,
                             default_strategy)
    {
        typedef typename point_type<Geometry>::type point_type;

        typedef typename strategy::distance::services::default_strategy
        <
            point_tag, segment_tag, point_type
        >::type ds_strategy_type;

        typedef strategy::simplify::douglas_peucker
        <
            point_type, ds_strategy_type
        > strategy_type;

        BOOST_CONCEPT_ASSERT(
            (concepts::SimplifyStrategy<strategy_type, point_type>)
        );

        apply(geometry, out, max_distance, strategy_type());
    }
};

struct simplify_insert
{
    template
    <
        typename Geometry,
        typename OutputIterator,
        typename Distance,
        typename Strategy
    >
    static inline void apply(Geometry const& geometry,
                             OutputIterator& out,
                             Distance const& max_distance,
                             Strategy const& strategy)
    {
        dispatch::simplify_insert<Geometry>::apply(geometry, out, max_distance, strategy);
    }

    template <typename Geometry, typename OutputIterator, typename Distance>
    static inline void apply(Geometry const& geometry,
                             OutputIterator& out,
                             Distance const& max_distance,
                             default_strategy)
    {
        typedef typename point_type<Geometry>::type point_type;

        typedef typename strategy::distance::services::default_strategy
        <
            point_tag, segment_tag, point_type
        >::type ds_strategy_type;

        typedef strategy::simplify::douglas_peucker
        <
            point_type, ds_strategy_type
        > strategy_type;

        BOOST_CONCEPT_ASSERT(
            (concepts::SimplifyStrategy<strategy_type, point_type>)
        );

        apply(geometry, out, max_distance, strategy_type());
    }
};

} // namespace resolve_strategy

namespace resolve_variant {

template <typename Geometry>
struct simplify
{
    template <typename Distance, typename Strategy>
    static inline void apply(Geometry const& geometry,
                             Geometry& out,
                             Distance const& max_distance,
                             Strategy const& strategy)
    {
        resolve_strategy::simplify::apply(geometry, out, max_distance, strategy);
    }
};

template <BOOST_VARIANT_ENUM_PARAMS(typename T)>
struct simplify<boost::variant<BOOST_VARIANT_ENUM_PARAMS(T)> >
{
    template <typename Distance, typename Strategy>
    struct visitor: boost::static_visitor<void>
    {
        Distance const& m_max_distance;
        Strategy const& m_strategy;

        visitor(Distance const& max_distance, Strategy const& strategy)
            : m_max_distance(max_distance)
            , m_strategy(strategy)
        {}

        template <typename Geometry>
        void operator()(Geometry const& geometry, Geometry& out) const
        {
            simplify<Geometry>::apply(geometry, out, m_max_distance, m_strategy);
        }
    };

    template <typename Distance, typename Strategy>
    static inline void
    apply(boost::variant<BOOST_VARIANT_ENUM_PARAMS(T)> const& geometry,
          boost::variant<BOOST_VARIANT_ENUM_PARAMS(T)>& out,
          Distance const& max_distance,
          Strategy const& strategy)
    {
        boost::apply_visitor(
            visitor<Distance, Strategy>(max_distance, strategy),
            geometry,
            out
        );
    }
};

} // namespace resolve_variant

/*!
\brief Simplify a geometry using a specified strategy
\ingroup simplify
\tparam Geometry \tparam_geometry
\tparam Distance A numerical distance measure
\tparam Strategy A type fulfilling a SimplifyStrategy concept
\param strategy A strategy to calculate simplification
\param geometry input geometry, to be simplified
\param out output geometry, simplified version of the input geometry
\param max_distance distance (in units of input coordinates) of a vertex
    to other segments to be removed
\param strategy simplify strategy to be used for simplification, might
    include point-distance strategy

\image html svg_simplify_country.png "The image below presents the simplified country"
\qbk{distinguish,with strategy}
*/
template<typename Geometry, typename Distance, typename Strategy>
inline void simplify(Geometry const& geometry, Geometry& out,
                     Distance const& max_distance, Strategy const& strategy)
{
    concepts::check<Geometry>();

    geometry::clear(out);

    resolve_variant::simplify<Geometry>::apply(geometry, out, max_distance, strategy);
}

/*!
\brief Simplify a geometry
\ingroup simplify
\tparam Geometry \tparam_geometry
\tparam Distance \tparam_numeric
\note This version of simplify simplifies a geometry using the default
    strategy (Douglas Peucker),
\param geometry input geometry, to be simplified
\param out output geometry, simplified version of the input geometry
\param max_distance distance (in units of input coordinates) of a vertex
    to other segments to be removed

\qbk{[include reference/algorithms/simplify.qbk]}
 */
template<typename Geometry, typename Distance>
inline void simplify(Geometry const& geometry, Geometry& out,
                     Distance const& max_distance)
{
    concepts::check<Geometry>();

    geometry::simplify(geometry, out, max_distance, default_strategy());
}

#ifndef DOXYGEN_NO_DETAIL
namespace detail { namespace simplify
{

/*!
\brief Simplify a geometry, using an output iterator
    and a specified strategy
\ingroup simplify
\tparam Geometry \tparam_geometry
\param geometry input geometry, to be simplified
\param out output iterator, outputs all simplified points
\param max_distance distance (in units of input coordinates) of a vertex
    to other segments to be removed
\param strategy simplify strategy to be used for simplification,
    might include point-distance strategy

\qbk{distinguish,with strategy}
\qbk{[include reference/algorithms/simplify.qbk]}
*/
template<typename Geometry, typename OutputIterator, typename Distance, typename Strategy>
inline void simplify_insert(Geometry const& geometry, OutputIterator out,
                            Distance const& max_distance, Strategy const& strategy)
{
    concepts::check<Geometry const>();

    resolve_strategy::simplify_insert::apply(geometry, out, max_distance, strategy);
}

/*!
\brief Simplify a geometry, using an output iterator
\ingroup simplify
\tparam Geometry \tparam_geometry
\param geometry input geometry, to be simplified
\param out output iterator, outputs all simplified points
\param max_distance distance (in units of input coordinates) of a vertex
    to other segments to be removed

\qbk{[include reference/algorithms/simplify_insert.qbk]}
 */
template<typename Geometry, typename OutputIterator, typename Distance>
inline void simplify_insert(Geometry const& geometry, OutputIterator out,
                            Distance const& max_distance)
{
    // Concept: output point type = point type of input geometry
    concepts::check<Geometry const>();
    concepts::check<typename point_type<Geometry>::type>();

    simplify_insert(geometry, out, max_distance, default_strategy());
}

}} // namespace detail::simplify
#endif // DOXYGEN_NO_DETAIL

}} // namespace boost::geometry

#endif // BOOST_GEOMETRY_ALGORITHMS_SIMPLIFY_HPP


// Copyright (C) 2011-2012 by the Bem++ Authors
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

#ifndef fiber_default_local_assembler_for_integral_operators_on_surfaces_hpp
#define fiber_default_local_assembler_for_integral_operators_on_surfaces_hpp

#include "../common/common.hpp"

#include "local_assembler_for_operators.hpp"

#include "_2d_array.hpp"
#include "accuracy_options.hpp"
#include "default_local_assembler_for_operators_on_surfaces_utilities.hpp"
#include "element_pair_topology.hpp"
#include "numerical_quadrature.hpp"
#include "parallelization_options.hpp"
#include "shared_ptr.hpp"
#include "test_kernel_trial_integrator.hpp"
#include "verbosity_level.hpp"

#include <boost/static_assert.hpp>
#include <boost/tuple/tuple_comparison.hpp>
#include <tbb/concurrent_unordered_map.h>
#include <cstring>
#include <climits>
#include <set>
#include <utility>
#include <vector>

namespace Fiber
{

/** \cond FORWARD_DECL */
class OpenClHandler;
template <typename CoordinateType> class CollectionOfBasisTransformations;
template <typename ValueType> class CollectionOfKernels;
template <typename BasisFunctionType, typename KernelType, typename ResultType>
class TestKernelTrialIntegral;
template <typename CoordinateType> class RawGridGeometry;
/** \endcond */

template <typename BasisFunctionType, typename KernelType,
          typename ResultType, typename GeometryFactory>
class DefaultLocalAssemblerForIntegralOperatorsOnSurfaces :
        public LocalAssemblerForOperators<ResultType>
{
public:
    typedef typename ScalarTraits<ResultType>::RealType CoordinateType;

    DefaultLocalAssemblerForIntegralOperatorsOnSurfaces(
            const shared_ptr<const GeometryFactory>& testGeometryFactory,
            const shared_ptr<const GeometryFactory>& trialGeometryFactory,
            const shared_ptr<const RawGridGeometry<CoordinateType> >& testRawGeometry,
            const shared_ptr<const RawGridGeometry<CoordinateType> >& trialRawGeometry,
            const shared_ptr<const std::vector<const Basis<BasisFunctionType>*> >& testBases,
            const shared_ptr<const std::vector<const Basis<BasisFunctionType>*> >& trialBases,
            const shared_ptr<const CollectionOfBasisTransformations<CoordinateType> >& testTransformations,
            const shared_ptr<const CollectionOfKernels<KernelType> >& kernel,
            const shared_ptr<const CollectionOfBasisTransformations<CoordinateType> >& trialTransformations,
            const shared_ptr<const TestKernelTrialIntegral<BasisFunctionType, KernelType, ResultType> >& integral,
            const shared_ptr<const OpenClHandler>& openClHandler,
            const ParallelizationOptions& parallelizationOptions,
            VerbosityLevel::Level verbosityLevel,
            bool cacheSingularIntegrals,
            const AccuracyOptionsEx& accuracyOptions);
    virtual ~DefaultLocalAssemblerForIntegralOperatorsOnSurfaces();

public:
    virtual void evaluateLocalWeakForms(
            CallVariant callVariant,
            const std::vector<int>& elementIndicesA,
            int elementIndexB,
            LocalDofIndex localDofIndexB,
            std::vector<arma::Mat<ResultType> >& result,
            CoordinateType nominalDistance = -1.);

    virtual void evaluateLocalWeakForms(
            const std::vector<int>& testElementIndices,
            const std::vector<int>& trialElementIndices,
            Fiber::_2dArray<arma::Mat<ResultType> >& result,
            CoordinateType nominalDistance = -1.);

    virtual void evaluateLocalWeakForms(
            const std::vector<int>& elementIndices,
            std::vector<arma::Mat<ResultType> >& result);

    virtual CoordinateType estimateRelativeScale(CoordinateType minDist) const;

private:
    /** \cond PRIVATE */
    typedef TestKernelTrialIntegrator<BasisFunctionType, KernelType, ResultType> Integrator;
    typedef typename Integrator::ElementIndexPair ElementIndexPair;
    typedef DefaultLocalAssemblerForOperatorsOnSurfacesUtilities<
    BasisFunctionType> Utilities;

    /** \brief Alternative comparison functor for pairs.
     *
     *  This functor can be used to sort pairs according to the second member
     *  and then, in case of equality, according to the first member. */
    template <typename T1, typename T2>
    struct alternative_less {
        bool operator() (const std::pair<T1, T2>& a, const std::pair<T1, T2>& b) const {
            return a.second < b.second
                || (!(b.second < a.second) && a.first < b.first);
        }
    };

    /** \brief Comparison functor for element-index pairs.
     *
     *  This functor sorts element index pairs first after the trial element
     *  index (second member) and then, in case of equality, after the test
     *  element index (first member) */
    typedef alternative_less<
        typename ElementIndexPair::first_type,
        typename ElementIndexPair::second_type> ElementIndexPairCompare;

    /** \brief Set of element index pairs.
     *
     *  The alternative sorting (first after the trial element index) is used
     *  because profiling has shown that evaluateLocalWeakForms is called more
     *  often in the TEST_TRIAL mode (with a single trial element index) than
     *  in the TRIAL_TEST mode. Therefore the singular integral cache is
     *  indexed with trial element index, and this sorting mode makes it easier
     *  to construct such cache. */
    typedef std::set<ElementIndexPair, ElementIndexPairCompare> ElementIndexPairSet;

    bool testAndTrialGridsAreIdentical() const;

    void cacheSingularLocalWeakForms();
    void findPairsOfAdjacentElements(ElementIndexPairSet& pairs) const;
    void cacheLocalWeakForms(const ElementIndexPairSet& elementIndexPairs);

    const Integrator& selectIntegrator(
            int testElementIndex, int trialElementIndex,
            CoordinateType nominalDistance = -1.);

    enum ElementType {
        TEST, TRIAL
    };

    const Integrator& getIntegrator(const DoubleQuadratureDescriptor& index);

    void getRegularOrders(int testElementIndex, int trialElementIndex,
                          int& testQuadOrder, int& trialQuadOrder,
                          CoordinateType nominalDistance) const;
    int singularOrder(int elementIndex, ElementType elementType) const;

    CoordinateType elementDistanceSquared(
            int testElementIndex, int trialElementIndex) const;

    void precalculateElementSizesAndCenters();

private:
    shared_ptr<const GeometryFactory> m_testGeometryFactory;
    shared_ptr<const GeometryFactory> m_trialGeometryFactory;
    shared_ptr<const RawGridGeometry<CoordinateType> > m_testRawGeometry;
    shared_ptr<const RawGridGeometry<CoordinateType> > m_trialRawGeometry;
    shared_ptr<const std::vector<const Basis<BasisFunctionType>*> > m_testBases;
    shared_ptr<const std::vector<const Basis<BasisFunctionType>*> > m_trialBases;
    shared_ptr<const CollectionOfBasisTransformations<CoordinateType> > m_testTransformations;
    shared_ptr<const CollectionOfKernels<KernelType> > m_kernels;
    shared_ptr<const CollectionOfBasisTransformations<CoordinateType> > m_trialTransformations;
    shared_ptr<const TestKernelTrialIntegral<BasisFunctionType, KernelType, ResultType> > m_integral;
    shared_ptr<const OpenClHandler> m_openClHandler;
    ParallelizationOptions m_parallelizationOptions;
    VerbosityLevel::Level m_verbosityLevel;
    AccuracyOptionsEx m_accuracyOptions;

    typedef tbb::concurrent_unordered_map<DoubleQuadratureDescriptor,
    Integrator*> IntegratorMap;
    IntegratorMap m_testKernelTrialIntegrators;

    enum { INVALID_INDEX = INT_MAX };
    typedef _2dArray<std::pair<int, arma::Mat<ResultType> > > Cache;
    /** \brief Singular integral cache.
     *
     *  This cache stores the preevaluated local weak forms expressed by
     *  singular integrals. A particular item it stored in r'th row and c'th
     *  column stores, in its second member, the local weak form calculated for
     *  the test element with index it.first and the trial element with index
     *  c. In each column, the items are sorted after increasing test element
     *  index. At the end of each column there can be unused items with test
     *  element index set to INVALID_INDEX (= INT_MAX, so that the sorting is
     *  preserved). */
    Cache m_cache;
    std::vector<CoordinateType> m_testElementSizesSquared;
    std::vector<CoordinateType> m_trialElementSizesSquared;
    arma::Mat<CoordinateType> m_testElementCenters;
    arma::Mat<CoordinateType> m_trialElementCenters;
    CoordinateType m_averageElementSize;

    // tbb::atomic<size_t> m_foundInCache;
    /** \endcond */
};

} // namespace Fiber

#include "default_local_assembler_for_integral_operators_on_surfaces_imp.hpp"

#endif