// Copyright (c) 2012-2017, The CryptoNote developers, The Bytecoin developers
// Copyright (c) 2016-2018  zawy12
// Copyright (c) 2016-2018, The Karbowanec developers
// Copyright (c) 2018-2021, The Qwertycoin Group.
//
// This file is part of Qwertycoin.
//
// Qwertycoin is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Qwertycoin is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with Qwertycoin.  If not, see <http://www.gnu.org/licenses/>.

#include <algorithm>
#include <cctype>
#include <cmath>
#include <boost/algorithm/string/trim.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/math/special_functions/round.hpp>
#include <Common/Base58.h>
#include <Common/int-util.h>
#include <Common/Math.h>
#include <Common/StringTools.h>
#include <CryptoNoteCore/Account.h>
#include <CryptoNoteCore/CryptoNoteBasicImpl.h>
#include <CryptoNoteCore/CryptoNoteFormatUtils.h>
#include <CryptoNoteCore/CryptoNoteTools.h>
#include <CryptoNoteCore/Currency.h>
#include <CryptoNoteCore/TransactionExtra.h>
#include <CryptoNoteCore/UpgradeDetector.h>
#include <Global/Constants.h>
#include <Global/CryptoNoteConfig.h>

#undef ERROR

using namespace Logging;
using namespace Common;
using namespace Qwertycoin;

namespace CryptoNote {

bool Currency::init()
{
    if (!generateGenesisBlock()) {
        logger(ERROR, BRIGHT_RED) << "Failed to generate genesis block";
        return false;
    }

    if (!get_block_hash(m_genesisBlock, m_genesisBlockHash)) {
        logger(ERROR, BRIGHT_RED) << "Failed to get genesis block hash";
        return false;
    }

    if (isTestnet()) {
        m_upgradeHeightV2 = 10;
        m_upgradeHeightV3 = 60;
        m_upgradeHeightV4 = 70;
        m_upgradeHeightV5 = 80;
        m_upgradeHeightV6 = 100;
        m_governancePercent = 10;
        m_governanceHeightStart = 1;
        m_governanceHeightEnd = 100;
        m_blocksFileName = "testnet_" + m_blocksFileName;
        m_blocksCacheFileName = "testnet_" + m_blocksCacheFileName;
        m_blockIndexesFileName = "testnet_" + m_blockIndexesFileName;
        m_txPoolFileName = "testnet_" + m_txPoolFileName;
        m_blockchainIndicesFileName = "testnet_" + m_blockchainIndicesFileName;
    }

    return true;
}

bool Currency::generateGenesisBlock()
{
    m_genesisBlock = boost::value_initialized<Block>();

    // Hard code coinbase tx in genesis block, because "tru" generating tx use random,
    // but genesis should be always the same
    std::string genesisCoinbaseTxHex = GENESIS_COINBASE_TX_HEX;
    BinaryArray minerTxBlob;

    bool r = fromHex(genesisCoinbaseTxHex, minerTxBlob)
             && fromBinaryArray(m_genesisBlock.baseTransaction, minerTxBlob);

    if (!r) {
        logger(ERROR, BRIGHT_RED) << "failed to parse coinbase tx from hard coded blob";
        return false;
    }

    m_genesisBlock.majorVersion = BLOCK_MAJOR_VERSION_1;
    m_genesisBlock.minorVersion = BLOCK_MINOR_VERSION_0;
    m_genesisBlock.timestamp = 0;
    m_genesisBlock.nonce = 70;
    if (m_testnet) {
        ++m_genesisBlock.nonce;
    }

    //miner::find_nonce_for_given_block(bl, 1, 0);

    return true;
}

size_t Currency::blockGrantedFullRewardZoneByBlockVersion(uint8_t blockMajorVersion) const
{
    return CryptoNote::parameters::CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE;
}

uint32_t Currency::upgradeHeight(uint8_t majorVersion) const
{
    if (majorVersion == BLOCK_MAJOR_VERSION_6) {
        return m_upgradeHeightV6;
    }
    else if (majorVersion == BLOCK_MAJOR_VERSION_5) {
        return m_upgradeHeightV5;
    }
    else if (majorVersion == BLOCK_MAJOR_VERSION_4) {
        return m_upgradeHeightV4;
    }
    else if (majorVersion == BLOCK_MAJOR_VERSION_3) {
        return m_upgradeHeightV3;
    }
    else if (majorVersion == BLOCK_MAJOR_VERSION_2) {
        return m_upgradeHeightV2;
    }
    else {
        return static_cast<uint32_t>(-1);
    }
}

bool Currency::getBlockReward(
    uint8_t blockMajorVersion,
    size_t medianSize,
    size_t currentBlockSize,
    uint64_t alreadyGeneratedCoins,
    uint64_t fee,
    uint64_t &reward,
    int64_t &emissionChange,
    uint32_t height,
    uint64_t blockTarget) const
{
    //assert(alreadyGeneratedCoins <= m_moneySupply);
    assert(m_emissionSpeedFactor > 0 && m_emissionSpeedFactor <= 8 * sizeof(uint64_t));

    // Consistency
    double consistency = 1.0;
    double exponent = 0.25;
    if (height >= CryptoNote::parameters::UPGRADE_HEIGHT_V1 && difficultyTarget() != 0) {
        // blockTarget is (Timestamp of New Block - Timestamp of Previous Block)
        consistency = (double) blockTarget / (double) difficultyTarget();

        // consistency range is 0..2
        if (consistency < 1.0) {
            consistency = std::max<double>(consistency, 0.0);
        }
        else if (consistency > 1.0) {
            consistency = pow(consistency, exponent);
            consistency = std::min<double>(consistency, 2.0);
        }
        else {
            consistency = 1.0;
        }
    }

    // Tail emission

    uint64_t baseReward = ((m_moneySupply - alreadyGeneratedCoins) >> m_emissionSpeedFactor) * consistency;
    if (alreadyGeneratedCoins + CryptoNote::parameters::TAIL_EMISSION_REWARD >= m_moneySupply
        || baseReward < CryptoNote::parameters::TAIL_EMISSION_REWARD) {
        baseReward = CryptoNote::parameters::TAIL_EMISSION_REWARD;
    }

    size_t blockGrantedFullRewardZone = blockGrantedFullRewardZoneByBlockVersion(blockMajorVersion);
    medianSize = std::max(medianSize, blockGrantedFullRewardZone);
    if (currentBlockSize > medianSize * UINT64_C(2)) {
        logger(TRACE)
            << "Block cumulative size is too big: "
            << currentBlockSize
            << ", expected less than "
            << medianSize * UINT64_C(2);
        return false;
    }

    uint64_t penalizedBaseReward = getPenalizedAmount(baseReward, medianSize, currentBlockSize);
    uint64_t penalizedFee = getPenalizedAmount(fee, medianSize, currentBlockSize);

    emissionChange = penalizedBaseReward - (fee - penalizedFee);
    reward = penalizedBaseReward + penalizedFee;

    return true;
}

size_t Currency::maxBlockCumulativeSize(uint64_t height) const
{
    assert(height <= std::numeric_limits<uint64_t>::max() / m_maxBlockSizeGrowthSpeedNumerator);

    size_t maxSize =
        static_cast<size_t>(m_maxBlockSizeInitial
        + (height * m_maxBlockSizeGrowthSpeedNumerator) / m_maxBlockSizeGrowthSpeedDenominator);

    assert(maxSize >= m_maxBlockSizeInitial);

    return maxSize;
}

bool Currency::isGovernanceEnabled(uint32_t height) const
{
    if (height >= m_governanceHeightStart && height <= m_governanceHeightEnd) {
        return true;
    }
    else {
        return false;
    }
}

uint64_t Currency::getGovernanceReward(uint64_t base_reward) const
{
    // minimum is 1% to avoid a zero amount and maximum is 50%
    uint16_t percent = (m_governancePercent < 1) ? 1 : (m_governancePercent > 50) ? 50 : m_governancePercent;
    return (uint64_t)(base_reward * (percent * 0.01));
}

bool Currency::validate_government_fee(const Transaction &baseTx) const
{
    AccountKeys governanceKeys;
    getGovernanceAddressAndKey(governanceKeys);

    Crypto::PublicKey txPublicKey = getTransactionPublicKeyFromExtra(baseTx.extra);

    Crypto::KeyDerivation derivation;
    if (!Crypto::generate_key_derivation( txPublicKey,
                                  governanceKeys.viewSecretKey,
                                  derivation)) {
        return false;
    }

    uint64_t minerReward = 0;
    uint64_t governmentFee = 0;
    for (size_t idx = 0; idx < baseTx.outputs.size(); ++idx) {
        minerReward += baseTx.outputs[idx].amount;
        if (baseTx.outputs[idx].target.type() != typeid(CryptoNote::KeyOutput))
            continue;
        Crypto::PublicKey outEphemeralKey;
        Crypto::derive_public_key(derivation, idx,
            governanceKeys.address.spendPublicKey, outEphemeralKey);
        if (outEphemeralKey == boost::get<KeyOutput>(baseTx.outputs[idx].target).key)
            governmentFee += baseTx.outputs[idx].amount;
    }

    return (governmentFee == getGovernanceReward(minerReward));
}

bool Currency::getGovernanceAddressAndKey(AccountKeys& governanceKeys) const
{
    std::string address       = GOVERNANCE_WALLET_ADDRESS;
    std::string viewSecretkey = GOVERNANCE_VIEW_SECRET_KEY;

    AccountPublicAddress governanceAddress = boost::value_initialized<AccountPublicAddress>();
    if (!parseAccountAddressString(address, governanceAddress)) {
        logger(Logging::ERROR)
            << "Failed to parse governance wallet address ("
            << address
            << "), "
            << "Check /lib/Global/CryptoNoteConfig.h";
        return false;
    }

    Crypto::SecretKey governanceViewSecretKey;
    if (!Common::podFromHex(viewSecretkey, governanceViewSecretKey)) {
        logger(Logging::ERROR)
            << "Failed to parse governance view secret key"
            << "Check /lib/Global/CryptoNoteConfig.h";
        return false;
    }

    governanceKeys.address = governanceAddress;
    governanceKeys.viewSecretKey = governanceViewSecretKey;

    return true;
}

bool Currency::constructMinerTx(
    uint8_t blockMajorVersion,
    uint32_t height,
    size_t medianSize,
    uint64_t alreadyGeneratedCoins,
    size_t currentBlockSize,
    uint64_t fee,
    const AccountPublicAddress &minerAddress,
    Transaction &tx,
    const BinaryArray &extraNonce /* = BinaryArray()*/,
    size_t maxOuts /* = 1*/,
    uint64_t blockTarget /* = 0xffffffffffffffff*/) const
{
    if (blockTarget == 0xffffffffffffffff)
        blockTarget = difficultyTarget();
    tx.inputs.clear();
    tx.outputs.clear();
    tx.extra.clear();

    KeyPair txkey = generateKeyPair();
    addTransactionPublicKeyToExtra(tx.extra, txkey.publicKey);
    if (!extraNonce.empty()) {
        if (!addExtraNonceToTransactionExtra(tx.extra, extraNonce)) {
            return false;
        }
    }

    BaseInput in;
    in.blockIndex = height;

    uint64_t governanceReward = 0;
    uint64_t totalRewardWGR = 0; //totalRewardWithoutGovernanceReward

    uint64_t blockReward;
    int64_t emissionChange;

    if (!getBlockReward(
            blockMajorVersion,
            medianSize,
            currentBlockSize,
            alreadyGeneratedCoins,
            fee,
            blockReward,
            emissionChange,
            height,
            blockTarget)
        ) {
        logger(INFO) << "Block is too big";
        return false;
    }

    totalRewardWGR = blockReward;

    // If Governance Fee blockReward is decreased by GOVERNANCE_PERCENT_FEE
    bool enableGovernance = isGovernanceEnabled(height);

    if (enableGovernance) {
        governanceReward = getGovernanceReward(blockReward);

        if (alreadyGeneratedCoins != 0) {
            blockReward -= governanceReward;
            totalRewardWGR  = blockReward + governanceReward;
        }
    }

    std::vector<uint64_t> outAmounts;
    decompose_amount_into_digits(
        blockReward,
        UINT64_C(0),
        [&outAmounts](uint64_t a_chunk) { outAmounts.push_back(a_chunk); },
        [&outAmounts](uint64_t a_dust) { outAmounts.push_back(a_dust); }
    );

    if (maxOuts < 1) {
        logger(ERROR, BRIGHT_RED) << "max_out must be non-zero";
        return false;
    }
    while (maxOuts < outAmounts.size()) {
        outAmounts[outAmounts.size() - 2] += outAmounts.back();
        outAmounts.resize(outAmounts.size() - 1);
    }

    uint64_t summaryAmounts = 0;
    for (size_t no = 0; no < outAmounts.size(); no++) {
        Crypto::KeyDerivation derivation = boost::value_initialized<Crypto::KeyDerivation>();
        Crypto::PublicKey outEphemeralPubKey = boost::value_initialized<Crypto::PublicKey>();

        bool r = Crypto::generate_key_derivation(
            minerAddress.viewPublicKey,
            txkey.secretKey,
            derivation
        );

        if (!(r)) {
            logger(ERROR, BRIGHT_RED)
                << "while creating outs: failed to generate_key_derivation("
                << minerAddress.viewPublicKey
                << ", "
                << txkey.secretKey
                << ")";
            return false;
        }

        r = Crypto::derive_public_key(derivation,no,minerAddress.spendPublicKey,outEphemeralPubKey);

        if (!(r)) {
            logger(ERROR, BRIGHT_RED)
                << "while creating outs: failed to derive_public_key("
                << derivation << ", "
                << no << ", "
                << minerAddress.spendPublicKey
                << ")";
            return false;
        }

        KeyOutput tk;
        tk.key = outEphemeralPubKey;

        TransactionOutput out;
        summaryAmounts += out.amount = outAmounts[no];
        out.target = tk;
        tx.outputs.push_back(out);
    }

    if (enableGovernance) {
        AccountKeys governanceKeys;
        getGovernanceAddressAndKey(governanceKeys);

        Crypto::KeyDerivation derivation = boost::value_initialized<Crypto::KeyDerivation>();
        Crypto::PublicKey outEphemeralPubKey = boost::value_initialized<Crypto::PublicKey>();

        bool r = Crypto::generate_key_derivation(governanceKeys.address.viewPublicKey, txkey.secretKey, derivation);
        if (!(r)) {
            logger(ERROR, BRIGHT_RED)
                << "while creating outs: failed to generate_key_derivation("
                << governanceKeys.address.viewPublicKey << ", " << txkey.secretKey << ")";
            return false;
        }
        size_t pos = tx.outputs.size();
        r = Crypto::derive_public_key(derivation, pos++, governanceKeys.address.spendPublicKey, outEphemeralPubKey);
        if (!(r)) {
            logger(ERROR, BRIGHT_RED)
                << "while creating outs: failed to derive_public_key("
                << derivation << ", " << 0 << ", "
                << governanceKeys.address.spendPublicKey << ")";
            return false;
        }
        KeyOutput tk;
        tk.key = outEphemeralPubKey;

        TransactionOutput out;
        summaryAmounts += out.amount = governanceReward;
        out.target = tk;
        tx.outputs.push_back(out);
    }

    if (summaryAmounts != totalRewardWGR) {
        logger(ERROR, BRIGHT_RED)
            << "Failed to construct miner tx, summaryAmounts = "
            << summaryAmounts
            << " not equal blockReward = "
            << blockReward;
        return false;
    }

    tx.version = CURRENT_TRANSACTION_VERSION;
    //lock
    tx.unlockTime = height + m_minedMoneyUnlockWindow;
    tx.inputs.push_back(in);

    return true;
}

bool Currency::isFusionTransaction(
    const std::vector<uint64_t> &inputsAmounts,
    const std::vector<uint64_t> &outputsAmounts,
    size_t size,
    uint32_t height) const
{
    // TODO: Simplify if (...) content.
    if (size > fusionTxMaxSize()) {
        logger(ERROR) << "Fusion transaction verification failed: size exceeded max allowed size.";
        return false;
    }

    if (inputsAmounts.size() < fusionTxMinInputCount()) {
        logger(ERROR)
            << "Fusion transaction verification failed: "
            << "inputs count is less than minimum.";
        return false;
    }

    if (inputsAmounts.size() < outputsAmounts.size() * fusionTxMinInOutCountRatio()) {
        logger(ERROR)
            << "Fusion transaction verification failed: "
            << "inputs to outputs count ratio is less than minimum.";
        return false;
    }

    uint64_t inputAmount = 0;
    for (auto amount : inputsAmounts) {
        inputAmount += amount;
    }

    std::vector<uint64_t> expectedOutputsAmounts;
    expectedOutputsAmounts.reserve(outputsAmounts.size());
    decomposeAmount(
        inputAmount,
        UINT64_C(0), expectedOutputsAmounts);
    std::sort(expectedOutputsAmounts.begin(), expectedOutputsAmounts.end());

    bool decompose = expectedOutputsAmounts == outputsAmounts;
    if (!decompose) {
        logger(ERROR)
            << "Fusion transaction verification failed: "
            << "decomposed output amounts do not match expected.";
        return false;
    }

    return true;
}

bool Currency::isFusionTransaction(const Transaction &transaction,size_t size,uint32_t height) const
{
    assert(getObjectBinarySize(transaction) == size);

    std::vector<uint64_t> outputsAmounts;
    outputsAmounts.reserve(transaction.outputs.size());
    for (const TransactionOutput &output : transaction.outputs) {
        outputsAmounts.push_back(output.amount);
    }

    return isFusionTransaction(getInputsAmounts(transaction), outputsAmounts, size, height);
}

bool Currency::isFusionTransaction(const Transaction &transaction, uint32_t height) const
{
    return isFusionTransaction(transaction, getObjectBinarySize(transaction), height);
}

bool Currency::isAmountApplicableInFusionTransactionInput(
    uint64_t amount,
    uint64_t threshold,
    uint32_t height) const
{
    uint8_t ignore;
    return isAmountApplicableInFusionTransactionInput(amount, threshold, ignore, height);
}

bool Currency::isAmountApplicableInFusionTransactionInput(
    uint64_t amount,
    uint64_t threshold,
    uint8_t &amountPowerOfTen,
    uint32_t height) const
{
    if (amount >= threshold) {
        return false;
    }

    auto it = std::lower_bound(
        Constants::PRETTY_AMOUNTS.begin(),
        Constants::PRETTY_AMOUNTS.end(),
        amount
    );
    if (it == Constants::PRETTY_AMOUNTS.end() || amount != *it) {
        return false;
    }

    amountPowerOfTen = static_cast<uint8_t>(std::distance(Constants::PRETTY_AMOUNTS.begin(), it)/9);

    return true;
}

std::string Currency::accountAddressAsString(const AccountBase &account) const
{
    return getAccountAddressAsStr(m_publicAddressBase58Prefix, account.getAccountKeys().address);
}

std::string Currency::accountAddressAsString(const AccountPublicAddress &accountPublicAddress) const
{
    return getAccountAddressAsStr(m_publicAddressBase58Prefix, accountPublicAddress);
}

bool Currency::parseAccountAddressString(const std::string &str, AccountPublicAddress &addr) const
{
    uint64_t prefix;
    if (!CryptoNote::parseAccountAddressString(prefix, addr, str)) {
        return false;
    }

    if (prefix != m_publicAddressBase58Prefix) {
        logger(DEBUGGING)
        << "Wrong address prefix: " << prefix
        << ", expected " << m_publicAddressBase58Prefix;
        return false;
    }

    return true;
}

std::string Currency::formatAmount(uint64_t amount) const
{
    std::string s = std::to_string(amount);
    if (s.size() < m_numberOfDecimalPlaces + 1) {
        s.insert(0, m_numberOfDecimalPlaces + 1 - s.size(), '0');
    }
    s.insert(s.size() - m_numberOfDecimalPlaces, ".");

    return s;
}

std::string Currency::formatAmount(int64_t amount) const
{
    std::string s = formatAmount(static_cast<uint64_t>(std::abs(amount)));

    if (amount < 0) {
        s.insert(0, "-");
    }

    return s;
}

bool Currency::parseAmount(const std::string &str, uint64_t &amount) const
{
    std::string strAmount = str;
    boost::algorithm::trim(strAmount);

    size_t pointIndex = strAmount.find_first_of('.');
    size_t fractionSize;
    if (std::string::npos != pointIndex) {
        fractionSize = strAmount.size() - pointIndex - 1;
        while (m_numberOfDecimalPlaces < fractionSize && '0' == strAmount.back()) {
            strAmount.erase(strAmount.size() - 1, 1);
            --fractionSize;
        }
        if (m_numberOfDecimalPlaces < fractionSize) {
            return false;
        }
        strAmount.erase(pointIndex, 1);
    } else {
        fractionSize = 0;
    }

    if (strAmount.empty()) {
        return false;
    }

    if (!std::all_of(strAmount.begin(), strAmount.end(), ::isdigit)) {
        return false;
    }

    if (fractionSize < m_numberOfDecimalPlaces) {
        strAmount.append(m_numberOfDecimalPlaces - fractionSize, '0');
    }

    return Common::fromString(strAmount, amount);
}

// Copyright (c) 2017-2018 Zawy
// http://zawy1.blogspot.com/2017/12/using-difficulty-to-get-constant-value.html
// Moore's law application by Sergey Kozlov
uint64_t Currency::getMinimalFee(
    uint64_t dailyDifficulty,
    uint64_t reward,
    uint64_t avgHistoricalDifficulty,
    uint64_t medianHistoricalReward,
    uint32_t height) const
{
    return CryptoNote::parameters::MINIMUM_FEE;
}

uint64_t Currency::roundUpMinFee(uint64_t minimalFee, int digits) const
{
    uint64_t ret(0);
    std::string minFeeString = formatAmount(minimalFee);
    double minFee = boost::lexical_cast<double>(minFeeString);
    double scale = pow(10., floor(log10(fabs(minFee))) + (1 - digits));
    double roundedFee = ceil(minFee / scale) * scale;
    std::stringstream ss;
    ss << std::fixed << std::setprecision(12) << roundedFee;
    std::string roundedFeeString = ss.str();
    parseAmount(roundedFeeString, ret);

    return ret;
}

difficulty_type Currency::nextDifficulty(
    uint8_t blockMajorVersion,
    std::vector<uint64_t> timestamps,
    std::vector<difficulty_type> cumulativeDifficulties,
    uint32_t height) const
{
    if(isTestnet()){
        return CryptoNote::parameters::DEFAULT_DIFFICULTY;
    }

    if (timestamps.empty()) {
        return CryptoNote::parameters::DEFAULT_DIFFICULTY;
    }
    // Dynamic difficulty calculation window
    uint32_t diffWindow = timestamps.size() - 1;

    difficulty_type nextDiffV6 = CryptoNote::parameters::DEFAULT_DIFFICULTY;
    difficulty_type min_difficulty = CryptoNote::parameters::DEFAULT_DIFFICULTY;
    // Condition #1 When starting a chain or a working testnet requiring
    // block sample gathering until enough blocks are available (Kick-off Scenario)
    // We shall set a baseline for the hashrate that is specific to mining
    // algo and I propose doing so in the config file.
    // *** During this initial sampling period, a best guess is the best strategy.
    // Consider this as a service or trial period.
    // With EPoW reward algo in place, we don't really need to worry about attackers
    // or large miners taking advanatage of our system.
    if (height < CryptoNote::parameters::UPGRADE_HEIGHT_V1 + diffWindow) {
        return nextDiffV6;
    }

    // check all values in input vectors greater than previous value to calc adjacent differences later
    if (std::adjacent_find(timestamps.begin(), timestamps.end(), std::greater<uint64_t>()) != timestamps.end()) {
        logger (ERROR) << "Invalid timestamps for difficulty calculation";
        return nextDiffV6;
    }
    if (std::adjacent_find(cumulativeDifficulties.begin(), cumulativeDifficulties.end(), std::greater_equal<difficulty_type>()) != cumulativeDifficulties.end()) {
        logger (ERROR) << "Invalid cumulativeDifficulties for difficulty calculation";
        return nextDiffV6;
    }

    uint64_t difficulty_target = CryptoNote::parameters::DIFFICULTY_TARGET;
    uint64_t window_target = difficulty_target * diffWindow;
    uint64_t window_time = timestamps.back() - timestamps.front(); // now subtraction is safe,  we know it is not negative

    // get solvetimes from timestamps
    std::vector<uint64_t> solveTimes;
    solveTimes.resize(timestamps.size());
    std::adjacent_difference(timestamps.begin(), timestamps.end(), solveTimes.begin()); // we check it before and all diffs are positive
    solveTimes.erase(solveTimes.begin());

    // get difficulties from the cumulative difficulties
    std::vector<difficulty_type> difficulties;
    difficulties.resize(cumulativeDifficulties.size());
    std::adjacent_difference(cumulativeDifficulties.begin(),
                             cumulativeDifficulties.end(),
                             difficulties.begin()); // we check it before and all diffs are positive
    difficulties.erase(difficulties.begin());
    difficulty_type prev_difficulty = difficulties.back();

    // calc stat values to detect outliers
    double avg_solvetime = Common::meanValue(solveTimes);
    double stddev_solvetime = Common::stddevValue(solveTimes);
    double solvetime_lowborder = 1.0;
    if(avg_solvetime > stddev_solvetime)
        solvetime_lowborder = avg_solvetime - stddev_solvetime;
    double solvetime_highborder = avg_solvetime + stddev_solvetime;
    size_t valid_solvetime_number = 0;
    uint64_t valid_solvetime_sum = 0;
    size_t invalid_solvetime_number = 0;
    uint64_t invalid_solvetime_sum = 0;
    std::for_each(solveTimes.begin(), solveTimes.end(),
                  [solvetime_lowborder, solvetime_highborder,
                          &valid_solvetime_number, &valid_solvetime_sum,
                          &invalid_solvetime_number, &invalid_solvetime_sum] (uint64_t st) {
                      if ((st >= solvetime_lowborder) && (st <= solvetime_highborder)) {
                          valid_solvetime_number++;
                          valid_solvetime_sum += st;
                      } else {
                          invalid_solvetime_number++;
                          invalid_solvetime_sum += st;
                      }
                  });

    // if there is no "invalid" solvetimes we can use previous difficulty value
    if (invalid_solvetime_number == 0) {
        return std::max(prev_difficulty, min_difficulty);
    }

    // process data with "invalid" solvetimes
    double valid_solvetime_mean = double(valid_solvetime_sum) / valid_solvetime_number;
    double invalid_solvetime_mean = double(invalid_solvetime_sum) / invalid_solvetime_number;

    if ( (window_time >= window_target * 0.97) &&
         (window_time <= window_target * 1.03) ) {
        if (valid_solvetime_mean >= invalid_solvetime_mean) {
            double coef = double(difficulty_target) / double(valid_solvetime_mean);
            if (valid_solvetime_mean < difficulty_target) {
                nextDiffV6 = prev_difficulty * std::min(1.01, coef) + 0.5;
            } else {
                nextDiffV6 = prev_difficulty * std::max(0.99, coef) + 0.5;
            }
        } else {
            double coef = double(difficulty_target) / double(invalid_solvetime_mean);
            if (invalid_solvetime_mean < difficulty_target) {
                nextDiffV6 = prev_difficulty * std::min(1.01, coef) + 0.5;
            } else {
                nextDiffV6 = prev_difficulty * std::max(0.99, coef) + 0.5;
            }
        }
    } else if (window_time < window_target * 0.97) {
        nextDiffV6 = prev_difficulty * 1.02 + 0.5;
    } else {
        nextDiffV6 = prev_difficulty * 0.98 + 0.5;
    }

    return std::max(nextDiffV6, min_difficulty);
}

difficulty_type Currency::getClifDifficulty(uint32_t height,
                                          uint8_t blockMajorVersion,
                                          difficulty_type last_difficulty,
                                          uint64_t last_timestamp,
                                          uint64_t currentSolveTime,
                                          lazy_stat_callback_type &lazy_stat_cb) const
{
    logger (INFO) << "CLIF difficulty inputs: height " << height <<
                     ", block version " << (int)blockMajorVersion <<
                     ", last difficulty " << last_difficulty <<
                     ", current solve time " << currentSolveTime;

    difficulty_type new_diff = last_difficulty;

    if (new_diff > CryptoNote::parameters::DEFAULT_DIFFICULTY) {
        uint64_t correction_interval = currentSolveTime -
                CryptoNote::parameters::CRYPTONOTE_CLIF_THRESHOLD;
        //below equation shall return quotient of the division.
        int decrease_counter = ((int)correction_interval / (int)CryptoNote::parameters::DIFFICULTY_TARGET) + 1;
        int round_counter = 1;

        new_diff = new_diff / 2;
        logger (INFO) << "CLIF decreased difficulty " << round_counter <<
            " times, intermediate difficulty is " << new_diff;
        difficulty_type mean_diff = lazy_stat_cb(IMinerHandler::stat_period::hour, last_timestamp);
        logger (INFO) << "Last hour average difficulty is " << mean_diff;
        if (mean_diff > 0)
            new_diff = std::min(mean_diff, new_diff);
        mean_diff = lazy_stat_cb(IMinerHandler::stat_period::day, last_timestamp);
        logger (INFO) << "Last day average difficulty is " << mean_diff;
        if (mean_diff > 0)
            new_diff = std::min(mean_diff, new_diff);
        mean_diff = lazy_stat_cb(IMinerHandler::stat_period::week, last_timestamp);
        logger (INFO) << "Last week average difficulty is " << mean_diff;
        if (mean_diff > 0)
            new_diff = std::min(mean_diff, new_diff);
        mean_diff = lazy_stat_cb(IMinerHandler::stat_period::month, last_timestamp);
        logger (INFO) << "Last month average difficulty is " << mean_diff;
        if (mean_diff > 0)
            new_diff = std::min(mean_diff, new_diff);
        mean_diff = lazy_stat_cb(IMinerHandler::stat_period::halfyear, last_timestamp);
        logger (INFO) << "Last halfyear average difficulty is " << mean_diff;
        if (mean_diff > 0)
            new_diff = std::min(mean_diff, new_diff);
        mean_diff = lazy_stat_cb(IMinerHandler::stat_period::year, last_timestamp);
        logger (INFO) << "Last year average difficulty is " << mean_diff;
        if (mean_diff > 0)
            new_diff = std::min(mean_diff, new_diff);

        if (decrease_counter > 1) {
            while (round_counter < decrease_counter) {
                new_diff = new_diff / 2;
                round_counter++;
                if (new_diff <= CryptoNote::parameters::DEFAULT_DIFFICULTY)
                    break;
            }
            logger (INFO) << "CLIF decreased difficulty " << round_counter <<
                " times, intermediate difficulty is " << new_diff;
        }

        new_diff = std::max(new_diff, difficulty_type(CryptoNote::parameters::DEFAULT_DIFFICULTY));
    }

    logger (INFO) << "CLIF difficulty result: " << new_diff;
    return new_diff;
}

bool Currency::checkProofOfWorkV1(
    Crypto::cn_context &context,
    const Block &block,
    difficulty_type currentDiffic,
    Crypto::Hash &proofOfWork) const
{
    if (BLOCK_MAJOR_VERSION_2 == block.majorVersion) {
        return false;
    }

    if (!get_block_longhash(context, block, proofOfWork)) {
        return false;
    }

    return check_hash(proofOfWork, currentDiffic);
}

bool Currency::checkProofOfWorkV2(
    Crypto::cn_context &context,
    const Block &block,
    difficulty_type currentDiffic,
    Crypto::Hash &proofOfWork) const
{
    if (block.majorVersion < BLOCK_MAJOR_VERSION_2) {
        return false;
    }

    if (!get_block_longhash(context, block, proofOfWork)) {
        return false;
    }

    if (!check_hash(proofOfWork, currentDiffic)) {
        return false;
    }

    TransactionExtraMergeMiningTag mmTag;
    if (!getMergeMiningTagFromExtra(block.parentBlock.baseTransaction.extra, mmTag)) {
        logger(ERROR)
            << "merge mining tag wasn't found in extra of the parent block miner transaction";
        return false;
    }

    if (8 * sizeof(m_genesisBlockHash) < block.parentBlock.blockchainBranch.size()) {
        return false;
    }

    Crypto::Hash auxBlockHeaderHash;
    if (!get_aux_block_header_hash(block, auxBlockHeaderHash)) {
        return false;
    }

    Crypto::Hash auxBlocksMerkleRoot;
    Crypto::tree_hash_from_branch(
        block.parentBlock.blockchainBranch.data(),
        block.parentBlock.blockchainBranch.size(),
        auxBlockHeaderHash,
        &m_genesisBlockHash,
        auxBlocksMerkleRoot
    );

    if (auxBlocksMerkleRoot != mmTag.merkleRoot) {
        logger(ERROR, BRIGHT_YELLOW) << "Aux block hash wasn't found in merkle tree";
        return false;
    }

    return true;
}

bool Currency::checkProofOfWork(
    Crypto::cn_context &context,
    const Block &block,
    difficulty_type currentDiffic,
    Crypto::Hash &proofOfWork) const
{
    switch (block.majorVersion) {
    case BLOCK_MAJOR_VERSION_1:
    case BLOCK_MAJOR_VERSION_3:
    case BLOCK_MAJOR_VERSION_4:
    case BLOCK_MAJOR_VERSION_5:
    case BLOCK_MAJOR_VERSION_6:
        return checkProofOfWorkV1(context, block, currentDiffic, proofOfWork);
    case BLOCK_MAJOR_VERSION_2:
        return checkProofOfWorkV2(context, block, currentDiffic, proofOfWork);
    }

    logger(ERROR, BRIGHT_RED)
        << "Unknown block major version: "
        << block.majorVersion
        << "."
        << block.minorVersion;

    return false;
}

size_t Currency::getApproximateMaximumInputCount(
    size_t transactionSize,
    size_t outputCount,
    size_t mixinCount) const
{
    const size_t KEY_IMAGE_SIZE = sizeof(Crypto::KeyImage);
    const size_t OUTPUT_KEY_SIZE = sizeof(decltype(KeyOutput::key));
    const size_t AMOUNT_SIZE = sizeof(uint64_t) + 2; // varint
    const size_t GLOBAL_INDEXES_VECTOR_SIZE_SIZE = sizeof(uint8_t); // varint
    const size_t GLOBAL_INDEXES_INITIAL_VALUE_SIZE = sizeof(uint32_t); // varint
    const size_t GLOBAL_INDEXES_DIFFERENCE_SIZE = sizeof(uint32_t); // varint
    const size_t SIGNATURE_SIZE = sizeof(Crypto::Signature);
    const size_t EXTRA_TAG_SIZE = sizeof(uint8_t);
    const size_t INPUT_TAG_SIZE = sizeof(uint8_t);
    const size_t OUTPUT_TAG_SIZE = sizeof(uint8_t);
    const size_t PUBLIC_KEY_SIZE = sizeof(Crypto::PublicKey);
    const size_t TRANSACTION_VERSION_SIZE = sizeof(uint8_t);
    const size_t TRANSACTION_UNLOCK_TIME_SIZE = sizeof(uint64_t);

    const size_t outputsSize = outputCount * (OUTPUT_TAG_SIZE + OUTPUT_KEY_SIZE + AMOUNT_SIZE);
    const size_t headerSize = TRANSACTION_VERSION_SIZE
                              + TRANSACTION_UNLOCK_TIME_SIZE
                              + EXTRA_TAG_SIZE
                              + PUBLIC_KEY_SIZE;
    const size_t inputSize = INPUT_TAG_SIZE
                             + AMOUNT_SIZE
                             + KEY_IMAGE_SIZE
                             + SIGNATURE_SIZE
                             + GLOBAL_INDEXES_VECTOR_SIZE_SIZE
                             + GLOBAL_INDEXES_INITIAL_VALUE_SIZE
                             + mixinCount * (GLOBAL_INDEXES_DIFFERENCE_SIZE + SIGNATURE_SIZE);

    return (transactionSize - headerSize - outputsSize) / inputSize;
}

CurrencyBuilder::CurrencyBuilder(Logging::ILogger &log)
    : m_currency(log)
{
    maxBlockNumber(parameters::CRYPTONOTE_MAX_BLOCK_NUMBER);
    maxBlockBlobSize(parameters::CRYPTONOTE_MAX_BLOCK_BLOB_SIZE);
    maxTxSize(parameters::CRYPTONOTE_MAX_TX_SIZE);
    publicAddressBase58Prefix(parameters::CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX);
    minedMoneyUnlockWindow(parameters::CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW);
    transactionSpendableAge(parameters::CRYPTONOTE_TX_SPENDABLE_AGE);
    safeTransactionSpendableAge(parameters::CRYPTONOTE_SAFE_TX_SPENDABLE_AGE);
    expectedNumberOfBlocksPerDay(parameters::EXPECTED_NUMBER_OF_BLOCKS_PER_DAY);

    timestampCheckWindow(parameters::BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW);
    timestampCheckWindow_v1(parameters::BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW_V2);
    blockFutureTimeLimit(parameters::CRYPTONOTE_BLOCK_FUTURE_TIME_LIMIT);
    blockFutureTimeLimit_v1(parameters::CRYPTONOTE_BLOCK_FUTURE_TIME_LIMIT_V2);

    moneySupply(parameters::MONEY_SUPPLY);
    emissionSpeedFactor(parameters::EMISSION_SPEED_FACTOR);
    cryptonoteCoinVersion(parameters::CRYPTONOTE_COIN_VERSION);

    rewardBlocksWindow(parameters::CRYPTONOTE_REWARD_BLOCKS_WINDOW);
    blockGrantedFullRewardZone(parameters::CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE);
    minerTxBlobReservedSize(parameters::CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE);
    maxTransactionSizeLimit(parameters::MAX_TRANSACTION_SIZE_LIMIT);

    governancePercent(parameters::GOVERNANCE_PERCENT_FEE);
    governanceHeightStart(parameters::GOVERNANCE_HEIGHT_START);
    governanceHeightEnd(parameters::GOVERNANCE_HEIGHT_END);

    minMixin(parameters::MIN_TX_MIXIN_SIZE);
    maxMixin(parameters::MAX_TX_MIXIN_SIZE);

    numberOfDecimalPlaces(parameters::CRYPTONOTE_DISPLAY_DECIMAL_POINT);

    minimumFee(parameters::MINIMUM_FEE);
    defaultDustThreshold(parameters::DEFAULT_DUST_THRESHOLD);

    difficultyTarget(parameters::DIFFICULTY_TARGET);
    difficultyWindow(parameters::DIFFICULTY_WINDOW);
    difficultyLag(parameters::DIFFICULTY_LAG);
    difficultyCut(parameters::DIFFICULTY_CUT);

    maxBlockSizeInitial(parameters::MAX_BLOCK_SIZE_INITIAL);
    maxBlockSizeGrowthSpeedNumerator(parameters::MAX_BLOCK_SIZE_GROWTH_SPEED_NUMERATOR);
    maxBlockSizeGrowthSpeedDenominator(parameters::MAX_BLOCK_SIZE_GROWTH_SPEED_DENOMINATOR);

    lockedTxAllowedDeltaSeconds(parameters::CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_SECONDS);
    lockedTxAllowedDeltaBlocks(parameters::CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_BLOCKS);

    mempoolTxLiveTime(parameters::CRYPTONOTE_MEMPOOL_TX_LIVETIME);
    mempoolTxFromAltBlockLiveTime(parameters::CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME);
    numberOfPeriodsToForgetTxDeletedFromPool(
        parameters::CRYPTONOTE_NUMBER_OF_PERIODS_TO_FORGET_TX_DELETED_FROM_POOL
    );

    fusionTxMaxSize(parameters::FUSION_TX_MAX_SIZE);
    fusionTxMinInputCount(parameters::FUSION_TX_MIN_INPUT_COUNT);
    fusionTxMinInOutCountRatio(parameters::FUSION_TX_MIN_IN_OUT_COUNT_RATIO);

    upgradeHeightV2(parameters::UPGRADE_HEIGHT_V2);
    upgradeHeightV3(parameters::UPGRADE_HEIGHT_V3);
    upgradeHeightV4(parameters::UPGRADE_HEIGHT_V4);
    upgradeHeightV5(parameters::UPGRADE_HEIGHT_V5);
    upgradeHeightV6(parameters::UPGRADE_HEIGHT_V6);
    upgradeVotingThreshold(parameters::UPGRADE_VOTING_THRESHOLD);
    upgradeVotingWindow(parameters::UPGRADE_VOTING_WINDOW);
    upgradeWindow(parameters::UPGRADE_WINDOW);

    blocksFileName(parameters::CRYPTONOTE_BLOCKS_FILENAME);
    blocksCacheFileName(parameters::CRYPTONOTE_BLOCKSCACHE_FILENAME);
    blockIndexesFileName(parameters::CRYPTONOTE_BLOCKINDEXES_FILENAME);
    txPoolFileName(parameters::CRYPTONOTE_POOLDATA_FILENAME);
    blockchainIndicesFileName(parameters::CRYPTONOTE_BLOCKCHAIN_INDICES_FILENAME);

    testnet(false);
    fix_difficulty(0);
}

Transaction CurrencyBuilder::generateGenesisTransaction()
{
    CryptoNote::Transaction tx;
    auto ac = boost::value_initialized<CryptoNote::AccountPublicAddress>();

    m_currency.constructMinerTx(1, 0, 0, 0, 0, 0, ac, tx); // zero fee in genesis

    return tx;
}

CurrencyBuilder& CurrencyBuilder::emissionSpeedFactor(unsigned int val)
{
    if (val <= 0 || val > 8 * sizeof(uint64_t)) {
        throw std::invalid_argument("val at emissionSpeedFactor()");
    }

    m_currency.m_emissionSpeedFactor = val;

    return *this;
}

CurrencyBuilder &CurrencyBuilder::numberOfDecimalPlaces(size_t val)
{
    m_currency.m_numberOfDecimalPlaces = val;
    m_currency.m_coin = 1;
    for (size_t i = 0; i < m_currency.m_numberOfDecimalPlaces; ++i) {
        m_currency.m_coin *= 10;
    }

    return *this;
}

CurrencyBuilder &CurrencyBuilder::difficultyWindow(size_t val)
{
    if (val < 2) {
        throw std::invalid_argument("val at difficultyWindow()");
    }

    m_currency.m_difficultyWindow = val;

    return *this;
}

CurrencyBuilder& CurrencyBuilder::upgradeVotingThreshold(unsigned int val)
{
    if (val <= 0 || val > 100) {
        throw std::invalid_argument("val at upgradeVotingThreshold()");
    }

    m_currency.m_upgradeVotingThreshold = val;

    return *this;
}

CurrencyBuilder &CurrencyBuilder::upgradeWindow(size_t val)
{
    if (val <= 0) {
        throw std::invalid_argument("val at upgradeWindow()");
    }

    m_currency.m_upgradeWindow = static_cast<uint32_t>(val);

    return *this;
}

} // namespace CryptoNote


/*
Copyright (c) 2019 Naomasa Matsubayashi (aka. Fadis)

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

#include <memory>
#include <tuple>
#include <vector>
#include <algorithm>
#include <iterator>
#include <sys/mman.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <boost/scope_exit.hpp>
#include <boost/spirit/include/qi.hpp>
#include <liblnn/buffer.h>
#include <liblnn/load_mnist.h>
#include <liblnn/exceptions.h>
namespace liblnn {
  mnist::mnist(
    const std::string &image_filename,
    const std::string &label_filename
  ) : current_image( 0 ), gen( std::random_device()() ) {
    {
      int fd = open( image_filename.c_str(), O_RDONLY );
      if( fd == -1 ) throw unable_to_load_file();
      size_t file_size = 0;
      {
        struct stat stat_;
        if( fstat( fd, &stat_ ) == -1 ) {
          close( fd );
          throw unable_to_load_file();
        }
        file_size = stat_.st_size;
      }
      void *addr = mmap( nullptr, file_size, PROT_READ, MAP_PRIVATE, fd, 0 );
      if( !addr ) {
        close( fd );
        throw unable_to_load_file();
      }
      images.reset(
        reinterpret_cast< unsigned char* >( addr ),
        [file_size,fd]( unsigned char *p ) {
          munmap( reinterpret_cast< void* >( p ), file_size );
          close( fd );
        }
      );
      {
        auto hbegin = images.get();
        const auto hend = std::next( hbegin, 16 );
        uint32_t magic;
        boost::spirit::qi::parse( hbegin, hend, boost::spirit::qi::big_dword, magic );
        if( magic != 0x803 ) throw corrupted_file();
        boost::spirit::qi::parse( hbegin, hend, boost::spirit::qi::big_dword, image_count );
        boost::spirit::qi::parse( hbegin, hend, boost::spirit::qi::big_dword, image_width );
        boost::spirit::qi::parse( hbegin, hend, boost::spirit::qi::big_dword, image_height );
        image_head = hend;
        if( file_size < ( 16 + image_count * image_width * image_height ) ) throw corrupted_file();
      }
    }
    {
      int fd = open( label_filename.c_str(), O_RDONLY );
      if( fd == -1 ) throw unable_to_load_file();
      size_t file_size = 0;
      {
        struct stat stat_;
        if( fstat( fd, &stat_ ) == -1 ) {
          close( fd );
          throw unable_to_load_file();
        }
        file_size = stat_.st_size;
      }
      void *addr = mmap( nullptr, file_size, PROT_READ, MAP_PRIVATE, fd, 0 );
      if( !addr ) {
        close( fd );
        throw unable_to_load_file();
      }
      labels.reset(
        reinterpret_cast< unsigned char* >( addr ),
        [file_size,fd]( unsigned char *p ) {
          munmap( reinterpret_cast< void* >( p ), file_size );
          close( fd );
        }
      );
      {
        auto hbegin = labels.get();
        const auto hend = std::next( hbegin, 8 );
        uint32_t magic;
        boost::spirit::qi::parse( hbegin, hend, boost::spirit::qi::big_dword, magic );
        if( magic != 0x801 ) throw corrupted_file();
	uint32_t label_count;
        boost::spirit::qi::parse( hbegin, hend, boost::spirit::qi::big_dword, label_count );
	if( label_count != image_count ) throw invalid_data_length();
        label_head = hend;
        if( file_size < ( 8 + label_count ) ) throw corrupted_file();
	label_width = 10;
      }
    }
  }
  void mnist::operator()(
    vk::CommandBuffer&,
    const std::shared_ptr< vk::Device >&,
    const std::shared_ptr< vk::Queue >&,
    const device_props&,
    const buffer_view< float > &image_buffer,
    const buffer_view< float > &label_buffer
  ) {
    if( image_buffer.size() % ( image_width * image_height ) ) throw invalid_data_length();
    uint32_t batch_size = image_buffer.size() / ( image_width * image_height );
    if( label_buffer.size() != batch_size * 10 ) throw invalid_data_length();
    std::uniform_int_distribution< uint32_t > dist( 0u, image_count - 1u );
    std::vector< uint32_t > selected_index;
    selected_index.reserve( batch_size );
    {
      auto mapped = image_buffer.map();
      auto obegin = mapped.get();
      for( uint32_t local_image_index = 0; local_image_index != batch_size; ++local_image_index ) {
        selected_index.push_back( dist( gen ) );
        uint32_t global_image_index = selected_index.back(); // ( current_image + local_image_index ) % image_count;
	auto ibegin = image_head + global_image_index * image_width * image_height;
	auto iend = image_head + ( global_image_index + 1 ) * image_width * image_height;
        std::transform( ibegin, iend, obegin, []( unsigned char v ) { return v / 255.f; } );
	obegin += image_width * image_height;
      }
    }
    {
      auto mapped = label_buffer.map();
      auto obegin = mapped.get();
      std::fill( obegin, std::next( obegin, label_buffer.size() ), 0.f );
      for( uint32_t local_label_index = 0; local_label_index != batch_size; ++local_label_index ) {
        uint32_t global_label_index = selected_index[ local_label_index ]; // ( current_image + local_label_index ) % image_count;
	auto ibegin = label_head + global_label_index;
	obegin[ int( *ibegin ) ] = 1.f;
	obegin += 10;
      }
    }
    current_image += batch_size;
    current_image %= image_count;
  }

  std::tuple< std::shared_ptr< buffer< float > >, uint32_t >
  load_mnist_images(
    const std::shared_ptr< VmaAllocator > &allocator,
    const std::string &filename
  ) {
    int fd = open( filename.c_str(), O_RDONLY );
    if( fd == -1 ) throw unable_to_load_file();
    BOOST_SCOPE_EXIT( &fd ) {
      close( fd );
    } BOOST_SCOPE_EXIT_END
    struct stat stat_;
    if( fstat( fd, &stat_ ) == -1 ) throw unable_to_load_file();
    const size_t file_size = stat_.st_size;
    void *addr = mmap( nullptr, file_size, PROT_READ, MAP_PRIVATE, fd, 0 );
    if( !addr ) throw unable_to_load_file();
    {
      BOOST_SCOPE_EXIT( &addr, &file_size ) {
        munmap( addr, file_size );
      } BOOST_SCOPE_EXIT_END
      auto hbegin = reinterpret_cast< const unsigned char* >( addr );
      const auto hend = std::next( hbegin, 16 );
      uint32_t magic;
      boost::spirit::qi::parse( hbegin, hend, boost::spirit::qi::big_dword, magic );
      if( magic != 0x803 ) throw corrupted_file();
      uint32_t data_count;
      boost::spirit::qi::parse( hbegin, hend, boost::spirit::qi::big_dword, data_count );
      uint32_t width = 0;
      boost::spirit::qi::parse( hbegin, hend, boost::spirit::qi::big_dword, width );
      if( width != 0x1c ) throw corrupted_file();
      uint32_t height = 0;
      boost::spirit::qi::parse( hbegin, hend, boost::spirit::qi::big_dword, height );
      if( height != 0x1c ) throw corrupted_file();
      auto dbegin = hend;
      size_t output_size = 32 * 32 * data_count;
      std::shared_ptr< liblnn::buffer< float > > output( new liblnn::buffer< float >( allocator, VMA_MEMORY_USAGE_CPU_TO_GPU,
        vk::BufferCreateInfo()
          .setSize( output_size * sizeof( float ) )
          .setUsage( vk::BufferUsageFlagBits::eStorageBuffer )
      ) );
      {
        auto mapped = output->map();
        auto obegin = mapped.get();
	auto oend = std::next( obegin, output_size );
	std::fill( obegin, oend, 0.f );
	for( uint32_t data_index = 0; data_index != data_count; ++data_index ) {
	  for( uint32_t y = 0; y != height; ++y ) {
	    for( uint32_t x = 0; x != width; ++x ) {
	      obegin[ data_index * 32 * 32 + ( y + 2 ) * width + x + 2 ] = dbegin[ data_index * width * height + y * width + x ] / 255.f;
	    }
	  }
	}
      }
      return std::make_tuple( output, data_count );
    }
  }
  std::tuple< std::shared_ptr< buffer< float > >, uint32_t >
  load_mnist_labels(
    const std::shared_ptr< VmaAllocator > &allocator,
    const std::string &filename
  ) {
    int fd = open( filename.c_str(), O_RDONLY );
    if( fd == -1 ) throw unable_to_load_file();
    BOOST_SCOPE_EXIT( &fd ) {
      close( fd );
    } BOOST_SCOPE_EXIT_END
    struct stat stat_;
    if( fstat( fd, &stat_ ) == -1 ) throw unable_to_load_file();
    const size_t file_size = stat_.st_size;
    void *addr = mmap( nullptr, file_size, PROT_READ, MAP_PRIVATE, fd, 0 );
    if( !addr ) throw unable_to_load_file();
    {
      BOOST_SCOPE_EXIT( &addr, &file_size ) {
        munmap( addr, file_size );
      } BOOST_SCOPE_EXIT_END
      auto hbegin = reinterpret_cast< const unsigned char* >( addr );
      const auto hend = std::next( hbegin, 16 );
      uint32_t magic;
      boost::spirit::qi::parse( hbegin, hend, boost::spirit::qi::big_dword, magic );
      if( magic != 0x801 ) throw corrupted_file();
      uint32_t data_count;
      boost::spirit::qi::parse( hbegin, hend, boost::spirit::qi::big_dword, data_count );
      auto dbegin = hend;
      size_t output_size = 10 * data_count;
      std::shared_ptr< liblnn::buffer< float > > output( new liblnn::buffer< float >( allocator, VMA_MEMORY_USAGE_CPU_TO_GPU,
        vk::BufferCreateInfo()
          .setSize( output_size * sizeof( float ) )
          .setUsage( vk::BufferUsageFlagBits::eStorageBuffer )
      ) );
      {
        auto mapped = output->map();
        auto obegin = mapped.get();
	auto oend = std::next( obegin, output_size );
	std::fill( obegin, oend, 0.f );
	for( uint32_t data_index = 0; data_index != data_count; ++data_index ) {
	  obegin[ data_index * 10 + dbegin[ data_index ] ] = 1.f;
	}
      }
      return std::make_tuple( output, data_count );
    }
  }
}


// Copyright (c) 2012-2016, NVIDIA CORPORATION. All rights reserved.
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
//  * Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
//  * Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
//  * Neither the name of NVIDIA CORPORATION nor the names of its
//    contributors may be used to endorse or promote products derived
//    from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include <dp/culling/ObjectBitSet.h>
#include <dp/culling/ResultBitSet.h>
#include <dp/culling/ManagerBitSet.h>
#include <dp/culling/GroupBitSet.h>
#include <dp/util/BitArray.h>
#include <boost/scoped_array.hpp>

#include <dp/util/FrameProfiler.h>

#include <limits>

// TODO verify alignment of vectors/matrices on linux
#if defined(DP_ARCH_X86_64) && defined(DP_OS_WINDOWS)
#define SSE
#endif

#if defined(SSE)
#include <dp/math/sse/Vecnt.h>
#include <dp/math/sse/Matmnt.h>
static bool useSSE = true;
#else
static bool useSSE = false;
#endif

#if defined(DP_ARCH_ARM_32)
#define NEON
#endif

#if defined(NEON)
#include <dp/math/neon/Vecnt.h>
#include <dp/math/neon/Matmnt.h>
static bool useNEON = true;
#else
static bool useNEON = false;
#endif

namespace dp
{
  namespace culling
  {

    ManagerBitSet::ManagerBitSet()
    {

    }
    ManagerBitSet::~ManagerBitSet()
    {

    }

    void ManagerBitSet::objectSetUserData( const ObjectSharedPtr& object, PayloadSharedPtr const& userData )
    {
      std::static_pointer_cast<ObjectBitSet>(object)->setUserData( userData );
    }

    PayloadSharedPtr const& ManagerBitSet::objectGetUserData( const ObjectSharedPtr& object )
    {
      return( std::static_pointer_cast<ObjectBitSet>(object)->getUserData() );
    }

    void ManagerBitSet::objectSetTransformIndex( const ObjectSharedPtr& object, size_t index )
    {
      ObjectBitSetSharedPtr objectImpl = std::static_pointer_cast<ObjectBitSet>(object);

      objectImpl->setTransformIndex( index );
      if ( objectImpl->getGroup() )
      {
        objectImpl->getGroup()->setOBBDirty( true );
      }
    }

    void ManagerBitSet::objectSetBoundingBox( const ObjectSharedPtr& object, const dp::math::Box3f& boundingBox )
    {
      ObjectBitSetSharedPtr objectImpl = std::static_pointer_cast<ObjectBitSet>(object);
      objectImpl->setLowerLeft( dp::math::Vec4f(boundingBox.getLower(), 1.0f ) );
      objectImpl->setExtent( dp::math::Vec4f( boundingBox.getSize(), 0.0f ) );

      if ( objectImpl->getGroup() )
      {
        objectImpl->getGroup()->setOBBDirty( true );
      }
    }

    void ManagerBitSet::groupAddObject( const GroupSharedPtr& group, const ObjectSharedPtr& object )
    {
      std::static_pointer_cast<GroupBitSet>(group)->addObject( std::static_pointer_cast<ObjectBitSet>(object) );
    }

    ObjectSharedPtr ManagerBitSet::groupGetObject( const GroupSharedPtr& group, size_t index )
    {
      return( std::static_pointer_cast<GroupBitSet>(group)->getObject( index ) );
    }

    void ManagerBitSet::groupRemoveObject( const GroupSharedPtr& group, const ObjectSharedPtr& objectHandle )
    {
      std::static_pointer_cast<GroupBitSet>(group)->removeObject( std::static_pointer_cast<ObjectBitSet>(objectHandle) );
    }

    size_t ManagerBitSet::groupGetCount( const GroupSharedPtr& group )
    {
      return( std::static_pointer_cast<GroupBitSet>(group)->getObjectCount() );
    }

    void ManagerBitSet::groupSetMatrices( const GroupSharedPtr& group, void const* matrices, size_t numberOfMatrices, size_t stride )
    {
      std::static_pointer_cast<GroupBitSet>(group)->setMatrices( matrices, numberOfMatrices, stride );
    }

    void ManagerBitSet::groupMatrixChanged( GroupSharedPtr const& group, size_t index )
    {
      std::static_pointer_cast<GroupBitSet>(group)->markMatrixDirty( index );
    }

    std::vector<ObjectSharedPtr> const & ManagerBitSet::resultGetChanged( const ResultSharedPtr& result )
    {
      return( std::static_pointer_cast<ResultBitSet>(result)->getChangedObjects() );
    }

    bool ManagerBitSet::resultObjectIsVisible( ResultSharedPtr const& result, ObjectSharedPtr const& object )
    {
      return( std::static_pointer_cast<ResultBitSet>(result)->isVisible( std::static_pointer_cast<ObjectBitSet>(object) ) );
    }

    dp::math::Box3f ManagerBitSet::getBoundingBox( const GroupSharedPtr& group ) const
    {
      dp::util::ProfileEntry p("cull::getBoundingBox");

      GroupBitSetSharedPtr groupImpl = std::static_pointer_cast<GroupBitSet>(group);
      if ( groupImpl->isBoundingBoxDirty() )
      {
        groupImpl->setBoundingBox( calculateBoundingBox( group ) );
        groupImpl->setBoundingBoxDirty( false );
      }
      return groupImpl->getBoundingBox();
    }

    dp::math::Box3f ManagerBitSet::calculateBoundingBox( const GroupSharedPtr& group ) const
    {
#if defined(SSE)
      if ( useSSE )
      {
        GroupBitSetSharedPtr groupImpl = std::static_pointer_cast<GroupBitSet>(group);

        __m128 minValue = _mm_set1_ps( std::numeric_limits<float>::signaling_NaN() );
        __m128 maxValue = _mm_set1_ps( std::numeric_limits<float>::signaling_NaN() );

        char const* basePtr = reinterpret_cast<char const*>(groupImpl->getMatrices());
        for ( size_t index = 0;index < groupImpl->getObjectCount(); ++index )
        {
          ObjectBitSetSharedPtr objectImpl = std::static_pointer_cast<ObjectBitSet>(groupImpl->getObject( index ));
          dp::math::sse::Mat44f const& modelView = *reinterpret_cast<dp::math::sse::Mat44f const*>(basePtr + objectImpl->getTransformIndex() * groupImpl->getMatricesStride());
          dp::math::Vec4f const& extent = objectImpl->getExtent();

          dp::math::sse::Vec4f vectors[8];
          vectors[0] = *reinterpret_cast<dp::math::sse::Vec4f const*>(&objectImpl->getLowerLeft()) * modelView;

          dp::math::sse::Vec4f x( extent[0] * modelView[0] );
          dp::math::sse::Vec4f y( extent[1] * modelView[1] );
          dp::math::sse::Vec4f z( extent[2] * modelView[2] );

          vectors[1] = vectors[0] + x;
          vectors[2] = vectors[0] + y;
          vectors[3] = vectors[1] + y;
          vectors[4] = vectors[0] + z;
          vectors[5] = vectors[1] + z;
          vectors[6] = vectors[2] + z;
          vectors[7] = vectors[3] + z;

          for ( unsigned int i = 0;i < 8; ++i )
          {
            minValue = _mm_min_ps( minValue, vectors[i].sse() );
            maxValue = _mm_max_ps( maxValue, vectors[i].sse() );
          }
        }

        dp::math::Vec3f minVec, maxVec;
        _MM_EXTRACT_FLOAT( minVec[0], minValue, 0);
        _MM_EXTRACT_FLOAT( minVec[1], minValue, 1);
        _MM_EXTRACT_FLOAT( minVec[2], minValue, 2);

        _MM_EXTRACT_FLOAT( maxVec[0], maxValue, 0);
        _MM_EXTRACT_FLOAT( maxVec[1], maxValue, 1);
        _MM_EXTRACT_FLOAT( maxVec[2], maxValue, 2);

        return dp::math::Box3f( minVec, maxVec );
      }
      else
#elif defined(NEON)
        if ( useNEON )
        {
          const GroupBitSetSharedPtr& groupImpl = std::static_pointer_cast<GroupBitSet>(group);

          float32x4_t minValue = vdupq_n_f32( std::numeric_limits<float>::max() );
          float32x4_t maxValue = vdupq_n_f32( -std::numeric_limits<float>::max() );

          char const* basePtr = reinterpret_cast<char const*>(groupImpl->getMatrices());
          for ( size_t index = 0;index < groupImpl->getObjectCount(); ++index )
          {
            const ObjectBitSetSharedPtr objectImpl = std::static_pointer_cast<ObjectBitSet>(groupImpl->getObject( index ));
            dp::math::neon::Mat44f const& modelView = *reinterpret_cast<dp::math::neon::Mat44f const*>(basePtr + objectImpl->getTransformIndex() * groupImpl->getMatricesStride());
            dp::math::Vec4f const& extent = objectImpl->getExtent();

            dp::math::neon::Vec4f vectors[8];
            vectors[0] = *reinterpret_cast<dp::math::neon::Vec4f const*>(&objectImpl->getLowerLeft()) * modelView;

            dp::math::neon::Vec4f x( extent[0] * modelView[0] );
            dp::math::neon::Vec4f y( extent[1] * modelView[1] );
            dp::math::neon::Vec4f z( extent[2] * modelView[2] );

            vectors[1] = vectors[0] + x;
            vectors[2] = vectors[0] + y;
            vectors[3] = vectors[1] + y;
            vectors[4] = vectors[0] + z;
            vectors[5] = vectors[1] + z;
            vectors[6] = vectors[2] + z;
            vectors[7] = vectors[3] + z;

            for ( unsigned int i = 0;i < 8; ++i )
            {
              minValue = vminq_f32( minValue, vectors[i].neon() );
              maxValue = vmaxq_f32( maxValue, vectors[i].neon() );
            }

          }

          dp::math::Vec3f minVec, maxVec;

          vst1q_lane_f32( &minVec[0], minValue, 0);
          vst1q_lane_f32( &minVec[1], minValue, 1);
          vst1q_lane_f32( &minVec[2], minValue, 2);

          vst1q_lane_f32( &maxVec[0], maxValue, 0);
          vst1q_lane_f32( &maxVec[1], maxValue, 1);
          vst1q_lane_f32( &maxVec[2], maxValue, 2);

          return dp::math::Box3f( minVec, maxVec );
        }
        else

#endif
      // CPU fallback
      {
        const GroupBitSetSharedPtr& groupImpl = std::static_pointer_cast<GroupBitSet>(group);

        dp::math::Box4f boundingBox;

        char const* basePtr = reinterpret_cast<char const*>(groupImpl->getMatrices());
        for ( size_t index = 0;index < groupImpl->getObjectCount(); ++index )
        {
          const ObjectBitSetSharedPtr objectImpl = std::static_pointer_cast<ObjectBitSet>(groupImpl->getObject( index ));
          dp::math::Mat44f const& modelView = reinterpret_cast<dp::math::Mat44f const&>(*(basePtr + objectImpl->getTransformIndex() * groupImpl->getMatricesStride()));
          dp::math::Vec4f const& extent = objectImpl->getExtent();

          dp::math::Vec4f vectors[8];
          vectors[0] = (objectImpl->getLowerLeft() * modelView);

          dp::math::Vec4f x( extent[0] * modelView.getPtr()[0], extent[0] * modelView.getPtr()[1], extent[0] * modelView.getPtr()[2], extent[0] * modelView.getPtr()[3] );
          dp::math::Vec4f y( extent[1] * modelView.getPtr()[4], extent[1] * modelView.getPtr()[5], extent[1] * modelView.getPtr()[6], extent[1] * modelView.getPtr()[7] );
          dp::math::Vec4f z( extent[2] * modelView.getPtr()[8], extent[2] * modelView.getPtr()[9], extent[2] * modelView.getPtr()[10], extent[2] * modelView.getPtr()[11] );

          vectors[1] = vectors[0] + x;
          vectors[2] = vectors[0] + y;
          vectors[3] = vectors[1] + y;
          vectors[4] = vectors[0] + z;
          vectors[5] = vectors[1] + z;
          vectors[6] = vectors[2] + z;
          vectors[7] = vectors[3] + z;

          for ( unsigned int i = 0;i < 8; ++i )
          {
            boundingBox.update( vectors[i] );
          }
        }

        dp::math::Vec4f lower = boundingBox.getLower();
        dp::math::Vec4f upper = boundingBox.getUpper();

        return dp::math::Box3f( dp::math::Vec3f( lower[0], lower[1], lower[2]), dp::math::Vec3f( upper[0], upper[1], upper[2]));
      }
    }

  } // namespace culling
} // namespace dp


/*****************************************************************************
*
* Copyright (c) 2003-2020 by The University of Queensland
* http://www.uq.edu.au
*
* Primary Business: Queensland, Australia
* Licensed under the Apache License, version 2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Development until 2012 by Earth Systems Science Computational Center (ESSCC)
* Development 2012-2013 by School of Earth Sciences
* Development from 2014-2017 by Centre for Geoscience Computing (GeoComp)
* Development from 2019 by School of Earth and Environmental Sciences
**
*****************************************************************************/

#include "FunctionSpace.h"

#include "Data.h"
#include "DataFactory.h"
#include "FunctionSpaceException.h"
#include "NullDomain.h"

#include <iostream>
#include <sstream>
#include <boost/smart_ptr.hpp>

using namespace std;
using namespace boost;

namespace escript {

  bool canInterpolate(FunctionSpace src, FunctionSpace dest)
  {
      return src.getDomain()->probeInterpolationOnDomain(src.getTypeCode(), dest.getTypeCode());
  }

namespace {
//
// Create a null domain for use with any default-constructed function space
// NullDomain const FunctionSpace::nullDomainValue;
const_Domain_ptr nullDomainValue(new NullDomain());
}

FunctionSpace::FunctionSpace():
//   m_domain(static_cast<const AbstractDomain*>(&nullDomainValue)),
  m_domain(nullDomainValue),
  m_functionSpaceType(dynamic_cast<const NullDomain*>(nullDomainValue.get())->getFunctionCode())
{
}

// FunctionSpace::FunctionSpace(const AbstractDomain& domain,
//                              int functionSpaceType):
// /*  m_domain(dynamic_cast<const AbstractDomain*>(&domain)),*/
//   m_domain(domain.getPtr()),
//   m_functionSpaceType(functionSpaceType)
// {
//   if (!m_domain->isValidFunctionSpaceType(functionSpaceType)) {
//     std::stringstream temp;
//     temp << "Invalid function space type: " << functionSpaceType
// 	 << " for domain: " << m_domain->getDescription();
//     throw FunctionSpaceException(temp.str());
//   }
// }

FunctionSpace::FunctionSpace(const_Domain_ptr domain,
                             int functionSpaceType):
/*  m_domain(dynamic_cast<const AbstractDomain*>(&domain)),*/
  m_domain(domain),
  m_functionSpaceType(functionSpaceType)
{
  if (!m_domain->isValidFunctionSpaceType(functionSpaceType)) {
    std::stringstream temp;
    temp << "Invalid function space type: " << functionSpaceType
	 << " for domain: " << m_domain->getDescription();
    throw FunctionSpaceException(temp.str());
  }
}

FunctionSpace::FunctionSpace(const FunctionSpace& other)
:m_domain(other.m_domain),
m_functionSpaceType(other.m_functionSpaceType)
{
}

std::pair<int,DataTypes::dim_t>
FunctionSpace::getDataShape() const
{
  return m_domain->getDataShape(m_functionSpaceType);
}

int
FunctionSpace::getTypeCode() const
{
  return  m_functionSpaceType;
}

// const
// AbstractDomain&
const_Domain_ptr
FunctionSpace::getDomain() const
{
  return m_domain;
}

Domain_ptr
FunctionSpace::getDomainPython() const
{
  // cast away the const-ness because python ignores it anyway
  return const_pointer_cast<AbstractDomain>(m_domain);
}

std::string
FunctionSpace::toString() const
{
  std::stringstream temp;
  temp << m_domain->functionSpaceTypeAsString(m_functionSpaceType)
       << " on " << m_domain->getDescription();

  return temp.str();
}

#ifdef DEBUG_PY_STRINGS
PyObject *
FunctionSpace::toPyString() const
{
  boost::python::to_python_value<const std::string &> cvtr;
  std::stringstream temp;

  temp << m_domain->functionSpaceTypeAsString(m_functionSpaceType)
       << " on " << m_domain->getDescription();

  return cvtr(temp.str());
}
#endif

int
FunctionSpace::getTagFromSampleNo(DataTypes::dim_t sampleNo) const
{
  return m_domain->getTagFromSampleNo(m_functionSpaceType,sampleNo);
}

int
FunctionSpace::getTagFromDataPointNo(DataTypes::dim_t dataPointNo) const
{
  //
  // Get the number of samples and data-points per sample
  int numSamples = getNumSamples();
  int numDataPointsPerSample = getNumDPPSample();
  int numDataPoints = numSamples * numDataPointsPerSample;

  if (numDataPointsPerSample==0) {
    throw DataException("FunctionSpace::getTagFromDataPointNo error: no data-points associated with this object.");
  }

  if (dataPointNo<0 || dataPointNo>=numDataPoints) {
    throw DataException("FunctionSpace::getTagFromDataPointNo error: invalid data-point number supplied.");
  }

  //
  // Determine the sample number which corresponds to this data-point number
  int sampleNo = dataPointNo / numDataPointsPerSample;

  //
  // Determine the tag number which corresponds to this sample number
  int tagNo = getTagFromSampleNo(sampleNo);

  //
  // return the tag number
  return(tagNo);
}

DataTypes::dim_t FunctionSpace::getReferenceIDFromDataPointNo(DataTypes::dim_t dataPointNo) const
{
     //
     // Get the number of samples and data-points per sample
    DataTypes::dim_t numSamples = getNumSamples();
    int numDataPointsPerSample = getNumDPPSample();
    const DataTypes::dim_t* referenceIDs= borrowSampleReferenceIDs();
    DataTypes::dim_t numDataPoints = numSamples * numDataPointsPerSample;

    if (numDataPointsPerSample==0) {
        throw DataException("FunctionSpace::getReferenceIDFromDataPointNo error: no data-points associated with this object.");
    }
    if (dataPointNo<0 || dataPointNo>numDataPoints) {
        throw DataException("FunctionSpace::getReferenceIDFromDataPointNo error: invalid data-point number supplied.");
    }
    DataTypes::dim_t sampleNo = dataPointNo / numDataPointsPerSample;
    return referenceIDs[sampleNo];
}

const DataTypes::dim_t*
FunctionSpace::borrowSampleReferenceIDs() const
{
  return m_domain->borrowSampleReferenceIDs(m_functionSpaceType);
}

// FunctionSpace instances should not be overwritten to point to different domains/types
// The only time this was actually used was in constructors and the copy constructor can deal with that
FunctionSpace&
FunctionSpace::operator=(const FunctionSpace& other)
{
  throw DataException("FunctionSpace::= should not be called. Programming Error.");
  // explicitly defined assignment operator to emphasise pointer copy
/*  m_functionSpaceType=other.m_functionSpaceType;
  m_domain=other.m_domain;
  return *this;*/
}

bool
FunctionSpace::operator==(const FunctionSpace& other) const
{
  return ((*(other.m_domain)==*(m_domain)) && (other.m_functionSpaceType==m_functionSpaceType));
}

bool
FunctionSpace::operator!=(const FunctionSpace& other) const
{
  return !(operator==(other));
}

escript::Data
FunctionSpace::getX() const
{
  Data out=escript::Vector(0,*this,true);
  getDomain()->setToX(out);
  out.setProtection();
  return out;
}

#ifdef ESYS_HAVE_BOOST_NUMPY
boost::python::numpy::ndarray
FunctionSpace::getNumpyX() const
{
    // Initialise boost numpy
    boost::python::numpy::initialize();

    // Get the data
    Data data=escript::Vector(0,*this,true);
    getDomain()->setToX(data);
    data.setProtection();

    // This is needed below in getSampleDataRW
    const DataTypes::real_t onlyreal = 0;

    // Work out how many data points there are
    int numDataPoints = data.getNumSamples();
    int dpps = data.getNumDataPointsPerSample();

    // Work out the data point shape
    std::vector<int> shape = data.getDataPointShape();

    // Work out how many spatial dimensions there are
    int dimensions = data.getShapeProduct();

    // Initialise the ndarray
    boost::python::tuple arrayshape = boost::python::make_tuple(dimensions, dpps * numDataPoints);
    boost::python::numpy::dtype datatype = boost::python::numpy::dtype::get_builtin<double>();
    boost::python::numpy::ndarray dataArray = boost::python::numpy::zeros(arrayshape, datatype);

    // Initialise variables
    std::string localmsg;
    std::vector<const DataTypes::real_t*> samplesR(1);

// #pragma omp parallel for
    for (int i = 0; i < numDataPoints; ++i) {
        for (int j = 0; j < shape[0]; j++) {
            dataArray[j][i] = *(data.getSampleDataRW(i, onlyreal)+j);
        }
    }

    // Print out the ndarray to the console - used during debugging
    // std::cout << "Finished array:\n" << bp::extract<char const *>(bp::str(dataArray)) << std::endl;

    return dataArray;
}
#endif

escript::Data
FunctionSpace::getNormal() const
{
  Data out=escript::Vector(0,*this,true);
  getDomain()->setToNormal(out);
  out.setProtection();
  return out;
}

escript::Data
FunctionSpace::getSize() const
{
  Data out=escript::Scalar(0,*this,true);
  getDomain()->setToSize(out);
  out.setProtection();
  return out;
}

void
FunctionSpace::setTags(const int newTag, const escript::Data& mask) const
{
   if (mask.getFunctionSpace()== *this) {
          m_domain->setTags(m_functionSpaceType,newTag,mask);
   } else {
          throw FunctionSpaceException("illegal function space of mask.");
   }
}

void
FunctionSpace::setTagsByString(const std::string& name, const escript::Data& mask) const
{
   int newTag=m_domain->getTag(name);
   if (mask.getFunctionSpace()== *this) {
          m_domain->setTags(m_functionSpaceType,newTag,mask);
   } else {
          throw FunctionSpaceException("illegal function space of mask.");
   }
}

int
FunctionSpace::getNumberOfTagsInUse() const
{
   return  m_domain->getNumberOfTagsInUse(m_functionSpaceType);
}

const int* FunctionSpace::borrowListOfTagsInUse() const
{
   return m_domain->borrowListOfTagsInUse(m_functionSpaceType);
}

std::list<int> FunctionSpace::getListOfTagsSTL() const
{
  const int* tags=borrowListOfTagsInUse();
  std::list<int> taglist(tags, tags+getNumberOfTagsInUse());
  return taglist;
}

boost::python::list FunctionSpace::getListOfTags() const
{
  const int* tags=borrowListOfTagsInUse();
  boost::python::list taglist;
  for (int i=0; i<getNumberOfTagsInUse(); ++i)
      taglist.append(tags[i]);
  return taglist;
}

bool
FunctionSpace::canTag() const
{
  return m_domain->canTag(m_functionSpaceType);
}

int
FunctionSpace::getApproximationOrder() const
{
   return m_domain->getApproximationOrder(m_functionSpaceType);
}

}  // end of namespace