/*
              __ __ __
             |__|__|  | __
             |  |  |  ||__|
  ___ ___ __ |  |  |  |
 |   |   |  ||  |  |  |    Ubiquitous Internet @ IIT-CNR
 |   |   |  ||  |  |  |    C++ edge computing libraries and tools
 |_______|__||__|__|__|    https://github.com/ccicconetti/serverlessonedge

Licensed under the MIT License <http://opensource.org/licenses/MIT>
Copyright (c) 2021 C. Cicconetti <https://ccicconetti.github.io/>

Permission is hereby  granted, free of charge, to any  person obtaining a copy
of this software and associated  documentation files (the "Software"), to deal
in the Software  without restriction, including without  limitation the rights
to  use, copy,  modify, merge,  publish, distribute,  sublicense, and/or  sell
copies  of  the Software,  and  to  permit persons  to  whom  the Software  is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE  IS PROVIDED "AS  IS", WITHOUT WARRANTY  OF ANY KIND,  EXPRESS OR
IMPLIED,  INCLUDING BUT  NOT  LIMITED TO  THE  WARRANTIES OF  MERCHANTABILITY,
FITNESS FOR  A PARTICULAR PURPOSE AND  NONINFRINGEMENT. IN NO EVENT  SHALL THE
AUTHORS  OR COPYRIGHT  HOLDERS  BE  LIABLE FOR  ANY  CLAIM,  DAMAGES OR  OTHER
LIABILITY, WHETHER IN AN ACTION OF  CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE  OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

#include "StateSim/network.h"

#include "Support/split.h"

#include <glog/logging.h>

#include <boost/graph/dijkstra_shortest_paths.hpp>

#include <cassert>
#include <fstream>
#include <sstream>

namespace uiiit {
namespace statesim {

struct NodeList : public std::vector<std::string> {
  static NodeList make(const std::string&             aString,
                       [[maybe_unused]] Counter<int>& aCounter) {
    const auto ret = support::split<NodeList>(aString, " ");
    if (ret.size() <= 1) {
      throw std::runtime_error("Invalid edge: " + aString);
    }
    return ret;
  }
};

template <class T>
std::vector<T> loadFile(const std::string& aPath, Counter<int>& aCounter) {
  std::vector<T> ret;
  std::ifstream  myFile(aPath);
  if (not myFile) {
    throw std::runtime_error("Cannot open file for reading: " + aPath);
  }
  std::string myLine;
  while (myFile) {
    std::getline(myFile, myLine);
    if (myLine.empty() or myLine[0] == '#') {
      continue;
    }
    ret.emplace_back(T::make(myLine, aCounter));
  }
  return ret;
}

std::vector<Node> loadNodes(const std::string& aPath, Counter<int>& aCounter) {
  return loadFile<Node>(aPath, aCounter);
}

std::vector<Link> loadLinks(const std::string& aPath, Counter<int>& aCounter) {
  return loadFile<Link>(aPath, aCounter);
}

std::vector<NodeList> loadNodeLists(const std::string& aPath) {
  Counter<int> myCounter;
  return loadFile<NodeList>(aPath, myCounter);
}

////////////////////////////////////////////////////////////////////////////////
// class Network

Network::Network(const std::string& aNodesPath,
                 const std::string& aLinksPath,
                 const std::string& aEdgesPath)
    : theMutex()
    , theNodes()
    , theLinks()
    , theElements()
    , theClients()
    , theProcessing()
    , theGraph()
    , theCloudParams(nullptr)
    , theCache()
    , theCentral(nullptr) {
  // read from files
  Counter<int> myCounter;
  auto         myNodes     = loadNodes(aNodesPath, myCounter);
  auto         myLinks     = loadLinks(aLinksPath, myCounter);
  const auto   myNodeLists = loadNodeLists(aEdgesPath);

  // copy into member maps
  for (const auto& myNode : myNodes) {
    theNodes.emplace(myNode.name(), myNode);
  }
  for (const auto& myLink : myLinks) {
    theLinks.emplace(myLink.name(), myLink);
  }

  // save pointers to processing nodes and clients into dedicated containers
  for (auto& elem : theNodes) {
    if (elem.second.client()) {
      theClients.emplace_back(&elem.second);
    }
    theProcessing.emplace_back(&elem.second);
  }

  // add to theNodes the non-processing nodes
  for (const auto& myNodeList : myNodeLists) {
    for (const auto& myName : myNodeList) {
      if (theNodes.find(myName) == theNodes.end() and
          theLinks.find(myName) == theLinks.end()) {
        [[maybe_unused]] const auto ret =
            theNodes.emplace(myName, Node(myName, myCounter()));
        assert(ret.second);
      }
    }
  }

  // create the list of edges (with weights)
  std::vector<Edge>  myEdges;
  std::vector<float> myWeights;
  for (const auto& myNodeList : myNodeLists) {
    assert(myNodeList.size() >= 2);
    auto       it               = myNodeList.begin();
    const auto mySourceId       = id(*it);
    const auto mySourceCapacity = capacity(*it);
    const auto mySourceLatency =
        mySourceCapacity > 0 ? (2.0f / mySourceCapacity) : 0;
    ++it;
    for (; it != myNodeList.end(); ++it) {
      const auto myTargetId       = id(*it);
      const auto myTargetCapacity = capacity(*it);

      // compute the weight as the sum of the half reciprocals of the source
      // and target, but only if they are links
      const auto myWeight =
          mySourceLatency +
          (myTargetCapacity > 0 ? (2.0f / myTargetCapacity) : 0);

      myEdges.emplace_back(Edge(mySourceId, myTargetId));
      myWeights.emplace_back(myWeight);

      VLOG(2) << "src " << mySourceId << " dst " << myTargetId << " weight "
              << myWeight;
    }
  }

  initElementsGraph(myEdges, myWeights);
}

Network::Network(const std::set<Node>&                               aNodes,
                 const std::set<Link>&                               aLinks,
                 const std::map<std::string, std::set<std::string>>& aEdges,
                 const std::set<std::string>&                        aClients)
    : theMutex()
    , theNodes()
    , theLinks()
    , theElements()
    , theClients()
    , theProcessing()
    , theGraph()
    , theCloudParams(nullptr)
    , theCache()
    , theCentral(nullptr) {
  // fill theNodes, theLinks, theClients, and theProcessing making sure that
  // names and numeric identifiers are unique
  std::set<size_t> myIds;
  for (const auto& myNode : aNodes) {
    const auto ret = theNodes.emplace(myNode.name(), myNode);
    if (not ret.second) {
      throw std::runtime_error("Duplicated node name: " + myNode.name());
    }
    if (not myIds.insert(myNode.id()).second) {
      throw std::runtime_error("Duplicated identifier: " +
                               std::to_string(myNode.id()));
    }
    assert(ret.first != theNodes.end());
    if (aClients.count(myNode.name()) > 0) {
      theClients.emplace_back(&ret.first->second);
    }
    if (myNode.type() == Node::Type::Processing) {
      theProcessing.emplace_back(&ret.first->second);
    }
  }
  for (const auto& myLink : aLinks) {
    const auto ret = theLinks.emplace(myLink.name(), myLink);
    if (not ret.second) {
      throw std::runtime_error("Duplicated link name: " + myLink.name());
    }
    if (not myIds.insert(myLink.id()).second) {
      throw std::runtime_error("Duplicated identifier: " +
                               std::to_string(myLink.id()));
    }
  }

  // make sure all the clients names exist
  if (theClients.size() != aClients.size()) {
    std::stringstream myStream;
    for (const auto& myClient : aClients) {
      auto myFound = false;
      for (const auto& myInnerClient : theClients) {
        if (myInnerClient->name() == myClient) {
          myFound = true;
          break;
        }
      }
      if (not myFound) {
        myStream << ' ' << myClient;
      }
    }
    throw std::runtime_error("Invalid clients passed: " + myStream.str());
  }

  // create the list of edges (with weights)
  std::vector<Edge>  myEdges;
  std::vector<float> myWeights;
  for (const auto& elem : aEdges) {
    const auto mySourceId       = id(elem.first);
    const auto mySourceCapacity = capacity(elem.first);
    const auto mySourceLatency =
        mySourceCapacity > 0 ? (2.0f / mySourceCapacity) : 0;
    for (const auto& myDst : elem.second) {
      const auto myTargetId       = id(myDst);
      const auto myTargetCapacity = capacity(myDst);

      // compute the weight as the sum of the half reciprocals of the source
      // and target, but only if they are links
      const auto myWeight =
          mySourceLatency +
          (myTargetCapacity > 0 ? (2.0f / myTargetCapacity) : 0);

      myEdges.emplace_back(Edge(mySourceId, myTargetId));
      myWeights.emplace_back(myWeight);

      VLOG(2) << mySourceId << ' ' << myTargetId << ' ' << myWeight;
    }
  }

  initElementsGraph(myEdges, myWeights);
}

void Network::initElementsGraph(const std::vector<Edge>&  aEdges,
                                const std::vector<float>& aWeights) {
  // fill the nodes/links vector indexed by the identifiers
  theElements.resize(theNodes.size() + theLinks.size());
  for (auto& myNode : theNodes) {
    assert(myNode.second.id() < theElements.size());
    theElements[myNode.second.id()] = &myNode.second;
  }
  for (auto& myLink : theLinks) {
    assert(myLink.second.id() < theElements.size());
    theElements[myLink.second.id()] = &myLink.second;
  }
  for ([[maybe_unused]] const auto& elem : theElements) {
    assert(elem != nullptr);
  }

  // create the network graph
  theGraph = Graph(aEdges.data(),
                   aEdges.data() + aEdges.size(),
                   aWeights.data(),
                   theNodes.size() + theLinks.size());
  VLOG(1) << "Created a network with " << theNodes.size() << " nodes and "
          << theLinks.size() << " links";
  assert(theElements.size() == boost::num_vertices(theGraph));
  theCache.resize(theElements.size(), {false, PredVec()});

  // print nodes and links, if verbose
  for (const auto myElement : theElements) {
    VLOG(2) << myElement->toString();
  }
}

void Network::cloud(const double aLatency, const double aRate) {
  if (aLatency < 0) {
    throw std::runtime_error("the cloud latency (" + std::to_string(aLatency) +
                             ") cannot be negative");
  }
  if (aRate <= 0) {
    throw std::runtime_error("the cloud rate (" + std::to_string(aRate) +
                             ") cannot be non-positive");
  }

  // overwrite previous settings, if any
  theCloudParams = std::make_unique<CloudParams>(aLatency, aRate);
}

std::pair<float, std::string> Network::nextHop(const std::string& aSrc,
                                               const std::string& aDst) {
  const auto myDstId      = id(aDst);
  auto&      myCacheEntry = cacheEntry(myDstId);
  const auto mySrcId      = id(aSrc);
  assert(mySrcId < myCacheEntry.second.size());

  const auto& myRet = myCacheEntry.second[mySrcId];
  assert(myRet.second < theElements.size());
  assert(theElements[myRet.second] != nullptr);
  return {myRet.first, theElements[myRet.second]->name()};
}

double
Network::txTime(const Node& aSrc, const Node& aDst, const size_t aBytes) {
  // short-cut for vanishing amount of data to transfer and self tx
  if (aBytes == 0 or &aSrc == &aDst) {
    return 0;
  }

  const auto& myCacheEntry = cacheEntry(aDst.id());
  assert(aSrc.id() < myCacheEntry.second.size());

  auto myTxTime = 0.0;
  auto myCur    = aSrc.id();
  while (myCacheEntry.second[myCur].second != aDst.id()) {
    myTxTime += theElements[myCacheEntry.second[myCur].second]->txTime(aBytes);
    myCur = myCacheEntry.second[myCur].second;
  }

  return myTxTime;
}

double Network::txTimeCloud(const Node& aNode, const size_t aBytes) const {
  // short-cut for vanishing amount of data, before even checking if the
  // cloud parameters have been set
  if (aBytes == 0) {
    return 0;
  }

  if (theCloudParams.get() == nullptr) {
    throw std::runtime_error("Cloud parameters not set");
  }

  return theCloudParams->theCloudLatency +
         (8 * aBytes) / (1e6 * theCloudParams->theCloudRate);
}

size_t Network::hops(const Node& aSrc, const Node& aDst) {
  const auto& myCacheEntry = cacheEntry(aDst.id());
  assert(aSrc.id() < myCacheEntry.second.size());

  size_t ret   = 0;
  auto   myCur = aSrc.id();
  while (myCacheEntry.second[myCur].second != aDst.id()) {
    const auto myNext = myCacheEntry.second[myCur].second;
    if (theElements[myNext]->device() == Element::Device::Link) {
      ret++;
    }
    myCur = myNext;
  }
  return ret;
}

Node* Network::central() {
  if (theCentral != nullptr) {
    return theCentral;
  }

  const size_t        N = 1000;
  std::vector<double> myTxTimes(theProcessing.size());
  double              myLastWorst = 0;
  for (const auto myCandidate : theProcessing) {
    size_t i = 0;
    for (const auto myTarget : theProcessing) {
      myTxTimes[i++] = std::max(txTime(*myCandidate, *myTarget, N),
                                txTime(*myTarget, *myCandidate, N));
    }
    const auto myWorst = *std::max_element(myTxTimes.begin(), myTxTimes.end());
    if (theCentral == nullptr or myWorst < myLastWorst) {
      myLastWorst = myWorst;
      theCentral  = myCandidate;
    }
  }
  return theCentral;
}

size_t Network::id(const std::string& aName) const {
  const auto it = theNodes.find(aName);
  if (it != theNodes.end()) {
    return it->second.id();
  }
  const auto jt = theLinks.find(aName);
  if (jt != theLinks.end()) {
    return jt->second.id();
  }
  throw std::runtime_error("Unknown node with name: " + aName);
}

float Network::capacity(const std::string& aName) const {
  const auto it = theNodes.find(aName);
  if (it != theNodes.end()) {
    return 0;
  }
  const auto jt = theLinks.find(aName);
  if (jt != theLinks.end()) {
    return jt->second.capacity();
  }
  throw std::runtime_error("Unknown node with name: " + aName);
}

const Network::Cache::value_type& Network::cacheEntry(const size_t aDstId) {
  assert(aDstId < theCache.size());

  auto& myCacheEntry = theCache[aDstId];

  {
    const std::lock_guard<std::mutex> myLock(theMutex);
    if (myCacheEntry.first == false) {
      assert(myCacheEntry.second.size() == 0);

      // create an entry in the cache for this destination

      std::vector<VertexDescriptor> myPred(theCache.size());
      std::vector<float>            myDist(theCache.size());

      boost::dijkstra_shortest_paths(
          theGraph,
          aDstId,
          boost::predecessor_map(
              boost::make_iterator_property_map(
                  myPred.begin(), get(boost::vertex_index, theGraph)))
              .distance_map(myDist.data()));

      myCacheEntry.first = true;
      myCacheEntry.second.resize(theCache.size());
      for (size_t i = 0; i < theCache.size(); i++) {
        myCacheEntry.second[i] = {myDist[i], myPred[i]};
      }
    }
  }

  return myCacheEntry;
}

const Network::Cache::value_type&
Network::cacheEntry(const size_t aDstId) const {
  assert(aDstId < theCache.size());
  const auto& myCacheEntry = theCache[aDstId];
  if (myCacheEntry.first == false) {
    assert(aDstId < theElements.size());
    throw std::runtime_error(
        "Routing cache entry does not exist for destination: " +
        theElements[aDstId]->name());
  }
  return myCacheEntry;
}
} // namespace statesim
} // namespace uiiit


//---------------------------------------------------------------------------//
// Copyright (c) 2018-2020 Mikhail Komarov <nemo@nil.foundation>
//
// MIT License
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//---------------------------------------------------------------------------//

#ifndef CRYPTO3_MAC_POLY1305_FUNCTIONS_HPP
#define CRYPTO3_MAC_POLY1305_FUNCTIONS_HPP

#include <boost/endian/arithmetic.hpp>
#include <boost/endian/conversion.hpp>

#include <nil/crypto3/detail/make_uint_t.hpp>

#include <nil/crypto3/mac/detail/poly1305/poly1305_policy.hpp>

namespace nil {
    namespace crypto3 {
        namespace mac {
            namespace detail {
                struct poly1305_functions : public poly1305_policy {
                    typedef poly1305_policy policy_type;

                    constexpr static const std::size_t word_bits = policy_type::word_bits;
                    typedef typename policy_type::word_type word_type;

                    constexpr static const std::size_t key_words = policy_type::key_words;
                    constexpr static const std::size_t key_bits = policy_type::key_bits;
                    typedef typename policy_type::key_type key_type;

                    constexpr static const std::size_t key_schedule_bits = policy_type::key_schedule_bits;
                    constexpr static const std::size_t key_schedule_words = policy_type::key_schedule_words;
                    typedef typename policy_type::key_schedule_type key_schedule_type;

                    static void poly1305_init(key_schedule_type &X, const key_type &key) {
                        using namespace nil::crypto3::detail;
                        /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */
                        const word_type t0 = boost::endian::native_to_little(
                            make_uint_t<word_bits>(key[0], key[1], key[2], key[3], key[4], key[5], key[6], key[7]));
                        const word_type t1 = boost::endian::native_to_little(make_uint_t<word_bits>(
                            key[8], key[9], key[10], key[11], key[12], key[13], key[14], key[15]));

                        X[0] = (t0)&0xffc0fffffff;
                        X[1] = ((t0 >> 44) | (t1 << 20)) & 0xfffffc0ffff;
                        X[2] = ((t1 >> 24)) & 0x00ffffffc0f;

                        /* h = 0 */
                        X[3] = 0;
                        X[4] = 0;
                        X[5] = 0;

                        /* save pad for later */
                        X[6] = boost::endian::native_to_little(make_uint_t<word_bits>(
                            key[16], key[17], key[18], key[19], key[20], key[21], key[22], key[23]));
                        X[7] = boost::endian::native_to_little(make_uint_t<word_bits>(
                            key[24], key[25], key[26], key[27], key[28], key[29], key[30], key[31]));
                    }

                    static void poly1305_blocks(key_schedule_type &X, const uint8_t *m, size_t blocks,
                                                bool is_final = false) {
                        const word_type hibit = is_final ? 0 : (static_cast<word_type>(1) << 40); /* 1 << 128 */

                        const word_type r0 = X[0];
                        const word_type r1 = X[1];
                        const word_type r2 = X[2];

                        word_type h0 = X[3 + 0];
                        word_type h1 = X[3 + 1];
                        word_type h2 = X[3 + 2];

                        const word_type s1 = r1 * (5 << 2);
                        const word_type s2 = r2 * (5 << 2);

                        while (blocks--) {
                            /* h += m[i] */
                            const word_type t0 = load_le<word_type>(m, 0);
                            const word_type t1 = load_le<word_type>(m, 1);

                            h0 += ((t0)&0xfffffffffff);
                            h1 += (((t0 >> 44) | (t1 << 20)) & 0xfffffffffff);
                            h2 += (((t1 >> 24)) & 0x3ffffffffff) | hibit;

                            /* h *= r */
                            uint128_t d0 = uint128_t(h0) * r0 + uint128_t(h1) * s2 + uint128_t(h2) * s1;
                            uint128_t d1 = uint128_t(h0) * r1 + uint128_t(h1) * r0 + uint128_t(h2) * s2;
                            uint128_t d2 = uint128_t(h0) * r2 + uint128_t(h1) * r1 + uint128_t(h2) * r0;

                            /* (partial) h %= p */
                            word_type c = carry_shift(d0, 44);
                            h0 = d0 & 0xfffffffffff;
                            d1 += c;
                            c = carry_shift(d1, 44);
                            h1 = d1 & 0xfffffffffff;
                            d2 += c;
                            c = carry_shift(d2, 42);
                            h2 = d2 & 0x3ffffffffff;
                            h0 += c * 5;
                            c = carry_shift(h0, 44);
                            h0 = h0 & 0xfffffffffff;
                            h1 += c;

                            m += 16;
                        }

                        X[3 + 0] = h0;
                        X[3 + 1] = h1;
                        X[3 + 2] = h2;
                    }

                    static void poly1305_finish(key_schedule_type &X, uint8_t mac[16]) {
                        /* fully carry h */
                        word_type h0 = X[3 + 0];
                        word_type h1 = X[3 + 1];
                        word_type h2 = X[3 + 2];

                        word_type c;
                        c = (h1 >> 44);
                        h1 &= 0xfffffffffff;
                        h2 += c;
                        c = (h2 >> 42);
                        h2 &= 0x3ffffffffff;
                        h0 += c * 5;
                        c = (h0 >> 44);
                        h0 &= 0xfffffffffff;
                        h1 += c;
                        c = (h1 >> 44);
                        h1 &= 0xfffffffffff;
                        h2 += c;
                        c = (h2 >> 42);
                        h2 &= 0x3ffffffffff;
                        h0 += c * 5;
                        c = (h0 >> 44);
                        h0 &= 0xfffffffffff;
                        h1 += c;

                        /* compute h + -p */
                        word_type g0 = h0 + 5;
                        c = (g0 >> 44);
                        g0 &= 0xfffffffffff;
                        word_type g1 = h1 + c;
                        c = (g1 >> 44);
                        g1 &= 0xfffffffffff;
                        word_type g2 = h2 + c - (static_cast<word_type>(1) << 42);

                        /* select h if h < p, or h + -p if h >= p */
                        c = (g2 >> ((sizeof(word_type) * 8) - 1)) - 1;
                        g0 &= c;
                        g1 &= c;
                        g2 &= c;
                        c = ~c;
                        h0 = (h0 & c) | g0;
                        h1 = (h1 & c) | g1;
                        h2 = (h2 & c) | g2;

                        /* h = (h + pad) */
                        const word_type t0 = X[6];
                        const word_type t1 = X[7];

                        h0 += ((t0)&0xfffffffffff);
                        c = (h0 >> 44);
                        h0 &= 0xfffffffffff;
                        h1 += (((t0 >> 44) | (t1 << 20)) & 0xfffffffffff) + c;
                        c = (h1 >> 44);
                        h1 &= 0xfffffffffff;
                        h2 += (((t1 >> 24)) & 0x3ffffffffff) + c;
                        h2 &= 0x3ffffffffff;

                        /* mac = h % (2^128) */
                        h0 = ((h0) | (h1 << 44));
                        h1 = ((h1 >> 20) | (h2 << 24));

                        store_le(mac, h0, h1);

                        /* zero out the state */
                        clear_mem(X.data(), X.size());
                    }
                };
            }    // namespace detail
        }        // namespace mac
    }            // namespace crypto3
}    // namespace nil

#endif    // CRYPTO3_POLY1305_FUNCTIONS_HPP


// Boost.Geometry (aka GGL, Generic Geometry Library)

// Copyright (c) 2007-2012 Barend Gehrels, Amsterdam, the Netherlands.
// Copyright (c) 2008-2012 Bruno Lalande, Paris, France.
// Copyright (c) 2009-2012 Mateusz Loskot, London, UK.
// Copyright (c) 2013 Adam Wulkiewicz, Lodz, Poland.

// This file was modified by Oracle on 2013, 2014.
// Modifications copyright (c) 2013, 2014, Oracle and/or its affiliates.

// Parts of Boost.Geometry are redesigned from Geodan's Geographic Library
// (geolib/GGL), copyright (c) 1995-2010 Geodan, Amsterdam, the Netherlands.

// Use, modification and distribution is subject to the Boost Software License,
// Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)

// Contributed and/or modified by Adam Wulkiewicz, on behalf of Oracle

#ifndef BOOST_GEOMETRY_ALGORITHMS_TOUCHES_HPP
#define BOOST_GEOMETRY_ALGORITHMS_TOUCHES_HPP

#include <deque>

#include <boost/variant/apply_visitor.hpp>
#include <boost/variant/static_visitor.hpp>
#include <boost/variant/variant_fwd.hpp>

#include <boost/geometry/geometries/concepts/check.hpp>
#include <boost/geometry/algorithms/detail/for_each_range.hpp>
#include <boost/geometry/algorithms/detail/overlay/overlay.hpp>
#include <boost/geometry/algorithms/detail/overlay/self_turn_points.hpp>
#include <boost/geometry/algorithms/disjoint.hpp>
#include <boost/geometry/algorithms/intersects.hpp>
#include <boost/geometry/algorithms/num_geometries.hpp>
#include <boost/geometry/algorithms/detail/sub_range.hpp>
#include <boost/geometry/policies/robustness/no_rescale_policy.hpp>

#include <boost/geometry/algorithms/relate.hpp>
#include <boost/geometry/algorithms/detail/relate/relate_impl.hpp>

namespace geofeatures_boost {} namespace boost = geofeatures_boost; namespace geofeatures_boost { namespace geometry
{

#ifndef DOXYGEN_NO_DETAIL
namespace detail { namespace touches
{

// Box/Box

template
<
    std::size_t Dimension,
    std::size_t DimensionCount
>
struct box_box_loop
{
    template <typename Box1, typename Box2>
    static inline bool apply(Box1 const& b1, Box2 const& b2, bool & touch)
    {
        typedef typename coordinate_type<Box1>::type coordinate_type1;
        typedef typename coordinate_type<Box2>::type coordinate_type2;

        coordinate_type1 const& min1 = get<min_corner, Dimension>(b1);
        coordinate_type1 const& max1 = get<max_corner, Dimension>(b1);
        coordinate_type2 const& min2 = get<min_corner, Dimension>(b2);
        coordinate_type2 const& max2 = get<max_corner, Dimension>(b2);

        // TODO assert or exception?
        //BOOST_GEOMETRY_ASSERT(min1 <= max1 && min2 <= max2);

        if ( max1 < min2 || max2 < min1 )
        {
            return false;
        }

        if ( max1 == min2 || max2 == min1 )
        {
            touch = true;
        }

        return box_box_loop
                <
                    Dimension + 1,
                    DimensionCount
                >::apply(b1, b2, touch);
    }
};

template
<
    std::size_t DimensionCount
>
struct box_box_loop<DimensionCount, DimensionCount>
{
    template <typename Box1, typename Box2>
    static inline bool apply(Box1 const& , Box2 const&, bool &)
    {
        return true;
    }
};

struct box_box
{
    template <typename Box1, typename Box2>
    static inline bool apply(Box1 const& b1, Box2 const& b2)
    {
        BOOST_STATIC_ASSERT((geofeatures_boost::is_same
                                <
                                    typename geometry::coordinate_system<Box1>::type,
                                    typename geometry::coordinate_system<Box2>::type
                                >::value
                           ));
        assert_dimension_equal<Box1, Box2>();

        bool touches = false;
        bool ok = box_box_loop
                    <
                        0,
                        dimension<Box1>::type::value
                    >::apply(b1, b2, touches);

        return ok && touches;
    }
};

// Areal/Areal

struct areal_interrupt_policy
{
    static bool const enabled = true;
    bool found_touch;
    bool found_not_touch;

    // dummy variable required by self_get_turn_points::get_turns
    static bool const has_intersections = false;

    inline bool result()
    {
        return found_touch && !found_not_touch;
    }

    inline areal_interrupt_policy()
        : found_touch(false), found_not_touch(false)
    {}

    template <typename Range>
    inline bool apply(Range const& range)
    {
        // if already rejected (temp workaround?)
        if ( found_not_touch )
            return true;

        typedef typename geofeatures_boost::range_iterator<Range const>::type iterator;
        for ( iterator it = geofeatures_boost::begin(range) ; it != geofeatures_boost::end(range) ; ++it )
        {
            if ( it->has(overlay::operation_intersection) )
            {
                found_not_touch = true;
                return true;
            }

            switch(it->method)
            {
                case overlay::method_crosses:
                    found_not_touch = true;
                    return true;
                case overlay::method_equal:
                    // Segment spatially equal means: at the right side
                    // the polygon internally overlaps. So return false.
                    found_not_touch = true;
                    return true;
                case overlay::method_touch:
                case overlay::method_touch_interior:
                case overlay::method_collinear:
                    if ( ok_for_touch(*it) )
                    {
                        found_touch = true;
                    }
                    else
                    {
                        found_not_touch = true;
                        return true;
                    }
                    break;
                case overlay::method_none :
                case overlay::method_disjoint :
                case overlay::method_error :
                    break;
            }
        }

        return false;
    }

    template <typename Turn>
    inline bool ok_for_touch(Turn const& turn)
    {
        return turn.both(overlay::operation_union)
            || turn.both(overlay::operation_blocked)
            || turn.combination(overlay::operation_union, overlay::operation_blocked)
            ;
    }
};

template<typename Geometry>
struct check_each_ring_for_within
{
    bool has_within;
    Geometry const& m_geometry;

    inline check_each_ring_for_within(Geometry const& g)
        : has_within(false)
        , m_geometry(g)
    {}

    template <typename Range>
    inline void apply(Range const& range)
    {
        typename geometry::point_type<Range>::type p;
        geometry::point_on_border(p, range);
        if ( !has_within && geometry::within(p, m_geometry) )
        {
            has_within = true;
        }
    }
};

template <typename FirstGeometry, typename SecondGeometry>
inline bool rings_containing(FirstGeometry const& geometry1,
                SecondGeometry const& geometry2)
{
    check_each_ring_for_within<FirstGeometry> checker(geometry1);
    geometry::detail::for_each_range(geometry2, checker);
    return checker.has_within;
}

template <typename Geometry1, typename Geometry2>
struct areal_areal
{
    static inline
    bool apply(Geometry1 const& geometry1, Geometry2 const& geometry2)
    {
        typedef detail::no_rescale_policy rescale_policy_type;
        typedef typename geometry::point_type<Geometry1>::type point_type;
        typedef detail::overlay::turn_info
            <
                point_type,
                typename segment_ratio_type<point_type, rescale_policy_type>::type
            > turn_info;

        std::deque<turn_info> turns;
        detail::touches::areal_interrupt_policy policy;
        rescale_policy_type robust_policy;
        geofeatures_boost::geometry::get_turns
                <
                    detail::overlay::do_reverse<geometry::point_order<Geometry1>::value>::value,
                    detail::overlay::do_reverse<geometry::point_order<Geometry2>::value>::value,
                    detail::overlay::assign_null_policy
                >(geometry1, geometry2, robust_policy, turns, policy);

        return policy.result()
            && ! geometry::detail::touches::rings_containing(geometry1, geometry2)
            && ! geometry::detail::touches::rings_containing(geometry2, geometry1);
    }
};

// P/*

struct use_point_in_geometry
{
    template <typename Point, typename Geometry>
    static inline bool apply(Point const& point, Geometry const& geometry)
    {
        return detail::within::point_in_geometry(point, geometry) == 0;
    }
};

}}
#endif // DOXYGEN_NO_DETAIL

#ifndef DOXYGEN_NO_DISPATCH
namespace dispatch {

// TODO: Since CastedTags are used is Reverse needed?

template
<
    typename Geometry1, typename Geometry2,
    typename Tag1 = typename tag<Geometry1>::type,
    typename Tag2 = typename tag<Geometry2>::type,
    typename CastedTag1 = typename tag_cast<Tag1, pointlike_tag, linear_tag, areal_tag>::type,
    typename CastedTag2 = typename tag_cast<Tag2, pointlike_tag, linear_tag, areal_tag>::type,
    bool Reverse = reverse_dispatch<Geometry1, Geometry2>::type::value
>
struct touches
    : not_implemented<Tag1, Tag2>
{};

// If reversal is needed, perform it
template
<
    typename Geometry1, typename Geometry2,
    typename Tag1, typename Tag2,
    typename CastedTag1, typename CastedTag2
>
struct touches<Geometry1, Geometry2, Tag1, Tag2, CastedTag1, CastedTag2, true>
    : touches<Geometry2, Geometry1, Tag2, Tag1, CastedTag2, CastedTag1, false>
{
    static inline bool apply(Geometry1 const& g1, Geometry2 const& g2)
    {
        return touches<Geometry2, Geometry1>::apply(g2, g1);
    }
};

// P/P

template <typename Geometry1, typename Geometry2, typename Tag1, typename Tag2>
struct touches<Geometry1, Geometry2, Tag1, Tag2, pointlike_tag, pointlike_tag, false>
{
    static inline bool apply(Geometry1 const& , Geometry2 const& )
    {
        return false;
    }
};

// P/*

template <typename Point, typename Geometry, typename Tag2, typename CastedTag2>
struct touches<Point, Geometry, point_tag, Tag2, pointlike_tag, CastedTag2, false>
    : detail::touches::use_point_in_geometry
{};

// TODO: support touches(MPt, Linear/Areal)

// Box/Box

template <typename Box1, typename Box2, typename CastedTag1, typename CastedTag2>
struct touches<Box1, Box2, box_tag, box_tag, CastedTag1, CastedTag2, false>
    : detail::touches::box_box
{};

template <typename Box1, typename Box2>
struct touches<Box1, Box2, box_tag, box_tag, areal_tag, areal_tag, false>
    : detail::touches::box_box
{};

// L/L

template <typename Linear1, typename Linear2, typename Tag1, typename Tag2>
struct touches<Linear1, Linear2, Tag1, Tag2, linear_tag, linear_tag, false>
    : detail::relate::relate_impl
    <
        detail::de9im::static_mask_touches_type,
        Linear1,
        Linear2
    >
{};

// L/A

template <typename Linear, typename Areal, typename Tag1, typename Tag2>
struct touches<Linear, Areal, Tag1, Tag2, linear_tag, areal_tag, false>
    : detail::relate::relate_impl
    <
        detail::de9im::static_mask_touches_type,
        Linear,
        Areal
    >
{};

// A/L
template <typename Linear, typename Areal, typename Tag1, typename Tag2>
struct touches<Linear, Areal, Tag1, Tag2, linear_tag, areal_tag, true>
    : detail::relate::relate_impl
    <
        detail::de9im::static_mask_touches_type,
        Areal,
        Linear
    >
{};

// A/A

template <typename Areal1, typename Areal2, typename Tag1, typename Tag2>
struct touches<Areal1, Areal2, Tag1, Tag2, areal_tag, areal_tag, false>
    : detail::touches::areal_areal<Areal1, Areal2>
{};

} // namespace dispatch
#endif // DOXYGEN_NO_DISPATCH

namespace resolve_variant {

template <typename Geometry1, typename Geometry2>
struct touches
{
    static bool apply(Geometry1 const& geometry1, Geometry2 const& geometry2)
    {
        concept::check<Geometry1 const>();
        concept::check<Geometry2 const>();

        return dispatch::touches<Geometry1, Geometry2>
                       ::apply(geometry1, geometry2);
    }
};

template <BOOST_VARIANT_ENUM_PARAMS(typename T), typename Geometry2>
struct touches<geofeatures_boost::variant<BOOST_VARIANT_ENUM_PARAMS(T)>, Geometry2>
{
    struct visitor: geofeatures_boost::static_visitor<bool>
    {
        Geometry2 const& m_geometry2;

        visitor(Geometry2 const& geometry2): m_geometry2(geometry2) {}

        template <typename Geometry1>
        bool operator()(Geometry1 const& geometry1) const
        {
            return touches<Geometry1, Geometry2>::apply(geometry1, m_geometry2);
        }
    };

    static inline bool
    apply(geofeatures_boost::variant<BOOST_VARIANT_ENUM_PARAMS(T)> const& geometry1,
          Geometry2 const& geometry2)
    {
        return geofeatures_boost::apply_visitor(visitor(geometry2), geometry1);
    }
};

template <typename Geometry1, BOOST_VARIANT_ENUM_PARAMS(typename T)>
struct touches<Geometry1, geofeatures_boost::variant<BOOST_VARIANT_ENUM_PARAMS(T)> >
{
    struct visitor: geofeatures_boost::static_visitor<bool>
    {
        Geometry1 const& m_geometry1;

        visitor(Geometry1 const& geometry1): m_geometry1(geometry1) {}

        template <typename Geometry2>
        bool operator()(Geometry2 const& geometry2) const
        {
            return touches<Geometry1, Geometry2>::apply(m_geometry1, geometry2);
        }
    };

    static inline bool
    apply(Geometry1 const& geometry1,
          geofeatures_boost::variant<BOOST_VARIANT_ENUM_PARAMS(T)> const& geometry2)
    {
        return geofeatures_boost::apply_visitor(visitor(geometry1), geometry2);
    }
};

template <BOOST_VARIANT_ENUM_PARAMS(typename T1),
          BOOST_VARIANT_ENUM_PARAMS(typename T2)>
struct touches<geofeatures_boost::variant<BOOST_VARIANT_ENUM_PARAMS(T1)>,
               geofeatures_boost::variant<BOOST_VARIANT_ENUM_PARAMS(T2)> >
{
    struct visitor: geofeatures_boost::static_visitor<bool>
    {
        template <typename Geometry1, typename Geometry2>
        bool operator()(Geometry1 const& geometry1,
                        Geometry2 const& geometry2) const
        {
            return touches<Geometry1, Geometry2>::apply(geometry1, geometry2);
        }
    };

    static inline bool
    apply(geofeatures_boost::variant<BOOST_VARIANT_ENUM_PARAMS(T1)> const& geometry1,
          geofeatures_boost::variant<BOOST_VARIANT_ENUM_PARAMS(T2)> const& geometry2)
    {
        return geofeatures_boost::apply_visitor(visitor(), geometry1, geometry2);
    }
};

template <typename Geometry>
struct self_touches
{
    static bool apply(Geometry const& geometry)
    {
        concept::check<Geometry const>();

        typedef detail::no_rescale_policy rescale_policy_type;
        typedef typename geometry::point_type<Geometry>::type point_type;
        typedef detail::overlay::turn_info
            <
                point_type,
                typename segment_ratio_type<point_type, rescale_policy_type>::type
            > turn_info;

        typedef detail::overlay::get_turn_info
        <
            detail::overlay::assign_null_policy
        > policy_type;

        std::deque<turn_info> turns;
        detail::touches::areal_interrupt_policy policy;
        rescale_policy_type robust_policy;
        detail::self_get_turn_points::get_turns
        <
            policy_type
        >::apply(geometry, robust_policy, turns, policy);

        return policy.result();
    }
};

template <BOOST_VARIANT_ENUM_PARAMS(typename T)>
struct self_touches<geofeatures_boost::variant<BOOST_VARIANT_ENUM_PARAMS(T)> >
{
    struct visitor: geofeatures_boost::static_visitor<bool>
    {
        template <typename Geometry>
        bool operator()(Geometry const& geometry) const
        {
            return self_touches<Geometry>::apply(geometry);
        }
    };

    static inline bool
    apply(geofeatures_boost::variant<BOOST_VARIANT_ENUM_PARAMS(T)> const& geometry)
    {
        return geofeatures_boost::apply_visitor(visitor(), geometry);
    }
};

} // namespace resolve_variant

/*!
\brief \brief_check{has at least one touching point (self-tangency)}
\note This function can be called for one geometry (self-tangency) and
    also for two geometries (touch)
\ingroup touches
\tparam Geometry \tparam_geometry
\param geometry \param_geometry
\return \return_check{is self-touching}

\qbk{distinguish,one geometry}
\qbk{[def __one_parameter__]}
\qbk{[include reference/algorithms/touches.qbk]}
*/
template <typename Geometry>
inline bool touches(Geometry const& geometry)
{
    return resolve_variant::self_touches<Geometry>::apply(geometry);
}

/*!
\brief \brief_check2{have at least one touching point (tangent - non overlapping)}
\ingroup touches
\tparam Geometry1 \tparam_geometry
\tparam Geometry2 \tparam_geometry
\param geometry1 \param_geometry
\param geometry2 \param_geometry
\return \return_check2{touch each other}

\qbk{distinguish,two geometries}
\qbk{[include reference/algorithms/touches.qbk]}
 */
template <typename Geometry1, typename Geometry2>
inline bool touches(Geometry1 const& geometry1, Geometry2 const& geometry2)
{
    return resolve_variant::touches<Geometry1, Geometry2>::apply(geometry1, geometry2);
}

}} // namespace geofeatures_boost::geometry

#endif // BOOST_GEOMETRY_ALGORITHMS_TOUCHES_HPP


/* $Id: MannWhitneyRanksum.cpp,v 1.6 2016/02/24 06:27:15 severin Exp $ */

/***

NAME - EEDB::SPStreams::MannWhitneyRanksum

SYNOPSIS

DESCRIPTION

 A simple signal procesor which is configured with a minimum expression value
 and which will only pass expressions which are greater than that value

CONTACT

Jessica Severin <severin@gsc.riken.jp>

LICENSE

 * Software License Agreement (BSD License)
 * ZENBU [eeDB] system
 * copyright (c) 2007-2015 Jessica Severin RIKEN OSC
 * All rights reserved.
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *     * Neither the name of Jessica Severin RIKEN OSC nor the
 *       names of its contributors may be used to endorse or promote products
 *       derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ''AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

APPENDIX

The rest of the documentation details each of the object methods. Internal methods are usually preceded with a _

***/

#include <stdio.h>
#include <stdlib.h>
#include <iostream>
#include <string.h>
#include <string>
#include <stdarg.h>
#include <rapidxml.hpp>  //rapidxml must be include before boost
#include <boost/algorithm/string.hpp>
#include <EEDB/Experiment.h>
#include <EEDB/Feature.h>
#include <EEDB/Expression.h>
#include <EEDB/Symbol.h>
#include <EEDB/SPStream.h>
#include <EEDB/SPStreams/MannWhitneyRanksum.h>

using namespace std;
using namespace MQDB;

const char*  EEDB::SPStreams::MannWhitneyRanksum::class_name = "EEDB::SPStreams::MannWhitneyRanksum";

//function prototypes
void _spstream_mannwhitney_delete_func(MQDB::DBObject *obj) {
  delete (EEDB::SPStreams::MannWhitneyRanksum*)obj;
}
void _spstream_mannwhitney_xml_func(MQDB::DBObject *obj, string &xml_buffer) {
  ((EEDB::SPStreams::MannWhitneyRanksum*)obj)->_xml(xml_buffer);
}
string _spstream_mannwhitney_display_desc_func(MQDB::DBObject *obj) {
  return ((EEDB::SPStreams::MannWhitneyRanksum*)obj)->_display_desc();
}
MQDB::DBObject* _spstream_mannwhitney_next_in_stream_func(EEDB::SPStream* node) {
  return ((EEDB::SPStreams::MannWhitneyRanksum*)node)->_next_in_stream();
}
bool _spstream_mannwhitney_stream_by_named_region_func(EEDB::SPStream* node, string assembly_name, string chrom_name, long int start, long int end) {
  return ((EEDB::SPStreams::MannWhitneyRanksum*)node)->_stream_by_named_region(assembly_name, chrom_name, start, end);
}
/*
void _spstream_mannwhitney_reset_stream_node_func(EEDB::SPStream* node) {
  ((EEDB::SPStreams::MannWhitneyRanksum*)node)->_reset_stream_node();
}
void _spstream_mannwhitney_stream_clear_func(EEDB::SPStream* node) {
  ((EEDB::SPStreams::MannWhitneyRanksum*)node)->_stream_clear();
}
void _spstream_mannwhitney_stream_data_sources_func(EEDB::SPStream* node, string classname, string filter_logic) {
  ((EEDB::SPStreams::MannWhitneyRanksum*)node)->_stream_data_sources(classname, filter_logic);
}
void _spstream_mannwhitney_reload_stream_data_sources_func(EEDB::SPStream* node) {
  ((EEDB::SPStreams::MannWhitneyRanksum*)node)->_reload_stream_data_sources();
}
*/

EEDB::SPStreams::MannWhitneyRanksum::MannWhitneyRanksum() {
  init();
}

EEDB::SPStreams::MannWhitneyRanksum::~MannWhitneyRanksum() {
}

void EEDB::SPStreams::MannWhitneyRanksum::init() {
  EEDB::SPStream::init();
  _classname                 = EEDB::SPStreams::MannWhitneyRanksum::class_name;
  _module_name               = "MannWhitneyRanksum";
  _funcptr_delete            = _spstream_mannwhitney_delete_func;
  _funcptr_xml               = _spstream_mannwhitney_xml_func;
  _funcptr_simple_xml        = _spstream_mannwhitney_xml_func;
  _funcptr_display_desc      = _spstream_mannwhitney_display_desc_func;

  //function pointer code
  //_funcptr_reset_stream_node           = _spstream_mannwhitney_reset_stream_node_func;
  //_funcptr_stream_clear                = _spstream_mannwhitney_stream_clear_func;
  //_funcptr_stream_data_sources         = _spstream_mannwhitney_stream_data_sources_func;
  //_funcptr_reload_stream_data_sources  = _spstream_mannwhitney_reload_stream_data_sources_func;
  _funcptr_next_in_stream              = _spstream_mannwhitney_next_in_stream_func;
  _funcptr_stream_by_named_region      = _spstream_mannwhitney_stream_by_named_region_func;

  //attribute variables
  _experiment_cache.clear();
  _experiment_filter.clear();
  _filter_mdkeys.clear();
  _min_zscore = 1.6449; //equiv of pvalue 0.05
}

void EEDB::SPStreams::MannWhitneyRanksum::set_mdata_key_filter(string filter) {
  //space separated list of keys
  _filter_mdkeys.clear();
  if(!filter.empty()) {
    char *str = strdup(filter.c_str());
    char* tok = strtok(str," \t\n");
    while(tok) {
      _filter_mdkeys[tok] = true;
      fprintf(stderr, "MannWhitneyRanksum::set_mdata_key_filter [%s]\n", tok);
      tok = strtok(NULL," \t\n");
    }
    free(str);
  }
}

void EEDB::SPStreams::MannWhitneyRanksum::add_mdata_key_filter(string key) {
  fprintf(stderr, "MannWhitneyRanksum::set_mdata_key_filter [%s]\n", key.c_str());
  _filter_mdkeys[key] = true;
}

void EEDB::SPStreams::MannWhitneyRanksum::set_experiment_filter(string filter) {
  _experiment_filter = filter;
  _experiment_cache.clear(); //clear the experiment filter cache so it can recalculate
  //clear _mdgroup_hash
  map<string, EEDB::MWGroup*>::iterator mdg_it;
  for(mdg_it = _mdgroup_hash.begin(); mdg_it != _mdgroup_hash.end(); mdg_it++) {
    EEDB::MWGroup *mdgroup = (*mdg_it).second;
    mdgroup->release();
  }
  _mdgroup_hash.clear();
}

void EEDB::SPStreams::MannWhitneyRanksum::set_min_zscore(double value) {
  _min_zscore = fabs(value);
}

////////////////////////////////////////////////////////////////////////////
//
//  creation from XML section
//
////////////////////////////////////////////////////////////////////////////

void EEDB::SPStreams::MannWhitneyRanksum::_xml(string &xml_buffer) {
  _xml_start(xml_buffer);  //from SPStream superclass
  char strbuf[256];

  snprintf(strbuf, 256, "<min_zscore>%1.6f</min_zscore>", _min_zscore);
  xml_buffer += strbuf;

  if(!_experiment_filter.empty()) {
    xml_buffer += "<experiment_filter>" + _experiment_filter + "</experiment_filter>";
  }

  if(!_filter_mdkeys.empty()) {
    xml_buffer += "<mdata_keys_filter>";
    map<string,bool>::iterator it1;
    for(it1=_filter_mdkeys.begin(); it1!=_filter_mdkeys.end(); it1++) {
      xml_buffer += (*it1).first+" ";
    }
    xml_buffer += "</mdata_keys_filter>";
  }

  _xml_end(xml_buffer);  //from superclass
}

string EEDB::SPStreams::MannWhitneyRanksum::mdgroup_xml() {
  string xml_buffer;
  char   strbuf[1024];

  snprintf(strbuf, 1024, "<metadata_groups count=\"%ld\">\n", _mdgroup_hash.size());
  xml_buffer = strbuf;

  map<string, EEDB::MWGroup*>::iterator mdg_it;
  for(mdg_it = _mdgroup_hash.begin(); mdg_it != _mdgroup_hash.end(); mdg_it++) {
    EEDB::MWGroup *mdgroup = (*mdg_it).second;
    mdgroup->xml(xml_buffer);
  }
  xml_buffer += "/<metadata_groups>\n";
  return xml_buffer;
}

bool _mannwhit_mwgroup_sort_func (EEDB::MWGroup *a, EEDB::MWGroup *b) {
  if(a == NULL) { return false; }  //a is NULL, so pick b
  if(b == NULL) { return true; }   //b is NULL, so pick a

  if(a->avg_zscore() < b->avg_zscore()) { return false; }
  if(a->avg_zscore() > b->avg_zscore()) { return true; }

  if(a->abundance_count < b->abundance_count) { return false; }
  if(a->abundance_count > b->abundance_count) { return true; }

  return false;
}

vector<EEDB::MWGroup*>  EEDB::SPStreams::MannWhitneyRanksum::mdgroups() {
  vector<EEDB::MWGroup*> mwgroup_array;

  map<string, EEDB::MWGroup*>::iterator mdg_it;
  for(mdg_it = _mdgroup_hash.begin(); mdg_it != _mdgroup_hash.end(); mdg_it++) {
    mwgroup_array.push_back((*mdg_it).second);
  }
  std::sort(mwgroup_array.begin(), mwgroup_array.end(), _mannwhit_mwgroup_sort_func);

  return mwgroup_array;
}

EEDB::SPStreams::MannWhitneyRanksum::MannWhitneyRanksum(void *xml_node) {
  //constructor using a rapidxml <spstream> description
  init();
  if(xml_node==NULL) { return; }

  rapidxml::xml_node<>      *root_node = (rapidxml::xml_node<>*)xml_node;
  rapidxml::xml_node<>      *node;

  if(string(root_node->name()) != "spstream") { return; }

  if((node = root_node->first_node("min_zscore")) != NULL) {
    _min_zscore = strtod(node->value(), NULL);
  }
  if((node = root_node->first_node("experiment_filter")) != NULL) {
    _experiment_filter = node->value();
  }
  if((node = root_node->first_node("mdata_keys_filter")) != NULL) {
    set_mdata_key_filter(node->value());
  }
}

string EEDB::SPStreams::MannWhitneyRanksum::_display_desc() {
  return "MannWhitneyRanksum";
}

////////////////////////////////////////////////////////////////////////////
//
// callback methods
//
////////////////////////////////////////////////////////////////////////////

/*
void EEDB::SPStreams::MannWhitneyRanksum::_stream_clear() {
  //re-initialize the stream-stack back to a clear/empty state
  if(source_stream() != NULL) { source_stream()->stream_clear(); }

  EEDB::SPStream::_reset_stream_node();
  fprintf(stderr, "MannWhitneyRanksum::_stream_clear\n");

  _region_start = -1;
  _region_end   = -1;
}

void EEDB::SPStreams::MannWhitneyRanksum::_reset_stream_node() {
  EEDB::SPStream::_reset_stream_node();
  fprintf(stderr, "MannWhitneyRanksum::_reset_stream_node\n");

  _region_start = -1;
  _region_end   = -1;
}

void EEDB::SPStreams::MannWhitneyRanksum::_stream_data_sources(string classname, string filter_logic) {
  fprintf(stderr, "MannWhitneyRanksum::_stream_data_sources\n");
}

void EEDB::SPStreams::MannWhitneyRanksum::_reload_stream_data_sources() {
  fprintf(stderr, "MannWhitneyRanksum::_reload_stream_data_sources\n");
}
*/

bool EEDB::SPStreams::MannWhitneyRanksum::_stream_by_named_region(string assembly_name, string chrom_name, long int start, long int end) {
  _region_start = start;
  _region_end   = end;

  fprintf(stderr,"MannWhitneyRanksum::_stream_by_named_region[%ld] %s %s %ld .. %ld\n", (long)this, assembly_name.c_str(), chrom_name.c_str(), start, end);

  loadprep_experiment_metadata();

  return source_stream()->stream_by_named_region(assembly_name, chrom_name, start, end);
}

MQDB::DBObject* EEDB::SPStreams::MannWhitneyRanksum::_next_in_stream() {
  if(_source_stream == NULL) { return NULL; }

  MQDB::DBObject *obj = _source_stream->next_in_stream();
  if(obj == NULL) { return NULL; }

  if(obj->classname() == EEDB::Feature::class_name) {
    EEDB::Feature *feature = (EEDB::Feature*)obj;
    analyze_feature(feature);
  }

  //everything else is just passed through
  return obj;
}

////////////////////////////////////////////////////////////////////////////
//
// MannWhitney ranksum analysis methods
//
////////////////////////////////////////////////////////////////////////////

bool _mw_expression_rank_sort_func (EEDB::Expression *a, EEDB::Expression *b) {
  // < function
  if(a == NULL) { return false; }  //a is NULL, so pick b
  if(b == NULL) { return true; }   //b is NULL, so pick a

  if(a->value() < b->value()) { return true; }
  if(a->value() > b->value()) { return false; }

  return false;
}

bool EEDB::SPStreams::MannWhitneyRanksum::loadprep_experiment_metadata() {
  if(!_experiment_cache.empty()) { return true; }

  struct timeval       starttime, endtime, time_diff;
  gettimeofday(&starttime, NULL);

  //make sure variables are clear
  _experiment_cache.clear();   // [srcID] = experiment :: to keep cache of all experiments
  _keyval_mdata.clear();       // [key::value] = mdata
  _keyval_enrichment.clear();  // [key::value] = double :: pvalue stat
  _keyval_experiment.clear();  // [key::value][expid] = bool :: mdata present in experiment

  EEDB::SPStream  *stream = source_stream();

  if(!_experiment_filter.empty()) {
    stream->stream_data_sources("Experiment", _experiment_filter);
  } else {
    stream->stream_data_sources("Experiment");
  }

  long sources_count=0;
  while(EEDB::Experiment *source = (EEDB::Experiment*)stream->next_in_stream()) {
    if(source->classname() != EEDB::Experiment::class_name) { continue; }

    source->metadataset()->remove_duplicates();
    sources_count++;
    vector<EEDB::Metadata*> mdlist = source->metadataset()->metadata_list();
    for(unsigned int i=0; i<mdlist.size(); i++) {
      EEDB::Metadata *md = mdlist[i];
      string key = md->type();
      if(key=="keyword") { continue; }
      if(!_filter_mdkeys.empty() && (_filter_mdkeys.find(key)==_filter_mdkeys.end())) { continue; }

      string value = md->data();
      //boost::algorithm::to_lower(value);
      string keyval = key+"::"+value;

      source->retain();
      _experiment_cache[source->db_id()] = source;
      _keyval_mdata[keyval] = md;
      _keyval_enrichment[keyval] = 0;
      _keyval_experiment[keyval][source->db_id()] = true;

      EEDB::DataSource::add_to_sources_cache(source);
    }
  }
  fprintf(stderr,"MannWhitneyRanksum::loadprep_experiment_metadata exps=%ld %ld  mdkeyvals=%ld\n",
          sources_count, _experiment_cache.size(), _keyval_mdata.size());

  //create unique experiment-groups based on source_id-list
  //loop on each metadata and generate a unique id-list to hash into a group
  map<string, EEDB::Metadata*>::iterator  md_it1;
  //fprintf(stderr, "ranksum %ld keyval_metadata\n", _keyval_mdata.size());
  for(md_it1 = _keyval_mdata.begin(); md_it1 != _keyval_mdata.end(); md_it1++) {
    EEDB::Metadata *mdata  = (*md_it1).second;
    string          keyval = (*md_it1).first;

    map<string, map<string, bool> >::iterator it7;
    it7 = _keyval_experiment.find(keyval);

    list<string> src_ids;
    map<string, EEDB::Experiment*>::iterator it4;
    for(it4=_experiment_cache.begin(); it4!=_experiment_cache.end(); it4++) {
      EEDB::Experiment *exp = (*it4).second;
      if((*it7).second[exp->db_id()]) {  //present
        src_ids.push_back(exp->db_id());
      }
    }
    if(src_ids.size() < 2) { continue; } //group need at least 2 experiments
    if(sources_count - src_ids.size() < 2) { continue; } //need at least 2 experiments in out-group

    //make unique src_id_key
    src_ids.sort();  //sort just to make sure it is unique reproducible
    string src_id_key;
    list<string>::iterator s_it;
    for(s_it=src_ids.begin(); s_it!=src_ids.end(); s_it++) {
      src_id_key += (*s_it);
    }
    //create mdgroup if missing or add new metadata if exists
    if(_mdgroup_hash.find(src_id_key) == _mdgroup_hash.end()) {
      //first time
      EEDB::MWGroup *mdgroup = new EEDB::MWGroup();
      mdgroup->src_id_key = src_id_key;
      mdgroup->add_metadata(mdata);
      for(s_it=src_ids.begin(); s_it!=src_ids.end(); s_it++) {
        mdgroup->add_experiment_id(*s_it);
      }
      mdgroup->retain();
      _mdgroup_hash[src_id_key] = mdgroup;
    } else {
      //append
      EEDB::MWGroup *mdgroup = (EEDB::MWGroup*)_mdgroup_hash[src_id_key];
      mdgroup->add_metadata(mdata);
    }
  }
  fprintf(stderr, "  %ld src_id_key groups\n", _mdgroup_hash.size());

  gettimeofday(&endtime, NULL);
  timersub(&endtime, &starttime, &time_diff);
  double   runtime  = (double)time_diff.tv_sec*1000.0 + ((double)time_diff.tv_usec)/1000.0;
  fprintf(stderr, "loadprep_experiment_metadata %1.3f msec\n", runtime);
  return true;
}

bool EEDB::SPStreams::MannWhitneyRanksum::analyze_feature(EEDB::Feature *feature) {
  struct timeval       starttime, endtime, time_diff;

  _debug=false;

  if(!feature) { return false; }
  gettimeofday(&starttime, NULL);

  if(_debug) { fprintf(stderr, "\n\nMannWhitneyRanksum::analyze_feature :: %s", feature->simple_xml().c_str()); }

  //fill in all the missing experiments to convert the sparse-matrix to a full-matrix
  vector<EEDB::Expression*>  feat_exp_array = feature->expression_array();
  vector<EEDB::Expression*>  express_array; //final array for analysis

  if(_debug) { fprintf(stderr, "%ld filtered experiments in cache\n", _experiment_cache.size()); }
  if(_debug) { fprintf(stderr, "%ld feature expressions\n", feat_exp_array.size()); }

  //first place the expression which matches the known/filtered experiments into express_hash
  map<string, EEDB::Expression*>  express_hash;   //experiment srcID to expression, fill in the missing
  EEDB::Datatype *dtype = NULL;
  for(unsigned i=0; i<feat_exp_array.size(); i++) {
    EEDB::Expression *expr = feat_exp_array[i];
    if(_experiment_cache.find(expr->experiment_dbid()) != _experiment_cache.end()) {
      express_hash[expr->experiment_dbid()] = expr;
      dtype = expr->datatype();
    }
  }
  if(_debug) { fprintf(stderr, "  %ld expression match the experiment filter\n", express_hash.size()); }

  //second fill in the missing expression with zero values
  map<string, EEDB::Experiment*>::iterator it4;
  for(it4=_experiment_cache.begin(); it4!=_experiment_cache.end(); it4++) {
    EEDB::Experiment *experiment = (*it4).second;
    if(express_hash.find(experiment->db_id()) == express_hash.end()) {
      //missing Expression
      EEDB::Expression *expr2 = EEDB::Expression::realloc();
      expr2->experiment(experiment);
      expr2->datatype(dtype);
      expr2->value(0);
      expr2->sig_error(0);
      express_hash[experiment->db_id()] = expr2;
    }
  }
  if(_debug) { fprintf(stderr, "  %ld after fill in\n",express_hash.size()); }

  //third covert express_hash to express_array
  map<string, EEDB::Expression*>::iterator it5;
  for(it5=express_hash.begin(); it5!=express_hash.end(); it5++) {
    EEDB::Expression *express = (*it5).second;
    express_array.push_back(express);
  }

  //ranksum sort the expression array and asign the rank number
  std::sort(express_array.begin(), express_array.end(), _mw_expression_rank_sort_func);

  unsigned pos=0;
  while(pos<express_array.size()) {
    express_array[pos]->sig_error(pos+1);

    unsigned pos2 = pos;
    double sumrank=pos+1;
    while((pos2+1<express_array.size()) &&
          (express_array[pos]->value() == express_array[pos2+1]->value())) {
      sumrank += (pos2+1)+1;
      pos2++;
    }
    if(pos != pos2) {  //duplicates
      sumrank = sumrank / (pos2-pos+1);
      for(unsigned i=pos; i<=pos2; i++) {
        express_array[i]->sig_error(sumrank);
      }
      pos = pos2;
    }
    pos++;
  }
  //rank stored in the ->sig_error()

  //loop on each mdgroup, foreach mdgroup check all expression/experiments to match
  //calc rank-sum statistic for each metadata
  long mdcount=0;
  map<string, EEDB::MWGroup*>::iterator mdg_it;
  for(mdg_it = _mdgroup_hash.begin(); mdg_it != _mdgroup_hash.end(); mdg_it++) {
    EEDB::MWGroup *mdgroup = (*mdg_it).second;
    mdgroup->reset_stats();

    long group_count = 0;
    long other_count = 0;
    //calculate the ranksums
    double group_sum=0.0, other_sum=0.0, totalRank=0.0;
    double group_mean=0.0, other_mean=0.0;
    vector<EEDB::Expression*>::iterator it6;
    for(it6=express_array.begin(); it6!=express_array.end(); it6++) {
      EEDB::Expression *expr = (*it6);

      if(mdgroup->has_experiment_id(expr->experiment_dbid())) {
        group_sum += expr->sig_error();
        group_count++;
        //group_mean += log(expr->value());
        group_mean += expr->value();
      } else {
        other_sum += expr->sig_error();
        other_count++;
        //other_mean += log(expr->value());
        other_mean += expr->value();
      }
      totalRank += expr->sig_error();

    }
    mdgroup->other_count = other_count;
    mdgroup->group_count = group_count;
    mdgroup->other_sum   = other_sum;
    mdgroup->group_sum   = group_sum;

    if(group_count < 2) { continue; } //need at least 2 experiments
    if(other_count < 2) { continue; } //need at least 2 experiments in out-group

    group_mean /= group_count;
    other_mean /= other_count;

    //calc the wilcoxon-mann-whitney u-test
    //redo from https://epilab.ich.ucl.ac.uk/coursematerial/statistics/non_parametric/wilcox_mann_whitney.html

    double n1 = group_count;
    double n2 = other_count;

    double u1 = group_sum - ((n1 * (n1+1)) / 2.0);
    //double u2 = other_sum - ((n2 * (n2+1)) / 2.0);

    //double Uobt= 0.0;
    //if(u1<u2) { Uobt = u1; } else { Uobt = u2; }
    double Uobt= u1;  //if I want to preserve the directionality, just use U1

    double oa = sqrt(n1*n2*(n1+n2+1)/12);

    double zscore = (Uobt -  (n1 * n2 / 2)) / oa;

    /*
     if(group_count < other_count) {
     na = group_count;
     wa = group_sum;
     nb = other_count;
     } else {
     na = other_count;
     wa = other_sum;
     nb = group_count;
     }

     double ua = na * (na + nb + 1) / 2;
     double oa = sqrt(na*nb*(na+nb+1)/12);

     double zscore = (wa-ua) / oa;
     */

    //TODO: need to do table lookup for when either n1 or n2 is less than 20

    //TODO: need to do zscore to pvalue calc/lookup
    double pvalue = zscore;

    mdgroup->zscore = zscore;
    mdgroup->pvalue = pvalue;

    //fprintf(stderr, "ok6\n");
    //fprintf(stderr, "%s", mdgroup->simple_xml().c_str());

    if(fabs(zscore)>_min_zscore) {
      if(_debug) {
        fprintf(stderr, "%s", mdgroup->simple_xml().c_str());
        fprintf(stderr, "SIGNIFICANT!!!!!\n");
      }
      mdcount++;

      mdgroup->other_count = other_count;
      mdgroup->group_count = group_count;
      mdgroup->other_sum = other_sum;
      mdgroup->group_sum = group_sum;
      mdgroup->zscore = zscore;
      mdgroup->pvalue = pvalue;
      mdgroup->abundance_count++;
      mdgroup->abundance_zscores.push_back(zscore);

      mdgroup->retain();
      feature->metadataset()->add_metadata(mdgroup);
    }
    /*
     EEDB::MetadataSet *mdset = experiment->metadataset();
     if(mdset->find_metadata(mdata->type(), mdata->data())) {
     _keyval_enrichment[keyval]++;
     }
     */
  }
  if(_debug) {
    gettimeofday(&endtime, NULL);
    timersub(&endtime, &starttime, &time_diff);
    double   runtime  = (double)time_diff.tv_sec*1000.0 + ((double)time_diff.tv_usec)/1000.0;
    fprintf(stderr, "found %ld significant mann whitney groups - %1.3f msec\n", mdcount, runtime);
  }

  return true;
}

////////////////////////////////////////////////////////////////////////////
//
// MWGroup section
//
////////////////////////////////////////////////////////////////////////////

const char*  EEDB::MWGroup::class_name = "EEDB::MWGroup";

//function prototypes
void _spstream_mwgroup_delete_func(MQDB::DBObject *obj) {
  delete (EEDB::MWGroup*)obj;
}
void _spstream_mwgroup_xml_func(MQDB::DBObject *obj, string &xml_buffer) {
  ((EEDB::MWGroup*)obj)->_xml(xml_buffer);
}
void _spstream_mwgroup_simple_xml_func(MQDB::DBObject *obj, string &xml_buffer) {
  ((EEDB::MWGroup*)obj)->_simple_xml(xml_buffer);
}

EEDB::MWGroup::MWGroup() {
  init();
}

EEDB::MWGroup::~MWGroup() {
  fprintf(stderr, "mwgroup delete\n");
}

void EEDB::MWGroup::init() {
  EEDB::Metadata::init();
  _classname                 = EEDB::MWGroup::class_name;
  _funcptr_delete            = _spstream_mwgroup_delete_func;
  _funcptr_xml               = _spstream_mwgroup_xml_func;
  _funcptr_simple_xml        = _spstream_mwgroup_simple_xml_func;

  //attribute variables
  _exp_id_hash.clear();
  src_id_key.clear();
  _mdata_list.clear();

  other_count = 0;
  group_count = 0;
  other_sum = 0.0;
  group_sum = 0.0;
  zscore = 0.0;
  pvalue = 0.0;
  abundance_count = 0;
}

void EEDB::MWGroup::reset_stats() {
  other_count = 0;
  group_count = 0;
  other_sum = 0.0;
  group_sum = 0.0;
  zscore = 0.0;
  pvalue = 0.0;
}

bool EEDB::MWGroup::has_experiment_id(string value) {
  if(_exp_id_hash.find(value) == _exp_id_hash.end()) { return false; }
  return _exp_id_hash[value];
}

void EEDB::MWGroup::add_experiment_id(string value) {
  _exp_id_hash[value] = true;
}

void  EEDB::MWGroup::add_metadata(EEDB::Metadata *mdata) {
  if(!mdata) { return; }
  if(_mdata_list.empty()) {
    //set type/data to the first element in the group
    _type = mdata->type();
    _data = mdata->data();
  }
  _mdata_list.push_back(mdata);
}

void EEDB::MWGroup::_simple_xml(string &xml_buffer) {
  if(_mdata_list.empty()) { return; }

  char strbuf[8192];
  EEDB::Metadata *mdata  = _mdata_list.front();

  long src_count=0;
  map<string,bool>::iterator it2;
  for(it2=_exp_id_hash.begin(); it2!=_exp_id_hash.end(); it2++) {
    if((*it2).second) { src_count++; }
  }

  snprintf(strbuf, 8192,
           "<mdgroup type=\"%s\" value=\"%s\" source_count=\"%ld\" other_count=\"%ld\" group_count=\"%ld\" other_ranksum=\"%f\" group_ranksum=\"%f\" zscore=\"%f\" pvalue=\"%f\" >",
           html_escape(mdata->type()).c_str(), html_escape(mdata->data()).c_str(),
           src_count, other_count, group_count, other_sum, group_sum, zscore, pvalue);
  xml_buffer += strbuf;
  xml_buffer += "</mdgroup>\n";
}

void EEDB::MWGroup::_xml(string &xml_buffer) {
  if(_mdata_list.empty()) { return; }

  char strbuf[8192];
  EEDB::Metadata *mdata  = _mdata_list.front();

  long src_count=0;
  map<string,bool>::iterator it2;
  for(it2=_exp_id_hash.begin(); it2!=_exp_id_hash.end(); it2++) {
    if((*it2).second) { src_count++; }
  }

  snprintf(strbuf, 8192,
          "<mdgroup type=\"%s\" value=\"%s\" source_count=\"%ld\" other_count=\"%ld\" group_count=\"%ld\" other_ranksum=\"%f\" group_ranksum=\"%f\" zscore=\"%f\" pvalue=\"%f\" >",
           html_escape(mdata->type()).c_str(), html_escape(mdata->data()).c_str(),
           src_count, other_count, group_count, other_sum, group_sum, zscore, pvalue);
  xml_buffer += strbuf;

  //_exp_id_hash to get unique source ids in this group
  for(it2=_exp_id_hash.begin(); it2!=_exp_id_hash.end(); it2++) {
    if((*it2).second) { xml_buffer += "<datasource id=\"" + (*it2).first + "\"/>"; }
  }

  //individual metadata which share exact same experiment group
  list<EEDB::Metadata*>::iterator s_it;
  for(s_it=_mdata_list.begin(); s_it!=_mdata_list.end(); s_it++) {
    (*s_it)->xml(xml_buffer);
  }
  xml_buffer += "</mdgroup>\n";
}

EEDB::MWGroup::MWGroup(void *xml_node) {
  //constructor using a rapidxml <spstream> description
  init();
  if(xml_node==NULL) { return; }

  rapidxml::xml_node<>      *root_node = (rapidxml::xml_node<>*)xml_node;
  //rapidxml::xml_node<>      *node;

  //TODO: need to do this to store/retrieve finished results from ZDX

  if(string(root_node->name()) != "spstream") { return; }
}

EEDB::MWGroup* EEDB::MWGroup::copy() {
  EEDB::MWGroup *t_copy = new EEDB::MWGroup();
  t_copy->src_id_key = src_id_key;
  t_copy->other_count = other_count;
  t_copy->group_count = group_count;
  t_copy->other_sum = other_sum;
  t_copy->group_sum = group_sum;
  t_copy->zscore = zscore;
  t_copy->pvalue = pvalue;
  t_copy->abundance_count = abundance_count;

  t_copy->abundance_zscores = abundance_zscores;
  t_copy->_mdata_list = _mdata_list;
  t_copy->_exp_id_hash = _exp_id_hash;
  return t_copy;
}

double  EEDB::MWGroup::avg_zscore() {
  double sum=0;
  for(unsigned i=0; i<abundance_zscores.size(); i++) {
    sum += fabs(abundance_zscores[i]);
  }
  return sum/abundance_zscores.size();
}

double  EEDB::MWGroup::max_zscore() {
  double max=0;
  for(unsigned i=0; i<abundance_zscores.size(); i++) {
    if(fabs(abundance_zscores[i]) > fabs(max)) { max = abundance_zscores[i]; }
  }
  return max;
}


/*
 * Python bindings for libroom
 * Copyright (C) 2019  Robin Scheibler, Cyril Cadoux
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 * You should have received a copy of the MIT License along with this program. If
 * not, see <https://opensource.org/licenses/MIT>.
 */

#include <string>
#include <vector>
#include <pybind11/pybind11.h>
#include <pybind11/eigen.h>
#include <pybind11/stl.h>
#include <Eigen/Dense>

#include "common.hpp"
#include "geometry.hpp"
#include "microphone.hpp"
#include "wall.hpp"
#include "room.hpp"

namespace py = pybind11;

float libroom_eps = 1e-4;  // epsilon is set to 0.1 millimeter (100 um)

PYBIND11_MODULE(libroom, m) {
  m.doc() = "Libroom room simulation extension plugin"; // optional module docstring

  // The 3D Room class
  py::class_<Room<3>>(m, "Room")
    .def(py::init<
        const std::vector<Wall<3>> &,
        const std::vector<int> &,
        const std::vector<Microphone<3>> &,
        float, int, float, float, float, float, bool
        >())
    .def(py::init<
        const Vectorf<3> &,
        const Eigen::Array<float,Eigen::Dynamic,6> &,
        const Eigen::Array<float,Eigen::Dynamic,6> &,
        const std::vector<Microphone<3>> &,
        float, int, float, float, float, float, bool
        >())
    .def("set_params", &Room<3>::set_params)
    .def("add_mic", &Room<3>::add_mic)
    .def("reset_mics", &Room<3>::reset_mics)
    .def("image_source_model", &Room<3>::image_source_model)
    .def("get_wall", &Room<3>::get_wall)
    .def("get_max_distance", &Room<3>::get_max_distance)
    .def("next_wall_hit", &Room<3>::next_wall_hit)
    .def("scat_ray", &Room<3>::scat_ray)
    .def("simul_ray", &Room<3>::simul_ray)
    .def("ray_tracing",
        (void (Room<3>::*)(
                             const Eigen::Matrix<float,2,Eigen::Dynamic> &angles,
                             const Vectorf<3> source_pos
                             )
        )
        &Room<3>::ray_tracing)
    .def("ray_tracing",
        (void (Room<3>::*)(
                             size_t nb_phis,
                             size_t nb_thetas,
                             const Vectorf<3> source_pos
                             )
        )
        &Room<3>::ray_tracing)
    .def("ray_tracing",
        (void (Room<3>::*)(
                             size_t nb_rays,
                             const Vectorf<3> source_pos
                             )
        )
        &Room<3>::ray_tracing)
    .def("contains", &Room<3>::contains)
    .def_property("is_hybrid_sim", &Room<3>::get_is_hybrid_sim, &Room<3>::set_is_hybrid_sim)
    .def_property_readonly_static("dim", [](py::object /* self */) { return 3; })
    .def_readonly("walls", &Room<3>::walls)
    .def_readonly("sources", &Room<3>::sources)
    .def_readonly("orders", &Room<3>::orders)
    .def_readonly("attenuations", &Room<3>::attenuations)
    .def_readonly("gen_walls", &Room<3>::gen_walls)
    .def_readonly("visible_mics", &Room<3>::visible_mics)
    .def_readonly("walls", &Room<3>::walls)
    .def_readonly("obstructing_walls", &Room<3>::obstructing_walls)
    .def_readonly("microphones", &Room<3>::microphones)
    .def_readonly("max_dist", &Room<3>::max_dist)
    ;

  // The 2D Room class
  py::class_<Room<2>>(m, "Room2D")
    //.def(py::init<py::list, py::list, const Eigen::MatrixXf &>())
    .def(py::init<
        const std::vector<Wall<2>> &,
        const std::vector<int> &,
        const std::vector<Microphone<2>> &,
        float, int, float, float, float, float, bool
        >())
    .def(py::init<
        const Vectorf<2> &,
        const Eigen::Array<float,Eigen::Dynamic,4> &,
        const Eigen::Array<float,Eigen::Dynamic,4> &,
        const std::vector<Microphone<2>> &,
        float, int, float, float, float, float, bool
        >())
    .def("set_params", &Room<2>::set_params)
    .def("add_mic", &Room<2>::add_mic)
    .def("reset_mics", &Room<2>::reset_mics)
    .def("image_source_model", &Room<2>::image_source_model)
    .def("get_wall", &Room<2>::get_wall)
    .def("get_max_distance", &Room<2>::get_max_distance)
    .def("next_wall_hit", &Room<2>::next_wall_hit)
    .def("scat_ray", &Room<2>::scat_ray)
    .def("simul_ray", &Room<2>::simul_ray)
    .def("ray_tracing",
        (void (Room<2>::*)(
                             const Eigen::Matrix<float,1,Eigen::Dynamic> &angles,
                             const Vectorf<2> source_pos
                            )
        )
        &Room<2>::ray_tracing)
    .def("ray_tracing",
        (void (Room<2>::*)(
                             size_t nb_phis,
                             size_t nb_thetas,
                             const Vectorf<2> source_pos
                            )
        )
        &Room<2>::ray_tracing)
    .def("ray_tracing",
        (void (Room<2>::*)(
                             size_t n_rays,
                             const Vectorf<2> source_pos
                            )
        )
        &Room<2>::ray_tracing)
    .def("contains", &Room<2>::contains)
    .def_property_readonly_static("dim", [](py::object /* self */) { return 2; })
    .def_property("is_hybrid_sim", &Room<2>::get_is_hybrid_sim, &Room<2>::set_is_hybrid_sim)
    .def_readonly("walls", &Room<2>::walls)
    .def_readonly("sources", &Room<2>::sources)
    .def_readonly("orders", &Room<2>::orders)
    .def_readonly("attenuations", &Room<2>::attenuations)
    .def_readonly("gen_walls", &Room<2>::gen_walls)
    .def_readonly("visible_mics", &Room<2>::visible_mics)
    .def_readonly("walls", &Room<2>::walls)
    .def_readonly("obstructing_walls", &Room<2>::obstructing_walls)
    .def_readonly("microphones", &Room<2>::microphones)
    .def_readonly("max_dist", &Room<2>::max_dist)
    ;

  // The Wall class
  py::class_<Wall<3>> wall_cls(m, "Wall");

  wall_cls
    .def(py::init<const Eigen::Matrix<float,3,Eigen::Dynamic> &, const Eigen::ArrayXf &, const Eigen::ArrayXf &, const std::string &>(),
        py::arg("corners"), py::arg("absorption") = Eigen::ArrayXf::Zero(1),
        py::arg("scattering") = Eigen::ArrayXf::Zero(1), py::arg("name") = "")
    .def("area", &Wall<3>::area)
    .def("intersection", &Wall<3>::intersection)
    .def("intersects", &Wall<3>::intersects)
    .def("side", &Wall<3>::side)
    .def("reflect", &Wall<3>::reflect)
    .def("normal_reflect", (Vectorf<3>(Wall<3>::*)(const Vectorf<3>&, const Vectorf<3>&, float) const)&Wall<3>::normal_reflect)
    .def("normal_reflect", (Vectorf<3>(Wall<3>::*)(const Vectorf<3>&) const)&Wall<3>::normal_reflect)
    .def("same_as", &Wall<3>::same_as)
    .def_property_readonly_static("dim", [](py::object /* self */) { return 3; })
    .def_readwrite("absorption", &Wall<3>::absorption)
    .def_readwrite("scatter", &Wall<3>::scatter)
    .def_readwrite("name", &Wall<3>::name)
    .def_readonly("corners", &Wall<3>::corners)
    .def_readonly("origin", &Wall<3>::origin)
    .def_readonly("normal", &Wall<3>::normal)
    .def_readonly("basis", &Wall<3>::basis)
    .def_readonly("flat_corners", &Wall<3>::flat_corners)
    ;

  py::enum_<Wall<3>::Isect>(wall_cls, "Isect")
    .value("NONE", Wall<3>::Isect::NONE)
    .value("VALID", Wall<3>::Isect::VALID)
    .value("ENDPT", Wall<3>::Isect::ENDPT)
    .value("BNDRY", Wall<3>::Isect::BNDRY)
    .export_values();

  // The Wall class
  py::class_<Wall<2>> wall2d_cls(m, "Wall2D");

  wall2d_cls
    .def(py::init<const Eigen::Matrix<float,2,Eigen::Dynamic> &, const Eigen::ArrayXf &, const Eigen::ArrayXf &, std::string &>(),
        py::arg("corners"), py::arg("absorption") = Eigen::ArrayXf::Zero(1),
        py::arg("scattering") = Eigen::ArrayXf::Zero(1), py::arg("name") = "")
    .def("area", &Wall<2>::area)
    .def("intersection", &Wall<2>::intersection)
    .def("intersects", &Wall<2>::intersects)
    .def("side", &Wall<2>::side)
    .def("reflect", &Wall<2>::reflect)
    .def("normal_reflect", (Vectorf<2>(Wall<2>::*)(const Vectorf<2>&, const Vectorf<2>&, float) const)&Wall<2>::normal_reflect)
    .def("normal_reflect", (Vectorf<2>(Wall<2>::*)(const Vectorf<2>&) const)&Wall<2>::normal_reflect)
    .def("same_as", &Wall<2>::same_as)
    .def_property_readonly_static("dim", [](py::object /* self */) { return 2; })
    .def_readwrite("absorption", &Wall<2>::absorption)
    .def_readwrite("scatter", &Wall<2>::scatter)
    .def_readwrite("name", &Wall<2>::name)
    .def_readonly("corners", &Wall<2>::corners)
    .def_readonly("origin", &Wall<2>::origin)
    .def_readonly("normal", &Wall<2>::normal)
    .def_readonly("basis", &Wall<2>::basis)
    .def_readonly("flat_corners", &Wall<2>::flat_corners)
    ;

  // The different wall intersection cases
  m.attr("WALL_ISECT_NONE") = WALL_ISECT_NONE;
  m.attr("WALL_ISECT_VALID") = WALL_ISECT_VALID;
  m.attr("WALL_ISECT_VALID_ENDPT") = WALL_ISECT_VALID_ENDPT;
  m.attr("WALL_ISECT_VALID_BNDRY") = WALL_ISECT_VALID_BNDRY;

  // The microphone class
  py::class_<Microphone<3>>(m, "Microphone")
    .def(py::init<const Vectorf<3> &, int, float, float>())
    .def_readonly("loc", &Microphone<3>::loc)
    .def_readonly("hits", &Microphone<3>::hits)
    .def_readonly("histograms", &Microphone<3>::histograms)
    ;

  py::class_<Microphone<2>>(m, "Microphone2D")
    .def(py::init<const Vectorf<2> &, int, float, float>())
    .def_readonly("loc", &Microphone<2>::loc)
    .def_readonly("hits", &Microphone<2>::hits)
    .def_readonly("histograms", &Microphone<2>::histograms)
    ;

  // The 2D histogram class
  py::class_<Histogram2D>(m, "Histogram2D")
    .def(py::init<int, int>())
    .def("log", &Histogram2D::log)
    .def("bin", &Histogram2D::bin)
    .def("get_hist", &Histogram2D::get_hist)
    .def("reset", &Histogram2D::reset)
    ;

  // Structure to hold detector hit information
  py::class_<Hit>(m, "Hit")
    .def(py::init<int>())
    .def(py::init<const float, const Eigen::ArrayXf &>())
    .def_readonly("transmitted", &Hit::transmitted)
    .def_readonly("distance", &Hit::distance)
    ;

  // getter and setter for geometric epsilon
  m.def("set_eps", [](const float &eps) { libroom_eps = eps; });
  m.def("get_eps", []() { return libroom_eps; });

  // Routines for the geometry packages
  m.def("ccw3p", &ccw3p, "Determines the orientation of three points");

  m.def("check_intersection_2d_segments",
      &check_intersection_2d_segments,
      "A function that checks if two line segments intersect");

  m.def("intersection_2d_segments",
      &intersection_2d_segments,
      "A function that finds the intersection of two line segments");

  m.def("intersection_3d_segment_plane",
      &intersection_3d_segment_plane,
      "A function that finds the intersection between a line segment and a plane");

  m.def("cross", &cross, "Cross product of two 3D vectors");

  m.def("is_inside_2d_polygon", &is_inside_2d_polygon,
      "Checks if a 2D point lies in or out of a planar polygon");

  m.def("area_2d_polygon", &area_2d_polygon,
      "Compute the signed area of a planar polygon");

  m.def("cos_angle_between", &cos_angle_between,
      "Computes the angle between two 2D or 3D vectors");

  m.def("dist_line_point", &dist_line_point,
      "Computes the distance between a point and an infinite line");

}


// This file is part of DM-HEOM (https://github.com/noma/dm-heom)
//
// Copyright (c) 2015-2019 Matthias Noack, Zuse Institute Berlin
//
// Licensed under the 3-clause BSD License, see accompanying LICENSE,
// CONTRIBUTORS.md, and README.md for further information.

#include "heom/instance.hpp"

#include <cmath>
#include <fstream>
#include <iostream>
#include <unordered_map>

#include <boost/functional/hash.hpp>
#include <boost/math/special_functions/binomial.hpp>

#include <noma/memory/memory.hpp>

#include "heom/constants.hpp"

namespace heom {

// constructor from heom_config
instance::instance(const config& cfg, sites_to_states_mode_t sts_mode, const hierarchy_graph& graph)
  : sites_to_states_mode_(sts_mode)
{
	// setup sizes
	states_ = sites_to_states(cfg.system_sites(), sts_mode); // NOTE: site to state conversion
	matsubaras_ = cfg.baths_matsubaras();
	baths_per_state_ = max_baths_per_state(cfg.baths_max_per_site(), sts_mode); // NOTE: site to state conversion
	baths_ = cfg.baths_number();

	ado_width_ = graph.width(); //baths_ * matsubaras_;
	ado_depth_ = graph.depth(); //cfg.system_ado_depth();

	matrices_ = graph.nodes(); // includes halo nodes
	first_non_plus_id_ = graph.first_non_plus_uid();
	assert(first_non_plus_id_ > 0);

	// allocate buffer memory
	allocate();

	// initialise low-level instance data structures from high_level graph partition data structure
	for (uid_t id = 0; id < graph.nodes(); ++id) {
		// copy tuples
		auto tuples = graph.tuples()[id].data();
		std::copy(tuples, tuples + ado_width_, &ado_tuples_[id * ado_width_]);
		// copy plus edges
		auto plus_edges = graph.plus_edges()[id].data();
		std::copy(plus_edges, plus_edges + ado_width_, &ado_plus_[id * ado_width_]);
		// copy minus edges
		auto minus_edges = graph.minus_edges()[id].data();
		std::copy(minus_edges, minus_edges + ado_width_, &ado_minus_[id * ado_width_]);
	}

	// initialise memory
	// NOTE: site to state conversion for the hamiltonian happens here
	set_hamiltonian(reinterpret_cast<const real_t*>(state_hamiltonian(cfg.system_hamiltonian(), sites_to_states_mode_).data()));

	// compute kernel input
	DEBUG_ONLY( std::cout << "instance::instance(..): begin precompute_baths_input" << std::endl; )
	// TODO: this call is annoyingly slow when testing
	precompute_baths_input(cfg);
	DEBUG_ONLY( std::cout << "instance::instance(..): end precompute_baths_input" << std::endl; )

	// initialize y_n (i.e. the initial value of our initial value problem)
	null_hierarchy();

	// NOTE: set_hierarchy_top() needs to be called to produce actual results, for testing: set (global) first element to 1.0
}

// constructor from saved state
instance::instance(const std::string& filename)
{
	allocate();
	read_file(filename);
}

// copy ctor
instance::instance(const instance& other)
{
	// copy shallow members
	version_ = other.version_;
	hierarchy_top_only_ = other.hierarchy_top_only_;
	hierarchy_layout_ = other.hierarchy_layout_;
	hamiltonian_layout_ = other.hamiltonian_layout_;
	sites_to_states_mode_ = other.sites_to_states_mode_;

	time_ = other.time_;
	first_non_plus_id_ = other.first_non_plus_id_;

	states_ = other.states_;
	matrices_ = other.matrices_;
	matsubaras_ = other.matsubaras_;
	baths_per_state_ = other.baths_per_state_;
	baths_ = other.baths_;
	ado_width_ = other.ado_width_;
	ado_depth_ = other.ado_depth_;

	allocate();

	// copy dynamic members
	std::copy(other.hierarchy_, other.hierarchy_ + other.size_hierarchy(), hierarchy_);
	std::copy(other.hamiltonian_, other.hamiltonian_ + other.size_hamiltonian(), hamiltonian_);

	std::copy(other.ado_tuples_, other.ado_tuples_ + other.size_ado_tuples(), ado_tuples_);
	std::copy(other.ado_plus_, other.ado_plus_ + other.size_ado_plus(), ado_plus_);
	std::copy(other.ado_minus_, other.ado_minus_ + other.size_ado_minus(), ado_minus_);

	std::copy(other.pf_same_sum_, other.pf_same_sum_ + other.size_pf_same_sum(), pf_same_sum_);
	std::copy(other.pf_same_, other.pf_same_ + other.size_pf_same(), pf_same_);
	std::copy(other.pf_circ_, other.pf_circ_ + other.size_pf_circ(), pf_circ_);
	std::copy(other.cbk_, other.cbk_ + other.size_cbk(), cbk_);
	std::copy(other.cbk_sqrt_, other.cbk_sqrt_ + other.size_cbk_sqrt(), cbk_sqrt_);
}

instance::~instance()
{
	free();
}

void instance::allocate()
{
	hierarchy_ = memory::aligned::instance().allocate<real_t>(size_hierarchy());
	hamiltonian_ = memory::aligned::instance().allocate<real_t>(size_hamiltonian());

	ado_tuples_ = memory::aligned::instance().allocate<std::int32_t>(size_ado_tuples());
	ado_plus_ = memory::aligned::instance().allocate<std::int32_t>(size_ado_plus());
	ado_minus_ = memory::aligned::instance().allocate<std::int32_t>(size_ado_minus());

	pf_same_sum_ = memory::aligned::instance().allocate<real_t>(size_pf_same_sum());
	pf_same_ = memory::aligned::instance().allocate<real_t>(size_pf_same());
	pf_circ_ = memory::aligned::instance().allocate<real_t>(size_pf_circ());
	cbk_ = memory::aligned::instance().allocate<real_t>(size_cbk());
	cbk_sqrt_ = memory::aligned::instance().allocate<real_t>(size_cbk_sqrt());
}

void instance::free()
{
	memory::aligned::instance().free(hierarchy_);
	memory::aligned::instance().free(hamiltonian_);

	memory::aligned::instance().free(ado_tuples_);
	memory::aligned::instance().free(ado_plus_);
	memory::aligned::instance().free(ado_minus_);

	memory::aligned::instance().free(pf_same_sum_);
	memory::aligned::instance().free(pf_same_);
	memory::aligned::instance().free(pf_circ_);
	memory::aligned::instance().free(cbk_);
	memory::aligned::instance().free(cbk_sqrt_);
}

void instance::precompute_baths_input(const config& cfg)
{

	complex_t* PFSAMESUMIn = reinterpret_cast<complex_t*>(this->pf_same_sum_);
	complex_t* PFSAMEIn = reinterpret_cast<complex_t*>(this->pf_same_);
	complex_t* PFCIRCIn = reinterpret_cast<complex_t*>(this->pf_circ_);
	complex_t* CBKIn = reinterpret_cast<complex_t*>(this->cbk_);
	real_t* CBKSqrtIn = reinterpret_cast<real_t*>(this->cbk_sqrt_);

	// FIXME: location of physical constants definitions?
	const real_t& hbar = constants::h_bar;

	real_t beta = 1.0 / (constants::k_b * cfg.baths_temperature());

	for(int_t b = 0; b < cfg.baths_number(); ++b)
	{
		complex_t aux;

		complex_t lambda_b { cfg.baths_lambda().at(b) }; // complexify
		complex_t nu_b { 1.0/cfg.baths_invnu().at(b), cfg.baths_uppercase_omega().at(b) }; // complexify

		PFCIRCIn[b] = nu_b * lambda_b / hbar;
		aux = 2.0 * lambda_b / (beta * hbar * hbar * nu_b);

		for (int_t k = 0; k < matsubaras_; ++k)
		{
			complex_t cbk, cbk0, nubk;
			// only the k=0 term differs with respect to complex conjugation
			if (k == 0)
			{
				//NOTE: with PADE
				complex_t auxsum=0.0;
				for(int_t j = 0; j < matsubaras_; ++j)
				{
					real_t eta = constants::eta_pade[(matsubaras_ - 1) * constants::max_pade + j];
					real_t xi = constants::xi_pade [(matsubaras_ - 1) * constants::max_pade + j];
					auxsum += (2.0 * eta * nu_b * nu_b) / (xi * xi / (beta * beta * hbar * hbar) - nu_b * nu_b);
				}
				cbk0 = 2.0 * lambda_b / (beta * hbar) * (1.0 - auxsum);

				// Alternative without PADE
				//cbk0  =lambda_b*nu_b*(1.0/tan(0.5*beta*hbar*nu_b));

				cbk = cbk0;
				nubk = nu_b;
			}
			else
			{

				//NOTE: with PADE
				real_t eta = constants::eta_pade[(matsubaras_ - 1) * constants::max_pade + k];
				real_t xi = constants::xi_pade[(matsubaras_ - 1) * constants::max_pade + k];
				cbk = 4.0 * lambda_b * nu_b / (beta * beta * hbar * hbar) * (eta * xi) / (xi * xi / (beta * beta * hbar * hbar) - nu_b * nu_b);
				nubk = xi / (beta*hbar);

				// Alternative without PADE
				//cbk=8.0*real_t(k)*M_PI*lambda_b*nu_b/(4.0*k*k*M_PI*M_PI-beta*beta*nu_b*nu_b*hbar*hbar);
				//nubk=2.0*real_t(k)*M_PI/(beta*hbar);

			}
			// FIXME we define a mapping here
			CBKIn[b * matsubaras_ + k] = cbk;
			CBKSqrtIn[b * matsubaras_ + k] = sqrt(abs(cbk));
			aux -= cbk / (hbar * nubk);
		}
		PFSAMEIn[b] = aux;
	}

	// pre-compute outer summation:
	// TODO: does not work with filtering, as it assumes all matrices are included in the propagation
	// TODO: do when coding post-filter compactify...

	for (uid_t id = 0; id < matrices_; ++id)
	{
		// long double needed due to large summation to avoid cancellation effects
		long double zr = 0.0L;
		long double zi = 0.0L;

		for (int b = 0; b < cfg.baths_number(); ++b)
		{
			complex_t nu_b { 1.0 / cfg.baths_invnu().at(b), cfg.baths_uppercase_omega().at(b) }; // complexify

			for (int k = 0; k < matsubaras_; ++k)
			{
				long double nubk_r,nubk_i;
				if (k==0)
				{
					nubk_r = static_cast<long double>(real(nu_b));
					nubk_i = static_cast<long double>(imag(nu_b));
				}
				else
				{
					// NOTE: with PADE
					nubk_r = constants::xi_pade[(matsubaras_ - 1) * constants::max_pade + k] / static_cast<long double>(beta*hbar);
					// Alternative without PADE
					//nubk_r=2.0L*(long double)M_PI*(long double)k/(beta*hbar);

					nubk_i = 0.0L;
				}
				// NOTE: we imply a mapping here
				int m = b * matsubaras_ + k;

				zr -= nubk_r * static_cast<long double>(ado_tuples_[id * ado_width_ + m]);
				zi -= nubk_i * static_cast<long double>(ado_tuples_[id * ado_width_ + m]);
			}
		}
		PFSAMESUMIn[id] = complex_t(zr, zi);
	} // sum over all HEOM matrices

	// output for comparison
//	DEBUG_ONLY( array_to_file("HAMIn", reinterpret_cast<complex_t*>(this->hamiltonian_), this->size_hamiltonian_byte() / 2 / sizeof(real_t)); )
//	DEBUG_ONLY( array_to_file("ADOTupleIn", this->ado_tuples_, this->size_ado_tuples_byte() / sizeof(std::int32_t)); )
//	DEBUG_ONLY( array_to_file("PlusIndexIn", this->ado_plus_, this->size_ado_plus_byte() / sizeof(std::int32_t)); )
//	DEBUG_ONLY( array_to_file("MinusIndexIn", this->ado_minus_, this->size_ado_minus_byte() / sizeof(std::int32_t)); )
//	DEBUG_ONLY( array_to_file("PFSAMESUMIn", PFSAMESUMIn, this->size_pf_same_sum_byte() / 2 / sizeof(real_t)); )
//	DEBUG_ONLY( array_to_file("PFSAMEIn", PFSAMEIn, this->size_pf_same_byte() / 2 / sizeof(real_t)); )
//	DEBUG_ONLY( array_to_file("PFCIRCIn", PFCIRCIn, this->size_pf_circ_byte() / 2 / sizeof(real_t)); )
//	DEBUG_ONLY( array_to_file("CBKIn", CBKIn, this->size_cbk_byte() / 2 / sizeof(real_t)); )
}

template<typename T>
std::ifstream::char_type* cast_to_char(T* ptr)
{
	return reinterpret_cast<std::ifstream::char_type*>(ptr);
}

bool instance::read_file(std::string filename)
{
	try {
		std::ifstream file;
		file.open(filename, std::ios::in | std::ios::binary);

		if (file.fail())
			throw config_error("could not open instance file: " + filename);

		// format specifics
		file.read(cast_to_char(&version_), sizeof version_);
		file.read(cast_to_char(&hierarchy_top_only_), sizeof hierarchy_top_only_);
		file.read(cast_to_char(&hierarchy_layout_), sizeof hierarchy_layout_);
		file.read(cast_to_char(&hamiltonian_layout_), sizeof hamiltonian_layout_);
		file.read(cast_to_char(&sites_to_states_mode_), sizeof sites_to_states_mode_);

		// variables
		file.read(cast_to_char(&time_), sizeof time_);
		file.read(cast_to_char(&first_non_plus_id_), sizeof first_non_plus_id_);

		// sizes
		file.read(cast_to_char(&states_), sizeof states_);
		file.read(cast_to_char(&matrices_), sizeof matrices_);
		file.read(cast_to_char(&matsubaras_), sizeof matsubaras_);
		file.read(cast_to_char(&baths_per_state_), sizeof baths_per_state_);
		file.read(cast_to_char(&baths_), sizeof baths_);
		file.read(cast_to_char(&ado_width_), sizeof ado_width_);
		file.read(cast_to_char(&ado_depth_), sizeof ado_depth_);

		// buffers
		file.read(cast_to_char(hierarchy_), hierarchy_top_only_ ? size_hierarchy_top_byte() : size_hierarchy_byte());
		file.read(cast_to_char(hamiltonian_), size_hamiltonian_byte());
		file.read(cast_to_char(ado_tuples_), size_ado_tuples_byte());
		file.read(cast_to_char(ado_plus_), size_ado_plus_byte());
		file.read(cast_to_char(ado_minus_), size_ado_minus_byte());
		file.read(cast_to_char(pf_same_sum_), size_pf_same_sum_byte());
		file.read(cast_to_char(pf_same_), size_pf_same_byte());
		file.read(cast_to_char(pf_circ_), size_pf_circ_byte());
		file.read(cast_to_char(cbk_), size_cbk_byte());
		file.read(cast_to_char(cbk_sqrt_), size_cbk_sqrt_byte());

		file.close();
	} catch (std::exception& e) {
		std::cerr << "instance::read_file(): Exception for file '" << filename << "': " << e.what() << std::endl;
		return false;
	}

	return true;

}

bool instance::write_file(std::string filename)
{
	try {
		std::ofstream file;
		file.open(filename, std::ios::out | std::ios::binary | std::ios::trunc);

		// format specifics
		file.write(cast_to_char(&version_), sizeof version_);
		file.write(cast_to_char(&hierarchy_top_only_), sizeof hierarchy_top_only_);
		file.write(cast_to_char(&hierarchy_layout_), sizeof hierarchy_layout_);
		file.write(cast_to_char(&hamiltonian_layout_), sizeof hamiltonian_layout_);
		file.write(cast_to_char(&sites_to_states_mode_), sizeof sites_to_states_mode_);

		// variables
		file.write(cast_to_char(&time_), sizeof time_);
		file.write(cast_to_char(&first_non_plus_id_), sizeof first_non_plus_id_);

		// sizes
		file.write(cast_to_char(&states_), sizeof states_);
		file.write(cast_to_char(&matrices_), sizeof matrices_);
		file.write(cast_to_char(&matsubaras_), sizeof matsubaras_);
		file.write(cast_to_char(&baths_per_state_), sizeof baths_per_state_);
		file.write(cast_to_char(&baths_), sizeof baths_);
		file.write(cast_to_char(&ado_width_), sizeof ado_width_);
		file.write(cast_to_char(&ado_depth_), sizeof ado_depth_);

		// buffers
		file.write(cast_to_char(hierarchy_), hierarchy_top_only_ ? size_hierarchy_top_byte() : size_hierarchy_byte());
		file.write(cast_to_char(hamiltonian_), size_hamiltonian_byte());
		file.write(cast_to_char(ado_tuples_), size_ado_tuples_byte());
		file.write(cast_to_char(ado_plus_), size_ado_plus_byte());
		file.write(cast_to_char(ado_minus_), size_ado_minus_byte());
		file.write(cast_to_char(pf_same_sum_), size_pf_same_sum_byte());
		file.write(cast_to_char(pf_same_), size_pf_same_byte());
		file.write(cast_to_char(pf_circ_), size_pf_circ_byte());
		file.write(cast_to_char(cbk_), size_cbk_byte());
		file.write(cast_to_char(cbk_sqrt_), size_cbk_sqrt_byte());

		file.close();
	} catch (std::exception& e) {
		std::cerr << "instance::write_file(): Exception for file '" << filename << "': " << e.what() << std::endl;
		return false;
	}

	return true;
}

size_t instance::size_hierarchy() const
{
	return states_ * states_ * matrices_ * 2;
}

size_t instance::size_hierarchy_top() const
{
	return states_ * states_ * 2; // just the top of the hierarchy
}

size_t instance::size_hamiltonian() const
{
	return states_ * states_ * 2;
}

size_t instance::size_ado_tuples() const
{
	return matrices_ * ado_width_;
}

size_t instance::size_ado_plus() const
{
	return matrices_ * ado_width_;
}

size_t instance::size_ado_minus() const
{
	return matrices_ * ado_width_;
}

size_t instance::size_pf_same_sum() const
{
	return matrices_ * 2;
}

size_t instance::size_pf_same() const
{
	return baths_ * 2;
}

size_t instance::size_pf_circ() const
{
	return baths_ * 2;
}

size_t instance::size_cbk() const
{
	return baths_ * matsubaras_ * 2;
}

size_t instance::size_cbk_sqrt() const
{
	return baths_ * matsubaras_;
}

size_t instance::allocated_byte() const
{
	return size_hierarchy_byte() +
	       size_hamiltonian_byte() +
	       size_ado_tuples_byte() +
	       size_ado_plus_byte() +
	       size_ado_minus_byte() +
	       size_pf_same_sum_byte() +
	       size_pf_same_byte() +
	       size_pf_circ_byte() +
	       size_cbk_byte() +
	       size_cbk_sqrt_byte();
}

void instance::null_hierarchy() const
{
	for (size_t i = 0; i < this->size_hierarchy_byte() / sizeof(real_t); ++i) {
		hierarchy_[i] = 0.0;
	}
}

void instance::set_hamiltonian(const complex_matrix_t& new_hamiltonian) const
{
	set_hamiltonian(reinterpret_cast<const real_t*>(new_hamiltonian.data()));
}

void instance::set_hamiltonian(const real_t* new_hamiltonian) const
{
	for (size_t j = 0; j < size_hamiltonian(); ++j) {
		hamiltonian_[j] = new_hamiltonian[j];
	}
}

void instance::set_hamiltonian(const std::valarray<complex_t> new_hamiltonian) const
{
	for (size_t j = 0; j < size_hamiltonian(); j += 2) {
		hamiltonian_[j] = new_hamiltonian[j / 2].real(); // NOTE: intended integer division
		hamiltonian_[j+1] = new_hamiltonian[j / 2].imag();
	}
}

void instance::get_hierarchy_top(complex_t* buffer) const
{
	// TODO: adapt for when other memory layouts are introduced, maybe use some overloaded []
	for (int_t j = 0; j < states_ * states_; j += 1) {
		buffer[j] = complex_t( hierarchy_[j * 2], hierarchy_[j * 2 + 1] );
	}
}

void instance::set_hierarchy_top() const
{
	hierarchy_[0] = 1.0;
}

void instance::set_hierarchy_top(const complex_t* new_hierarchy_top) const
{
	// TODO: adapt for when other memory layouts are introduced, maybe use some overloaded []
	for (int_t j = 0; j < states_ * states_; ++j) {
		hierarchy_[2 * j] = new_hierarchy_top[j].real();
		hierarchy_[2 * j + 1] = new_hierarchy_top[j].imag();
	}
}

} // namespace heom