#include "account.h"

#include "numbers.h"
#include "../blake2/blake2.h"
#include "../duthomhas/csprng.hpp"
#include "../ed25519-donna/ed25519.h"
#include "../qrcode/QrCode.hpp"

#include "core/io/file_access_encrypted.h"

#include <boost/multiprecision/cpp_int.hpp>
#include <vector>

const char base32_characters[33] = "13456789abcdefghijkmnopqrstuwxyz";
const char hex_characters[17] = "0123456789ABCDEF";
char const * account_reverse ("~0~1234567~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~89:;<=>?@AB~CDEFGHIJK~LMNO~~~~~");

int key_string_to_bytes(String const & in, std::array<uint8_t, 32> & out){
    // Seed, Private Key, and Public Key should all be 64 Hexadecimal characters (representing 32 bytes of data)
    ERR_FAIL_COND_V_MSG(in.length() != 64, 1, "String has incorrect length: " + itos(in.length()) + " String: " + in);
    for(int i = 0; i < in.length(); i+=2){
        out[i/2] = in.substr(i, 2).hex_to_int(false);
    }
    return 0;
}

void key_string_to_bytes16(String const & in, std::array<uint8_t, 16> & out){
    // Balance should be 32 Hexadecimal characters (representing 16 bytes of data)
    ERR_FAIL_COND_MSG(in.length() != 32, "String has incorrect length: " + itos(in.length()) + " String: " + in);
    for(int i = 0; i < in.length(); i+=2){
        out[i/2] = in.substr(i, 2).hex_to_int(false);
    }
}

String uint8_t_to_hex(uint8_t const & in) {
    char hex[3] = {hex_characters[in/16], hex_characters[in%16], '\0'};
    return String(hex);
}

template <typename Iter>
String bytes_to_key_string(Iter first, Iter last) {
    String out;
    for(; first != last; ++first){
        out += uint8_t_to_hex(*first);
    }
    return out;
}

String encode_base32(uint8_t* bytes, size_t length) {
    int leftover = (length * 8) % 5;
    int offset = leftover == 0 ? 0 : 5 - leftover;

    int value = 0, bits = 0;

    String output = "";

    for (size_t i = 0; i < length; i++) {
        value = (value << 8) | bytes[i];
        bits += 8;
        while (bits >= 5) {
            output += (base32_characters[(value >> (bits + offset - 5)) & 31]);
            bits -= 5;
        }
    }

    if (bits > 0) {
        output += (base32_characters[(value << (5 - (bits + offset))) & 31]);
    }

    return output;
}

uint8_t account_decode (char value)
{
    ERR_FAIL_COND_V_MSG(value < '0' || value >= '~', '~', "Invalid address: checksum value out of bounds with " + itos(value));
	auto result (account_reverse[value - 0x30]);
	if (result != '~')
	{
		result -= 0x30;
	}
	return result;
}

int NanoAccount::set_address(String const & a) {
    if(this->address.empty()){
        this->address = a;

        String encoded_val = "";
        if(address.begins_with("nano_")){
            ERR_FAIL_COND_V_MSG(address.length() != 65, 1, "Invalid nano address");
            encoded_val = address.substr(5);
        } else if(address.begins_with("xrb_")){
            ERR_FAIL_COND_V_MSG(address.length() != 64, 1, "Invalid xrb address");
            encoded_val = address.substr(4);
        } else ERR_FAIL_V_MSG(1, "Address is invalid.");

        ERR_FAIL_COND_V_MSG(encoded_val[0] != '1' && encoded_val[0] != '3', 1, "Invalid address");

        boost::multiprecision::uint512_t decoded;
        for(int i = 0; i < encoded_val.length(); i++){
            uint8_t byte = account_decode(encoded_val[i]);
            ERR_FAIL_COND_V_MSG(byte == '~', 1, "Invalid address");
            decoded <<= 5;
            decoded += byte;
        }

        boost::multiprecision::uint256_t a = (decoded >> 40).convert_to<boost::multiprecision::uint256_t>();
        boost::multiprecision::export_bits(a, public_key.rbegin(), 8, false);
        uint64_t checksum (decoded & static_cast<uint64_t>(0xffffffffff));

        uint64_t validation (0);
        blake2b_state hash;
        blake2b_init (&hash, 5);
        blake2b_update (&hash, public_key.data(), public_key.size());
        blake2b_final (&hash, reinterpret_cast<uint8_t *> (&validation), 5);
        ERR_FAIL_COND_V_MSG(checksum != validation, 1, "Checksum does not match");
    }
    return 0;
}

NanoAccount::NanoAccount() {
    boost::multiprecision::uint256_t preamble_num(6);

    for (auto i (preamble.rbegin ()), n (preamble.rend ()); i != n; ++i)
	{
		*i = static_cast<uint8_t> (preamble_num & static_cast<uint8_t> (0xff));
		preamble_num >>= 8;
	}
}

void NanoAccount::generate_keys_and_address() {
    blake2b_state hash;
    blake2b_init(&hash, private_key.size());
    blake2b_update(&hash, seed.data(), seed.size());
    blake2b_update(&hash, reinterpret_cast<uint8_t *> (&index), sizeof(uint32_t));
    blake2b_final(&hash, private_key.data(), private_key.size());

    // Create public key from private key
    ed25519_publickey(private_key.data(), public_key.data());

    // Create address from public key
    std::array<uint8_t, 5> checksum;
    blake2b(checksum.data(), 5, public_key.data(), 32, NULL, 0);
    std::reverse(checksum.begin(), checksum.end());

    address = "nano_" + encode_base32(public_key.data(), 32) + encode_base32(checksum.data(), 5);
}

void NanoAccount::initialize_with_new_seed() {
    duthomhas::csprng rng;
    rng(seed);

    generate_keys_and_address();
}

int NanoAccount::set_seed(String const & s) {
    return set_seed_and_index(s, 0);
}

int NanoAccount::set_seed_and_index(String const & s, uint32_t index) {
    if(key_string_to_bytes(s, seed)) return 1;
    this->index = index;

    generate_keys_and_address();
    return 0;
}

Ref<ImageTexture> get_qr_code_for_text(String text) {
    const char * c_text = text.utf8().ptr();
    qrcodegen::QrCode qr = qrcodegen::QrCode::encodeText(c_text, qrcodegen::QrCode::Ecc::MEDIUM);

    int size = qr.getSize();

    Ref<Image> image(memnew(Image));
    image->create(size, size, false, Image::Format::FORMAT_L8);
    image->lock();
    for(int x = 0; x < size; x++) {
        for(int y = 0; y < size; y++) {
            if(qr.getModule(x, y)) {
                image->set_pixel(x, y, Color(0, 0, 0));
            } else {
                image->set_pixel(x, y, Color(1, 1, 1));
            }
        }
    }
    image->unlock();

    Ref<ImageTexture> texture(memnew(ImageTexture));
    texture->create_from_image(image, 0); // 0 means no flags: no mipmap, filter, or repeat (none of which we would want on a qr code)
    return texture;
}

Ref<ImageTexture> NanoAccount::get_qr_code() {
    return get_qr_code_for_text("nano:" + address);
}

Ref<ImageTexture> NanoAccount::get_qr_code_with_amount(Ref<NanoAmount> amount) {
    String text = "nano:" + address;
    text += "?amount=" + amount->get_raw_amount();
    return get_qr_code_for_text(text);
}

uint256_union NanoAccount::internal_block_hash(String previous, Ref<NanoAccount> representative, Ref<NanoAmount> balance, String link) {
    nano::uint256_union prev_u(previous);
    nano::uint256_union link_u(link);

    nano::uint256_union result;
	blake2b_state hash_l;
	auto status (blake2b_init (&hash_l, sizeof (result.bytes)));
    nano::uint256_union preamble (6);
	blake2b_update (&hash_l, preamble.bytes.data (), preamble.bytes.size ());
    blake2b_update (&hash_l, public_key.data (), sizeof (public_key));
	blake2b_update (&hash_l, prev_u.bytes.data (), sizeof (prev_u.bytes));
	blake2b_update (&hash_l, representative->public_key.data (), sizeof (representative->public_key));
	blake2b_update (&hash_l, balance->get_amount().bytes.data (), sizeof (balance->get_amount().bytes));
	blake2b_update (&hash_l, link_u.bytes.data (), sizeof (link_u.bytes));
    status = blake2b_final (&hash_l, result.bytes.data (), sizeof (result.bytes));
    ERR_FAIL_COND_V_MSG(status, uint256_union(), "Block hashing failed");
	return result;
}

String NanoAccount::block_hash(String previous, Ref<NanoAccount> representative, Ref<NanoAmount> balance, String link) {
    return internal_block_hash(previous, representative, balance, link).to_string();
}

String NanoAccount::sign(String previous, Ref<NanoAccount> representative, Ref<NanoAmount> balance, String link) {
    uint256_union message = internal_block_hash(previous, representative, balance, link);

    uint512_union result;
	ed25519_sign (message.bytes.data (), sizeof (message.bytes), private_key.data (), public_key.data (), result.bytes.data ());
	return result.to_string();
}

String NanoAccount::get_seed() {
    return bytes_to_key_string(seed.begin(), seed.end());
}

String NanoAccount::get_private_key() {
    return bytes_to_key_string(private_key.begin(), private_key.end());
}

String NanoAccount::get_public_key() {
    return bytes_to_key_string(public_key.begin(), public_key.end());
}

void NanoAccount::_bind_methods() {
    ClassDB::bind_method(D_METHOD("set_address", "address"), &NanoAccount::set_address);
    ClassDB::bind_method(D_METHOD("get_address"), &NanoAccount::get_address);
    ADD_PROPERTY(PropertyInfo(Variant::STRING, "address", PROPERTY_HINT_TYPE_STRING, ""), "set_address", "get_address");

    ClassDB::bind_method(D_METHOD("initialize_with_new_seed"), &NanoAccount::initialize_with_new_seed);
    ClassDB::bind_method(D_METHOD("set_seed", "seed"), &NanoAccount::set_seed);
    ClassDB::bind_method(D_METHOD("set_seed_and_index", "seed", "index"), &NanoAccount::set_seed_and_index);
    ClassDB::bind_method(D_METHOD("get_seed"), &NanoAccount::get_seed);
    ADD_PROPERTY(PropertyInfo(Variant::STRING, "seed", PROPERTY_HINT_TYPE_STRING, ""), "set_seed", "get_seed");

    ClassDB::bind_method(D_METHOD("get_private_key"), &NanoAccount::get_private_key);
    ClassDB::bind_method(D_METHOD("get_public_key"), &NanoAccount::get_public_key);
    ClassDB::bind_method(D_METHOD("get_index"), &NanoAccount::get_index);

    ClassDB::bind_method(D_METHOD("get_qr_code"), &NanoAccount::get_qr_code);
    ClassDB::bind_method(D_METHOD("get_qr_code_with_amount", "amount"), &NanoAccount::get_qr_code_with_amount);

    ClassDB::bind_method(D_METHOD("block_hash", "previous", "representative", "balance", "link"), &NanoAccount::block_hash);
    ClassDB::bind_method(D_METHOD("sign", "previous", "representative", "balance", "link"), &NanoAccount::sign);
}


//
// Copyright Jason Rice 2015
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
//
#ifndef NBDL_MPDEF_COMPOSE_COLLECTORS_HPP
#define NBDL_MPDEF_COMPOSE_COLLECTORS_HPP

#include <mpdef/list.hpp>

#include <boost/hana.hpp>
#include <utility>

namespace mpdef {

namespace hana = boost::hana;

struct compose_collectors_fn
{
  template<std::size_t... i, typename... Collector>
  constexpr auto helper(std::index_sequence<i...>, Collector&&... collector) const
  {
    return hana::demux(mpdef::make_list)
    (
      hana::demux(std::forward<Collector>(collector))
      (
        hana::demux(hana::reverse_partial(hana::at, hana::int_c<i>))(hana::arg<1>),
        hana::arg<2>
      )...
    );
  }

  template<typename... Collector>
  constexpr auto operator()(Collector&&... collector) const
  {
    return helper(std::make_index_sequence<sizeof...(Collector)>{},
        std::forward<Collector>(collector)...);
  }
};
constexpr compose_collectors_fn compose_collectors{};

}//mpdef
#endif


#pragma once

#include <string>
#include <memory>
#include <map>
#include <boost/numeric/ublas/vector.hpp>
#include "SAMSON.hpp"
#include "SBCReferencePointer.hpp"
#include "SBCReferencePointerIndexer.hpp"

namespace ublas = boost::numeric::ublas;

/* Using smart pointers */
//template <typename T>
//using ADNPointer = std::shared_ptr<T>;

template <typename T>
using ADNPointer = SBCReferencePointer<T>;

//template <typename T>
//using ADNWeakPointer = std::weak_ptr<T>;

template <typename T>
using ADNWeakPointer = SBCReferencePointer<T>;

//template<typename T>
//using CollectionMap = std::map<int, ADNPointer<T>>;

template<typename T>
using CollectionMap = SBPointerIndexer<T>;

//using Position3D = ublas::vector<double>;
using Position3D = SBPosition3;

class Nameable {
public:
  Nameable() = default;
  ~Nameable() = default;
  Nameable(const Nameable& other);

  Nameable& operator=(const Nameable& other);

  void SetName(std::string name);
  std::string GetName() const;
private:
  std::string name_;
};

class Positionable {
public:
  Positionable() = default;
  ~Positionable() = default;
  Positionable(const Positionable& other);

  Positionable& operator=(const Positionable& other);

  void SetPosition(ublas::vector<double> pos);
  ublas::vector<double> GetPosition() const;
private:
  ublas::vector<double> position_;
};

class Identifiable {
public:
  Identifiable() = default;
  ~Identifiable() = default;
  Identifiable(const Identifiable& other);

  Identifiable& operator=(const Identifiable& other);

  void SetId(int id);
  int GetId() const;
private:
  int id_ = -1;
};

class Orientable {
public:
  Orientable();
  ~Orientable() = default;
  Orientable(const Orientable& other);

  Orientable& operator=(const Orientable& other);

  void SetE1(ublas::vector<double> e1);
  void SetE2(ublas::vector<double> e2);
  void SetE3(ublas::vector<double> e3);
  ublas::vector<double> GetE1() const;
  ublas::vector<double> GetE2() const;
  ublas::vector<double> GetE3() const;
private:
  ublas::vector<double> e1_;
  ublas::vector<double> e2_;
  ublas::vector<double> e3_;
};

template <class T>
class Collection {
public:
  Collection() = default;
  ~Collection() = default;
  Collection(const Collection<T>& other);

  Collection<T>& operator=(const Collection<T>& other);

  void AddElement(ADNPointer<T> elem, int id = -1);
  CollectionMap<T> GetCollection() const;
  void DeleteElement(int id);
  int GetLastKey();
  ADNPointer<T> GetElement(int id) const;
private:
  CollectionMap<T> collection_;
};

template<class T>
inline Collection<T>::Collection(const Collection<T>& other)
{
  *this = other;
}

template<class T>
inline Collection<T> & Collection<T>::operator=(const Collection<T>& other)
{
  if (&other == this) {
    return *this;
  }

  collection_ = other.GetCollection();

  return *this;
}

template<class T>
inline void Collection<T>::AddElement(ADNPointer<T> elem, int id)
{
  if (id == -1) {
    id = 0;
    if (collection_.size() > 0) id = collection_.rbegin()->first + 1;
  }
  collection_.insert(std::make_pair(id,elem));
}

template<class T>
inline CollectionMap<T> Collection<T>::GetCollection() const
{
  return collection_;
}

template<class T>
inline void Collection<T>::DeleteElement(int id)
{
  collection_.erase(id);
}

template<class T>
inline int Collection<T>::GetLastKey()
{
  int id = -1;
  if (collection_.size() > 0) {
    id = collection_.rbegin()->first;
  }
  return id;
}

template<class T>
inline ADNPointer<T> Collection<T>::GetElement(int id) const
{
  return collection_.at(id);
}


#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic> test_float_D_D(const Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic> & M)
{
	return M;
}
void export_float_D_D()
{
	boost::python::def("test_float_D_D",test_float_D_D);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, Eigen::Dynamic, 1> test_float_D_1(const Eigen::Matrix<float, Eigen::Dynamic, 1> & M)
{
	return M;
}
void export_float_D_1()
{
	boost::python::def("test_float_D_1",test_float_D_1);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, Eigen::Dynamic, 2> test_float_D_2(const Eigen::Matrix<float, Eigen::Dynamic, 2> & M)
{
	return M;
}
void export_float_D_2()
{
	boost::python::def("test_float_D_2",test_float_D_2);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, Eigen::Dynamic, 3> test_float_D_3(const Eigen::Matrix<float, Eigen::Dynamic, 3> & M)
{
	return M;
}
void export_float_D_3()
{
	boost::python::def("test_float_D_3",test_float_D_3);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, Eigen::Dynamic, 4> test_float_D_4(const Eigen::Matrix<float, Eigen::Dynamic, 4> & M)
{
	return M;
}
void export_float_D_4()
{
	boost::python::def("test_float_D_4",test_float_D_4);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, Eigen::Dynamic, 5> test_float_D_5(const Eigen::Matrix<float, Eigen::Dynamic, 5> & M)
{
	return M;
}
void export_float_D_5()
{
	boost::python::def("test_float_D_5",test_float_D_5);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, Eigen::Dynamic, 6> test_float_D_6(const Eigen::Matrix<float, Eigen::Dynamic, 6> & M)
{
	return M;
}
void export_float_D_6()
{
	boost::python::def("test_float_D_6",test_float_D_6);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 1, Eigen::Dynamic> test_float_1_D(const Eigen::Matrix<float, 1, Eigen::Dynamic> & M)
{
	return M;
}
void export_float_1_D()
{
	boost::python::def("test_float_1_D",test_float_1_D);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 1, 1> test_float_1_1(const Eigen::Matrix<float, 1, 1> & M)
{
	return M;
}
void export_float_1_1()
{
	boost::python::def("test_float_1_1",test_float_1_1);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 1, 2> test_float_1_2(const Eigen::Matrix<float, 1, 2> & M)
{
	return M;
}
void export_float_1_2()
{
	boost::python::def("test_float_1_2",test_float_1_2);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 1, 3> test_float_1_3(const Eigen::Matrix<float, 1, 3> & M)
{
	return M;
}
void export_float_1_3()
{
	boost::python::def("test_float_1_3",test_float_1_3);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 1, 4> test_float_1_4(const Eigen::Matrix<float, 1, 4> & M)
{
	return M;
}
void export_float_1_4()
{
	boost::python::def("test_float_1_4",test_float_1_4);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 1, 5> test_float_1_5(const Eigen::Matrix<float, 1, 5> & M)
{
	return M;
}
void export_float_1_5()
{
	boost::python::def("test_float_1_5",test_float_1_5);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 1, 6> test_float_1_6(const Eigen::Matrix<float, 1, 6> & M)
{
	return M;
}
void export_float_1_6()
{
	boost::python::def("test_float_1_6",test_float_1_6);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 2, Eigen::Dynamic> test_float_2_D(const Eigen::Matrix<float, 2, Eigen::Dynamic> & M)
{
	return M;
}
void export_float_2_D()
{
	boost::python::def("test_float_2_D",test_float_2_D);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 2, 1> test_float_2_1(const Eigen::Matrix<float, 2, 1> & M)
{
	return M;
}
void export_float_2_1()
{
	boost::python::def("test_float_2_1",test_float_2_1);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 2, 2> test_float_2_2(const Eigen::Matrix<float, 2, 2> & M)
{
	return M;
}
void export_float_2_2()
{
	boost::python::def("test_float_2_2",test_float_2_2);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 2, 3> test_float_2_3(const Eigen::Matrix<float, 2, 3> & M)
{
	return M;
}
void export_float_2_3()
{
	boost::python::def("test_float_2_3",test_float_2_3);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 2, 4> test_float_2_4(const Eigen::Matrix<float, 2, 4> & M)
{
	return M;
}
void export_float_2_4()
{
	boost::python::def("test_float_2_4",test_float_2_4);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 2, 5> test_float_2_5(const Eigen::Matrix<float, 2, 5> & M)
{
	return M;
}
void export_float_2_5()
{
	boost::python::def("test_float_2_5",test_float_2_5);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 2, 6> test_float_2_6(const Eigen::Matrix<float, 2, 6> & M)
{
	return M;
}
void export_float_2_6()
{
	boost::python::def("test_float_2_6",test_float_2_6);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 3, Eigen::Dynamic> test_float_3_D(const Eigen::Matrix<float, 3, Eigen::Dynamic> & M)
{
	return M;
}
void export_float_3_D()
{
	boost::python::def("test_float_3_D",test_float_3_D);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 3, 1> test_float_3_1(const Eigen::Matrix<float, 3, 1> & M)
{
	return M;
}
void export_float_3_1()
{
	boost::python::def("test_float_3_1",test_float_3_1);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 3, 2> test_float_3_2(const Eigen::Matrix<float, 3, 2> & M)
{
	return M;
}
void export_float_3_2()
{
	boost::python::def("test_float_3_2",test_float_3_2);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 3, 3> test_float_3_3(const Eigen::Matrix<float, 3, 3> & M)
{
	return M;
}
void export_float_3_3()
{
	boost::python::def("test_float_3_3",test_float_3_3);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 3, 4> test_float_3_4(const Eigen::Matrix<float, 3, 4> & M)
{
	return M;
}
void export_float_3_4()
{
	boost::python::def("test_float_3_4",test_float_3_4);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 3, 5> test_float_3_5(const Eigen::Matrix<float, 3, 5> & M)
{
	return M;
}
void export_float_3_5()
{
	boost::python::def("test_float_3_5",test_float_3_5);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 3, 6> test_float_3_6(const Eigen::Matrix<float, 3, 6> & M)
{
	return M;
}
void export_float_3_6()
{
	boost::python::def("test_float_3_6",test_float_3_6);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 4, Eigen::Dynamic> test_float_4_D(const Eigen::Matrix<float, 4, Eigen::Dynamic> & M)
{
	return M;
}
void export_float_4_D()
{
	boost::python::def("test_float_4_D",test_float_4_D);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 4, 1> test_float_4_1(const Eigen::Matrix<float, 4, 1> & M)
{
	return M;
}
void export_float_4_1()
{
	boost::python::def("test_float_4_1",test_float_4_1);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 4, 2> test_float_4_2(const Eigen::Matrix<float, 4, 2> & M)
{
	return M;
}
void export_float_4_2()
{
	boost::python::def("test_float_4_2",test_float_4_2);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 4, 3> test_float_4_3(const Eigen::Matrix<float, 4, 3> & M)
{
	return M;
}
void export_float_4_3()
{
	boost::python::def("test_float_4_3",test_float_4_3);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 4, 4> test_float_4_4(const Eigen::Matrix<float, 4, 4> & M)
{
	return M;
}
void export_float_4_4()
{
	boost::python::def("test_float_4_4",test_float_4_4);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 4, 5> test_float_4_5(const Eigen::Matrix<float, 4, 5> & M)
{
	return M;
}
void export_float_4_5()
{
	boost::python::def("test_float_4_5",test_float_4_5);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 4, 6> test_float_4_6(const Eigen::Matrix<float, 4, 6> & M)
{
	return M;
}
void export_float_4_6()
{
	boost::python::def("test_float_4_6",test_float_4_6);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 5, Eigen::Dynamic> test_float_5_D(const Eigen::Matrix<float, 5, Eigen::Dynamic> & M)
{
	return M;
}
void export_float_5_D()
{
	boost::python::def("test_float_5_D",test_float_5_D);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 5, 1> test_float_5_1(const Eigen::Matrix<float, 5, 1> & M)
{
	return M;
}
void export_float_5_1()
{
	boost::python::def("test_float_5_1",test_float_5_1);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 5, 2> test_float_5_2(const Eigen::Matrix<float, 5, 2> & M)
{
	return M;
}
void export_float_5_2()
{
	boost::python::def("test_float_5_2",test_float_5_2);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 5, 3> test_float_5_3(const Eigen::Matrix<float, 5, 3> & M)
{
	return M;
}
void export_float_5_3()
{
	boost::python::def("test_float_5_3",test_float_5_3);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 5, 4> test_float_5_4(const Eigen::Matrix<float, 5, 4> & M)
{
	return M;
}
void export_float_5_4()
{
	boost::python::def("test_float_5_4",test_float_5_4);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 5, 5> test_float_5_5(const Eigen::Matrix<float, 5, 5> & M)
{
	return M;
}
void export_float_5_5()
{
	boost::python::def("test_float_5_5",test_float_5_5);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 5, 6> test_float_5_6(const Eigen::Matrix<float, 5, 6> & M)
{
	return M;
}
void export_float_5_6()
{
	boost::python::def("test_float_5_6",test_float_5_6);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 6, Eigen::Dynamic> test_float_6_D(const Eigen::Matrix<float, 6, Eigen::Dynamic> & M)
{
	return M;
}
void export_float_6_D()
{
	boost::python::def("test_float_6_D",test_float_6_D);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 6, 1> test_float_6_1(const Eigen::Matrix<float, 6, 1> & M)
{
	return M;
}
void export_float_6_1()
{
	boost::python::def("test_float_6_1",test_float_6_1);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 6, 2> test_float_6_2(const Eigen::Matrix<float, 6, 2> & M)
{
	return M;
}
void export_float_6_2()
{
	boost::python::def("test_float_6_2",test_float_6_2);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 6, 3> test_float_6_3(const Eigen::Matrix<float, 6, 3> & M)
{
	return M;
}
void export_float_6_3()
{
	boost::python::def("test_float_6_3",test_float_6_3);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 6, 4> test_float_6_4(const Eigen::Matrix<float, 6, 4> & M)
{
	return M;
}
void export_float_6_4()
{
	boost::python::def("test_float_6_4",test_float_6_4);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 6, 5> test_float_6_5(const Eigen::Matrix<float, 6, 5> & M)
{
	return M;
}
void export_float_6_5()
{
	boost::python::def("test_float_6_5",test_float_6_5);
}

#include <Eigen/Core>

#include <numpy_eigen/boost_python_headers.hpp>
Eigen::Matrix<float, 6, 6> test_float_6_6(const Eigen::Matrix<float, 6, 6> & M)
{
	return M;
}
void export_float_6_6()
{
	boost::python::def("test_float_6_6",test_float_6_6);
}


#include "srges.h"

#include <boost/filesystem.hpp>
#include <boost/filesystem/fstream.hpp>

#define UPDATE_TIME UNIXTIME_MINUTE
/*
	srges is a rating method of a peer into a decentralized network.
	structure:
	- Toplist vector of pair cpubkey, peerrate
	- Mypeer cpubkey
	- PeerRate Class
		- memvector
		- pointers to memvector (rate, firstmeet, lastseen)
		- GetSerializedSize
		- Serialize
		- Unserialize
	- Leveldeb interface read / write into srges.dat
	- Addpeer method
	- Erase peer method (Clean)
	- Sort toplist
	- Standard rate Gain rating method
	- Standard rate Gain Curve
	- Endorse System broadcast method
	- Endorse System broadcast check
	- CheckSig method
*/

bool srges::Init(){
	/* please add here a cpubkey of god user */
	//unsigned char rnd[65]; // {0xff}
	//CPubKey rndpk(&rnd[0], (&rnd[64]+ sizeof(unsigned char)));
	//PeerRate pr;
	//toplist.push_back(std::pair(rndpk, pr));
	//toplist.sort(sort_toplist);
	if(!dbRead("toplist",toplist) && !dbWrite("toplist",toplist)){
		printf("srges::Init() : can't write toplist to db\n");
		return false;
	}
	if(!this->dbWrite("timerank", GetAdjustedTime())){
		printf("srges::Init() : can't write timerank to db\n");
		return false;
	}
	CPubKey null;
	std::pair<int, CPubKey> zero;
	zero.first = 5;
	zero.second = null;
	if(!this->dbWrite("endorse", zero)){
		printf("srges::Init() : can't write endorse zero to db\n");
		return false;
	}
	return true;
}

CPubKey srges::me(){
	CKey tme;
	dbRead("mysrges", tme);
	return tme.GetPubKey();
}

CKey srges::pme(){
	CKey tme;
	dbRead("mysrges", tme);
	return tme;
}

bool srges::AddPeer(const CPubKey& peer){
	printf("srges::AddPeer() : adding a peer in toplist\n");
	PeerRate prate;
	PeerRecord pr;
	pr.first = peer;
	pr.second = prate;
	toplist.push_back(pr);
	int64 timerank;
	if(!this->dbRead("timerank",timerank))
		return false;
	if(GetAdjustedTime() - timerank > UNIXTIME_HOUR)
		std::sort(toplist.begin(), toplist.end(), srges::sort_toplist);
	return true;
}

bool srges::Find(const CPubKey& peer, int &rank){
	tlist::iterator it = toplist.begin();
	int k = 1;
	while(it != toplist.end()){
		if(it->first == peer){
			rank = k;
			return true;
		}
		it++;
		k++;
	}
	rank = toplist.size();
	return true;
}
/*
	bool srges::Find(const CPubKey& peer, int &rank){
		int k = 1;
		BOOST_FOREACH(PeerRecord record,  toplist){
			if(it->first == peer){
				rank = k;
				return true;
			}
			k++;
		}
		rank = toplist.size();
		return true;
	}
*/
void srges::StandardRateGain(PeerRate* &pr){ // SHOULD BE OK
	if(pr->rate > AVG_RATE){
		if(pr->rate - FIXED_DECREASE < AVG_RATE)
			pr->rate = AVG_RATE;
		else
			pr->rate -= FIXED_DECREASE;
	}else if(pr->rate < AVG_RATE){
		int nextRate;
		srgCurve(pr->firstmeet, nextRate);
		pr->rate += nextRate;
	}
}

// srg Curve
void srges::srgCurve(const int &firstmeet, int&nextRate){
	int x = (GetAdjustedTime() - firstmeet) / UPDATE_TIME;
	printf("srges::srgCurve() : x value : %d\n", x);
	nextRate = SRG_COSTANT*abs(SRG_ELEVATOR*sin(x/SRG_OFFSET+SRG_PI)*log(x)/(x*sqrt(SRG_BASE)+SRG_OFFSET));
	printf("srges::srgCurve() : nextrate sum value : %d\n", nextRate);
}

bool srges::IsAffidable(const CPubKey& peer){
	if(!peer.IsValid())
		return false;
	if(toplist.size() > 1 )
	{
		int64 age;
		if(!this->dbRead("timerank", age))
			return false;

		if(GetAdjustedTime()-age > UPDATE_TIME &&
			this->LocalUpdateEndorse() &&
			this->dbWrite("timerank", GetAdjustedTime()) &&
			!this->Rank()
			)
			return false;
	}
	int rank;
	if(!Find(peer, rank)){
		printf("srges::IsAffidable() : cannot find peer in toplist!\n");
		if(!this->AddPeer(peer))
			return false;
		if(toplist.size() == 1)
			return true;
		else
			return false;
	}
	Refresh(peer);
	if(rank > 3 && toplist[rank-1].second.rate < SECOND_LVL)
		return false;
	printf("srges::IsAffidable() : Peer rate : %d\n", toplist[0].second.rate);
	return true;
}

bool srges::EndorsementSystem( const CPubKey& target, unsigned char &value, const bool& CorruptedAction){ /// FIXME
	tlist::iterator it = toplist.begin();
	std::pair<int, CPubKey> tmp;
	if(!dbRead("endorse", tmp)){
		value = 0xff;
		return false;
	}
	if( tmp.first-- <= 0 || tmp.second == target ){
		value = 0xff;
		return false;
	}
	if(!dbWrite("endorse", tmp))
		return false;

	int k = 1;
	do{
		if(it->first != target)
			continue;
		PeerRate& pr = it->second;
		if(pr.rate == MAX_RATE)
			value = 0;
		else if( !CorruptedAction )
			value = (int) 100/toplist.size() + (int)abs(k-toplist.size()/2);
		else
			value = ((int) 100/toplist.size() + (int)abs(k-toplist.size()/2))*(-1);
		value += 101;
		return true;
	}while(++it != toplist.end());
	return false;
}

bool srges::ProcessEndorse(const CPubKey &target, const CPubKey &source, const unsigned char&value, const std::vector<unsigned char> &vchSig){
	if(!source.IsValid() || !target.IsValid())
		return false;
	unsigned char temp[ target.size()*2 + sizeof(unsigned char) ];
    memcpy(temp, target.begin(), target.size());
    memcpy(temp+source.size(), &value, sizeof(unsigned char));
    memcpy(temp+source.size()+sizeof(unsigned char), source.begin(), source.size());
    uint256 thash = Hash(temp, temp+source.size()*2 + sizeof(unsigned char));

    if(!source.Verify(thash, vchSig))
    	return false;

    PeerRate* targetPr;
    if(!Find(target, targetPr))
    	return false;

    PeerRate* sourcePr;
    if(!Find(source, sourcePr) || sourcePr->freeEndorsement <= 0)
    	return false;

    int rank;
    if(!Find(source, rank) || ( rank > 3 && sourcePr->rate < SECOND_LVL ))
    	return false;

    double percentage;
    OnWholeRate(sourcePr, percentage);
    percentage *= (((int)value)-101)/100;
    sourcePr->freeEndorsement --;

    targetPr->rate += percentage*MAX_ENDORSEMENT_RATE_VALUE;

	int64 age;
	if(!this->dbRead("timerank", age))
		return false;
	if(GetAdjustedTime()-age > UNIXTIME_MINUTE && !this->Rank())
		return false;
	if(!this->dbWrite("timerank", GetAdjustedTime()))
		return false;

	return true;
}

bool srges::Clean(PeerRate*&pr){
	if( GetAdjustedTime() - pr->lastseen > UNIXTIME_DAY ){
		tlist::iterator it = toplist.begin();
		while(it != toplist.end()){
			if(it->second == *pr){
				toplist.erase(it);
				return true;
			}
			it++;
		}
	}
	return false;
}

bool srges::Refresh(const CPubKey& peer){
	PeerRate* pr;
	if(!this->Find(peer, pr)){
		AddPeer(peer);
		return true;
	}
	if(Clean(pr))
		return false;
	printf("srges::Refresh() : GetAdjustedTime : %lld\n", GetAdjustedTime());
	printf("srges::Refresh() : pr->lastUpdateRate : %lld\n", pr->lastUpdateRate);
	if(GetAdjustedTime()- pr->lastUpdateRate > UNIXTIME_MINUTE && GetAdjustedTime() - pr->lastseen < UNIXTIME_DAY){
		printf("srges::Refresh() : casting srg\n");
		StandardRateGain(pr);
		pr->freeEndorsement = MAX_ENDORSEMENT_PER_UPDATE;
	}
	return true;
}

bool srges::Rank(){
	tlist toplist;
	if(!this->dbRead("toplist", toplist))
		return false;
	std::sort(toplist.begin(), toplist.end(), srges::sort_toplist);
	return true;
}

// method used for sorting toplist vector
bool srges::sort_toplist(PeerRecord x, PeerRecord y){
	return x.second.rate >= y.second.rate;
}
/*
bool srges::Find(const CPubKey& peer, PeerRate* &pr){
	tlist::iterator it = toplist.begin();
	int k = 1;
	while(it != toplist.end()){
		if(it->first == peer){
			pr = &it->second;
			return true;
		}
		it++;
		k++;
	}
	pr = NULL;
	return false;
}
*/

bool srges::Find(const CPubKey& peer, PeerRate* &pr){
	tlist::iterator it = toplist.begin();
	int k = 1;
	while(it != toplist.end()){
		if(it->first == peer){
			pr = &it->second;
			return true;
		}
		it++;
		k++;
	}
	pr = NULL;
	return false;
}
bool srges::WholeRate( int64& WholeRate ){
	tlist::iterator it = toplist.begin();
	do{
		WholeRate += it->second.rate;
	}while(++it != toplist.end());
	return true;
}

bool srges::OnWholeRate( PeerRate* &peer, double &percentage){
	int64 wr;
	this->WholeRate(wr);
	percentage = peer->rate/wr;
	return true;
}

bool srges::CanEndorse( const CPubKey & peer ){
	PeerRate * pr;
	if(!Find(peer, pr))
		return false;
	if(pr->freeEndorsement <= 0)
		return false;
	return true;
}

bool srges::LocalUpdateEndorse(){
	std::pair<int, CPubKey> tmp;
	if(!dbRead("endorse", tmp))
		return false;
	tmp.first = 5;
	if(!dbWrite("endorse", tmp))
		return false;
	return true;
}
	/*
	PeerRate pr;
	if(!Read(sender, pr)
		return false;
	// if cannot rate push him to 0
	if(pr.rate <= AVG_RATE){
		pr.rate = MIN_RATE;
		Write(sender, pr);
		return false;
	}

	if(!Read(target, targetValue)
		return false;

	if(pr.rate = MAX_RATE)
		if(isPositive)
			targetValue.rate += NEXT_LVL;
		else
			targetValue.rate = MIN_RATE;
	else if(pr.rate >= FIFTH_LVL)
		if(isPositive)
			targetValue.rate += FIFTH_LVL_INC;
		else
			targetValue.rate -= FIFTH_LVL_INC;
	else if(pr.rate >= FOURTH_LVL)
		if(isPositive)
			targetValue.rate += FOURTH_LVL_INC;
		else
			targetValue.rate -= FOURTH_LVL_INC;
	else if(pr.rate >= THIRD_LVL)
		if(isPositive)
			targetValue.rate += THIRD_LVL_INC;
		else
			targetValue.rate -= THIRD_LVL_INC;
	else if(pr.rate >= SECOND_LVL)
		if(isPositive)
			targetValue.rate += SECOND_LVL_INC;
		else
			targetValue.rate -= SECOND_LVL_INC;
	else if(pr.rate >= FIRST_LVL)
		if(isPositive)
			targetValue.rate += FIRST_LVL_INC;
		else
			targetValue.rate -= FIRST_LVL_INC;
	if(!Write(target, targetValue))
		return false;*/


#pragma once
#include <Eigen/Core>
#include <Eigen/Sparse>
#include <algorithm>
#include <atomic>
#include <cstddef>
#include <cstdint>
#include <future>
#include <iostream>
#include <random>
#include <stdexcept>
#include <thread>
#include <tuple>
#include <vector>

#include "argcheck.hpp"

namespace irspack {
namespace sparse_util {

template <typename Real>
using CSRMatrix = Eigen::SparseMatrix<Real, Eigen::RowMajor>;

template <typename Real>
using RowMajorMatrix =
    Eigen::Matrix<Real, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>;

template <typename Real>
using ColMajorMatrix =
    Eigen::Matrix<Real, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor>;

template <typename Real>
using DenseVector = Eigen::Matrix<Real, Eigen::Dynamic, 1>;

template <typename Real>
using DenseColVector = Eigen::Matrix<Real, 1, Eigen::Dynamic, Eigen::RowMajor>;

template <typename Real>
using CSCMatrix = Eigen::SparseMatrix<Real, Eigen::ColMajor>;

template <typename Real>
inline RowMajorMatrix<Real>
parallel_sparse_product(const CSRMatrix<Real> &left,
                        const CSCMatrix<Real> &right, const size_t n_threads) {
  RowMajorMatrix<Real> result(left.rows(), right.cols());
  result.array() = 0;
  check_arg(n_threads > 0, "n_thraed must be > 0");
  const int64_t n_row = left.rows();
  std::atomic<int64_t> cursor(0);
  std::vector<std::thread> workers;
  for (int i = 0; i < static_cast<int>(n_threads); i++) {
    workers.emplace_back([&left, &right, &cursor, n_row, &result]() {
      const int64_t chunk_size = 16;
      while (true) {
        auto current_position = cursor.fetch_add(chunk_size);
        if (current_position >= n_row) {
          break;
        }
        auto block_size =
            std::min(current_position + chunk_size, n_row) - current_position;
        result.middleRows(current_position, block_size) +=
            left.middleRows(current_position, block_size) * right;
      }
    });
  }
  for (auto &worker : workers) {
    worker.join();
  }
  return result;
}

template <typename Real, class Derived, typename Integer = int64_t>
struct SplitFunction {
  template <typename... Args>
  static std::pair<CSRMatrix<Real>, CSRMatrix<Real>>
  split_imple(const CSRMatrix<Real> &X, std::int64_t random_seed,
              Args... args) {
    Derived::check_args(args...);
    using Triplet = Eigen::Triplet<Integer>;
    std::mt19937 random_state(random_seed);
    std::vector<Integer> col_buffer;
    std::vector<Real> data_buffer;
    std::vector<uint64_t> index_;
    std::vector<Triplet> train_data, test_data;
    for (int row = 0; row < X.outerSize(); ++row) {
      col_buffer.clear(); // does not change capacity
      data_buffer.clear();
      index_.clear();
      Integer cnt = 0;
      for (typename CSRMatrix<Real>::InnerIterator it(X, row); it; ++it) {
        index_.push_back(cnt);
        col_buffer.push_back(it.col());
        data_buffer.push_back(it.value());
        cnt += 1;
      }
      std::shuffle(index_.begin(), index_.end(), random_state);
      size_t n_test = Derived::get_n_test(cnt, args...);
      for (size_t i = 0; i < n_test; i++) {
        test_data.emplace_back(row, col_buffer[index_[i]],
                               data_buffer[index_[i]]);
      }
      for (size_t i = n_test; i < col_buffer.size(); i++) {
        train_data.emplace_back(row, col_buffer[index_[i]],
                                data_buffer[index_[i]]);
      }
    }
    CSRMatrix<Real> X_train(X.rows(), X.cols()), X_test(X.rows(), X.cols());
    auto dupfunction = [](const Integer &a, const Integer &b) { return a + b; };
    X_train.setFromTriplets(train_data.begin(), train_data.end(), dupfunction);
    X_test.setFromTriplets(test_data.begin(), test_data.end(), dupfunction);
    X_train.makeCompressed();
    X_test.makeCompressed();
    return {X_train, X_test};
  }
};

template <typename Real, typename Integer = int64_t>
struct SplitByRatioFunction
    : SplitFunction<Real, SplitByRatioFunction<Real, Integer>, Integer> {
  using Base =
      SplitFunction<Real, SplitByRatioFunction<Real, Integer>, Integer>;
  static void check_args(double test_ratio, bool n_test_ceil) {
    check_arg(((test_ratio <= 1.0 && (test_ratio >= 0.0))),
              "test_ratio must be within [0.0, 1.0]");
  }

  static size_t get_n_test(size_t nnz_row, double test_ratio,
                           bool n_test_ceil) {
    if (n_test_ceil) {
      return std::ceil(nnz_row * test_ratio);
    } else {
      return std::floor(nnz_row * test_ratio);
    }
  };

  static std::pair<CSRMatrix<Real>, CSRMatrix<Real>>
  split(const CSRMatrix<Real> &X, std::int64_t random_seed, Real heldout_ratio,
        bool n_test_ceil) {
    return Base::split_imple(X, random_seed, heldout_ratio, n_test_ceil);
  }
};

template <typename Real, typename Integer = int64_t>
struct SplitFixedN : SplitFunction<Real, SplitFixedN<Real, Integer>, Integer> {
  using Base = SplitFunction<Real, SplitFixedN<Real, Integer>, Integer>;
  static void check_args(size_t n_held_out) {}

  static size_t get_n_test(size_t nnz_row, size_t n_held_out) {
    return std::min(nnz_row, n_held_out);
  };

  static std::pair<CSRMatrix<Real>, CSRMatrix<Real>>
  split(const CSRMatrix<Real> &X, std::int64_t random_seed, size_t n_held_out) {
    return Base::split_imple(X, random_seed, n_held_out);
  }
};

template <typename Real>
CSRMatrix<Real> okapi_BM_25_weight(const CSRMatrix<Real> &X, Real k1, Real b) {
  CSRMatrix<Real> result(X);
  using itertype = typename CSRMatrix<Real>::InnerIterator;
  const int N = X.rows();
  result.makeCompressed();
  DenseVector<Real> idf(X.cols());
  DenseVector<Real> doc_length(N);
  idf.array() = 0;
  doc_length.array() = 0;

  for (int i = 0; i < N; i++) {
    for (itertype iter(X, i); iter; ++iter) {
      idf(iter.col()) += 1;
      doc_length(i) += iter.value();
    }
  }
  Real avgdl = doc_length.sum() / N;
  idf.array() =
      (N / (idf.array() + static_cast<Real>(1)) + static_cast<Real>(1)).log();
  for (int i = 0; i < N; i++) {
    Real regularizer = k1 * (1 - b + b * doc_length(i) / avgdl);
    for (itertype iter(result, i); iter; ++iter) {
      iter.valueRef() = idf(iter.col()) * (iter.valueRef() * (k1 + 1)) /
                        (iter.valueRef() + regularizer);
    }
  }
  return result;
}

template <typename Real>
CSRMatrix<Real> tf_idf_weight(const CSRMatrix<Real> &X, bool smooth) {
  CSRMatrix<Real> result(X);
  using itertype = typename CSRMatrix<Real>::InnerIterator;
  const int N = X.rows();
  result.makeCompressed();
  DenseVector<Real> idf(X.cols());
  idf.array() = 0;

  for (int i = 0; i < N; i++) {
    for (itertype iter(X, i); iter; ++iter) {
      idf(iter.col()) += 1;
    }
  }
  idf.array() = (N / (idf.array() + static_cast<Real>(smooth))).log();
  for (int i = 0; i < N; i++) {
    for (itertype iter(result, i); iter; ++iter) {
      iter.valueRef() *= idf(iter.col());
    }
  }
  return result;
}

template <typename Real>
CSRMatrix<Real> remove_diagonal(const CSRMatrix<Real> &X) {
  check_arg(X.rows() == X.cols(), "X must be square");
  CSRMatrix<Real> result(X);
  using itertype = typename CSRMatrix<Real>::InnerIterator;
  const int N = X.rows();
  result.makeCompressed();
  for (int i = 0; i < N; i++) {
    for (itertype iter(result, i); iter; ++iter) {
      if (i == iter.col()) {
        iter.valueRef() = static_cast<Real>(0);
      }
    }
  }
  return result;
}

template <typename Real, bool positive_only = false,
          int block_size = Eigen::internal::packet_traits<Real>::size>
inline CSCMatrix<Real> SLIM(const CSRMatrix<Real> &X, size_t n_threads,
                            size_t n_iter, Real l2_coeff, Real l1_coeff,
                            Real tol, int64_t top_k) {
  check_arg(n_threads > 0, "n_threads must be > 0.");
  check_arg(n_iter > 0, "n_iter must be > 0.");
  check_arg(l2_coeff >= 0, "l2_coeff must be > 0.");
  check_arg(l1_coeff >= 0, "l1_coeff must be > 0.");
  using MatrixType =
      Eigen::Matrix<Real, block_size, Eigen::Dynamic, Eigen::ColMajor>;
  using VectorType = Eigen::Matrix<Real, block_size, 1>;

  CSCMatrix<Real> X_csc(X);
  X_csc.makeCompressed();
  using TripletType = Eigen::Triplet<Real>;
  using CSCIter = typename CSCMatrix<Real>::InnerIterator;
  using RealAndIndex = std::pair<Real, int64_t>;
  std::vector<std::future<std::vector<TripletType>>> workers;
  std::atomic<int64_t> cursor(0);
  for (size_t th = 0; th < n_threads; th++) {
    workers.emplace_back(std::async(std::launch::async, [&cursor, &X_csc,
                                                         l2_coeff, l1_coeff,
                                                         n_iter, tol, top_k] {
      const int64_t F = X_csc.cols();
      std::mt19937 gen(0);
      std::vector<int64_t> indices(F);
      for (int64_t i = 0; i < F; i++) {
        indices[i] = i;
      }
      MatrixType remnants(block_size, X_csc.rows());
      MatrixType coeffs(block_size, F);
      VectorType coeff_temp(block_size);
      VectorType diff(block_size);
      VectorType linear(block_size);
      VectorType linear_plus(block_size);
      VectorType linear_minus(block_size);

      std::vector<RealAndIndex> argsort_buffer;
      if (top_k >= 0) {
        argsort_buffer.resize(F);
      }

      std::vector<TripletType> local_results;
      while (true) {
        int64_t current_cursor = cursor.fetch_add(block_size);
        if (current_cursor >= F) {
          break;
        }

        int64_t block_begin = current_cursor;
        int64_t block_end = std::min(block_begin + block_size, F);
        int64_t valid_block_size = block_end - block_begin;
        remnants.setZero();
        coeffs.setZero();

        for (int64_t f_cursor = block_begin; f_cursor < block_end; f_cursor++) {
          const int64_t internal_col_position = f_cursor - block_begin;
          for (CSCIter iter(X_csc, f_cursor); iter; ++iter) {
            remnants(internal_col_position, iter.row()) = -iter.value();
          }
        }

        for (size_t cd_iteration = 0; cd_iteration < n_iter; cd_iteration++) {
          std::shuffle(indices.begin(), indices.end(), gen);
          Real delta = 0;
          for (int64_t feature_index = 0; feature_index < F; feature_index++) {
            int64_t shuffled_feature_index = indices[feature_index];
            coeff_temp = coeffs.col(shuffled_feature_index);
            diff = coeff_temp;
            linear.setZero();
            Real x2_sum = 0.0;
            for (CSCIter nnz_iter(X_csc, shuffled_feature_index); nnz_iter;
                 ++nnz_iter) {
              Real x = nnz_iter.value();
              x2_sum += x * x;
              linear.noalias() += x * remnants.col(nnz_iter.row());
            }
            linear.noalias() -= x2_sum * coeff_temp;

            Real quadratic = x2_sum + l2_coeff;
            linear_plus.array() = (-linear.array() - l1_coeff) / quadratic;
            if (!positive_only) {
              linear_minus.array() = (-linear.array() + l1_coeff) / quadratic;
            }

            Real *ptr_location =
                coeffs.data() + shuffled_feature_index * block_size;
            Real *lp_ptr = linear_plus.data();
            Real *lm_ptr = linear_minus.data();

            for (int64_t inner_cursor_position = 0;
                 inner_cursor_position < block_size; inner_cursor_position++) {
              Real lplus = *(lp_ptr++);
              Real lminus = *(lm_ptr++);
              int64_t original_cursor_position =
                  inner_cursor_position + block_begin;
              if (original_cursor_position == shuffled_feature_index) {
                *(ptr_location++) = 0.0;
                continue;
              }
              if (positive_only) {
                if (lplus > 0) {
                  *(ptr_location++) = lplus;
                } else {
                  *(ptr_location++) = static_cast<Real>(0.0);
                }

              } else {
                if (lplus > 0) {
                  *(ptr_location++) = lplus;
                } else {
                  if (lminus < 0) {
                    *(ptr_location++) = lminus;
                  } else {
                    *(ptr_location++) = static_cast<Real>(0.0);
                  }
                }
              } // allow nagative block
            }
            coeff_temp.noalias() =
                coeffs.col(shuffled_feature_index) - coeff_temp;

            if (!coeff_temp.isZero()) {
              for (CSCIter nnz_iter(X_csc, shuffled_feature_index); nnz_iter;
                   ++nnz_iter) {
                const int64_t row = nnz_iter.row();
                remnants.col(row).noalias() += nnz_iter.valueRef() * coeff_temp;
              }
              delta = std::max(delta, coeff_temp.cwiseAbs().array().maxCoeff());
            }
          }
          if (delta < tol) {
            break;
          }
        }

        if (top_k < 0) {
          for (int64_t f = 0; f < F; f++) {
            for (int64_t inner_cursor_position = 0;
                 inner_cursor_position < valid_block_size;
                 inner_cursor_position++) {
              int64_t original_location = inner_cursor_position + block_begin;
              Real c = coeffs(inner_cursor_position, f);
              if (c != 0.0) {
                local_results.emplace_back(f, original_location, c);
              }
            }
          }
        } else {
          for (int64_t inner_cursor_position = 0;
               inner_cursor_position < valid_block_size;
               inner_cursor_position++) {
            int64_t original_location = inner_cursor_position + block_begin;
            auto iter = argsort_buffer.begin();
            int64_t nnz = 0;
            for (int64_t f = 0; f < F; f++) {
              Real c = coeffs(inner_cursor_position, f);
              if (c != 0.0) {
                iter->first = c;
                iter->second = f;
                iter++;
                nnz++;
              }
            }
            int64_t n_taken_coeffs = nnz;
            if (nnz > top_k) {
              std::sort(argsort_buffer.begin(), argsort_buffer.end(),
                        [](RealAndIndex &val1, RealAndIndex &val2) {
                          return val1.first > val2.first;
                        });
              n_taken_coeffs = top_k;
            }
            for (int64_t i = 0; i < n_taken_coeffs; i++) {
              local_results.emplace_back(argsort_buffer[i].second,
                                         original_location,
                                         argsort_buffer[i].first);
            }
          }
        }
      }
      return local_results;
    }));
  }
  std::vector<TripletType> nnzs;
  for (auto &fres : workers) {
    auto result = fres.get();
    for (const auto &w : result) {
      nnzs.emplace_back(w);
    }
  }

  CSCMatrix<Real> result(X.cols(), X.cols());
  result.setFromTriplets(nnzs.begin(), nnzs.end());
  result.makeCompressed();
  return result;
}

template <typename Real>
inline std::vector<std::vector<std::pair<int64_t, float>>>
retrieve_recommend_from_score(
    const RowMajorMatrix<Real> &score,
    const std::vector<std::vector<int64_t>> &allowed_indices,
    const size_t cutoff, size_t n_threads) {
  using score_and_index = std::pair<int64_t, float>;
  check_arg(n_threads > 0, "n_threads must not be 0.");
  check_arg(
      (score.rows() == static_cast<int64_t>(allowed_indices.size())) ||
          allowed_indices.empty(),
      "allowed_indices, if not empty, must have a size equal to X.rows()");
  std::vector<std::vector<score_and_index>> result(score.rows());
  std::vector<std::future<void>> workers;
  std::atomic<size_t> cursor(0);
  const size_t n_users = static_cast<size_t>(score.rows());
  for (size_t thread = 0; thread < std::min(n_threads, n_users); thread++) {
    workers.emplace_back(
        std::async([&score, cutoff, &allowed_indices, &cursor, &result]() {
          const int64_t n_rows = score.rows();
          const int64_t n_items = score.cols();
          std::vector<score_and_index> index_holder;
          index_holder.reserve(n_items);

          while (true) {
            int64_t current = cursor.fetch_add(1);
            if (current >= n_rows) {
              break;
            }

            std::vector<score_and_index> inserted;
            const Real *score_ptr = score.data() + n_items * current;

            index_holder.clear();
            if (!allowed_indices.empty()) {
              for (auto item_index : allowed_indices.at(current)) {
                if ((item_index < n_items) && (item_index >= 0)) {
                  index_holder.emplace_back(item_index, score_ptr[item_index]);
                }
              }
            } else {
              for (int64_t i = 0; i < n_items; i++) {
                index_holder.emplace_back(i, *(score_ptr++));
              }
            }
            std::partial_sort(
                index_holder.begin(),
                index_holder.begin() + std::min(cutoff, index_holder.size()),
                index_holder.end(), [](score_and_index i1, score_and_index i2) {
                  return i1.second > i2.second;
                });

            size_t items_recommended = 0;
            for (auto item_index : index_holder) {
              if (items_recommended >= cutoff) {
                break;
              }

              if (item_index.second == -std::numeric_limits<Real>::infinity()) {
                break;
              }
              result[current].emplace_back(item_index);
              items_recommended++;
            }
          }
        }));
  }
  workers.clear();
  return result;
}

} // namespace sparse_util
} // namespace irspack


#include <GL/glew.h>
#include <GL/glu.h>
#include <ImathVec.h>
#include <possumwood_sdk/app.h>
#include <possumwood_sdk/node_implementation.h>

#include <boost/algorithm/string/join.hpp>

#include "cgal/datatypes/meshes.h"
#include "datatypes/vertex_data.inl"

namespace possumwood {

template <typename T>
struct VBOTraits<CGAL::Point_3<CGAL::Simple_cartesian<T>>> {
	typedef T element;
	static constexpr std::size_t width() {
		return 3;
	};
};

template <typename T>
struct VBOTraits<std::vector<T>> {
	typedef T element;
	static constexpr std::size_t width() {
		return 1;
	};
};
}  // namespace possumwood

namespace {

void addVerticesVBO(possumwood::VertexData& vd,
                    const std::string& name,
                    const possumwood::Meshes& meshes,
                    std::size_t triangleCount) {
	vd.addVBO<possumwood::CGALPolyhedron::Point_3>(
	    name, triangleCount * 3, possumwood::VertexData::kStatic,
	    [meshes, triangleCount](
	        possumwood::Buffer<typename possumwood::VBOTraits<possumwood::CGALPolyhedron::Point_3>::element>& buffer,
	        const possumwood::ViewportState& vs) {
		    // TODO: use index buffer instead
		    std::size_t index = 0;

		    for(const auto& m : meshes) {
			    for(auto fit = m.polyhedron().facets_begin(); fit != m.polyhedron().facets_end(); ++fit)
				    if(fit->facet_degree() > 2) {
					    auto vit = fit->facet_begin();

					    const auto& p = vit->vertex()->point();
					    ++vit;

					    for(std::size_t v = 2; v < fit->facet_degree(); ++v) {
						    const auto& p1 = vit->vertex()->point();
						    ++vit;
						    const auto& p2 = vit->vertex()->point();

						    buffer.element(index++) = p;
						    buffer.element(index++) = p1;
						    buffer.element(index++) = p2;
					    }
				    }
		    }

		    assert(index == triangleCount * 3);
	    });
}

template <typename T>
const T& identity(const T& val) {
	return val;
}

template <typename T, typename EXTRACT = std::function<T(const T&)>>
void addPerPointVBO(possumwood::VertexData& vd,
                    const std::string& name,
                    const std::string& propertyName,
                    std::size_t triangleCount,
                    const possumwood::Meshes& mesh,
                    EXTRACT extract = identity<T>) {
	vd.addVBO<T>(name, triangleCount * 3, possumwood::VertexData::kStatic,
	             [mesh, propertyName, extract](possumwood::Buffer<typename possumwood::VBOTraits<T>::element>& buffer,
	                                           const possumwood::ViewportState& vs) {
		             std::size_t index = 0;

		             for(auto& m : mesh) {
			             // vertex props, handled by polygon-triangle
			             if(m.vertexProperties().hasProperty(propertyName)) {
				             auto& vertProp = m.vertexProperties().property<T>(propertyName);

				             for(auto fit = m.polyhedron().facets_begin(); fit != m.polyhedron().facets_end(); ++fit) {
					             if(fit->facet_degree() > 2) {
						             auto it = fit->facet_begin();

						             const auto& val0 = extract(vertProp.get(it->vertex()->property_key()));
						             ++it;

						             for(unsigned ctr = 2; ctr < fit->facet_degree(); ++ctr) {
							             const auto& val1 = extract(vertProp.get(it->vertex()->property_key()));
							             ++it;
							             const auto& val2 = extract(vertProp.get(it->vertex()->property_key()));

							             buffer.element(index++) = val0;
							             buffer.element(index++) = val1;
							             buffer.element(index++) = val2;
						             }
					             }
				             }
			             }

			             // face props, constant for all triangles of a face
			             else if(m.faceProperties().hasProperty(propertyName)) {
				             auto& faceProp = m.faceProperties().property<T>(propertyName);

				             for(auto fit = m.polyhedron().facets_begin(); fit != m.polyhedron().facets_end(); ++fit) {
					             if(fit->facet_degree() > 2) {
						             auto n = extract(faceProp.get(fit->property_key()));

						             for(unsigned i = 2; i < fit->facet_degree(); ++i)
							             for(unsigned a = 0; a < 3; ++a)
								             buffer.element(index++) = n;
					             }
				             }
			             }

			             // halfedge props
			             else if(m.halfedgeProperties().hasProperty(propertyName)) {
				             auto& heProp = m.halfedgeProperties().property<T>(propertyName);

				             for(auto fit = m.polyhedron().facets_begin(); fit != m.polyhedron().facets_end(); ++fit) {
					             if(fit->facet_degree() > 2) {
						             auto it = fit->facet_begin();

						             const auto& val0 = extract(heProp.get(it->property_key()));
						             ++it;

						             for(unsigned ctr = 2; ctr < fit->facet_degree(); ++ctr) {
							             const auto& val1 = extract(heProp.get(it->property_key()));
							             ++it;
							             const auto& val2 = extract(heProp.get(it->property_key()));

							             buffer.element(index++) = val0;
							             buffer.element(index++) = val1;
							             buffer.element(index++) = val2;
						             }
					             }
				             }
			             }

			             else
				             throw std::runtime_error("Property " + propertyName +
				                                      " not found as halfedge, vertex or face property.");
		             }

		             assert(index == buffer.vertexCount());
	             });
}

using possumwood::CGALPolyhedron;
using possumwood::Meshes;

dependency_graph::OutAttr<possumwood::VertexData> a_vd;
dependency_graph::InAttr<Meshes> a_mesh;
dependency_graph::InAttr<std::string> a_pointsName;

struct PropertyKey {
	std::string name;
	std::type_index type;

	bool operator<(const PropertyKey& pk) const {
		if(name != pk.name)
			return name < pk.name;
		return type < pk.type;
	}
};

dependency_graph::State compute(dependency_graph::Values& data) {
	dependency_graph::State result;

	const Meshes mesh = data.get(a_mesh);

	// we're drawing triangles
	possumwood::VertexData vd(GL_TRIANGLES);

	// first, figure out how many triangles we have
	//   there has to be a better way to do this, come on
	std::size_t triangleCount = 0;
	for(auto& m : mesh)
		for(auto it = m.polyhedron().facets_begin(); it != m.polyhedron().facets_end(); ++it)
			if(it->facet_degree() > 2)
				triangleCount += (it->facet_degree() - 2);

	// and build the buffers
	if(triangleCount > 0) {
		// transfer the position data
		addVerticesVBO(vd, data.get(a_pointsName), mesh, triangleCount);

		// count the properties - only properties consistent between all meshes can be
		// transfered. We don't care about the type of the properties, though - face or
		// vertex are both fine
		std::map<PropertyKey, std::size_t> propertyCounters;
		for(auto& m : mesh) {
			auto vertPropList = m.vertexProperties();
			for(auto& vp : vertPropList)
				++propertyCounters[PropertyKey{vp.name(), vp.type()}];

			auto facePropList = m.faceProperties();
			for(auto& fp : facePropList)
				++propertyCounters[PropertyKey{fp.name(), fp.type()}];

			auto halfedgePropList = m.halfedgeProperties();
			for(auto& hp : halfedgePropList)
				++propertyCounters[PropertyKey{hp.name(), hp.type()}];
		}

		// and transfer all properties that are common to all meshes
		std::set<std::string> ignoredProperties, inconsistentProperties;
		for(auto& pc : propertyCounters)
			if(pc.second == mesh.size()) {
				// yep, hardcoded for now, sorry :(
				if(pc.first.type == typeid(std::array<float, 3>))
					addPerPointVBO<std::array<float, 3>>(vd, pc.first.name, pc.first.name, triangleCount, mesh);

				else if(pc.first.type == typeid(std::array<float, 2>))
					addPerPointVBO<std::array<float, 2>>(vd, pc.first.name, pc.first.name, triangleCount, mesh);

				else if(pc.first.type == typeid(float))
					addPerPointVBO<float>(vd, pc.first.name, pc.first.name, triangleCount, mesh);

				else if(pc.first.type == typeid(int))
					addPerPointVBO<int>(vd, pc.first.name, pc.first.name, triangleCount, mesh);

				// else if(name.type == "float")
				// 	for(std::size_t i = 0; i < name.arraySize; ++i) {
				// 		auto fn = [i](const std::vector<float>& v) -> float { return v[i]; };

				// 		addPerPointVBO<std::vector<float>>(vd, name.name + "[" + std::to_string(i) + "]", pc.first,
				// 		                                   triangleCount, mesh, fn);
				// 	}

				// else if(name.type == "int")
				// 	for(std::size_t i = 0; i < name.arraySize; ++i) {
				// 		auto fn = [i](const std::vector<int>& v) -> int { return v[i]; };

				// 		addPerPointVBO<std::vector<int>>(vd, name.name + "[" + std::to_string(i) + "]", pc.first,
				// 		                                 triangleCount, mesh, fn);
				// 	}

				else
					ignoredProperties.insert(pc.first.name);
			}
			else
				inconsistentProperties.insert(pc.first.name + "(" + std::to_string(pc.second) + "/" +
				                              std::to_string(mesh.size()) + ")");

		if(!inconsistentProperties.empty())
			result.addWarning(
			    "The following properties were not common to all meshes, and had to be "
			    "ignored:\n" +
			    boost::algorithm::join(inconsistentProperties, ", "));

		if(!ignoredProperties.empty())
			result.addWarning("The following property types were not recognized:\n" +
			                  boost::algorithm::join(ignoredProperties, ", "));
	}

	data.set(a_vd, std::move(vd));

	return result;
}

void init(possumwood::Metadata& meta) {
	meta.addAttribute(a_mesh, "mesh", possumwood::Meshes(), possumwood::AttrFlags::kVertical);
	meta.addAttribute(a_pointsName, "p_attr_name", std::string("P"));
	meta.addAttribute(a_vd, "vertex_data");

	meta.addInfluence(a_mesh, a_vd);
	meta.addInfluence(a_pointsName, a_vd);

	meta.setCompute(&compute);
}

possumwood::NodeImplementation s_impl("render/vertex_data/cgal", init);
}  // namespace


//          Copyright Gavin Band 2008 - 2012.
// Distributed under the Boost Software License, Version 1.0.
//    (See accompanying file LICENSE_1_0.txt or copy at
//          http://www.boost.org/LICENSE_1_0.txt)

#include <iostream>
#include <iomanip>
#include <sstream>
#include <vector>
#include <utility>
#include <algorithm>
#include <boost/iterator/counting_iterator.hpp>
#include <boost/bind.hpp>
#include "metro/regression/Design.hpp"
#include "metro/intersect_ranges.hpp"

// #define DEBUG_REGRESSIONDESIGN 1

namespace metro {
	namespace regression {
		Design::UniquePtr Design::create(
			Matrix const& outcome, SampleRanges const& nonmissing_outcome, std::vector< std::string > const& outcome_names,
			Matrix const& covariates, SampleRanges const& nonmissing_covariate, std::vector< std::string > const& covariate_names,
			std::vector< std::string > const& predictor_names,
			Transform transform,
			std::vector< int > const& interacting_covariates
		) {
			return Design::UniquePtr(
				new Design(
					outcome, nonmissing_outcome, outcome_names,
					covariates, nonmissing_covariate, covariate_names,
					predictor_names,
					transform,
					interacting_covariates
				)
			) ;
		}

		Design::UniquePtr Design::create(
			Matrix const& outcome, SampleRanges const& nonmissing_outcome, std::vector< std::string > const& outcome_names,
			std::vector< std::string > const& predictor_names,
			Transform transform
		) {
			return Design::UniquePtr(
				new Design(
					outcome, nonmissing_outcome, outcome_names,
					predictor_names,
					transform
				)
			) ;
		}

		Design::Design(
			Matrix const& outcome, SampleRanges const& nonmissing_outcome,  std::vector< std::string > const& outcome_names,
			Matrix const& covariates, SampleRanges const& nonmissing_covariates, std::vector< std::string > const& covariate_names,
			std::vector< std::string > const& predictor_names,
			Transform transform,
			std::vector< int > const& interacting_covariates
		):
			// outcome variables
		 	m_outcome( outcome ),
			m_outcome_names( outcome_names ),
		 	m_nonmissing_outcome( nonmissing_outcome ),
			// predictor variables
			m_predictor_level_probabilities( Matrix::Zero( outcome.rows(), 0 )),
			m_predictor_levels( Matrix::Zero( 0, predictor_names.size() )),
			m_predictor_names( predictor_names ),
			m_nonmissing_predictors( SampleRanges() ),
			// covariate variables
			m_covariates( 1, covariates ),
			m_covariate_names( covariate_names ),
			m_nonmissing_covariates( nonmissing_covariates ),
			// predictor-covariate interactions
			m_predictor_covariate_interactions( interacting_covariates ),
			// Design matrix column transform
			m_transform( transform )
		{
			assert( covariates.rows() == m_outcome.rows() ) ;
			assert( m_outcome_names.size() == std::size_t( m_outcome.cols() )) ;
			assert( m_covariate_names.size() == std::size_t( m_covariates[0].cols() )) ;
			recalculate() ;
		}

		Design::Design(
			Matrix const& outcome, SampleRanges const& nonmissing_outcome,  std::vector< std::string > const& outcome_names,
			std::vector< std::string > const& predictor_names,
			Transform transform
		):
			// outcome variables
		 	m_outcome( outcome ),
			m_outcome_names( outcome_names ),
		 	m_nonmissing_outcome( nonmissing_outcome ),
			// predictor variables
			m_predictor_level_probabilities( Matrix::Zero( outcome.rows(), 0 )),
			m_predictor_levels( Matrix::Zero( 0, predictor_names.size() )),
			m_predictor_names( predictor_names ),
			m_nonmissing_predictors( SampleRanges() ),
			// covariate variables
			m_covariates(),
			m_covariate_names(),
			m_nonmissing_covariates( 1, SampleRange( 0, m_outcome.rows() )),
			// predictor-covariate interactions
			m_predictor_covariate_interactions(),
			// Design matrix column transform
			m_transform( transform )
		{
			assert( m_outcome_names.size() == m_outcome.cols() ) ;
			recalculate() ;
		}

		// add a single continuous covariate
		// missing values are encoded by NaN
		void Design::add_single_covariate(
			std::string const& name,
			boost::function< double( std::size_t ) > const& values
		) {
			Vector data = Vector::Constant( m_outcome.rows(), 0 ) ;
			std::vector< SampleRange > nonmissingness ;
			int last_nonmissing_sample_i = 0 ;
			for( int i = 0; i < m_outcome.rows(); ++i ) {
				double v = values(i) ;
				if( v == v ) {
					data(i) = v ;
				} else {
					if( i > last_nonmissing_sample_i ) {
						nonmissingness.push_back( SampleRange( last_nonmissing_sample_i, i )) ;
					}
					last_nonmissing_sample_i = i+1 ;
				}
			}
			if( last_nonmissing_sample_i < m_outcome.rows() ) {
				nonmissingness.push_back( SampleRange( last_nonmissing_sample_i, m_outcome.rows() )) ;
			}

			// Ok push it.
			m_covariates.push_back( data ) ;
			m_covariate_names.push_back( name ) ;
			m_nonmissing_covariates = impl::intersect_ranges( m_nonmissing_covariates, nonmissingness ) ;

			recalculate() ;
		}

		// add a single discrete covariate (which expands into multiple columns of zeroes and ones)
		// levels are specified by non-negative integers.
		// missing values are encoded by -1
		void Design::add_discrete_covariate(
			std::string const& name,
			boost::function< int( std::size_t ) > const& values,
			boost::function< std::string( int ) > const& levelNames,
			int numberOfLevels
		) {
			// column for level 0 is not provided as will be a linear combination of the others.
			Matrix data = Matrix::Constant( m_outcome.rows(), numberOfLevels - 1, 0.0 ) ;
			std::vector< SampleRange > nonmissingness ;
			int last_nonmissing_sample_i = 0 ;
			std::vector< std::string > covariateLevelNames ;
			for( int i = 0; i < m_outcome.rows(); ++i ) {
				int level = values( i ) ;
				if( level > 0 ) {
					data( i, level-1 ) = 1.0 ;
				} else if( level == -1 ) {
					if( i > last_nonmissing_sample_i ) {
						nonmissingness.push_back( SampleRange( last_nonmissing_sample_i, i )) ;
					}
					last_nonmissing_sample_i = i+1 ;
				}
			}
			if( last_nonmissing_sample_i < m_outcome.rows() ) {
				nonmissingness.push_back( SampleRange( last_nonmissing_sample_i, m_outcome.rows() )) ;
			}
			m_covariates.push_back( data ) ;
			for( int i = 1; i < numberOfLevels; ++i ) {
				m_covariate_names.push_back( name + "=" + levelNames(i) ) ;
			}
			m_nonmissing_covariates = impl::intersect_ranges( m_nonmissing_covariates, nonmissingness ) ;

			recalculate() ;
		}

		std::string const& Design::get_predictor_name( std::size_t i ) const {
			assert( i < m_design_matrix_column_names.size() ) ;
			return m_design_matrix_column_names[i] ;
		}

		std::vector< std::string > const& Design::design_matrix_column_names() const {
			return m_design_matrix_column_names ;
		}

		std::string const& Design::get_outcome_name( std::size_t i ) const {
			assert( i < m_outcome_names.size() ) ;
			return m_outcome_names[i] ;
		}

		std::string Design::get_summary() const {
			std::ostringstream out ;

			int const N = matrix().rows() ;
			int const D = matrix().cols() ;
			int const P = number_of_predictors() + number_of_interaction_terms() ;

			// Compute mean predictors
			Eigen::MatrixXd mean_predictors = Eigen::MatrixXd::Zero( N, P ) ;
			{
				for( int level = 0; level < m_predictor_level_probabilities.cols(); ++level ) {
					mean_predictors += m_predictor_level_probabilities.col(level).asDiagonal() * m_predictor_levels.block( 0, P*level, N, P ) ;
				}
			}

			std::vector< std::size_t > widths( D + 1 ) ;
			widths[0] = std::max( std::size_t( outcome().cols() * 4 - 1 ), 8ul ) ;
			std::size_t const n_top = 8 ;
			out << "       " << "   " << std::setw( widths[0] ) << "outcome" << "  " ;
			for( int j = 0; j < D; ++j ) {
				std::string const name = m_design_matrix_column_names[j]  ;
				widths[j+1] = std::max( name.size(), 7ul ) ;
				out << " " << std::setw( widths[j+1] ) << name ;
			}
			out << "\n" ;
			for( int i = 0; i < N; ++i ) {
				out << std::setw(7) << i ;
				out << "   " ;
				if( outcome().row(i).sum() == outcome().row(i).sum() ) {
					std::size_t outcomesWidth = 4 * outcome().cols() - 1 ;
					if( outcomesWidth < widths[0] ) {
						out << std::string( widths[0] - outcomesWidth, ' ' ) ;
					}
					for( int j = 0; j < outcome().cols(); ++j ) {
						out << ((j>0) ? " " : "" ) << std::setw(3) << outcome()(i,j) ;
					}
				} else {
					out << std::setw(widths[0]) << "NA" ;
				}
				out << (( i == n_top ) ? " ~ " : "   " ) ;
				for( int j = 0; j < D; ++j ) {
					if( j > 0 ) {
						out << std::setw(1) << " " ;
					}
					out << std::setw( widths[j+1] ) << std::defaultfloat << std::setprecision(3) ;
					if( j > 0 & j <= P ) {
						out << mean_predictors(i,j-1) ;
					} else if( matrix()(i,j) == matrix()(i,j) ){
						out << matrix()(i,j) ;
					} else {
						out << "NA" ;
					}
				}
				out << "\n" ;

				if( i == n_top && N > 8 ) {
					for( int i2 = 0; i2 < 3; ++i2 ) {
						out << std::setw(8) << " " ;
						out << std::setw(widths[0]) << "." ;
						out << "  " ;
						for( int j2 = 0; j2 < D; ++j2 ) {
							if( j2 > 0 ) {
								out << std::setw(1) << " " ;
							}
							out << std::setw( widths[j2+1] ) << "." ;
						}
						out << "\n" ;
					}
					i = N - n_top + 1 ;
				}
			}
			return out.str() ;
		}

		Design::SampleRanges Design::compute_nonmissing_samples(
			SampleRanges const& nonmissing_outcome,
			SampleRanges const& nonmissing_predictors,
			SampleRanges const& nonmissing_covariates
		) const {
			SampleRanges result = impl::intersect_ranges( nonmissing_outcome, nonmissing_predictors ) ;
			return impl::intersect_ranges( result, nonmissing_covariates ) ;
		}

		void Design::recalculate() {
			calculate_design_matrix(
				m_outcome.rows(),
				m_covariates,
				m_covariate_names,
				m_predictor_names,
				m_predictor_covariate_interactions,
				&m_design_matrix,
				&m_design_matrix_interaction_columns,
				&m_design_matrix_column_names
			) ;
			//
			m_nonmissing_samples = compute_nonmissing_samples(
				m_nonmissing_outcome, m_nonmissing_predictors, m_nonmissing_covariates
			) ;
			assert( m_design_matrix_column_names.size() == m_design_matrix.cols() ) ;
		}

		void Design::calculate_design_matrix(
			int const number_of_samples,
			std::vector< Matrix > const& covariates,
			std::vector< std::string > const& covariate_names,
			std::vector< std::string > const& predictor_names,
			std::vector< int > const& interacting_covariates,
			Matrix* result,
			std::vector< int >* design_matrix_interaction_cols,
			std::vector< std::string >* design_matrix_column_names
		) const {
			assert( result != 0 ) ;
			assert( design_matrix_interaction_cols != 0 ) ;
			assert( design_matrix_interaction_cols->size() == 0 ) ;
			std::size_t const number_of_predictors = predictor_names.size() ;
			// Compute the number of covariates
			std::size_t numberOfCovariates = 0 ;
			for( std::size_t i = 0; i < covariates.size(); ++i ) {
				numberOfCovariates += covariates[i].cols() ;
			}

			// design matrix is layed out as:
			// baseline column of 1's
			// columns for predictors with values set later
			// columns for predictor vs. covariate interactions with values set later
			// columns for covariates with values set here.
			std::size_t numberOfDesignMatrixColumns
				= 1 + number_of_predictors * ( 1 + interacting_covariates.size() ) + numberOfCovariates ;

			result->setZero( number_of_samples, numberOfDesignMatrixColumns ) ;
			design_matrix_column_names->clear() ;

			// baseline column
			result->leftCols( 1 ).setOnes() ;
			design_matrix_column_names->push_back( "baseline" ) ;

			// predict columns
			design_matrix_column_names->insert(
				design_matrix_column_names->end(),
				m_predictor_names.begin(),
				m_predictor_names.end()
			) ;

			// predictor-covariate interactions
			if( interacting_covariates.size() > 0 ) {
				for( std::size_t i = 0; i < interacting_covariates.size(); ++i ) {
					design_matrix_interaction_cols->push_back( interacting_covariates[i] + result->cols() - numberOfCovariates ) ;
					for( std::size_t predictor = 0; predictor < number_of_predictors; ++predictor ) {
						design_matrix_column_names->push_back( m_predictor_names[predictor] + "x" + covariate_names[interacting_covariates[i]] ) ;
					}
				}
			}

			// covariates
			if( numberOfCovariates > 0 ) {
				int col = numberOfDesignMatrixColumns - numberOfCovariates ;
				for( std::size_t i = 0; i < covariates.size(); ++i ) {
					result->block(
						0, col,
						result->rows(), covariates[i].cols()
					) = covariates[i] ;
					col += covariates[i].cols() ;
				}
				design_matrix_column_names->insert(
					design_matrix_column_names->end(),
					covariate_names.begin(),
					covariate_names.end()
				) ;
			}
	#if DEBUG_REGRESSIONDESIGN
			std::cerr << "Design::calculate_design_matrix():"
				<< "result has " << result->cols() << " columns, and " << design_matrix_column_names->size() << " names.\n" ;
	#endif

			assert( design_matrix_column_names->size() == result->cols() ) ;
		}

		Design& Design::set_outcome(
			Matrix const& outcome,
			SampleRanges const& nonmissingness,
			std::vector< std::string > const& names
		) {
			assert( outcome.rows() == m_design_matrix.rows() ) ;
			assert( names.size() == outcome.cols() ) ;

			m_outcome = outcome ;
			m_outcome_names = names ;
			if( nonmissingness != m_nonmissing_outcome ) {
				m_nonmissing_outcome = nonmissingness ;
				m_nonmissing_samples = compute_nonmissing_samples( m_nonmissing_outcome, m_nonmissing_predictors, m_nonmissing_covariates ) ;
			}
			return *this ;
		}

		Design& Design::set_predictors(
			Matrix const& levels,
			Matrix const& probabilities,
			std::vector< metro::SampleRange > const& nonmissingness
		) {
			assert( levels.cols() == number_of_predictors() ) ;
			assert( probabilities.cols() == levels.rows() ) ;
			assert( probabilities.rows() == m_design_matrix.rows() ) ;

			// Set probabilities
			m_predictor_level_probabilities = probabilities ;

			// Set levels
			{
				// This is complexified by the need to handle possible interactions between
				// predictors and covariates.
				// There is one column per predictor and one column per predictor per interacting covariate.
				// The levels go in a matrix of dimension N x ((#predictors & interactions) x (#levels)).
				int const N = probabilities.rows() ;
				int const number_of_predictors_and_interactions = number_of_predictors() + number_of_interaction_terms() ;
				m_predictor_levels = Matrix::Zero( N, number_of_predictors_and_interactions * levels.rows() ) ;

				// Lay this out as follows.
				// For predictor level level_i, we fill in the block of m_predictor_levels with indices
				// [0,N) x [ level_i*npi,(level_i+1)*npi )
				// where npi is the total number of predictor and interaction terms.
				for( int level_i = 0; level_i < levels.rows(); ++level_i ) {
					m_predictor_levels.block(
						0, level_i * number_of_predictors_and_interactions,
						N, number_of_predictors()
					).rowwise() = levels.row(level_i) ;
					for( std::size_t i = 0; i < m_design_matrix_interaction_columns.size(); ++i ) {
						m_predictor_levels.block(
							0, level_i * number_of_predictors_and_interactions + (i+1) * number_of_predictors(),
							N, number_of_predictors()
						) = m_design_matrix.col( m_design_matrix_interaction_columns[i] ) * levels.row(level_i) ;
					}
				}

				// Set nonmissingness if necessary
				if( nonmissingness != m_nonmissing_predictors ) {
					m_nonmissing_predictors = nonmissingness ;
					m_nonmissing_samples = compute_nonmissing_samples( m_nonmissing_outcome, m_nonmissing_predictors, m_nonmissing_covariates ) ;
				}

#if DEBUG_REGRESSIONDESIGN
				std::cerr << "Design::set_predictors(): levels is:\n"
					<< m_predictor_levels.block( 0, 0, 10, m_predictor_levels.cols() ) << "\n" ;
#endif
				if( m_transform == eMeanCentre ) {
					mean_centre_predictor_levels(
						m_predictor_level_probabilities,
						&m_predictor_levels,
						m_nonmissing_samples
					) ;
				}
#if DEBUG_REGRESSIONDESIGN
				std::cerr << "Design::set_predictors(): after mean centring, levels is:\n"
					<< m_predictor_levels.block( 0, 0, 10, m_predictor_levels.cols() ) << "\n" ;
#endif
			}

			return *this ;
		}

		Design& Design::set_predictors(
			Matrix const& predictors,
			std::vector< metro::SampleRange > const& nonmissingness
		) {
			int const N = predictors.rows() ;
			int const D = predictors.cols() ;
			assert( D == number_of_predictors() ) ;
			assert( N == m_design_matrix.rows() ) ;

			// Set probabilities
			m_predictor_level_probabilities = Matrix::Ones( N, 1 ) ;

			// Set levels
			{
				// This is complexified by the need to handle possible interactions.
				// There is one column per predictor and one column per predictor per interacting covariate.
				// The levels go in a matrix of dimension N x ((#predictors & interactions) x (#levels)).
				int const number_of_predictors_and_interactions = number_of_predictors() + number_of_interaction_terms() ;
				m_predictor_levels = Matrix::Zero( N, number_of_predictors_and_interactions ) ;

				// Lay this out as follows.
				// For predictor level level_i, we fill in the block of m_predictor_levels with indices
				// [0,N) x [ level_i*npi,(level_i+1)*npi )
				// where npi is the total number of predictor and interaction terms.
				m_predictor_levels.block(
					0, 0,
					N, D
				) = predictors ;
				for( std::size_t i = 0; i < m_design_matrix_interaction_columns.size(); ++i ) {
					for( std::size_t d = 0; d < D; ++d ) {
						m_predictor_levels.col( D * (1+i) + d )
							= m_design_matrix.col( m_design_matrix_interaction_columns[i] ).array()
								* predictors.col(d).array() ;
					}
				}
			}

			// Set nonmissingness if necessary
			if( nonmissingness != m_nonmissing_predictors ) {
				m_nonmissing_predictors = nonmissingness ;
				m_nonmissing_samples = compute_nonmissing_samples( m_nonmissing_outcome, m_nonmissing_predictors, m_nonmissing_covariates ) ;
			}

			if( m_transform == eMeanCentre ) {
				mean_centre_predictor_levels(
					m_predictor_level_probabilities,
					&m_predictor_levels,
					m_nonmissing_samples
				) ;
			}

			return *this ;
		}

		void Design::mean_centre_predictor_levels(
			Matrix const& probs,
			Matrix* predictor_levels,
			SampleRanges const& included_samples
		) const {
			int const number_of_predictors_and_interactions = number_of_predictors() + number_of_interaction_terms() ;

			// Compute mean of each predictor
			RowVector means = RowVector::Zero( number_of_predictors_and_interactions ) ;
			int const number_of_predictor_levels = probs.cols() ;
			double total = 0 ;
			for( std::size_t i = 0; i < included_samples.size(); ++i ) {
				metro::SampleRange const& range = included_samples[i] ;
				for( int level_i = 0; level_i < number_of_predictor_levels; ++level_i ) {
					means += (
						( probs.col( level_i ).segment( range.begin(), range.size() ) ).asDiagonal()
						* predictor_levels->block(
							range.begin(), number_of_predictors_and_interactions * level_i,
							range.size(), number_of_predictors_and_interactions
						)
					).colwise().sum() ;
				}
				total += probs.block( range.begin(), 0, range.size(), probs.cols() ).sum() ;
			}
			means /= total ;

#if DEBUG_REGRESSIONDESIGN
			std::cerr << "Design::mean_centre_predictor_levels(): nonmissingnes = " << included_samples << ".\n" ;
			std::cerr << "Design::mean_centre_predictor_levels(): covariate nonmissingness = " << m_nonmissing_covariates << ".\n" ;
			std::cerr << "Design::mean_centre_predictor_levels(): means = " << means << ".\n" ;
#endif
			// Subtract means from the predictor levels
			for( int level_i = 0; level_i < number_of_predictor_levels; ++level_i ) {
				predictor_levels->block(
					0, number_of_predictors_and_interactions * level_i,
					predictor_levels->rows(), number_of_predictors_and_interactions
				).rowwise() -= means ;
			}
		}

		Design& Design::set_predictor_level( int level ) {
			if( m_predictor_levels.cols() > 0 ) {
				set_predictor_level( level, metro::SampleRange( 0, m_design_matrix.rows() )) ;
			}
			return *this ;
		}

		Design& Design::set_predictor_level( int level, metro::SampleRange const& range ) {
			if( m_predictor_levels.cols() > 0 ) {
				int const number_of_predictors_and_interactions = ( 1 + m_design_matrix_interaction_columns.size() ) * number_of_predictors() ;
				m_design_matrix.block(
					range.begin(), 1,
					range.size(), number_of_predictors_and_interactions
				) = m_predictor_levels.block(
					range.begin(), level * number_of_predictors_and_interactions,
					range.size(), number_of_predictors_and_interactions
				) ;
			}
			return *this ;
		}

		Design& Design::set_predictor_level( int level, std::vector< metro::SampleRange > const& ranges ) {
			if( m_predictor_levels.cols() > 0 ) {
				for( std::size_t i = 0; i < ranges.size(); ++i ) {
					set_predictor_level( level, ranges[i] ) ;
				}
			}
			return *this ;
		}
	}
}