#include <turbodbc/make_description.h>

#include <turbodbc/descriptions.h>
#include <sqlext.h>

#include <boost/variant/apply_visitor.hpp>

#include <stdexcept>
#include <sstream>
#include <algorithm>

namespace turbodbc {

namespace {

SQLULEN const digits_representable_by_64_bit_integer = 18;

/*
 * This function returns a buffer size for the given string
 * which leaves room for future, larger strings.
 *
 * The intent is to waste little space for small strings while
 * keeping the number of required buffer rebinds small.
 */
std::size_t size_after_growth_strategy(std::size_t const & size)
{
    std::size_t const minimum_size = 10;
    if (size < minimum_size) {
        return minimum_size;
    }
    return std::ceil(size * 1.2);
}

std::unique_ptr<description const> make_small_decimal_description(cpp_odbc::column_description const & source)
{
    if (source.decimal_digits == 0) {
        return std::unique_ptr<description>(new integer_description(source.name, source.allows_null_values));
    } else {
        return std::unique_ptr<description>(new floating_point_description(source.name, source.allows_null_values));
    }
}

std::unique_ptr<description const> make_large_decimal_description(cpp_odbc::column_description const & source, turbodbc::options const & options)
{
    if (options.large_decimals_as_64_bit_types) {
        return make_small_decimal_description(source);
    } else {
        // fall back to strings; add two characters for decimal point and sign!
        return std::unique_ptr<description>(new string_description(source.name,
                                                                   source.allows_null_values,
                                                                   source.size + 2));
    }
}

std::unique_ptr<description const> make_decimal_description(cpp_odbc::column_description const & source, turbodbc::options const & options)
{
    if (source.size <= digits_representable_by_64_bit_integer) {
        return make_small_decimal_description(source);
    } else {
        return make_large_decimal_description(source, options);
    }
}

using description_ptr = description const *;

struct description_by_value : public boost::static_visitor<description_ptr> {
    description_ptr operator()(int64_t const &) const
    {
        return new integer_description;
    }

    description_ptr operator()(double const &) const
    {
        return new floating_point_description;
    }

    description_ptr operator()(bool const &) const
    {
        return new boolean_description;
    }

    description_ptr operator()(boost::gregorian::date const &) const
    {
        return new date_description;
    }

    description_ptr operator()(boost::posix_time::ptime const &) const
    {
        return new timestamp_description;
    }

    description_ptr operator()(std::string const & s) const
    {
        auto const target_size = size_after_growth_strategy(s.size());
        return new string_description(target_size);
    }
};

template <typename Description>
std::unique_ptr<description> make_character_description(cpp_odbc::column_description const & source,
                                                        turbodbc::options const & options)
{
    int const capacity_multiplier = (options.force_extra_capacity_for_unicode ? Description::max_code_units_per_unicode_code_point : 1);
    std::size_t const sanitized_size = (source.size == 0 ? options.varchar_max_character_limit : source.size);
    std::size_t const limited_size = (options.limit_varchar_results_to_max ?
                                      std::min(sanitized_size, options.varchar_max_character_limit) :
                                      sanitized_size) * capacity_multiplier;
    return std::unique_ptr<description>(new Description(source.name,
                                                        source.allows_null_values,
                                                        limited_size));
}

}

std::unique_ptr<description const> make_description(cpp_odbc::column_description const & source,
                                                    turbodbc::options const & options)
{
    switch (source.data_type) {
        case SQL_CHAR:
        case SQL_VARCHAR:
        case SQL_LONGVARCHAR:
            if (options.prefer_unicode) {
                return make_character_description<unicode_description>(source, options);
            } else {
                return make_character_description<string_description>(source, options);
            }
        case SQL_WVARCHAR:
        case SQL_WLONGVARCHAR:
        case SQL_WCHAR:
            if (options.fetch_wchar_as_char) {
                return make_character_description<string_description>(source, options);
            } else {
                return make_character_description<unicode_description>(source, options);
            }
        case SQL_INTEGER:
        case SQL_SMALLINT:
        case SQL_BIGINT:
        case SQL_TINYINT:
            return std::unique_ptr<description>(new integer_description(source.name, source.allows_null_values));
        case SQL_REAL:
        case SQL_FLOAT:
        case SQL_DOUBLE:
            return std::unique_ptr<description>(new floating_point_description(source.name, source.allows_null_values));
        case SQL_BIT:
            return std::unique_ptr<description>(new boolean_description(source.name, source.allows_null_values));
        case SQL_NUMERIC:
        case SQL_DECIMAL:
            return make_decimal_description(source, options);
        case SQL_TYPE_DATE:
            return std::unique_ptr<description>(new date_description(source.name, source.allows_null_values));
        case SQL_TYPE_TIMESTAMP:
            return std::unique_ptr<description>(new timestamp_description(source.name, source.allows_null_values));
        default:
            std::ostringstream message;
            message << "Error! Unsupported type identifier for column " << source << ")";
            throw std::runtime_error(message.str());
    }
}

std::unique_ptr<description const> make_description(field const & value)
{
    return std::unique_ptr<description const>(boost::apply_visitor(description_by_value{}, value));
}

std::unique_ptr<description const> make_description(type_code type, std::size_t size)
{
    switch (type) {
        case type_code::floating_point:
            return std::unique_ptr<description const>(new floating_point_description);
        case type_code::boolean:
            return std::unique_ptr<description const>(new boolean_description);
        case type_code::date:
            return std::unique_ptr<description const>(new date_description);
        case type_code::timestamp:
            return std::unique_ptr<description const>(new timestamp_description);
        case type_code::string:
            return std::unique_ptr<description const>(new string_description(size_after_growth_strategy(size)));
        case type_code::unicode:
            return std::unique_ptr<description const>(new unicode_description(size_after_growth_strategy(size)));
        default:
            return std::unique_ptr<description const>(new integer_description);
    }
}

}


/**
 * @file blmc_joint_module.hpp
 * @author Manuel Wuthrich
 * @author Maximilien Naveau (maximilien.naveau@gmail.com)
 * @author Julian Viereck (jviereck@tuebingen.mpg.de)
 * @license License BSD-3-Clause
 * @copyright Copyright (c) 2019, New York University and Max Planck
 * Gesellschaft.
 * @date 2020-02-17
 */
#pragma once

#include <math.h>
#include <Eigen/Eigen>
#include <array>
#include <iostream>
#include <stdexcept>

#include "blmc_drivers/devices/motor.hpp"
#include "solo/common_header.hpp"

#include "master_board_sdk/defines.h"
#include "master_board_sdk/master_board_interface.h"

#include <stdexcept>

namespace solo
{
/**
 * @brief This class defines an interface to a collection of BLMC joints. It
 * creates a BLMCJointModule for every blmc_driver::MotorInterface provided.
 */
template <int COUNT>
class SpiJointModules
{
public:
    /**
     * @brief Defines a static Eigen vector type in order to define the
     * interface.
     */
    typedef Eigen::Matrix<double, COUNT, 1> Vector;

    /**
     * @brief Construct a new SpiJointModules object.
     */
    SpiJointModules(std::shared_ptr<MasterBoardInterface> robot_if,
                    std::array<int, COUNT>& motor_to_card_index,
                    std::array<int, COUNT>& motor_to_card_port_index,
                    const Vector& motor_constants,
                    const Vector& gear_ratios,
                    const Vector& zero_angles,
                    const Vector& max_currents,
                    std::array<bool, COUNT> reverse_polarities)
    {
        robot_if_ = robot_if;

        // Setup the motor vectores based on the card and port mapping.
        for (int i = 0; i < COUNT; i++)
        {
            int driver_idx = motor_to_card_index[i];
            if (motor_to_card_port_index[i] == 0)
            {
                motors_[i] = (robot_if->motor_drivers[driver_idx].motor1);
            }
            else
            {
                motors_[i] = (robot_if->motor_drivers[driver_idx].motor2);
            }

            polarities_[i] = reverse_polarities[i] ? -1. : 1.;
        }

        motor_constants_ = motor_constants;
        gear_ratios_ = gear_ratios;
        zero_angles_ = zero_angles;
        max_currents_ = max_currents;

        motor_to_card_index_ = motor_to_card_index;

        index_angles_.fill(0.);
    }

    /**
     * @brief Enable all motors and motor drivers used by the joint module.
     */
    void enable()
    {
        for (int i = 0; i < COUNT; i++)
        {
            int driver_idx = motor_to_card_index_[i];
            robot_if_->motor_drivers[driver_idx].motor1->SetCurrentReference(0);
            robot_if_->motor_drivers[driver_idx].motor2->SetCurrentReference(0);
            robot_if_->motor_drivers[driver_idx].motor1->Enable();
            robot_if_->motor_drivers[driver_idx].motor2->Enable();
            robot_if_->motor_drivers[driver_idx].EnablePositionRolloverError();
            robot_if_->motor_drivers[driver_idx].SetTimeout(5);
            robot_if_->motor_drivers[driver_idx].Enable();
        }
        robot_if_->SendCommand();
    }

    /**
     * @brief Checks if all motors report ready.
     *
     * @return True if all motors are ready, false otherwise.
     */
    bool is_ready()
    {
        for (int i = 0; i < COUNT; i++)
        {
            if (!motors_[i]->IsEnabled() || !motors_[i]->IsReady())
            {
                return false;
            }
        }
        return true;
    }

    std::array<bool, COUNT> get_motor_enabled()
    {
        std::array<bool, COUNT> motor_enabled;
        for (int i = 0; i < COUNT; i++)
        {
            motor_enabled[i] = motors_[i]->IsEnabled();
        }
        return motor_enabled;
    }

    std::array<bool, COUNT> get_motor_ready()
    {
        std::array<bool, COUNT> motor_ready;
        for (int i = 0; i < COUNT; i++)
        {
            motor_ready[i] = motors_[i]->IsReady();
        }
        return motor_ready;
    }

    /**
     * @brief Send the registered torques to all modules.
     */
    void send_torques()
    {
        robot_if_->SendCommand();
    }

    /**
     * @brief Updates the measurements based on the lastest package from
     * the master board.
     */
    void acquire_sensors()
    {
        robot_if_->ParseSensorData();

        // Keep tack of the first recorded encoder.
        Vector positions = get_measured_angles();
        for (int i = 0; i < COUNT; i++)
        {
            if (saw_index_[i] == false && motors_[i]->HasIndexBeenDetected())
            {
                saw_index_[i] = true;
                index_angles_[i] = positions(i);
            }
        }
    }

    /**
     * @brief Register the joint torques to be sent for all modules.
     *
     * @param desired_torques (Nm)
     */
    void set_torques(const Vector& desired_torques)
    {
        Vector desired_current = polarities_.cwiseProduct(desired_torques)
                                     .cwiseQuotient(gear_ratios_)
                                     .cwiseQuotient(motor_constants_);

        // Current clamping.
        desired_current = desired_current.cwiseMin(max_currents_);
        desired_current = desired_current.cwiseMax(-max_currents_);
        // Vector desired_current;
        // desired_current.setZero();

        for (std::size_t i = 0; i < motors_.size(); i++)
        {
            motors_[i]->SetCurrentReference(desired_current(i));
        }
    }

    /**
     * @brief Get the maximum admissible joint torque that can be applied.
     *
     * @return Vector (N/m)
     */
    Vector get_max_torques()
    {
        return max_currents_.cwiseProduct(gear_ratios_)
            .cwiseProduct(motor_constants_);
    }

    /**
     * @brief Get the previously sent torques.
     *
     * @return Vector (Nm)
     */
    Vector get_sent_torques() const
    {
        Vector torques;
        for (size_t i = 0; i < COUNT; i++)
        {
            torques(i) = motors_[i]->current_ref;
        }
        torques = torques.cwiseProduct(polarities_)
                      .cwiseProduct(gear_ratios_)
                      .cwiseProduct(motor_constants_);
        return torques;
    }

    /**
     * @brief Get the measured joint torques.
     *
     * @return Vector (Nm)
     */
    Vector get_measured_torques() const
    {
        Vector torques;
        for (size_t i = 0; i < COUNT; i++)
        {
            torques(i) = motors_[i]->GetCurrent();
        }
        torques = torques.cwiseProduct(polarities_)
                      .cwiseProduct(gear_ratios_)
                      .cwiseProduct(motor_constants_);
        return torques;
    }

    /**
     * @brief Get the measured joint angles.
     *
     * @return Vector (rad)
     */
    Vector get_measured_angles() const
    {
        Vector positions;
        for (size_t i = 0; i < COUNT; i++)
        {
            positions(i) = motors_[i]->GetPosition();
        }
        positions =
            positions.cwiseProduct(polarities_).cwiseQuotient(gear_ratios_) -
            zero_angles_;
        return positions;
    }

    /**
     * @brief Get the measured joint velocities.
     *
     * @return Vector (rad/s)
     */
    Vector get_measured_velocities() const
    {
        Vector velocities;
        for (size_t i = 0; i < COUNT; i++)
        {
            velocities(i) = motors_[i]->GetVelocity();
        }
        velocities =
            velocities.cwiseProduct(polarities_).cwiseQuotient(gear_ratios_);
        return velocities;
    }

    /**
     * @brief Set the zero_angles. These are the joint angles between the
     * starting pose and the zero theoretical pose of the urdf.
     *
     * @param zero_angles (rad)
     */
    void set_zero_angles(const Vector& zero_angles)
    {
        zero_angles_ = zero_angles;
    }
    /**
     * @brief Get the zero_angles. These are the joint angles between the
     * starting pose and the zero theoretical pose of the urdf.
     *
     * @return Vector (rad)
     */
    Vector get_zero_angles() const
    {
        return zero_angles_;
    }
    /**
     * @brief Get the index_angles. There is one index per motor rotation so
     * there are gear_ratio indexes per joint rotation.
     *
     * @return Vector (rad)
     */
    Vector get_measured_index_angles() const
    {
        return index_angles_;
    }

private:
    Vector motor_constants_;
    Vector gear_ratios_;
    Vector max_currents_;
    Vector zero_angles_;
    Vector polarities_;

    std::array<int, COUNT> motor_to_card_index_;

    Vector index_angles_;
    std::array<bool, COUNT> saw_index_;

    /**
     * @brief Holds the motors in the joint order.
     */
    std::array<Motor*, COUNT> motors_;

    std::shared_ptr<MasterBoardInterface> robot_if_;
};

}  // namespace solo


/*
Copyright 2015-2016 Joanna Hulboj <j@hulboj.org>
Copyright 2016 Milosz Hulboj <m@hulboj.org>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

#include "vanilla_date.h"

#include "util/parse.h"

#include <boost/spirit/include/karma.hpp>

#include <contrib/date/date.h>
#include <chrono>
#include <functional>
#include <iterator>

namespace cornelich
{

std::int32_t cycle_formatter::cycle_from_date(const std::string & date) const
{
    return cycle_from_date_impl(date);
}

std::string cycle_formatter::date_from_cycle(std::int64_t cycle) const
{
    // TODO: Is it worth it...?
    auto hash = std::hash<std::int64_t>()(cycle) % (CACHE_SIZE - 1);
    auto cached = std::atomic_load(&m_cache[hash]);
    if(cached && cached->m_cycle == cycle)
    {
        // Cache hit - return the string
        return cached->m_data;
    }
    // Cache miss -> evaluate and cache the string
    auto node = std::make_shared<node_t>(cycle, date_from_cycle_impl(cycle));
    std::atomic_store(&m_cache[hash], node);
    return node->m_data;
}

using namespace date;
using namespace std::chrono;
using namespace boost::spirit;

static const day_point unix_epoch = day(1)/jan/1970;

bool parse_date(const std::string & date, day_point & dp)
{
    int y = 0, m = 0, d = 0;
    if(BOOST_UNLIKELY(
                !util::parse_number(date.begin(), date.begin() + 4, y) ||
                !util::parse_number(date.begin() + 4, date.begin() + 6, m) ||
                !util::parse_number(date.begin() + 6, date.begin() + 8, d)))
    {
        return false;
    }
    auto ymd = year(y)/m/d;
    dp = ymd;
    return ymd.ok();
}

std::int64_t get_epoch_seconds(const system_clock::time_point & ts)
{
    day_point today = time_point_cast<days>(ts);
    system_clock::time_point this_morning = today;
    days days_since_epoch = today - unix_epoch;
    auto s = ts - this_morning;
    std::int64_t unix_timestamp = (days_since_epoch + s) / seconds(1);
    return unix_timestamp;
}

std::int32_t cycle_formatter_yyyymmdd::cycle_from_date_impl(const std::string & date) const
{
    thread_local day_point dp;
    if(BOOST_UNLIKELY(date.length() != 8))
        return -1;
    if(BOOST_UNLIKELY(!parse_date(date, dp)))
        return -1;
    system_clock::time_point ts = dp;
    auto unix_timestamp = get_epoch_seconds(ts);

    return unix_timestamp * 1000 / m_cycle_length;
}

std::string cycle_formatter_yyyymmdd::date_from_cycle_impl(std::int64_t cycle) const
{
    const auto s = static_cast<std::time_t>(cycle * m_cycle_length / 1000);
    auto ts = system_clock::time_point(seconds(s));
    auto daypoint = floor<days>(ts);
    auto ymd = year_month_day(daypoint);

    using namespace boost::spirit;

    std::string result;
    std::back_insert_iterator<std::string> sink(result);
    karma::generate(sink,
                    karma::right_align(4, '0')[karma::int_] << karma::right_align(2, '0')[karma::uint_] << karma::right_align(2, '0')[karma::uint_],
            int(ymd.year()), unsigned(ymd.month()), unsigned(ymd.day()));
    return result;
}

std::int32_t cycle_formatter_yyyymmddhh::cycle_from_date_impl(const std::string & date) const
{
    thread_local day_point dp;
    int h = 0;
    if(BOOST_UNLIKELY(date.length() != 10))
        return -1;
    if(BOOST_UNLIKELY(
                !parse_date(date, dp) ||
                !util::parse_number(date.begin() + 8, date.begin() + 10, h) || h > 24))
        return -1;
    system_clock::time_point ts = dp + hours(h);
    auto unix_timestamp = get_epoch_seconds(ts);

    return unix_timestamp * 1000 / m_cycle_length;
}

std::string cycle_formatter_yyyymmddhh::date_from_cycle_impl(std::int64_t cycle) const
{
    const auto s = static_cast<std::time_t>(cycle * m_cycle_length / 1000);
    auto ts = system_clock::time_point(seconds(s));
    auto daypoint = floor<days>(ts);
    auto ymd = year_month_day(daypoint);
    auto tod = make_time(ts - daypoint);

    using namespace boost::spirit;

    std::string result;
    std::back_insert_iterator<std::string> sink(result);
    karma::generate(sink,
                    karma::right_align(4, '0')[karma::int_] << karma::right_align(2, '0')[karma::uint_] << karma::right_align(2, '0')[karma::uint_] << karma::right_align(2, '0')[karma::uint_],
            int(ymd.year()), unsigned(ymd.month()), unsigned(ymd.day()), tod.hours().count());
    return result;
}

std::int32_t cycle_formatter_yyyymmddhhmm::cycle_from_date_impl(const std::string & date) const
{
    thread_local day_point dp;
    int h = 0, min = 0;
    if(BOOST_UNLIKELY(date.length() != 12))
        return -1;
    if(BOOST_UNLIKELY(
                !parse_date(date, dp) ||
                !util::parse_number(date.begin() + 8, date.begin() + 10, h) || h > 24 ||
                !util::parse_number(date.begin() + 10, date.begin() + 12, min) || min > 60
                ))
        return -1;

    system_clock::time_point ts = dp + hours(h) + minutes(min);
    auto unix_timestamp = get_epoch_seconds(ts);

    return unix_timestamp * 1000 / m_cycle_length;
}

std::string cycle_formatter_yyyymmddhhmm::date_from_cycle_impl(std::int64_t cycle) const
{
    const auto s = static_cast<std::time_t>(cycle * m_cycle_length / 1000);
    auto ts = system_clock::time_point(seconds(s));
    auto daypoint = floor<days>(ts);
    auto ymd = year_month_day(daypoint);
    auto tod = make_time(ts - daypoint);

    using namespace boost::spirit;

    std::string result;
    std::back_insert_iterator<std::string> sink(result);
    karma::generate(sink,
                    karma::right_align(4, '0')[karma::int_] << karma::right_align(2, '0')[karma::uint_] << karma::right_align(2, '0')[karma::uint_] << karma::right_align(2, '0')[karma::uint_] << karma::right_align(2, '0')[karma::uint_],
            int(ymd.year()), unsigned(ymd.month()), unsigned(ymd.day()), tod.hours().count(), tod.minutes().count());
    return result;
}

}


/*
 * File:   score_calculator.cpp
 * Author: malone
 *
 * Created on November 24, 2012, 5:46 PM
 */

#include <algorithm>
#include <math.h>

#include <boost/dynamic_bitset.hpp>
#include <limits>

#include <boost/thread.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>

#include "score_calculator.h"
#include "scoring_function.h"

scoring::ScoreCalculator::ScoreCalculator(scoring::ScoringFunction *scoringFunction, int maxParents, int variableCount, int runningTime, scoring::Constraints *constraints) {
    this->scoringFunction = scoringFunction;
    this->maxParents = maxParents;
    this->variableCount = variableCount;
    this->runningTime = runningTime;
    this->constraints = constraints;
}

void scoring::ScoreCalculator::timeout(const boost::system::error_code& /*e*/) {
    printf("Out of time\n");
    outOfTime = true;
}

void scoring::ScoreCalculator::calculateScores(int variable, FloatMap &cache, varset & neighbors) {
    this->outOfTime = false;

    //io.reset();
    //boost::asio::deadline_timer *t;
    boost::asio::io_service io_t;
    t = new boost::asio::deadline_timer(io_t);
    if (runningTime > 0) {
        printf("I am using a timer in the calculation function.\n");
        t->expires_from_now(boost::posix_time::seconds(runningTime));
        t->async_wait(boost::bind(&scoring::ScoreCalculator::timeout, this, boost::asio::placeholders::error));
        boost::thread workerThread(boost::bind(&scoring::ScoreCalculator::calculateScores_internal, this, variable, boost::ref(cache), boost::ref(neighbors)));
        io_t.run();
        workerThread.join();
        io_t.stop();
//        t.cancel();
    } else {
        calculateScores_internal(variable, cache, neighbors);
    }
}

void scoring::ScoreCalculator::calculateScores_internal(int variable, FloatMap &cache, varset & neighbors) {
    // calculate the initial score
    VARSET_NEW(empty, variableCount);
    float score = scoringFunction->calculateScore(variable, empty, cache);

    if (score < 1) {
        cache[empty] = score;
    }

    int prunedCount = 0;
    // Ni added, record the neighbors' indices
    std::vector<int> neighbor_indices;
    for(int  i = 0; i  < variableCount; i++)
    {
        if(VARSET_GET(neighbors,  i))
        {
            //printf("Will map compact bit # %d to neighbor variable #%d, for variable %d\n", int(neighbor_indices.size()),  i, variable);
            neighbor_indices.push_back(i);
        }
    }
    const int num_neighbors = neighbor_indices.size();

    //printf("calculateScores_internal, variable %d has %d neighbors, total variable count  %d\n", variable, num_neighbors, variableCount);

    for (int layer = 1; layer <= maxParents && !outOfTime; layer++) {
#ifdef DEBUG
        printf("layer: %d, prunedCount: %d\n", layer, prunedCount);
#endif

        VARSET_NEW(compact_variables, num_neighbors);
        for (int i = 0; i < layer; i++) {
            VARSET_SET(compact_variables, i);
        }

        VARSET_NEW(max, num_neighbors);
        VARSET_SET(max, num_neighbors);

        while (VARSET_LESS_THAN(compact_variables, max) && !outOfTime) {
            //Ni added, translate shorter bitset compact_variables to full-length bitset variables
            VARSET_NEW(variables, variableCount);
            for(int i = 0; i < num_neighbors; i++)
            {
                if(VARSET_GET(compact_variables, i))
                    VARSET_SET(variables, neighbor_indices[i]);
            }

            if (!VARSET_GET(variables, variable)) {

                score = scoringFunction->calculateScore(variable, variables, cache);

#ifdef VERBOSE_DEBUG
                printf("%d, %" PRIu64 ": %f\n", variable, variables, score);
#endif

                // only store the score if it was not pruned
                if (score < 0) {
                    cache[variables] = score;
                } else {
                    prunedCount++;
                }
            }

            // find the next combination
            compact_variables = nextPermutation(compact_variables);
        }

        if (!outOfTime) highestCompletedLayer = layer;
    }

//    io.stop();
    t->cancel();
//    io.reset();

#ifdef VERBOSE_DEBUG
    printf("Scores\n");
    for (auto it = cache.begin(); it != cache.end(); it++) {
        printf("%d, %d: %f\n", variable, (*it).first, (*it).second);
    }
#endif
}

struct compareSecond {

    bool operator()(std::pair<varset, float> lhs, std::pair<varset, float> rhs) const {
        float val = lhs.second - rhs.second;

        if (fabs(val) > 2 * std::numeric_limits<float>::epsilon()) {
            return val > 0;
        }

        return lhs.first < rhs.first;
    }
} comparator;

void scoring::ScoreCalculator::prune(FloatMap &cache) {
    std::vector< std::pair<varset, float> > pairs;
    for (auto pair = cache.begin();
            pair != cache.end();
            pair++) {
        pairs.push_back(*pair);
    }

    std::sort(pairs.begin(), pairs.end(), comparator);

#ifdef VERBOSE_DEBUG
    printf("Sorted Scores\n");
    for (auto it = pairs.begin(); it != pairs.end(); it++) {
        printf("%d: %f\n", (*it).first, (*it).second);
    }
#endif

    // keep track of the ones that have been pruned
    boost::dynamic_bitset<> prunedSets(pairs.size());
    for (int i = 0; i < pairs.size(); i++) {
        if (prunedSets.test(i)) {
            continue;
        }

        varset pi = pairs[i].first;

        // make sure this variable set is not in an incomplete last layer
        if (cardinality(pi) > highestCompletedLayer) {
            prunedSets.set(i);
            continue;
        }

        for (int j = i + 1; j < pairs.size(); j++) {
            if (prunedSets.test(j)) {
                continue;
            }

            // check if parents[i] is a subset of parents[j]
            varset pj = pairs[j].first;

            if (VARSET_IS_SUBSET_OF(pi, pj)) {
                // then we can prune pj
                prunedSets.set(j);
                cache.erase(pj);
            }
        }
    }
}


// Boost.Geometry (aka GGL, Generic Geometry Library)

// Copyright (c) 2013 Barend Gehrels, Amsterdam, the Netherlands.

// Use, modification and distribution is subject to the Boost Software License,
// Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)

#ifndef BOOST_GEOMETRY_POLICIES_ROBUSTNESS_SEGMENT_RATIO_HPP
#define BOOST_GEOMETRY_POLICIES_ROBUSTNESS_SEGMENT_RATIO_HPP

#include <boost/config.hpp>
#include <boost/rational.hpp>

#include <boost/geometry/core/assert.hpp>
#include <boost/geometry/util/math.hpp>
#include <boost/geometry/util/promote_floating_point.hpp>

namespace geofeatures_boost {} namespace boost = geofeatures_boost; namespace geofeatures_boost { namespace geometry
{

namespace detail { namespace segment_ratio
{

template
<
    typename Type,
    bool IsIntegral = geofeatures_boost::is_integral<Type>::type::value
>
struct less {};

template <typename Type>
struct less<Type, true>
{
    template <typename Ratio>
    static inline bool apply(Ratio const& lhs, Ratio const& rhs)
    {
        return geofeatures_boost::rational<Type>(lhs.numerator(), lhs.denominator())
             < geofeatures_boost::rational<Type>(rhs.numerator(), rhs.denominator());
    }
};

template <typename Type>
struct less<Type, false>
{
    template <typename Ratio>
    static inline bool apply(Ratio const& lhs, Ratio const& rhs)
    {
        BOOST_GEOMETRY_ASSERT(lhs.denominator() != 0);
        BOOST_GEOMETRY_ASSERT(rhs.denominator() != 0);
        return lhs.numerator() * rhs.denominator()
             < rhs.numerator() * lhs.denominator();
    }
};

template
<
    typename Type,
    bool IsIntegral = geofeatures_boost::is_integral<Type>::type::value
>
struct equal {};

template <typename Type>
struct equal<Type, true>
{
    template <typename Ratio>
    static inline bool apply(Ratio const& lhs, Ratio const& rhs)
    {
        return geofeatures_boost::rational<Type>(lhs.numerator(), lhs.denominator())
            == geofeatures_boost::rational<Type>(rhs.numerator(), rhs.denominator());
    }
};

template <typename Type>
struct equal<Type, false>
{
    template <typename Ratio>
    static inline bool apply(Ratio const& lhs, Ratio const& rhs)
    {
        BOOST_GEOMETRY_ASSERT(lhs.denominator() != 0);
        BOOST_GEOMETRY_ASSERT(rhs.denominator() != 0);
        return geometry::math::equals
            (
                lhs.numerator() * rhs.denominator(),
                rhs.numerator() * lhs.denominator()
            );
    }
};

}}

//! Small class to keep a ratio (e.g. 1/4)
//! Main purpose is intersections and checking on 0, 1, and smaller/larger
//! The prototype used Boost.Rational. However, we also want to store FP ratios,
//! (so numerator/denominator both in float)
//! and Boost.Rational starts with GCD which we prefer to avoid if not necessary
//! On a segment means: this ratio is between 0 and 1 (both inclusive)
//!
template <typename Type>
class segment_ratio
{
public :
    typedef Type numeric_type;

    // Type-alias for the type itself
    typedef segment_ratio<Type> thistype;

    inline segment_ratio()
        : m_numerator(0)
        , m_denominator(1)
        , m_approximation(0)
    {}

    inline segment_ratio(const Type& nominator, const Type& denominator)
        : m_numerator(nominator)
        , m_denominator(denominator)
    {
        initialize();
    }

    inline Type const& numerator() const { return m_numerator; }
    inline Type const& denominator() const { return m_denominator; }

    inline void assign(const Type& nominator, const Type& denominator)
    {
        m_numerator = nominator;
        m_denominator = denominator;
        initialize();
    }

    inline void initialize()
    {
        // Minimal normalization
        // 1/-4 => -1/4, -1/-4 => 1/4
        if (m_denominator < 0)
        {
            m_numerator = -m_numerator;
            m_denominator = -m_denominator;
        }

        m_approximation =
            m_denominator == 0 ? 0
            : geofeatures_boost::numeric_cast<double>
                (
                    geofeatures_boost::numeric_cast<fp_type>(m_numerator) * scale()
                  / geofeatures_boost::numeric_cast<fp_type>(m_denominator)
                );
    }

    inline bool is_zero() const { return math::equals(m_numerator, 0); }
    inline bool is_one() const { return math::equals(m_numerator, m_denominator); }
    inline bool on_segment() const
    {
        // e.g. 0/4 or 4/4 or 2/4
        return m_numerator >= 0 && m_numerator <= m_denominator;
    }
    inline bool in_segment() const
    {
        // e.g. 1/4
        return m_numerator > 0 && m_numerator < m_denominator;
    }
    inline bool on_end() const
    {
        // e.g. 0/4 or 4/4
        return is_zero() || is_one();
    }
    inline bool left() const
    {
        // e.g. -1/4
        return m_numerator < 0;
    }
    inline bool right() const
    {
        // e.g. 5/4
        return m_numerator > m_denominator;
    }

    inline bool near_end() const
    {
        if (left() || right())
        {
            return false;
        }

        static fp_type const small_part_of_scale = scale() / 100.0;
        return m_approximation < small_part_of_scale
            || m_approximation > scale() - small_part_of_scale;
    }

    inline bool close_to(thistype const& other) const
    {
        return geometry::math::abs(m_approximation - other.m_approximation) < 2;
    }

    inline bool operator< (thistype const& other) const
    {
        return close_to(other)
            ? detail::segment_ratio::less<Type>::apply(*this, other)
            : m_approximation < other.m_approximation;
    }

    inline bool operator== (thistype const& other) const
    {
        return close_to(other)
            && detail::segment_ratio::equal<Type>::apply(*this, other);
    }

    static inline thistype zero()
    {
        static thistype result(0, 1);
        return result;
    }

    static inline thistype one()
    {
        static thistype result(1, 1);
        return result;
    }

#if defined(BOOST_GEOMETRY_DEFINE_STREAM_OPERATOR_SEGMENT_RATIO)
    friend std::ostream& operator<<(std::ostream &os, segment_ratio const& ratio)
    {
        os << ratio.m_numerator << "/" << ratio.m_denominator
           << " (" << (static_cast<double>(ratio.m_numerator)
                        / static_cast<double>(ratio.m_denominator))
           << ")";
        return os;
    }
#endif

private :
    typedef typename promote_floating_point<Type>::type fp_type;

    Type m_numerator;
    Type m_denominator;

    // Contains ratio on scale 0..1000000 (for 0..1)
    // This is an approximation for fast and rough comparisons
    // Boost.Rational is used if the approximations are close.
    // Reason: performance, Boost.Rational does a GCD by default and also the
    // comparisons contain while-loops.
    fp_type m_approximation;

    static inline fp_type scale()
    {
        return 1000000.0;
    }
};

}} // namespace geofeatures_boost::geometry

#endif // BOOST_GEOMETRY_POLICIES_ROBUSTNESS_SEGMENT_RATIO_HPP


/*
 SuperCollider real time audio synthesis system
 Copyright (c) 2002 James McCartney. All rights reserved.
 http://www.audiosynth.com

 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
 the Free Software Foundation; either version 2 of the License, or
 (at your option) any later version.

 This program is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.

 You should have received a copy of the GNU General Public License
 along with this program; if not, write to the Free Software
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 */

//UGen by Nick Collins
//Released under the GNU GPL as extensions for SuperCollider 3

//cmake -DSC_PATH=/data/gitprojects/SuperCollider -DCMAKE_OSX_ARCHITECTURES='x86_64' ..

//NOTE: there are certain assumptions in this code about the size of the input and output (currently 2048 frames of spectral data). However, more general UGens are possible based on adapting more of Kerasify, etc. The current code should work with deeper nets as long as input and output at size 2048
//It is also worth saying that realtime deep learning with smaller spectral frames gets very intensive; with the NRT thread it will never keep up, but there is a possibility of adapting KerasifyA

//uses Kerasify, thank you Robert W. Rose
//https://github.com/moof2k/kerasify
//note that this dependency could probably be removed, e.g. I've rolled my own neural net here since Kerasify itself wasn't realtime memory management efficient, but use Kerasify for export from python and loading a model in the UGen, to save writing my own export/import too.

//if use NRT via DoAsynchronousCommand, too slow
//if try to call model in RT, too much load for one block

//two solutions are possible
//amortiseflag = 0) PV_Kerasify: otherwise must write own thread, see DiskIO_UGens.cpp, potential issues with setting thread priority, but in practice does work out OK (just don't use DoAsynchronousCommand, that NRT thread on the server is far too slow)
//https://stackoverflow.com/questions/18884510/portable-way-of-setting-stdthread-priority-in-c11
//https://chromium.googlesource.com/chromium/src/base/+/master/threading/platform_thread_mac.mm
//https://stackoverflow.com/questions/10876342/equivalent-of-setthreadpriority-on-linux-pthreads

//amortiseflag = 1) Can only amortise over blocks if write own model calculations (going a layer at a time per block, etc). Did try going a layer at a time in Kerasify itself, but it was still too slow. So wrote own net implementation. First block sets up data, subsequent blocks calculate a layer of the network at a time
//assumes network not so deep that more layers than available blocks...

#include "keras_model.h"

#include "SC_PlugIn.h"
#include <math.h>
#include <stdlib.h>
#include "FFT_UGens.h"
#include <time.h>

//int messagecounter = 0;

clock_t start, end;
double cpu_time_used;

InterfaceTable *ft;

//those supported by kerasify, keras has a few more
//https://keras.io/activations/
enum {
    activation_linear,
    activation_relu,
    activation_softplus,
    activation_sigmoid,
    activation_tanh,
    activation_hardsigmoid
};

//public by default
struct NCLayer {

    int inputsize;
    int numunits;
    float * connectionmatrix;//rows by columns, as flat array
    float * bias;
    int activationtype;
    float * output;

};

//assumes sizes all fine
void CalculateNCLayer(NCLayer* layer, float * input) {

    int i,j;
    float unitsum;

    int inputsize = layer->inputsize;
    float * matrix = layer->connectionmatrix;
    float * bias = layer->bias;
    int activation = layer->activationtype;
    float calc;
    float * output = layer->output;

    for (i=0; i<layer->numunits; ++i) {

        unitsum = 0.0f;
        float * weightsnow = matrix + (i*inputsize);

        for (j=0; j<inputsize; ++j)
        {
            unitsum += weightsnow[j] * input[j];

        }

        unitsum += bias[i];

        //nothing to do for linear?
        //activation function
        switch(activation) {

            case activation_relu:
            if(unitsum<0.0) unitsum = 0.0;
            break;

            case activation_softplus:
            unitsum = log(1.0 + exp(unitsum));
            break;
            case activation_sigmoid: {
                float x = unitsum;

                if (x >= 0) {
                    unitsum = 1.0 / (1.0 + exp(-x));
                } else {
                    float z = exp(x);
                    unitsum = z / (1.0 + z);
                }
            }
            break;
            case activation_tanh:
            unitsum = tanh(unitsum);
            break;
            case activation_hardsigmoid: {

                float x = (unitsum * 0.2) + 0.5;

                if (x <= 0) {
                    unitsum = 0.0;
                } else if (x >= 1) {
                    unitsum = 1.0;
                } else {
                    unitsum = x;
                }

            }
            break;

            default:
            break;
        }

        output[i] = unitsum;

    }

}

void CalculateNCLayerAmort(NCLayer* layer, float * input, int startunit, int unitstodo) {

    int i,j;
    float unitsum;

    int inputsize = layer->inputsize;
    float * matrix = layer->connectionmatrix;
    float * bias = layer->bias;
    int activation = layer->activationtype;
    float calc;
    float * output = layer->output;

    int endunit = startunit+unitstodo;

    if(endunit>layer->numunits) {

        endunit = layer->numunits;
    }

    for (i=startunit; i<endunit; ++i) {

        unitsum = 0.0f;
        float * weightsnow = matrix + (i*inputsize);

        for (j=0; j<inputsize; ++j)
        {
            unitsum += weightsnow[j] * input[j];

        }

        unitsum += bias[i];

        //nothing to do for linear?
        //activation function
        switch(activation) {

            case activation_relu:
            if(unitsum<0.0) unitsum = 0.0;
            break;

            case activation_softplus:
            unitsum = log(1.0 + exp(unitsum));
            break;
            case activation_sigmoid: {
                float x = unitsum;

                if (x >= 0) {
                    unitsum = 1.0 / (1.0 + exp(-x));
                } else {
                    float z = exp(x);
                    unitsum = z / (1.0 + z);
                }
            }
            break;
            case activation_tanh:
            unitsum = tanh(unitsum);
            break;
            case activation_hardsigmoid: {

                float x = (unitsum * 0.2) + 0.5;

                if (x <= 0) {
                    unitsum = 0.0;
                } else if (x >= 1) {
                    unitsum = 1.0;
                } else {
                    unitsum = x;
                }

            }
            break;

            default:
            break;
        }

        output[i] = unitsum;

    }

}

//KerasModel * g_model;

struct PV_Kerasify : public Unit
{
    //int m_n;
    //float * m_topn;
    //int * m_topnindices;

    char * path;

    KerasModel * model;

    float * phases; //[2048];
    float * spectrumnow;

    //no longer needed, rolled own network below
    // Create a 1D Tensor on length 10 for input data.
    //Tensor * in; //(2048);
    //float datafortensor[2048];
    // Run prediction.
    //Tensor * out;

    bool modelready;
    bool newoutput;

    //roll own neural network
    float * input;
    int numlayers;
    NCLayer * layers;
    float * output;

    int amortisationflag;
    int amortisationcounter;

    float * currentinputpointer; //used in amortisation to shift active input data
    //int amortschedulelayer[100];
    //int amortscheduleindexstart[100];
    //int amortscheduleunitstodo[100];

};

#include "SC_SyncCondition.h"
#include <atomic>
//#include <new>
//#include <functional>
#include <thread>
//#include "SC_Lock.h"
#include <boost/lockfree/queue.hpp>
#include <boost/lockfree/spsc_queue.hpp>

enum {
    kCmd_Ctor,
    kCmd_Run,
    kCmd_Dtor,
};

struct KerasifyMsg
{
    //        World *mWorld;
    int16 mCommand;
    int counter;
    //        int16 mChannels;
    //        int32 mBufNum;
    //        int32 mPos;
    //        int32 mFrames;

    PV_Kerasify * unit;

    void Perform();
};

void KerasifyMsg::Perform()
{
    //PV_Kerasify* unit = (PV_Kerasify*)unit;

    switch (mCommand) {
        case kCmd_Run : {

            //printf("before apply \n");

            // Run prediction.
            //Tensor out(2048); // &out

            //out.PrintShape();
            //printf("after print \n");

            //printf("thread which %d model %p unit %p %p %p\n",counter, (void*)g_model, (void*)unit, (void*)unit->in, (void*)unit->out);

            //printf("unit %f %f %f\n", unit->spectrumnow[0],unit->spectrumnow[1],unit->spectrumnow[2]);

            //unit->in->PrintShape();
            //unit->out->PrintShape();

            //start = clock();
            //... /* Do the work. */

            //unit->model->Apply(unit->in, unit->out);
            //end = clock();
            //cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC;

            //printf("took %f seconds to execute \n", cpu_time_used);

            float * input = unit->input;

            for (int k=0; k<unit->numlayers; ++k) {

                CalculateNCLayer(&unit->layers[k],input);

                input = unit->layers[k].output;

            }

            //final input pointer points to final layer output
            for (int i=0; i<2048; ++i) {

                unit->spectrumnow[i] = input[i]; //((i*i)%17)/17.0;
            }

            //printf("unit %f %f %f\n", unit->spectrumnow[0],unit->spectrumnow[1],unit->spectrumnow[2]);

            unit->newoutput = true;

        }
        break;
        case kCmd_Ctor :
        {

            // Initialize model.

            //unit->in = new Tensor(2048);
            //unit->out = new Tensor(2048);

            // Create a 1D Tensor on length 10 for input data.
            //Tensor in(2048);
            //std::vector<float> data_;
            //float data[2048];

            //for (int i=0; i<2048; ++i) {

            //  data[i] = 0.0f; //((i*i)%17)/17.0;
            //}

            //https://stackoverflow.com/questions/259297/how-do-you-copy-the-contents-of-an-array-to-a-stdvector-in-c-without-looping
            //in.data_.insert(in.data_.end(), &data[0], &data[2048]);

            //unit->in->data_.insert(unit->in->data_.end(), &data[0], &data[2048]);

            //unit->out->data_.insert(unit->out->data_.end(), &data[0], &data[2048]);

            //in.data_ = data;
            //in.data_ = {{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}};

            //g_model = new KerasModel();
            //g_model->LoadModel(unit->path); //"DNN1.model");

            unit->model = new KerasModel();

            unit->model->LoadModel(unit->path); //"DNN1.model");

            //unit->in.data_.insert(unit->in.data_.end(), &(unit->datafortensor[0]), &(unit->datafortensor[2048]));

            printf("loaded %s\n",unit->path);

            //copy model data to my own local struct

            unit->input = new float[2048];

            unit->numlayers = unit->model->layers_.size();

            unit->layers = new NCLayer[unit->numlayers];

            for (int k=0; k<unit->numlayers; ++k) {

                //inputsize  * outputsize
                Tensor * weights_ = &(((KerasLayerDense*)unit->model->layers_[k])->weights_);
                Tensor * biases_ = &(((KerasLayerDense*)unit->model->layers_[k])->biases_);

                float * matrix = new float[weights_->dims_[0] * weights_->dims_[1]];
                float * bias = new float[biases_->dims_[0]];

                int inputsize = weights_->dims_[0];

                unit->layers[k].inputsize = inputsize;
                unit->layers[k].numunits = weights_->dims_[1];

                for (int i = 0; i < weights_->dims_[1]; i++) {

                    float * target = matrix + (inputsize*i);

                    for (int j = 0; j < weights_->dims_[0]; j++) {
                        target[j] = (*weights_)(j, i);
                    }
                }

                for (int i = 0; i < biases_->dims_[0]; i++) {
                    bias[i] = (*biases_)(i);
                }

                unit->layers[k].connectionmatrix = matrix;
                unit->layers[k].bias = bias;

                unit->layers[k].output = new float[biases_->dims_[0]];

                int type = ((KerasLayerDense*)unit->model->layers_[k])->activation_.activation_type_;
                //KerasLayerActivation::ActivationType

                unit->layers[k].activationtype = activation_linear;

                if(type == KerasLayerActivation::ActivationType::kRelu) unit->layers[k].activationtype = activation_relu;

                if(type == KerasLayerActivation::ActivationType::kSoftPlus) unit->layers[k].activationtype = activation_softplus;

                if(type == KerasLayerActivation::ActivationType::kSigmoid) unit->layers[k].activationtype = activation_sigmoid;

                if(type == KerasLayerActivation::ActivationType::kTanh) unit->layers[k].activationtype = activation_tanh;

                if(type == KerasLayerActivation::ActivationType::kHardSigmoid) unit->layers[k].activationtype = activation_hardsigmoid;

                printf("layer number %d type %d inputsize %d numunits %d \n",k, type, inputsize, unit->layers[k].numunits);

            }

            printf("Set up internal model data structure with %d layers\n", unit->numlayers);

            // Run prediction.
            //Tensor out;

            //            start = clock();
            //            //... /* Do the work. */
            //            unit->model->Apply(unit->in, unit->out);
            //            end = clock();
            //            cpu_time_used = ((double) (end - start)) / CLOCKS_PER_SEC;
            //
            //            printf("first time took %f seconds to execute \n", cpu_time_used);
            //

            unit-> modelready = true;

        }
        break;
        case kCmd_Dtor :
        {
            //problem if unit already deallocated by this point
            //delete unit->in;
            //delete unit->out;
            delete unit->model;

            for (int k=0; k<unit->numlayers; ++k) {

                delete [] unit->layers[k].connectionmatrix;
                delete [] unit->layers[k].bias;
                delete [] unit->layers[k].output;

            }

            delete [] unit->layers;
            delete [] unit->input;

        }
        break;
    }

}

struct KerasifyThread
{
    SC_SyncCondition mKerasifyFifoHasData;

#ifdef SUPERNOVA
    boost::lockfree::queue<KerasifyMsg, boost::lockfree::capacity<256> > mKerasifyFifo;
#else
    boost::lockfree::spsc_queue<KerasifyMsg, boost::lockfree::capacity<256> > mKerasifyFifo;
#endif

    std::atomic<bool> mRunning;
    std::thread mThread;

    KerasifyThread():
    mRunning(false)
    {}

    ~KerasifyThread()
    {
        if (mRunning) {
            mRunning.store(false);
            mKerasifyFifoHasData.Signal();
            mThread.join();
        }
    }

    void launchThread()
    {
        using namespace std;
        mRunning.store(true);

        mThread = thread( bind(&KerasifyThread::kerasifyThreadFunc, this) ) ;
    }

    bool Run(KerasifyMsg& data)
    {
        bool pushSucceeded = mKerasifyFifo.push(data);
        if (pushSucceeded)
        mKerasifyFifoHasData.Signal();
        return pushSucceeded;
    }

    void kerasifyThreadFunc()
    {
        while (mRunning.load()) {
            mKerasifyFifoHasData.WaitEach();

            KerasifyMsg msg;
            bool popSucceeded = mKerasifyFifo.pop(msg);

            if (popSucceeded)
            msg.Perform();
        }
    }
};

KerasifyThread *gKerasify;

extern "C" {

    void PV_Kerasify_next(PV_Kerasify* unit, int inNumSamples);
    void PV_Kerasify_Ctor(PV_Kerasify* unit);
    void PV_Kerasify_Dtor(PV_Kerasify* unit);

}

void PV_Kerasify_Ctor( PV_Kerasify* unit ) {

    //printf("PV_Kerasify_Ctor /n hello \n");

    unit-> amortisationflag = (int)ZIN0(1);

    unit-> amortisationcounter = 0;

    unit->modelready = false;
    unit->newoutput = false;

    World *world = unit->mWorld;

    unit->phases = (float * ) RTAlloc(unit->mWorld, 2048*sizeof(float));
    unit->spectrumnow = (float * ) RTAlloc(unit->mWorld, 2048*sizeof(float));

    for (int i=0; i<2048; ++i) {

        unit->phases[i] = 0.0f;
        unit->spectrumnow[i] = 0.0f;
    }

    int pathsize = (int) ZIN0(2);

    unit->path = (char *) RTAlloc(unit->mWorld,sizeof(char)*(pathsize+1));

    for(int i=0; i<pathsize; ++i) {
        unit->path[i] = (char)ZIN0(3+i);
    }

    unit->path[pathsize] = 0;

    printf("constructor for PV_Kerasify loading %s\n",unit->path);

    // send a message to side thread
    KerasifyMsg msg;
    msg.unit = unit;
    //msg.counter = messagecounter;
    msg.mCommand = kCmd_Ctor;
    gKerasify->Run(msg);

    SETCALC(PV_Kerasify_next);
    ZOUT0(0) = ZIN0(0);

}

void PV_Kerasify_Dtor( PV_Kerasify* unit ) {

    RTFree(unit->mWorld, unit->path);

    RTFree(unit->mWorld, unit->phases);
    RTFree(unit->mWorld, unit->spectrumnow);

    // send a message to side thread
    KerasifyMsg msg;
    msg.unit = unit;
    //msg.counter = messagecounter;
    msg.mCommand = kCmd_Dtor;
    gKerasify->Run(msg);

}

void PV_Kerasify_next( PV_Kerasify *unit, int inNumSamples ) {

    int i,j,k;

    if(unit-> amortisationcounter<=0) {

        float fbufnum = ZIN0(0);

        //if (fbufnum < 0.f) return;

        if (fbufnum < 0.f) { ZOUT0(0) = -1.f; return; }
        ZOUT0(0) = fbufnum;

        int ibufnum = (uint32)fbufnum;

        World *world = unit->mWorld;
        SndBuf *buf;

        if (ibufnum >= world->mNumSndBufs) {
            int localBufNum = ibufnum - world->mNumSndBufs;
            Graph *parent = unit->mParent;
            if(localBufNum <= parent->localBufNum) {
                buf = parent->mLocalSndBufs + localBufNum;
            } else {
                buf = world->mSndBufs;
            }
        } else {
            buf = world->mSndBufs + ibufnum;
        }

        LOCK_SNDBUF(buf);

        if(unit-> modelready) {

            int numbins = (buf->samples - 2) >> 1;

            float * data = buf->data; //just use it, it is in form dc, nyquist then real,imag pairs per ascending band

            //SCComplexBuf* complex = ToComplexApx(buf);
            //SCComplex * data = complex->bin;
            //also dc, nyquist

            float real, imag;

            real = data[0]; //DC

            //unit->in->data_[0] = 0.09163326695 * log((real*real) + 1);
            //unit->spectrumnow[0] = 0.09163326695 * log((real*real) + 1);
            unit->input[0] = 0.09163326695 * log((real*real) + 1);

            unit->phases[0] = 0.0;

            //printf("before prep data \n");

            for  (j=1;j<numbins; ++j) {

                int index = 2*j;

                real = data[index];
                imag = data[index+1];

                //0.5*Math.log(power+1)*scalefactor; //(1/5.456533600026138)
                //float ampnow = sqrt((real*real) + (imag*imag));

                //relates to scale factor and log power encoding used for training neural net in first place with spectral data in range [0,1]

                //unit->in->data_[j] = 0.09163326695 * log((real*real) + (imag*imag) + 1);

                //unit->spectrumnow[j] = 0.09163326695 * log((real*real) + (imag*imag) + 1);

                unit->input[j] = 0.09163326695 * log((real*real) + (imag*imag) + 1);

                unit->phases[j] = atan2(imag, real);

            }

            unit->currentinputpointer = unit->input;

            if(unit->newoutput) {
                //musn't be replaced mid use, so need to only transfer over from a completed run
                //previous output data potentially
                //power spectrum back to complex (eg no phase, real only)

                float magnitude, phase;

                for (i = 0; i < numbins; ++i) {

                    //out.data_[i] unit->out->data_[i] unit->out->data_[i]
                    magnitude = exp((unit->spectrumnow[i])*5.456533600026138)-1;

                    phase = unit->phases[i];

                    //fftdata[2*i] = outputspectra.w[i]; //act.w[i]; //
                    //return to magnitude not power
                    data[2*i] =  magnitude * cos(phase);//Math.sqrt(Math.abs(outputspectra.w[i]));
                    data[2*i+1] = magnitude * sin(phase); //0.0;

                    //if(i<10) fftdata[2*i] = 0.0;

                }

                unit->newoutput = false; //or not needed at all?
            }

            //printf("pre message %d model %p unit %p %p %p\n", messagecounter,(void*)g_model,(void*)unit, (void*)unit->in, (void*)unit->out);

            if(unit->amortisationflag==0) {

                // send a message to side thread
                KerasifyMsg msg;
                msg.unit = unit;
                //msg.counter = messagecounter;
                msg.mCommand = kCmd_Run;

                //++messagecounter;

                //printf("sendMessage %d  %d %d %d\n", msg.mBufNum, msg.mPos, msg.mFrames, msg.mChannels);
                gKerasify->Run(msg);

            } else
            unit->amortisationcounter = 1; //unit->model->layers_.size();

        }

        return;

    } else {

        //amortise layer by layer
        //could refine to calculate parts of layers for further amortisation
        //can create a schedule based on available blocks spread over calcs per layer and layers

        //printf("amortise flag %d counter %d layer %d of %d \n",unit->amortisationflag, unit->amortisationcounter, unit->amortisationcounter-1, unit->numlayers);

        //available amort periods before new spectral frame = (2048/64 - 1) = 31
        //assuming 2048 hop, 64 sample block size

        int stepsperlayer = unit->amortisationflag; //let user set this

        float * input = unit->currentinputpointer;

        int layernum = (unit->amortisationcounter-1)/stepsperlayer; //split each layer into stepsperlayer
        NCLayer * layernow = &unit->layers[layernum];
        int layerstep = (unit->amortisationcounter-1)%stepsperlayer;

        int unitsavailableinlayer = layernow->numunits;
        int totalperamortstep = unitsavailableinlayer/stepsperlayer; //assumes divisible by stepsperlayer, else add 1 to compensate

        if((totalperamortstep*stepsperlayer) < unitsavailableinlayer) {
            totalperamortstep = totalperamortstep + 1;
        }

        int startposforamortstep = totalperamortstep * layerstep;

        //printf("amortise flag %d counter %d layer %d of %d layernum %d layerste %d  unitsavailableinlayer %d totalperamortstep %d startposforamortstep %d \n",unit->amortisationflag, unit->amortisationcounter, unit->amortisationcounter-1, unit->numlayers, layernum, layerstep, unitsavailableinlayer, totalperamortstep, startposforamortstep);

        //all at once too high peak CPU
        //CalculateNCLayer(layernow,input);

        CalculateNCLayerAmort(layernow,input,startposforamortstep,totalperamortstep);

        //if((startposforamortstep+totalperamortstep)>=unitsavailableinlayer)
        if(layerstep==(stepsperlayer-1)) //just did final step of 4 per layer
        {
            unit->currentinputpointer = layernow->output;
            input = unit->currentinputpointer;
        }

        ++unit->amortisationcounter;

        //unit->amortisationcounter>unit->numlayers
        if(unit->amortisationcounter>(unit->numlayers*stepsperlayer)) {
            unit->amortisationcounter = 0;

            //final input pointer points to final layer output
            for (int i=0; i<2048; ++i) {

                unit->spectrumnow[i] = input[i];
            }

            unit->newoutput = true;
        }

        //safety
        ZOUT0(0) = -1.f;
        return;

    }

    /*

     //amortise by hacking KerasModel class
     //layers_ made public rather than private
     //call from outside, a step at a time over layers

     //bool KerasModel::Apply(Tensor* in, Tensor* out) {

     Tensor *temp_in, *temp_out;

     //for (unsigned int i = 0; i < layers_.size(); i++) {

     if (unit->amortisationcounter == 1) {
     temp_in = unit->in;
     temp_out = unit->out;
     } else {

     temp_in = unit->out;
     temp_out = unit->in;
     }
     //printf("Apply layer %d of %d layer(s)\n", unit->amortisationcounter-1, unit->model->layers_.size());

     bool result = unit->model->layers_[unit->amortisationcounter-1]->Apply(temp_in, temp_out);

     if(!result) printf("Failed to apply layer %d \n", unit->amortisationcounter-1);

     *(unit->out) = *temp_out;

     //        Tensor temp_in, temp_out;
     //
     //        //for (unsigned int i = 0; i < layers_.size(); i++) {
     //
     //            if (unit->amortisationcounter == 1) {
     //                temp_in = *(unit->in);
     //            } else {
     //
     //                temp_in = *(unit->out);
     //            }
     //
     //
     //    printf("Apply layer %d of %d layer(s)\n", unit->amortisationcounter-1, unit->model->layers_.size());
     //    bool result = unit->model->layers_[unit->amortisationcounter-1]->Apply(&temp_in, &temp_out);
     //
     //    if(!result) printf("Failed to apply layer %d \n", unit->amortisationcounter-1);
     //
     //            //temp_in = temp_out;
     //        //}
     //
     //        //*out = temp_out;
     //
     //        *(unit->out) = temp_out;
     //
     ++unit->amortisationcounter;

     if(unit->amortisationcounter>unit->model->layers_.size()) {
     unit->amortisationcounter = 0;
     unit->newoutput = true;
     }

     //}
     */

    //NRT thread too slow
    /*
     //run model on latest data, will lag at least one spectral frame behind, but only way to amortise
     CmdData* cmd = (CmdData*)RTAlloc(unit->mWorld, sizeof(CmdData));
     //cmd->samplingrate_ = unit->mRate->mSampleRate;
     cmd->unit = (Unit *)unit;
     cmd->nrtallocated = NULL; //will be allocated in NRT thread
     cmd->type = CmdData::NRTModelRunPV_Kerasify;

     DoAsynchronousCommand(unit->mWorld, 0, "", (void*)cmd,
     (AsyncStageFn)cmdStage2,
     (AsyncStageFn)cmdStage3,
     NULL,
     cmdCleanup,
     0, 0);

     */

}

#define DefinePVUnit(name) \
(*ft->fDefineUnit)(#name, sizeof(PV_Unit), (UnitCtorFunc)&name##_Ctor, 0, 0);

C_LINKAGE SC_API_EXPORT void unload(InterfaceTable *inTable)
{
    delete gKerasify;
}

PluginLoad(PV_Kerasify)
{

    init_SCComplex(inTable);

    ft = inTable;

    gKerasify = new KerasifyThread();
    gKerasify->launchThread();

    DefinePVUnit(PV_Kerasify);

}


#ifndef ISOLATOR_ANALYZE_HPP
#define ISOLATOR_ANALYZE_HPP

#include <boost/numeric/ublas/matrix.hpp>
#include <map>
#include <set>
#include <string>
#include <vector>
#include <cstdio>

#include "hdf5.hpp"
#include "fragment_model.hpp"
#include "queue.hpp"
#include "sampler.hpp"
#include "transcripts.hpp"

class SamplerTickThread;
class ConditionMeanShapeSamplerThread;
class ExperimentMeanShapeSamplerThread;
class GammaBetaSampler;
class AlphaSampler;
class BetaSampler;
class GammaNormalSigmaSampler;
class GammaStudentTSigmaSampler;
class ConditionSpliceMuSigmaEtaSamplerThread;
class ExperimentSpliceMuSigmaSamplerThread;
class GammaShapeSampler;
typedef std::pair<int, int> IdxRange;

class Analyze
{
    public:
        Analyze(unsigned int rng_seed,
                size_t burnin,
                size_t num_samples,
                TranscriptSet& ts,
                const char* genome_filename,
                bool run_seqbias_correction,
                bool run_gc_correction,
                bool run_3p_correction,
                bool run_frag_correction,
                bool collect_qc_data,
                bool nopriors,
                std::set<std::string> excluded_seqs,
                std::set<std::string> bias_training_seqnames,
                double experiment_shape_alpha,
                double experiment_shape_beta,
                double experiment_splice_sigma_alpha,
                double experiment_splice_sigma_beta,
                double condition_shape_alpha,
                double condition_shape_beta_a,
                double condition_shape_beta_b,
                double condition_splice_alpha,
                double condition_splice_beta_a,
                double condition_splice_beta_b);
        ~Analyze();

        // Add a replicate under a particular condition
        void add_sample(const char* condition_name,
                        const char* filename);

        void run(hid_t file_id, bool dryrun);
        void cleanup();

    private:
        void setup_samplers();
        void setup_output(hid_t output_file_id);
        void sample(bool optimize_state);
        void write_output(size_t sample_num);

        void qsampler_update_hyperparameters();

        void compute_ts();
        void compute_xs();

        void choose_initial_values();
        void compute_scaling();

        // number of burnin samples
        size_t burnin;

        // number of samples to generate
        size_t num_samples;

        // transcript set
        TranscriptSet& transcripts;

        // File name of a fasta file containing the reference genome sequence
        // against which the reads are aligned.
        const char* genome_filename;

        // True if SeqBias correction should be used.
        bool run_seqbias_correction;

        // True if GC content correction should be used.
        bool run_gc_correction;

        // True if 3' bias should be corrected
        bool run_3p_correction;

        // True if fragmentation bias should be corrected
        bool run_frag_correction;

        // Sequences on which aligned reads should be ignored
        std::set<std::string> excluded_seqs;

        // If non-empty, contains names of sequence to which bias training
        // should be restricted.
        std::set<std::string> bias_training_seqnames;

        // True if extra extra QC data should be collected
        bool collect_qc_data;

        // True if priors should not be applie during quantification
        bool nopriors;

        // file names for the BAM/SAM file corresponding to each
        std::vector<std::string> filenames;

        // condition index to sample indexes
        std::vector<std::vector<int> > condition_samples;

        // fragment models for each sample
        std::vector<FragmentModel*> fms;

        // quantification samplers for each sample
        std::vector<Sampler*> qsamplers;

        // threads used for iterating samplers
        std::vector<SamplerTickThread*> qsampler_threads;
        std::vector<ConditionMeanShapeSamplerThread*> meanshape_sampler_threads;
        std::vector<ExperimentMeanShapeSamplerThread*> experiment_meanshape_sampler_threads;
        GammaBetaSampler* gamma_beta_sampler;
        BetaSampler* invgamma_beta_sampler;
        GammaNormalSigmaSampler* gamma_normal_sigma_sampler;
        GammaShapeSampler* gamma_shape_sampler;

        std::vector<ConditionSpliceMuSigmaEtaSamplerThread*> splice_mu_sigma_sampler_threads;
        std::vector<ExperimentSpliceMuSigmaSamplerThread*>
            experiment_splice_mu_sigma_sampler_threads;

        // queues to send work to sampler threads, and be notified on completion
        // of ticks.
        Queue<int> qsampler_tick_queue, qsampler_notify_queue;

        // work is doled out in block for these. Otherwise threads can starve
        // when there are few sample in the experiment
        Queue<IdxRange> meanshape_sampler_tick_queue,
                        experiment_meanshape_sampler_tick_queue,
                        splice_mu_sigma_sampler_tick_queue,
                        experiment_splice_mu_sigma_sampler_tick_queue;

        Queue<int> meanshape_sampler_notify_queue,
                   experiment_meanshape_sampler_notify_queue,
                   splice_mu_sigma_sampler_notify_queue,
                   experiment_splice_mu_sigma_sampler_notify_queue;

        // We maintain a different rng for every unit of work for threads.
        // That way we can actually make isolator run reproducible.
        std::vector<rng_t> transcript_rng_pool;
        std::vector<rng_t> splice_rng_pool;

        // matrix containing relative transcript abundance samples, indexed by:
        //   sample -> transcript (tid)
        boost::numeric::ublas::matrix<float> Q;

        // transcript mean parameter, indexed by condition -> transcript
        boost::numeric::ublas::matrix<float> condition_mean;

        // transcript shape parameter, indexed by transcript
        std::vector<float> condition_shape;

        // parameters of the inverse gamma prior on condition_splice_sigma
        double condition_splice_alpha, condition_splice_beta;

        // parameters of the inverse gamma prior on condition_shape
        double condition_shape_alpha, condition_shape_beta;

        // experiment-wise transcript position paremeter, indexed by transcript
        std::vector<float> experiment_mean;

        // experiment-wide transcript scale parameter
        double experiment_shape;

        // gamma hypeparameters for prior on experiment_shape
        double experiment_shape_alpha;
        double experiment_shape_beta;

        // parameters for normal prior over experiment_mean
        double experiment_mean0, experiment_shape0;

        // tids belonging to each tgroup (indexed by tgroup)
        std::vector<std::vector<unsigned int> > tgroup_tids;

        // sorted indexes of tgroups with multiple transcripts
        std::vector<unsigned int> spliced_tgroup_indexes;

        // condition slice mean indexed by condition, spliced tgroup, transcript
        // according to spliced_tgroup_indexes and tgroup_tids
        std::vector<std::vector<std::vector<float> > > condition_splice_mu;

        // per-spliced-tgroup experiment wide logistic-normal mean
        std::vector<std::vector<float> > experiment_splice_mu;

        // prior parameters for experimient_splice_mu
        double experiment_splice_nu, experiment_splice_mu0, experiment_splice_sigma0;

        // experiment std. dev.
        double experiment_splice_sigma;

        // gamma hypeparameters for prior on experiment_splice_sigma
        double experiment_splice_sigma_alpha;
        double experiment_splice_sigma_beta;

        // splicing precision, indexed by spliced tgroup
        std::vector<std::vector<float> > condition_splice_sigma;

        // overparameterization to unstick stuck samplers
        std::vector<std::vector<float> > condition_splice_eta;

        // flattened condition_splice_sigma used for sampling alpha, beta
        // params.
        std::vector<float> condition_splice_sigma_work;
        std::vector<float> experiment_splice_sigma_work;

        // paramaters for the inverse gamma priors on splice_alpha and
        // splice_beta
        double condition_splice_beta_a,
               condition_splice_beta_b;

        // Condition index corresponding to the given name
        std::map<std::string, int> condition_index;

        // Condition index of sample i
        std::vector<int> condition;

        // normalization constant for each sample
        std::vector<float> scale;

        // temporary space used for computing scale
        std::vector<float> scale_work;

        // number of sequenced samples
        unsigned int K;

        // number of conditions
        unsigned int C;

        // number of transcripts
        unsigned int N;

        // number of tgroups
        unsigned int T;

        // Hyperparams for inverse gamma prior on tgroup_alpha/tgroup_beta
        double condition_shape_beta_a,
               condition_shape_beta_b;

        // RNG used for alpha/beta samplers
        unsigned int rng_seed;
        rng_t rng;

        // HDF5 dataspace ids, for output purposes

        // dataspaces
        hid_t h5_experiment_mean_dataspace;
        hid_t h5_condition_mean_dataspace;
        hid_t h5_condition_mean_mem_dataspace;
        hid_t h5_row_mem_dataspace;
        hid_t h5_sample_quant_dataspace;
        hid_t h5_sample_quant_mem_dataspace;
        hid_t h5_experiment_splice_dataspace;
        hid_t h5_condition_splice_mu_dataspace;
        hid_t h5_condition_splice_sigma_dataspace;
        hid_t h5_splicing_mem_dataspace;
        hid_t h5_sample_scaling_dataspace;
        hid_t h5_sample_scaling_mem_dataspace;

        // datasets
        hid_t h5_experiment_mean_dataset;
        hid_t h5_condition_mean_dataset;
        hid_t h5_condition_shape_dataset;
        hid_t h5_sample_quant_dataset;
        hid_t h5_experiment_splice_mu_dataset;
        hid_t h5_experiment_splice_sigma_dataset;
        hid_t h5_condition_splice_mu_dataset;
        hid_t h5_condition_splice_sigma_dataset;
        hid_t h5_sample_scaling_dataset;

        // variable length array for splicing paramaters
        hid_t h5_splice_param_type;

        // structure for the ragged-array splicing data
        hvl_t* h5_splice_work;

        // a write buffer for hdf5 output
        std::vector<float> row_data;

        friend void write_qc_data(FILE* fout, Analyze& analyze);
        friend void compare_seqbias(Analyze& analyze, TranscriptSet& ts,
                                    const char* genome_filename);
};

#endif


/*
 Copyright (C) 2017 Sascha Meiers
 Distributed under the MIT software license, see the accompanying
 file LICENSE.md or http://www.opensource.org/licenses/mit-license.php.
 */

#include <iostream>
#include <fstream>
#include <vector>
#include <unordered_map>
#include <tuple>

#include <boost/program_options/cmdline.hpp>
#include <boost/program_options/options_description.hpp>
#include <boost/program_options/parsers.hpp>
#include <boost/program_options/variables_map.hpp>
#include <boost/tokenizer.hpp>
#include <boost/filesystem.hpp>
#include <boost/progress.hpp>
#include <boost/iostreams/stream.hpp>
#include <boost/iostreams/stream_buffer.hpp>
#include <boost/iostreams/device/file.hpp>
#include <boost/iostreams/filtering_stream.hpp>
#include <boost/iostreams/filter/zlib.hpp>
#include <boost/iostreams/filter/gzip.hpp>
#include <htslib/sam.h>

#include "version.hpp"
#include "intervals.hpp"
#include "counter.hpp"
#include "distribution.hpp"
#include "hmm.hpp"
#include "iocounts.hpp"

/**
 * @file
 * @defgroup count Bin, count and classify W/C reads.
 *
 * Summary of how Strand-seq data is binned, counted and classified.
 *
 * ## Strand-seq read counting
 *
 * @todo write documentation about counting.
*/

using interval::Interval;
using count::TGenomeCounts;
using count::Counter;

struct Conf {
    std::vector<boost::filesystem::path> f_in;
    boost::filesystem::path f_out;
    boost::filesystem::path f_bins;
    boost::filesystem::path f_excl;
    boost::filesystem::path f_info;
    boost::filesystem::path f_sample_info;
    boost::filesystem::path f_removed_bins;
    boost::filesystem::path f_segments;
    int minMapQual;
    unsigned int window;
    std::string mode;
};

/**
 *
 */
void run_standard_HMM(std::vector<TGenomeCounts> & counts,
                      std::vector<unsigned> const & good_cells,
                      std::vector<CellInfo>  & cells,
                      std::vector<unsigned> const & good_bins,
                      std::vector<int32_t> const & good_map,
                      std::unordered_map<std::string, SampleInfo> const & samples,
                      float p_trans)
{
    // Set up and run HMM:
    hmm::HMM<unsigned, hmm::MultiVariate<hmm::NegativeBinomial> > hmm({"CC", "WC", "WW"});
    hmm.set_initials({0.3333, 0.3333, 0.3333});
    hmm.set_transitions({1-2*p_trans, p_trans,     p_trans,    \
        p_trans,     1-2*p_trans, p_trans,    \
        p_trans,     p_trans,     1-2*p_trans});

    for (auto i = good_cells.begin(); i != good_cells.end(); ++i)
    {
        // set NB(n,p) parameters according to `p` of sample and mean of cell.
        float p = samples.at(cells[*i].sample_name).p;
        float n = (float)cells[*i].mean_bin_count * p / (1-p);
        float a = 0.1;
        cells[*i].nb_p = p;
        cells[*i].nb_r = n;
        cells[*i].nb_a = a;

        //std::cout << "NB parameters for cell <?>" << ": p=" << p << "\tn=" << n << "\tz=" << z << std::endl;

        hmm.set_emissions( {\
            hmm::MultiVariate<hmm::NegativeBinomial>({hmm::NegativeBinomial(p, (1-a)*n), hmm::NegativeBinomial(p, a*n)}), // CC
            hmm::MultiVariate<hmm::NegativeBinomial>({hmm::NegativeBinomial(p, n/2),     hmm::NegativeBinomial(p, n/2)}), // WC
            hmm::MultiVariate<hmm::NegativeBinomial>({hmm::NegativeBinomial(p, a*n),     hmm::NegativeBinomial(p, (1-a)*n)})  // WW
        });
        run_HMM(hmm, counts[*i], good_bins, good_map);
    }
}

int main_count(int argc, char **argv)
{

    // Command line options
    Conf conf;
    boost::program_options::options_description generic("Generic options");
    generic.add_options()
    ("help,?", "show help message")
    ("verbose,v", "Be more verbose in the output")
    ("mapq,q", boost::program_options::value<int>(&conf.minMapQual)->default_value(10), "min mapping quality")
    ("window,w", boost::program_options::value<unsigned int>(&conf.window)->default_value(500000), "window size of fixed windows")
    ("out,o", boost::program_options::value<boost::filesystem::path>(&conf.f_out)->default_value("out.txt.gz"), "output file for counts + strand state (gz)")
    ("bins,b", boost::program_options::value<boost::filesystem::path>(&conf.f_bins), "BED file with manual bins (disables -w). See also 'makebins'")
    ("exclude,x", boost::program_options::value<boost::filesystem::path>(&conf.f_excl), "Exclude chromosomes and regions")
    ("info,i", boost::program_options::value<boost::filesystem::path>(&conf.f_info), "Write info about samples")
    ("do-not-filter-by-WC", "When black-listing bins, only consider coverage and not WC/WW/CC states")
    ("do-not-blacklist-hmm", "Do not output a blacklist (None bins). Bins will be blacklisted for parameter estimation, but not during HMM")
    ;
    // Note: Currently the blacklisting is done after counting via R/norm.R. A better way would be to
    // input the blacklist + normalization into MosaiCatcher during the counting, then the HMM could be run
    // only on non-blacklisted bins. Now it is run through many bad bin that potentially affect the quality of results.

    boost::program_options::options_description hidden("Hidden options");
    hidden.add_options()
    ("input-file", boost::program_options::value<std::vector<boost::filesystem::path> >(&conf.f_in), "input bam file(s)")
    ("sample_info,S", boost::program_options::value<boost::filesystem::path>(&conf.f_sample_info),   "write info per sample")
    ("removed_bins,R", boost::program_options::value<boost::filesystem::path>(&conf.f_removed_bins), "bins that were removed (bed file)")
    ;

    boost::program_options::positional_options_description pos_args;
    pos_args.add("input-file", -1);

    boost::program_options::options_description cmdline_options;
    cmdline_options.add(generic).add(hidden);
    boost::program_options::options_description visible_options;
    visible_options.add(generic);
    boost::program_options::variables_map vm;

    boost::program_options::store(boost::program_options::command_line_parser(argc, argv).options(cmdline_options).positional(pos_args).run(), vm);
    boost::program_options::notify(vm);

    // Check command line arguments
    if (!vm["window"].defaulted() && vm.count("bins")) {
        std::cerr << "[Error] -w and -b cannot be specified together" << std::endl << std::endl;
        goto print_usage_and_exit;
    }
    if (vm.count("bins") && vm.count("exclude")) {
        std::cerr << "[Error] Exclude chromosomes (-x) have no effect when -b is specified. Stop" << std::endl << std::endl;
        goto print_usage_and_exit;
    }

    if (vm.count("help") || !vm.count("input-file"))
    {
    print_usage_and_exit:
        std::cout << std::endl;
        std::cout << "Mosaicatcher " << STRINGIFYMACRO(MOSAIC_VERSION_MAJOR);
        std::cout << "." << STRINGIFYMACRO(MOSAIC_VERSION_MINOR) << std::endl;
        std::cout << "> Count reads from Strand-seq BAM files." << std::endl;
        std::cout << std::endl;
        std::cout << "Usage:   " << argv[0] << " [OPTIONS] <cell1.bam> <cell2.bam> ..." << std::endl << std::endl;
        std::cout << visible_options << std::endl;
        std::cout << "Notes:" << std::endl;
        std::cout << "  * writes a table of bin counts and state classifcation as a gzip file (default: out.txt.gz)" << std::endl;
        std::cout << "  * Reads are counted by start position" << std::endl;
        std::cout << "  * One cell per BAM file, including SM tag in header" << std::endl;
        std::cout << "  * For paired-end data, only read 1 is counted" << std::endl;
        return vm.count("help") ? 0 : 1;
    }

    /////////////////////////////////////////////////////////// global variables
    /* leave one BAM header open to get chrom names & lengths */
    bam_hdr_t* hdr = NULL;

    /* regarding each cell */
    std::vector<CellInfo>       cells(conf.f_in.size());
    std::vector<TGenomeCounts>  counts(conf.f_in.size());
    std::vector<unsigned>       good_cells;

    /* regarding each sample */
    std::unordered_map<std::string, SampleInfo> samples;

    /* regarding bins */
    std::vector<Interval>       bins;
    std::vector<int32_t>        chrom_map;
    std::vector<unsigned>       good_bins;
    std::vector<int32_t>        good_map;
    ////////////////////////////////////////////////////////////////////////////

    //
    // Chapter: Binning & counting
    // ===========================
    //

    // Read sample names from headers.
    // Keep one header throughout the program.
    if (vm.count("verbose")) std::cout << "[Info] Exploring SAM headers..." << std::endl;
    for(unsigned i = 0; i < conf.f_in.size(); ++i)
    {
        cells[i].id = (int32_t)i;
        cells[i].bam_file = conf.f_in[i].string();
        samFile* samfile = sam_open(conf.f_in[i].string().c_str(), "r");
        if (samfile == NULL) {
            std::cerr << "[Error] Fail to open file " << conf.f_in[i].string() << std::endl;
            return 1;
        }
        hdr = sam_hdr_read(samfile);
        if (!get_RG_tag("SM", hdr->text, cells[i].sample_name)) {
            std::cerr << "[Error] Each BAM file has to have exactly one RG tag. Group cells " << std::endl;
            std::cerr << "        belonging to the same sample by the SM tag." << std::endl;
            std::cerr << "        Problematic file: " << conf.f_in[i].string() << std::endl << std::endl;
            goto print_usage_and_exit;
        }
        if (!get_RG_tag("ID", hdr->text, cells[i].cell_name, /*allow_multiple_matches = */ true)) {
            std::cerr << "[Error] Each BAM file has to have exactly one RG tag." << std::endl;
            std::cerr << "        Problematic file: " << conf.f_in[i].string() << std::endl;
            goto print_usage_and_exit;
        }
        sam_close(samfile);
    }

    // Bin the genome
    unsigned median_binsize;
    chrom_map = std::vector<int32_t>(hdr->n_targets, -1);
    if (vm.count("bins"))
    {
        if (!read_dynamic_bins(bins,
                               chrom_map,
                               conf.f_bins.string().c_str(),
                               hdr))
            return 1;
        TMedianAccumulator<unsigned> med_acc;
        for (Interval const & b : bins)
            med_acc(b.end - b.start);
        median_binsize = boost::accumulators::median(med_acc);
        std::cout << "[Info] Reading " << bins.size() << " variable-width bins with median bin size of " << round(median_binsize/1000) << "kb" << std::endl;
    }
    else
    {
        std::vector<Interval> exclude;
        if (vm.count("exclude")) {
            read_exclude_file(conf.f_excl.string(), hdr, exclude, vm.count("verbose"));
            sort(exclude.begin(), exclude.end(), interval::invt_less);
        }
        std::cout << "[Info] Creating " << round(conf.window/1000) << "kb bins with " << exclude.size() << " excluded regions" << std::endl;
        create_fixed_bins(bins,
                          chrom_map,
                          conf.window,
                          exclude,
                          hdr->n_targets,
                          hdr->target_len);
        median_binsize = conf.window;
    }
    // add last element for easy calculation of number of bins
    chrom_map.push_back((int32_t)bins.size());

    // Count in bins. If A bam file cannot be read, the cell is ignored and
    //     the respective entry in `counts` and `cells` will be erased.
    std::cout << "[Info] Reading " << conf.f_in.size() <<  " BAM files...";
    boost::progress_display show_progress1(conf.f_in.size());
    for (unsigned i = 0, i_f = 0; i_f < conf.f_in.size(); ++i, ++i_f)
    {
        if (!count_sorted_reads(conf.f_in[i_f].string(),
                                bins,
                                chrom_map,
                                hdr,
                                conf.minMapQual,
                                counts[i],
                                cells[i]))
        {
            std::cerr << "[Warning] Ignoring cell " << conf.f_in[i_f].string() << std::endl;
            counts.erase(counts.begin()+i);
            cells.erase(cells.begin()+i);
            --i;
        }
        ++show_progress1;
    }

    //
    // Chapter: Filter cells and bins and estimate NB parameter p
    // ==========================================================
    //

    // median per cell
    count::set_median_per_cell(counts, cells);

    // filter cells with low counts
    good_cells = count::get_good_cells(counts, cells);

    // filter bins with abnormal counts
    if (good_cells.size() < 5) {
        std::cerr << "[Warning] Only few cells with sufficient coverage. I will not filter bad bins" << std::endl;
        good_bins.resize(bins.size());
        std::iota(good_bins.begin(), good_bins.end(), 0); // fill with 0,1,2,...
    } else {
        good_bins = count::get_good_bins(counts,
                                         cells,
                                         good_cells,
                                         vm.count("verbose"),
                                         !vm.count("do-not-filter-by-WC"));
        if (vm.count("verbose")) std::cout << "[Info] Filtered out " << bins.size() - good_bins.size() << " bad bins from " << bins.size() << std::endl;
    }

    // build chrom_map for good bins
    good_map = std::vector<int32_t>(chrom_map.size() - 1, -1);
    int32_t pos = 0;
    for (int32_t chr = 0; chr < static_cast<int32_t>(good_map.size()); ++chr) {
        while (pos < good_bins.size() && bins[good_bins[pos]].chr < chr)
            ++pos;
        // now goodit is either at first occurence of chr, or at the end.
        if (pos >= good_bins.size()) good_map[chr] = (int32_t)good_bins.size();
        else good_map[chr] = pos;
    }
    // add last element for easy calculation of number of bins
    good_map.push_back((int32_t)good_bins.size());

    // calculate cell means and cell variances, grouped by sample (not cell)
    calculate_new_cell_mean(samples, cells, counts, good_cells, good_bins);

    // Estimation of parameter p per sample (should work even with one cell only)
    for (auto it = samples.begin(); it != samples.end(); ++it) {
        SampleInfo & s = it->second;
        s.p = std::inner_product(s.means.begin(), s.means.end(), s.means.begin(), 0.0f) \
        / std::inner_product(s.means.begin(), s.means.end(), s.vars.begin(), 0.0f);
    }

    // Write sample information to file
    if (vm.count("sample_info")) {
        if (vm.count("verbose")) std::cout << "[Write] sample information: " << conf.f_sample_info.string() << std::endl;
        std::ofstream out(conf.f_sample_info.string());
        if (out.is_open()) {
            out << "sample\tcells\tp\tmeans\tvars" << std::endl;
            for (auto it = samples.begin(); it != samples.end(); ++it) {
                SampleInfo const & s = it->second;
                out << it->first << "\t" << s.means.size() << "\t" << s.p << "\t" << s.means[0];
                for (unsigned k=1; k<s.means.size(); ++k) out << "," << s.means[k];
                out << "\t" << s.vars[0];
                for (unsigned k=1; k<s.vars.size(); ++k) out << "," << s.vars[k];
                out << std::endl;
            }
        } else {
            std::cerr << "[Warning] Cannot write to " << conf.f_sample_info.string() << std::endl;
        }
    }

    //
    // Chapter: Run HMM
    // ================
    //
    if(vm.count("do-not-blacklist-hmm")) {
        if (vm.count("verbose")) std::cout << "[Info] Previous filters are not used during HMM phase" << std::endl;
        std::vector<unsigned> normal_bins(bins.size());
        std::iota(normal_bins.begin(), normal_bins.end(), 0);
        run_standard_HMM(counts,
                         good_cells,
                         cells,
                         normal_bins,
                         chrom_map,
                         samples,
                         10.0f / bins.size());
    } else {
        run_standard_HMM(counts,
                         good_cells,
                         cells,
                         good_bins,
                         good_map,
                         samples,
                         10.0f / good_bins.size());
    }

    // Print cell information:
    if (vm.count("info")) {
        if (vm.count("verbose")) std::cout << "[Write] Cell summary: " << conf.f_info.string() << std::endl;
        write_cell_info(conf.f_info.string(), cells);
    }

    // Write final counts + classification
    std::cout << "[Write] count table: " << conf.f_out.string() << std::endl;
    {
        // TODO: why do I pass vector<pair>? I could make it two separate vectors. Just check where else the io function is called.
        struct sample_cell_name_wrapper {
            std::vector<CellInfo> const & cells;
            sample_cell_name_wrapper(std::vector<CellInfo> const & cells) : cells(cells)
            {}
            std::pair<std::string,std::string> operator[](size_t i) const {
                return std::make_pair(cells[i].sample_name, cells[i].cell_name);
            }
        };

        if (!io::write_counts_gzip(conf.f_out.string(),
                                   counts,
                                   bins,
                                   hdr->target_name,
                                   sample_cell_name_wrapper(cells)) )
            return 1;
    }

    return 0;
}