//
// Created by veyry_p on 7/11/15.
//

#include <iostream>
#include <chrono>
#include <numeric>
#include <SFML/Graphics/Sprite.hpp>
#include <SFML/Graphics/Texture.hpp>
#include <SFML/Graphics/RectangleShape.hpp>
#include <SFML/Graphics/RenderTexture.hpp>
#include <tbb/tbb.h>
#include <boost/filesystem.hpp>
#include <boost/algorithm/string.hpp>
#include "strong-classifier.h"
#include "../features/four-rectangles-feature.h"
#include "../features/three-horizontal-rectangles-feature.h"
#include "../config.h"
#include "../features/two-vertical-rectangles-feature.h"
#include "../features/two-horizontal-rectangles-feature.h"
#include "../features/three-vertical-rectangles-feature.h"

static tbb::mutex mutex;
namespace violajones
{
  StrongClassifier::StrongClassifier(std::vector<WeakClassifier> classifiers)
          : classifiers_{classifiers}
  {
    long double galpha = 0;
    for (auto& classifier : classifiers)
      galpha += classifier.alpha_;

    global_alpha_ = galpha;
  }

  bool StrongClassifier::check(Window win, std::shared_ptr<IntegralImage> image)
  {
    long double sumvalues = 0.0;
    for (auto& weakclass : classifiers_)
      sumvalues += weakclass.get_value(win, image);

    bool tmp = sumvalues >= (global_alpha_ / 2.0);
    if (Config::debug_classifier_check)
      std::cout << "StrongClassifier::check " << tmp
      << " - sumvalues: " << sumvalues
      << " global_alpha: " << global_alpha_ << std::endl;

    return tmp;
  }

  void StrongClassifier::save(std::string path)
  {
    std::ofstream fs;
    fs.open(path);
    for (auto& classif : classifiers_)
      fs << classif.alpha_ << " "
      << classif.threshold_ << " "
      << (int)classif.parity_ << " "
      << classif.feature_->get_type() << " "
      << classif.feature_->frame.top_left.x << " "
      << classif.feature_->frame.top_left.y << " "
      << classif.feature_->frame.width << " "
      << classif.feature_->frame.height << std::endl;
    fs.close();
  }

  StrongClassifier StrongClassifier::load_from_file(std::string path)
  {
    std::function<WeakClassifier(std::string&)> restore_classifier =
            [](std::string& s) {
              std::vector<std::string> tokens;
              boost::split(tokens, s, boost::is_any_of(" ;"));
              double alpha = std::stod(tokens[0]);
              int threshold = std::stoi(tokens[1]);
              char parity = (char) std::stoi(tokens[2]);
              std::string feature_type = tokens[3];

              int featurex = std::stoi(tokens[4]);
              int featurey = std::stoi(tokens[5]);
              int featurewidth = std::stoi(tokens[6]);
              int featureheight = std::stoi(tokens[7]);

              Rectangle feature_frame(Point(featurex, featurey), featurewidth, featureheight);
              if (feature_type == "FourRectanglesFeature")
                return WeakClassifier(alpha, threshold, parity, std::make_shared<FourRectanglesFeature>(feature_frame));
              else if (feature_type == "ThreeHorizontalRectanglesFeature")
                return WeakClassifier(alpha, threshold, parity,
                                      std::make_shared<ThreeHorizontalRectanglesFeature>(feature_frame));
              else if (feature_type == "ThreeVerticalRectanglesFeature")
                return WeakClassifier(alpha, threshold, parity,
                                      std::make_shared<ThreeVerticalRectanglesFeature>(feature_frame));
              else if (feature_type == "TwoHorizontalRectanglesFeature")
                return WeakClassifier(alpha, threshold, parity,
                                      std::make_shared<TwoHorizontalRectanglesFeature>(feature_frame));
              else if (feature_type == "TwoVerticalRectanglesFeature")
                return WeakClassifier(alpha, threshold, parity,
                                      std::make_shared<TwoVerticalRectanglesFeature>(feature_frame));

              std::cerr << "Fatal Error: feature_type " << feature_type << std::endl;
              return WeakClassifier(alpha, threshold, parity, std::make_shared<FourRectanglesFeature>(feature_frame));
            };

    std::ifstream infile(path);
    std::string line;
    std::vector<WeakClassifier> classifiers;
    while (std::getline(infile, line))
      classifiers.push_back(restore_classifier(line));
    return StrongClassifier(classifiers);
  }

  StrongClassifier StrongClassifier::train(std::string tests_dir)
  {
    std::cout << "Initializing Trainer ...\n"
    << "Loading trainer tests ..." << std::endl;
    auto start = std::chrono::steady_clock::now();
    auto tests_set = load_tests_set(tests_dir);
    auto& tests = tests_set.first;
    auto& features_values = tests_set.second;

    tbb::atomic<unsigned long> ncached_features = 0;

    if (Config::parallelized)
      tbb::parallel_for_each(features_values.begin(), features_values.end(),
                             [&ncached_features](FeatureValues& ftvalues) {
                               if (ftvalues.values_.size())
                                 ++ncached_features;
                             });
    else
      for (FeatureValues& ftvalues : features_values)
        if (ftvalues.values_.size())
          ++ncached_features;

    auto end = std::chrono::steady_clock::now();
    std::chrono::duration<double> diff = end - start;
    std::cout << "Tests loaded in " << diff.count() << " seconds ("
    << ncached_features * 100 / features_values.size()
    << "% cached)\n Launching training..." << std::endl;

    std::vector<WeakClassifier> classifiers;
    unsigned ipass = 1;
    auto global_start = std::chrono::steady_clock::now();
    while (ipass <= Config::learn_pass)
    {
      start = std::chrono::steady_clock::now();
      std::cout << ipass << "/" << Config::learn_pass << " trainer pass..." << std::endl;
      double weightsum = std::accumulate(tests.begin(), tests.end(), 0.0,
                                         [](double acc, TestImage& t) { return t.weight_ + acc; });

      double validweight = 0.0;
      for (size_t i = 0; i < tests.size(); ++i)
      {
        tests[i].weight_ = tests[i].weight_ / weightsum;
        if (tests[i].valid_)
          validweight += tests[i].weight_;
      }

      TestWeakClassifier best(features_values[0], 0, 1, std::numeric_limits<double>::max());
      if (Config::parallelized)
      {
        tbb::parallel_for_each(features_values.begin(), features_values.end(),
                               [&](FeatureValues& fv) {
                                 auto new_classifier = TestWeakClassifier::train(tests, validweight, fv);
                                 tbb::mutex::scoped_lock lock;
                                 lock.acquire(mutex);
                                 if (best.errors_ > new_classifier.errors_)
                                   best = new_classifier;
                                 lock.release();
                               });
      }
      else
        std::for_each(features_values.begin(), features_values.end(),
                      [&](FeatureValues& fv) {
                        auto new_classifier = TestWeakClassifier::train(tests, validweight, fv);
                        if (best.errors_ > new_classifier.errors_)
                          best = new_classifier;
                      });

      end = std::chrono::steady_clock::now();
      diff = end - start;

      std::cout << "New weak classifier selected in " << diff.count() << " seconds (error score : "
      << best.errors_ << ")\n"
      << "X: " << best.feature_.feature_->frame.top_left.x
      << " Y: " << best.feature_.feature_->frame.top_left.y
      << " - Width: " << best.feature_.feature_->frame.width
      << " Height: " << best.feature_.feature_->frame.height << std::endl;

      double beta = best.errors_ / (1.0 - best.errors_);
      if (beta < 1.0 / 100000000)
        beta = 1.0 / 100000000;

      for (FeatureValue& featurevalue : best.feature_.values_)
        if (best.check(featurevalue.value_) == tests[featurevalue.test_index_].valid_)
          tests[featurevalue.test_index_].weight_ *= beta;

      auto alpha = std::log(1.0 / beta);
      classifiers.push_back(best.get_classifier(alpha));
      ++ipass;
    }

    auto global_end = std::chrono::steady_clock::now();
    diff = global_end - global_start;
    std::cout << "Training finished in " << diff.count()
    << " secs." << std::endl;
    return StrongClassifier(classifiers);
  }

  std::pair<std::vector<TestImage>,
          std::vector<FeatureValues> > StrongClassifier::load_tests_set(std::string tests_dir)
  {
    std::string gooddir = tests_dir + "/good";
    std::string baddir = tests_dir + "/bad";
    auto good = load_images(gooddir);
    auto bad = load_images(baddir);

    double goodweight = 1.0 / (2 * good.size());
    double badweight = 1.0 / (2 * bad.size());

    std::vector<TestImage> tests;
    for (size_t i = 0; i < good.size(); ++i)
      tests.push_back(TestImage(*good[i], goodweight, true));

    for (auto i = good.size(); i < good.size() + bad.size(); ++i)
      tests.push_back(TestImage(*bad[i - good.size()], badweight, false));

    auto features_values = compute_features_values(tests);
    return std::pair<std::vector<TestImage>, std::vector<FeatureValues>>(tests, features_values);
  }

  std::vector<FeatureValues> StrongClassifier::compute_features_values(std::vector<TestImage> tests)
  {
    std::vector<std::shared_ptr<Feature> > features = Window::list_features();
    std::vector<FeatureValues> features_values;
    for (size_t i = 0; i < features.size(); ++i)
    {
      auto values = FeatureValue::compute_all_values_sorted(tests, *features[i]);
      features_values.push_back(FeatureValues(features[i], values));
    }
    return features_values;
  }

  std::shared_ptr<GreyImage> StrongClassifier::load_image(std::string imagepath)
  {
    sf::Image sfimage;
    sfimage.loadFromFile(imagepath);
    if (sfimage.getSize().x != Config::window_width || sfimage.getSize().y != Config::window_height)
    {
      std::cerr << "Invalid image size (must be "
      << Config::window_width << "*"
      << Config::window_height << "): "
      << imagepath << std::endl;
      exit(1);
    }
    return std::make_shared<GreyImage>(GreyImage(sfimage));
  }

  tbb::concurrent_vector<std::shared_ptr<GreyImage>> StrongClassifier::load_images(std::string dir)
  {
    namespace fs = boost::filesystem;
    fs::path path(dir);
    fs::directory_iterator end_itr;
    tbb::concurrent_vector<std::shared_ptr<GreyImage>> images;

    size_t number_loaded = 0;
    if (Config::parallelized)
    {
      std::vector<std::string> strings;
      for (fs::directory_iterator iter(path);
           iter != end_itr && (number_loaded < Config::number_load || !Config::number_load);
           ++iter, ++number_loaded)
        if (fs::is_regular_file(iter->status()))
          strings.push_back(iter->path().string());

      tbb::parallel_for_each(strings.begin(), strings.end(),
                             [&](std::string string) {
                               images.push_back(load_image(string));
                             });
    }
    else if (fs::exists(path) && fs::is_directory(path))
      for (fs::directory_iterator iter(path);
           iter != end_itr && (number_loaded < Config::number_load || !Config::number_load);
           ++iter, ++number_loaded)
        if (fs::is_regular_file(iter->status()))
          images.push_back(load_image(iter->path().string()));

    std::cout << "Loaded " << images.size() << " in " << dir << std::endl;
    if (!images.size())
    {
      std::cerr << "Error: no images has been loaded from: " << dir << std::endl;
      exit(0);
    }

    return images;
  }
}


//---------------------------------------------------------------------------//
// Copyright (c) 2018-2021 Mikhail Komarov <nemo@nil.foundation>
// Copyright (c) 2020-2021 Nikita Kaskov <nbering@nil.foundation>
//
// MIT License
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//---------------------------------------------------------------------------//

#ifndef CRYPTO3_ALGEBRA_TYPE_TRAITS_HPP
#define CRYPTO3_ALGEBRA_TYPE_TRAITS_HPP

#include <complex>

#include <boost/type_traits.hpp>
#include <boost/tti/tti.hpp>
#include <boost/mpl/placeholders.hpp>
#include <boost/type_traits/is_same.hpp>

namespace nil {
    namespace crypto3 {
        namespace algebra {

            using namespace boost::mpl::placeholders;

            BOOST_TTI_HAS_TYPE(iterator)
            BOOST_TTI_HAS_TYPE(const_iterator)

            BOOST_TTI_HAS_TYPE(extension_policy)
            BOOST_TTI_HAS_TYPE(curve_type)
            BOOST_TTI_HAS_TYPE(field_type)
            BOOST_TTI_HAS_TYPE(value_type)
            BOOST_TTI_HAS_TYPE(integral_type)
            BOOST_TTI_HAS_TYPE(base_field_type)
            BOOST_TTI_HAS_TYPE(modular_type)
            BOOST_TTI_HAS_TYPE(scalar_field_type)
            BOOST_TTI_HAS_TYPE(g1_type)
            BOOST_TTI_HAS_TYPE(g2_type)
            BOOST_TTI_HAS_TYPE(gt_type)

            BOOST_TTI_HAS_TYPE(group_type)

            BOOST_TTI_HAS_STATIC_MEMBER_DATA(value_bits)
            BOOST_TTI_HAS_STATIC_MEMBER_DATA(modulus_bits)
            BOOST_TTI_HAS_STATIC_MEMBER_DATA(base_field_modulus)
            BOOST_TTI_HAS_STATIC_MEMBER_DATA(scalar_field_modulus)
            BOOST_TTI_HAS_STATIC_MEMBER_DATA(arity)
            BOOST_TTI_HAS_STATIC_MEMBER_DATA(p)
            BOOST_TTI_HAS_STATIC_MEMBER_DATA(q)

            BOOST_TTI_HAS_FUNCTION(to_affine)
            BOOST_TTI_HAS_FUNCTION(to_special)
            BOOST_TTI_HAS_FUNCTION(is_special)

            BOOST_TTI_HAS_STATIC_MEMBER_FUNCTION(zero)
            BOOST_TTI_HAS_STATIC_MEMBER_FUNCTION(one)
            BOOST_TTI_HAS_FUNCTION(is_zero)
            BOOST_TTI_HAS_FUNCTION(is_well_formed)
            BOOST_TTI_HAS_FUNCTION(doubled)

            template<typename T>
            struct is_curve {
                static const bool value = has_type_base_field_type<T>::value && has_type_scalar_field_type<T>::value &&
                                          has_type_g1_type<T>::value && has_type_g2_type<T>::value &&
                                          has_type_gt_type<T>::value;
                typedef T type;
            };

            // TODO: we should add some other params to curve group policy to identify it more clearly
            template<typename T>
            struct is_curve_group {
                static const bool value = has_type_value_type<T>::value && has_type_field_type<T>::value &&
                                          has_static_member_data_value_bits<T, const std::size_t>::value &&
                                          has_type_curve_type<T>::value;
                typedef T type;
            };

            template<typename T>
            struct is_field {
                static const bool value =
                    has_type_value_type<T>::value && has_static_member_data_value_bits<T, const std::size_t>::value &&
                    has_type_integral_type<T>::value &&
                    has_static_member_data_modulus_bits<T, const std::size_t>::value &&
                    has_type_modular_type<T>::value && has_static_member_data_arity<T, const std::size_t>::value;
                typedef T type;
            };

            template<typename T>
            struct is_extended_field {
                static const bool value = has_type_value_type<T>::value &&
                                          has_static_member_data_value_bits<T, const std::size_t>::value &&
                                          has_type_integral_type<T>::value &&
                                          has_static_member_data_modulus_bits<T, const std::size_t>::value &&
                                          has_type_modular_type<T>::value &&
                                          has_static_member_data_modulus_bits<T, const std::size_t>::value &&
                                          has_type_extension_policy<T>::value;
                typedef T type;
            };

            template<typename T>
            struct is_group_element{
                static const bool value = has_type_field_type<T>::value &&
                                          has_type_group_type<T>::value &&
                                          has_static_member_function_zero<T, T>::value &&
                                          has_static_member_function_one<T, T>::value &&
                                          has_function_is_zero<T, bool>::value &&
                                          has_function_is_well_formed<T, bool>::value &&
                                          has_function_doubled<T, T>::value;
            };

            template<typename T>
            struct is_g1_group_element {
                static const bool value =
                    boost::is_same<typename T::group_type::curve_type::template g1_type<typename T::coordinates,
                        typename T::form>, typename T::group_type>::value;
            };

            template<typename T>
            struct is_g2_group_element {
                static const bool value =
                    boost::is_same<typename T::group_type::curve_type::template g2_type<typename T::coordinates,
                        typename T::form>, typename T::group_type>::value;
            };

            template<typename T>
            struct is_complex : std::false_type { };
            template<typename T>
            struct is_complex<std::complex<T>> : std::true_type { };
            template<typename T>
            constexpr bool is_complex_v = is_complex<T>::value;

            template<typename T>
            struct remove_complex {
                using type = T;
            };
            template<typename T>
            struct remove_complex<std::complex<T>> {
                using type = T;
            };
            template<typename T>
            using remove_complex_t = typename remove_complex<T>::type;
        }    // namespace algebra
    }        // namespace crypto3
}    // namespace nil

#endif    // CRYPTO3_ALGEBRA_TYPE_TRAITS_HPP


#include <boost/archive/text_iarchive.hpp>
#include <boost/archive/text_oarchive.hpp>
#include <boost/serialization/vector.hpp>
#include <boost/serialization/map.hpp>
#include <boost/serialization/string.hpp>
#include <boost/serialization/base_object.hpp>

#include "ProgramBddContext.hpp"

#include "glog/logging.h"
#include "buddy/fdd.h"

#include <cstdlib>
#include <ctime>

#include <boost/lexical_cast.hpp>

using namespace std;
using namespace  wali::domains::binrel;

// ////////////////////////////////////////
//     TRASH IN --- TRASH OUT            //
// ////////////////////////////////////////

using namespace wali::domains::binrel;

namespace wali
{
  namespace domains
  {
    namespace binrel
    {
      extern RevBddContext idx2Name;
    }
  }
}

namespace details
{
  void
  interleave_all_fdds()
  {
    vector<int> new_order;

    for (int bit_num=0; bit_num<bdd_varnum(); ++bit_num) {
      for (int fdd_num=0; fdd_num<fdd_domainnum(); ++fdd_num) {
        if (bit_num < fdd_varnum(fdd_num)) {
          int * vars = fdd_vars(fdd_num);
          new_order.push_back(vars[bit_num]);
        }
      }
    }

    assert(new_order.size() == static_cast<size_t>(bdd_varnum()));

    bdd_setvarorder(&new_order[0]);
  }

  std::map<int, std::string>
  get_partial_mapping_for_fdd(std::string const & basename, int fdd_no)
  {
    int num_bits = fdd_varnum(fdd_no);
    int * var_nums = fdd_vars(fdd_no); // [0] = lsb
    std::map<int, std::string> ret;

    for (int i=0; i<num_bits; ++i) {
      ret[var_nums[i]] =
        basename + "_" + boost::lexical_cast<std::string>(i)
        + "(" + boost::lexical_cast<std::string>(var_nums[i]) + ")";
    }

    return ret;
  }

  void accumulate(std::map<int, std::string> & out,
                  std::map<int, std::string> const & more)
  {
    for (std::map<int, std::string>::const_iterator iter = more.begin();
         iter != more.end(); ++iter)
    {
      bool added = out.insert(*iter).second;
      assert(added);
    }
  }

  std::map<int, std::string>
  get_partial_mapping_for_var(std::string const & name, BddInfo const & info)
  {
    std::map<int, std::string> vars;
    accumulate(vars, get_partial_mapping_for_fdd(name, info.baseLhs));
    accumulate(vars, get_partial_mapping_for_fdd(name + "\'", info.baseRhs));
    accumulate(vars, get_partial_mapping_for_fdd(name + "\'\'", info.baseExtra));
    accumulate(vars, get_partial_mapping_for_fdd(name + "_t1", info.tensor1Lhs));
    accumulate(vars, get_partial_mapping_for_fdd(name + "_t1\'", info.tensor1Rhs));
    accumulate(vars, get_partial_mapping_for_fdd(name + "_t1\'\'", info.tensor1Extra));
    accumulate(vars, get_partial_mapping_for_fdd(name + "_t2", info.tensor2Lhs));
    accumulate(vars, get_partial_mapping_for_fdd(name + "_t2\'", info.tensor2Rhs));
    accumulate(vars, get_partial_mapping_for_fdd(name + "_t2\'\'", info.tensor2Extra));
    return vars;
  }

  std::map<int, std::string>
  get_mapping()
  {
    std::map<int, std::string> mapping;
    for (RevBddContext::const_iterator iter = idx2Name.begin(); iter != idx2Name.end(); ++iter)
    {
      accumulate(mapping, get_partial_mapping_for_fdd(iter->second, iter->first));
    }
    return mapping;
  }

  void
  print_bdd_variable_order(std::ostream & os)
  {
    std::map<int, std::string> var_num_to_name = get_mapping();

    int num = bdd_varnum();
    for (int i=0; i<num; ++i) {
      os << "  " << i << ": " << var_num_to_name[bdd_level2var(i)] << "\n";
    }
    os << "\n";
  }

  bdd make_bdd_high_bits_zero(int fdd_no, int fdd_vars[], int start_at_bit)
  {
    int total_bits = fdd_varnum(fdd_no);

    bdd ret = bddtrue;
    for (int bit = start_at_bit; bit < total_bits; ++bit) {
      int result_var = fdd_vars[bit];
      ret = ret & bdd_nithvar(result_var);
    }
    return ret;
  }

  bdd make_adder_box(int in1_varno, int in2_varno, int out_varno,
                     bdd t0, bdd t1, bdd t2, bool inv)
  {
    bdd a = bdd_ithvar(in1_varno);
    bdd b = bdd_ithvar(in2_varno);
    bdd c = bdd_ithvar(out_varno);

    if (inv) {
      // If there's a carry in, then the result is the opposite of what it
      // "should" be.
      c = !c;

      // Furthermore, if a+b=1, we carry to the next bit.
      t1 = t2;
    }

    // The following is a truth-table for a 1-bit adder, which also conjoins
    // in the appropriate box from the next level.
    return ((   a  &   b  & (!c) & t2)
            |(  a  & (!b) &   c  & t1)
            |((!a) &   b  &   c  & t1)
            |((!a) & (!b) & (!c) & t0));
  }

  bdd make_adder(int fdd_lhs, int fdd_rhs, int fdd_result, int num_bits)
  {
    //int num_bits = fdd_varnum(fdd_lhs);
    //assert(num_bits == fdd_varnum(fdd_rhs));
    //assert(num_bits == fdd_varnum(fdd_result));

    //std::cerr << "  make_adder: num_bits is " << num_bits << "\n";
    //std::cerr << "              lhs fdd has " << fdd_varnum(fdd_lhs) << " vars\n";
    //std::cerr << "              rhs fdd has " << fdd_varnum(fdd_rhs) << " vars\n";
    //std::cerr << "              result fdd has " << fdd_varnum(fdd_result) << " vars\n";

    int * lhs_vars = fdd_vars(fdd_lhs);
    int * rhs_vars = fdd_vars(fdd_rhs);
    int * result_vars = fdd_vars(fdd_result);

    bdd left_box = bddtrue;
    bdd right_box = bddtrue;

    for (int level=0; level<num_bits; ++level) {
      // IMPORTANT: level counts from 0=msb to num_bits-1=lsb. However,
      // *_vars have index 0 as the lsb.
      int lhs_var = lhs_vars[num_bits-1 - level];
      int rhs_var = rhs_vars[num_bits-1 - level];
      int result_var = result_vars[num_bits-1 - level];

      bdd new_left_box = make_adder_box(lhs_var, rhs_var, result_var,
                                        left_box, left_box, right_box, false);
      bdd new_right_box = make_adder_box(lhs_var, rhs_var, result_var,
                                         left_box, right_box, right_box, true);

      left_box = new_left_box;
      right_box = new_right_box;
    }

    bdd high_bits_zero = make_bdd_high_bits_zero(fdd_lhs, lhs_vars, num_bits)
                         & make_bdd_high_bits_zero(fdd_rhs, rhs_vars, num_bits)
                         & make_bdd_high_bits_zero(fdd_result, result_vars, num_bits);

    return left_box & high_bits_zero;
  }

  // http://www.exploringbinary.com/ten-ways-to-check-if-an-integer-is-a-power-of-two-in-c/
  bool is_power_of_two(unsigned x)
  {
    return ((x != 0) && ((x & (~x + 1)) == x));
  }

  int log2_ceiling(unsigned x)
  {
    int num_shifts = 0;
    while ((1u << num_shifts) <= x) {
      ++num_shifts;
    }
    return num_shifts;
  }
}

// ////////////////////////////////////////////////////////////////////////////
// The interface assumes that the google logging library is initialized before
// calling any of its functions.

ProgramBddContext::~ProgramBddContext()
{
  regAInfo = regBInfo = NULL;
  baseId = bddtrue;
  BinRel::reset();
}

void ProgramBddContext::addBoolVar(std::string name)
{
  addIntVar(name,2);
}

void ProgramBddContext::setIntVars(const std::map<std::string, int>& inflatvars)
{
  // copy inflatvars because we mess with it.
  std::map<std::string, int> flatvars = inflatvars;

  // Remember creation history
  CreationCall cc;
  cc.cf = SET_INT_MAP;
  cc.arg.push_back(flatvars);
  creationHistory.push_back(cc);

  // Add extra variables at the end of the variables passed in.
  unsigned size = 0;
  for(std::map<std::string, int>::const_iterator ci = flatvars.begin(); ci != flatvars.end(); ++ci)
    size = ((unsigned) ci->second) > size ? ci->second : size;
  size = size < 2 ? 2 : size; // The least size handled is 2. There must be at least one bool variable.
  maxSize = size;
  flatvars["__regA"] = size;
  flatvars["__regB"] = size;
  flatvars["__regSize"] = size + 1; //only baseLhs will be used. Rest of the levels will be wasted. Ah, what the hell!

  std::vector<std::map<std::string, int> > vars;
  vars.push_back(flatvars);

  createIntVars(vars);

  //createExtraVars(); <-- not needed.
  // Now extract the information about extra variables.
  regAInfo = (*this)["__regA"];
  regBInfo = (*this)["__regB"];
  bddinfo_t regSizeInfo = (*this)["__regSize"];
  sizeInfo = regSizeInfo->baseLhs;
  // Now delete entries for these extra variables from the vocabulary.
  this->erase(this->find("__regA"));
  this->erase(this->find("__regB"));
  this->erase(this->find("__regSize")); //will get deleted because no one references it anymore.

  //details::interleave_all_fdds(); //speeds up ProgramBddContext operations.
  BddContext::setupCachedBdds();
  setupCachedBdds();
}

void ProgramBddContext::setIntVars(const std::vector<std::map<std::string, int> >& vars)
{
  //Remember Creation history
  CreationCall cc;
  cc.cf = SET_INT_VECT;
  for(vector< map< string, int > >::const_iterator cit = vars.begin(); cit != vars.end(); ++cit)
    cc.arg.push_back(*cit);
  creationHistory.push_back(cc);

  createIntVars(vars);
  createExtraVars();
  BddContext::setupCachedBdds();
  setupCachedBdds();
}

void ProgramBddContext::createExtraVars()
{
  // Compute the size of register needed.
  unsigned size = 0;
  for(std::map<const std::string, bddinfo_t>::const_iterator ci = this->begin(); ci != this->end(); ++ci)
    size = ci->second->maxVal > size ? ci->second->maxVal : size;
  size = size < 2 ? 2 : size; // The least size handled is 2. There must be at least one bool variable.
  maxSize = size;

  // Now create some buddy levels for scratchpad
  int isize = (int) size;
  int domains[7] = {isize + 1, isize, isize, isize, isize, isize, isize};
  int base = fdd_extdomain(domains,7);

  // Now create the corresponding bddinfo
  sizeInfo = (unsigned) base++;
  regAInfo = new BddInfo();
  regAInfo->baseLhs = (unsigned) base++;
  regAInfo->baseRhs = (unsigned) base++;
  regAInfo->baseExtra = (unsigned) base++;

  regBInfo = new BddInfo();
  regBInfo->baseLhs = (unsigned) base++;
  regBInfo->baseRhs = (unsigned) base++;
  regBInfo->baseExtra = (unsigned) base++;

  //To pretty print during testing, we add some names for this extra register
  //Currently, idx2Name is a global variable in wali::domains::binrel
  idx2Name[sizeInfo] = "__regSize";
  idx2Name[regAInfo->baseLhs] = "__regA";
  idx2Name[regAInfo->baseRhs] = "__regA'";
  idx2Name[regAInfo->baseExtra] = "__regA''";
  idx2Name[regBInfo->baseLhs] = "__regB";
  idx2Name[regBInfo->baseRhs] = "__regB'";
  idx2Name[regBInfo->baseExtra] = "__regB''";
}

void ProgramBddContext::setupCachedBdds()
{
  // Create cached identity
  baseId = bddtrue;
  int * baseLhs = new int[this->size()];
  int * baseRhs = new int[this->size()];
  int vari = 0;
  for(BddContextIter varIter = this->begin(); varIter != this->end();  ++varIter){
    bddinfo_t varInfo = (*varIter).second;
    baseId = baseId & fdd_equals(varInfo->baseLhs, varInfo->baseRhs);
    baseLhs[vari] = varInfo->baseLhs;
    baseRhs[vari] = varInfo->baseRhs;
    ++vari;
  }
  fdd_setpairs(baseLhs2Rhs.get(), baseLhs, baseRhs, this->size());
  delete [] baseLhs;
  delete [] baseRhs;
}

void ProgramBddContext::addIntVar(std::string name, unsigned siz)
{
  //Remember Creation history
  CreationCall cc;
  cc.cf = ADD_INT;
  map< string, int > m;
  m[name] = siz;
  cc.arg.push_back(m);
  creationHistory.push_back(cc);

  BddContext::addIntVar(name,siz);
  if(siz > maxSize){
    if(maxSize == 0){
      //This is when we *create* the extra levels needed
      int domains[1] = {(int)siz+1};
      int retSizeInfo = fdd_extdomain(domains,1);
      if(retSizeInfo < 0)
        LOG(FATAL) << "[ERROR-BuDDy initialization] \"" << bdd_errstring(retSizeInfo) << "\"" << endl
          << "    Aborting." << endl;
      else
        sizeInfo = (unsigned) retSizeInfo;
      //To pretty print during testing, we add some names for this regsiter size fdd level.
      idx2Name[sizeInfo] = "__regSize";
      //Now create two extra bdd levels, and the corresponding BddInfo entries to be used for manipulation
      //inside ProgramBddContext.
      //We will create indices such that we get a default variable ordering where
      //baseLhs, baseRhs, baseExtra are mixed.
      {
        int domains2[3] = {(int)siz, (int)siz, (int)siz};
        regAInfo = new BddInfo();
        //Create fdds for base
        int base = fdd_extdomain(domains2,3);
        if (base < 0)
          LOG(ERROR) << "[ERROR-BuDDy initialization] \"" << bdd_errstring(base) << "\"" << endl
            << "    Aborting." << endl;

        regAInfo->baseLhs = (unsigned) base;
        regAInfo->baseRhs = (unsigned) base + 1;
        regAInfo->baseExtra = (unsigned) base + 2;

        //To pretty print during testing, we add some names for this extra register
        //Currently, idx2Name is a global variable in wali::domains::binrel
        idx2Name[regAInfo->baseLhs] = "__regA";
        idx2Name[regAInfo->baseRhs] = "__regA'";
        idx2Name[regAInfo->baseExtra] = "__regA''";
      }
      {
        int domains2[3] = {(int)siz, (int)siz, (int)siz};
        regBInfo = new BddInfo();
        //Create fdds for base
        int base = fdd_extdomain(domains2,3);
        if (base < 0)
          LOG(ERROR) << "[ERROR-BuDDy initialization] \"" << bdd_errstring(base) << "\"" << endl
            << "    Aborting." << endl;
        regBInfo->baseLhs = (unsigned) base;
        regBInfo->baseRhs = (unsigned )base + 1;
        regBInfo->baseExtra = (unsigned) base + 2;

        //To pretty print during testing, we add some names for this extra register
        //Currently, idx2Name is a global variable in wali::domains::binrel
        idx2Name[regBInfo->baseLhs] = "__regB";
        idx2Name[regBInfo->baseRhs] = "__regB'";
        idx2Name[regBInfo->baseExtra] = "__regB''";
      }
    }else{
      //This is when we *enlarge* the extra levels needed
      int maxVal = siz - maxSize;

      int domains[1] = {maxVal};
      int retSizeInfo = fdd_extdomain(domains,1);
      if(retSizeInfo < 0)
        LOG(FATAL) << "[ERROR-BuDDy initialization] \"" << bdd_errstring(retSizeInfo) << "\"" << endl
          << "    Aborting." << endl;
      retSizeInfo = fdd_overlapdomain(sizeInfo,retSizeInfo);
      if(retSizeInfo < 0)
        LOG(FATAL) << "[ERROR-BuDDy initialization] \"" << bdd_errstring(retSizeInfo) << "\"" << endl
          << "    Aborting." << endl;
      sizeInfo = retSizeInfo;

      //To pretty print during testing, we add some names for this regsiter size fdd level.
      idx2Name[sizeInfo] = "__regSize";
      //Now create two extra bdd levels, and the corresponding BddInfo entries to be used for manipulation
      //inside ProgramBddContext.
      //We will create indices such that we get a default variable ordering where
      //baseLhs, baseRhs, baseExtra are mixed.
      {
        int domains2[3] = {maxVal, maxVal, maxVal};
        //Create fdds for base
        int base = fdd_extdomain(domains2,3);
        if (base < 0)
          LOG(ERROR) << "[ERROR-BuDDy initialization] \"" << bdd_errstring(base) << "\"" << endl
            << "    Aborting." << endl;
        int retbase;
        retbase = fdd_overlapdomain(regAInfo->baseLhs,base);
        regAInfo->baseLhs = retbase;
        if (base < 0)
          LOG(ERROR) << "[ERROR-BuDDy initialization] \"" << bdd_errstring(base) << "\"" << endl
            << "    Aborting." << endl;
        retbase = fdd_overlapdomain(regAInfo->baseRhs,base+1);
        regAInfo->baseRhs = retbase;
        if (base < 0)
          LOG(ERROR) << "[ERROR-BuDDy initialization] \"" << bdd_errstring(base) << "\"" << endl
            << "    Aborting." << endl;
        retbase = fdd_overlapdomain(regAInfo->baseExtra,base+2);
        regAInfo->baseExtra = retbase;
        if (base < 0)
          LOG(ERROR) << "[ERROR-BuDDy initialization] \"" << bdd_errstring(base) << "\"" << endl
            << "    Aborting." << endl;
        //To pretty print during testing, we add some names for this extra register
        //Currently, idx2Name is a global variable in wali::domains::binrel
        idx2Name[regAInfo->baseLhs] = "__regA";
        idx2Name[regAInfo->baseRhs] = "__regA'";
        idx2Name[regAInfo->baseExtra] = "__regA''";
      }
      {
        int domains2[3] = {maxVal, maxVal, maxVal};
        //Create fdds for base
        int base = fdd_extdomain(domains2,3);
        if (base < 0)
          LOG(ERROR) << "[ERROR-BuDDy initialization] \"" << bdd_errstring(base) << "\"" << endl
            << "    Aborting." << endl;
        int retbase;
        retbase = fdd_overlapdomain(regBInfo->baseLhs,base);
        regBInfo->baseLhs = retbase;
        if (base < 0)
          LOG(ERROR) << "[ERROR-BuDDy initialization] \"" << bdd_errstring(base) << "\"" << endl
            << "    Aborting." << endl;
        retbase = fdd_overlapdomain(regBInfo->baseRhs,base+1);
        regBInfo->baseRhs = retbase;
        if (base < 0)
          LOG(ERROR) << "[ERROR-BuDDy initialization] \"" << bdd_errstring(base) << "\"" << endl
            << "    Aborting." << endl;
        retbase = fdd_overlapdomain(regBInfo->baseExtra,base+2);
        regBInfo->baseExtra = retbase;
        if (base < 0)
          LOG(ERROR) << "[ERROR-BuDDy initialization] \"" << bdd_errstring(base) << "\"" << endl
            << "    Aborting." << endl;
        //To pretty print during testing, we add some names for this extra register
        //Currently, idx2Name is a global variable in wali::domains::binrel
        idx2Name[regBInfo->baseLhs] = "__regB";
        idx2Name[regBInfo->baseRhs] = "__regB'";
        idx2Name[regBInfo->baseExtra] = "__regB''";
      }
    }
    maxSize = siz;
  }

    //update cached identity and base pair
    bddinfo_t varInfo = this->find(name)->second;
    baseId = baseId & fdd_equals(varInfo->baseLhs, varInfo->baseRhs);
    fdd_setpair(baseLhs2Rhs.get(), varInfo->baseLhs, varInfo->baseRhs);
}

ProgramBddContext::ProgramBddContext(int bddMemSize, int cacheSize) :
  BddContext(bddMemSize, cacheSize),
  sizeInfo(0),
  maxSize(0),
  regAInfo(NULL),
  regBInfo(NULL),
  baseId(bddtrue),
  baseLhs2Rhs(BddPairPtr(bdd_newpair()))
{}

ProgramBddContext::ProgramBddContext(const ProgramBddContext& other) :
  BddContext(other),
  sizeInfo(other.sizeInfo),
  maxSize(other.maxSize),
  regAInfo(other.regAInfo),
  regBInfo(other.regBInfo),
  baseId(other.baseId),
  baseLhs2Rhs(other.baseLhs2Rhs)
{}

ProgramBddContext& ProgramBddContext::operator = (const ProgramBddContext& other)
{
  if(this != &other){
    BddContext::operator=(other);
    sizeInfo=other.sizeInfo;
    maxSize=other.maxSize;
    regAInfo=other.regAInfo;
    regBInfo=other.regBInfo;
  }
  return *this;
}

ProgramBddContext::ProgramBddContext(const std::map<std::string, int>& vars, int bddMemSize, int cacheSize) :
  BddContext(bddMemSize, cacheSize),
  sizeInfo(0),
  maxSize(0),
  regAInfo(NULL),
  regBInfo(NULL),
  baseId(bddtrue),
  baseLhs2Rhs(BddPairPtr(bdd_newpair()))
{
  setIntVars(vars);
}

ProgramBddContext::ProgramBddContext(const std::vector<std::map<std::string, int> >& vars, int bddMemSize, int cacheSize) :
  BddContext(bddMemSize, cacheSize),
  sizeInfo(0),
  maxSize(0),
  regAInfo(NULL),
  regBInfo(NULL),
  baseId(bddtrue),
  baseLhs2Rhs(BddPairPtr(bdd_newpair()))
{
  setIntVars(vars);
}

std::ostream& ProgramBddContext::print(std::ostream& o) const
{
  o << "ProgramBddContext dump:" << std::endl;
  o << "sizeInfo: " << sizeInfo << std::endl;
  o << "maxSize: " << maxSize << std::endl;
  regAInfo->print(o << "regAInfo: ") << std::endl;
  regBInfo->print(o << "regBInfo: ") << std::endl;
  return o;
}

bdd ProgramBddContext::From(std::string var) const
{
  bdd ret = bddfalse;
  ProgramBddContext::const_iterator iter = (this->find(var));
  if(iter == this->end())
    LOG(FATAL) << "attempted from () on \"" << var << "\". i don't recognize this name\n";
  const bddinfo_t bi = iter->second;
  for(unsigned i = 0; i < bi->maxVal; ++i)
    ret = ret | (fdd_ithvar(bi->baseLhs, i) & fdd_ithvar(regAInfo->baseRhs, i));
  ret = bdd_exist(ret, fdd_ithset(sizeInfo));
  ret = ret & fdd_ithvar(sizeInfo, bi->maxVal);
  return ret;
}

bdd ProgramBddContext::True() const
{
  return fdd_ithvar(regAInfo->baseRhs,1) & fdd_ithvar(sizeInfo, 2);
}

bdd ProgramBddContext::False() const
{
  return fdd_ithvar(regAInfo->baseRhs,0) & fdd_ithvar(sizeInfo, 2);
}

bdd ProgramBddContext::Const(unsigned val) const
{
  if(val >= maxSize){
    LOG(ERROR) << "[Const] Attempted to create a constant value larger "
      << "than maxVal";
    assert(0);
  }
  return fdd_ithvar(regAInfo->baseRhs, val) & fdd_ithvar(sizeInfo, maxSize);
}

bdd ProgramBddContext::NonDet() const
{
  bdd ret = bddtrue;
  //ret = bdd_exist(ret, fdd_ithset(sizeInfo));
  ret = ret & fdd_ithvar(sizeInfo, maxSize);
  return ret;
}

bdd ProgramBddContext::applyBinOp(bdd lexpr, bdd rexpr, bdd op) const
{
  unsigned old1Size = getRegSize(lexpr);
  unsigned old2Size = getRegSize(rexpr);
  // If the sizes are different, we don't want that part of
  // the bdd interfering in actual operation.
  if(old1Size != old2Size){
    lexpr = bdd_exist(lexpr, fdd_ithset(sizeInfo));
    rexpr = bdd_exist(rexpr, fdd_ithset(sizeInfo));
  }
  bddPair *regA2regB = bdd_newpair();
  fdd_setpair(regA2regB, regAInfo->baseRhs, regBInfo->baseRhs);
  rexpr = bdd_replace(rexpr, regA2regB);
  bdd_freepair(regA2regB);

  lexpr = lexpr & rexpr;

  //IMP: fddsets are unioned by & of the representing bdds (Not | as I had
  //earlier expected.
  lexpr = bdd_relprod(
      lexpr,
      op,
      fdd_ithset(regAInfo->baseRhs) & fdd_ithset(regBInfo->baseRhs)
      );

  bddPair *regAExtra2Rhs = bdd_newpair();
  fdd_setpair(regAExtra2Rhs, regAInfo->baseExtra, regAInfo->baseRhs);
  lexpr = bdd_replace(lexpr, regAExtra2Rhs);
  bdd_freepair(regAExtra2Rhs);

  //update the regsize
  if(old1Size != old2Size){
    unsigned minSize = (old1Size < old2Size)? old1Size : old2Size;
    lexpr = lexpr & fdd_ithvar(sizeInfo, minSize);
    getRegSize(lexpr);
  }
  return lexpr;
}

bdd ProgramBddContext::applyUnOp(bdd expr, bdd op) const
{
  expr = bdd_relprod(
      expr,
      op,
      fdd_ithset(regAInfo->baseRhs)
      );

  bddPair *regAExtra2Rhs = bdd_newpair();
  fdd_setpair(regAExtra2Rhs, regAInfo->baseExtra, regAInfo->baseRhs);
  expr = bdd_replace(expr, regAExtra2Rhs);
  bdd_freepair(regAExtra2Rhs);

  return expr;
}

bdd ProgramBddContext::And(bdd lexpr, bdd rexpr) const
{
  return applyBinOp(lexpr, rexpr, bddAnd());
}

bdd ProgramBddContext::Or(bdd lexpr, bdd rexpr) const
{
  return applyBinOp(lexpr, rexpr, bddOr());
}

bdd ProgramBddContext::Not(bdd expr) const
{
  return applyUnOp(expr, bddNot());
}

bdd ProgramBddContext::Plus(bdd lexpr, bdd rexpr) const
{
  unsigned in1 = getRegSize(lexpr);
  unsigned in2 = getRegSize(rexpr);
  return applyBinOp(lexpr, rexpr, bddPlus(in1,in2));
}

#ifdef BINREL_I_WANT_MINUS_TIMES_AND_DIV_EVEN_THOUGH_THEY_CAN_BE_EXPONENTIALLY_SLOW
bdd ProgramBddContext::Minus(bdd lexpr, bdd rexpr) const
{
  unsigned in1 = getRegSize(lexpr);
  unsigned in2 = getRegSize(rexpr);
  return applyBinOp(lexpr, rexpr, bddMinus(in1,in2));
}

bdd ProgramBddContext::Times(bdd lexpr, bdd rexpr) const
{
  unsigned in1 = getRegSize(lexpr);
  unsigned in2 = getRegSize(rexpr);
  return applyBinOp(lexpr, rexpr, bddTimes(in1,in2));
}

bdd ProgramBddContext::Div(bdd lexpr, bdd rexpr) const
{
  unsigned in1 = getRegSize(lexpr);
  unsigned in2 = getRegSize(rexpr);
  return applyBinOp(lexpr, rexpr, bddDiv(in1,in2));
}
#endif

bdd ProgramBddContext::setPre(std::string var) const
{
  bdd ret;
  ProgramBddContext::const_iterator iter = (this->find(var));
  if(iter == this->end())
    LOG(FATAL) << "attempted setPre () on \"" << var << "\". i don't recognize this name\n";
  const bddinfo_t bi = iter->second;
  assert(bi->maxVal == 2);
  ret = fdd_ithvar(bi->baseLhs, 1);
  ret = bdd_exist(ret, fdd_ithset(sizeInfo));
  return ret;
}

bdd ProgramBddContext::unsetPre(std::string var) const
{
  bdd ret;
  ProgramBddContext::const_iterator iter = (this->find(var));
  if(iter == this->end())
    LOG(FATAL) << "attempted unsetPre () on \"" << var << "\". i don't recognize this name\n";
  const bddinfo_t bi = iter->second;
  assert(bi->maxVal == 2);
  ret = fdd_ithvar(bi->baseLhs, 0);
  ret = bdd_exist(ret, fdd_ithset(sizeInfo));
  return ret;
}
bdd ProgramBddContext::setPre(std::string var, unsigned val) const
{
  bdd ret;
  ProgramBddContext::const_iterator iter = (this->find(var));
  if(iter == this->end())
    LOG(FATAL) << "attempted setPre () on \"" << var << "\". i don't recognize this name\n";
  const bddinfo_t bi = iter->second;
  if(val >= bi->maxVal)
    LOG(FATAL) << "setPre: val too large\n";
  ret = fdd_ithvar(bi->baseLhs, val);
  ret = bdd_exist(ret, fdd_ithset(sizeInfo));
  return ret;
}
bdd ProgramBddContext::setPost(std::string var) const
{
  bdd ret;
  ProgramBddContext::const_iterator iter = (this->find(var));
  if(iter == this->end())
    LOG(FATAL) << "attempted setPoist () on \"" << var << "\". i don't recognize this name\n";
  const bddinfo_t bi = iter->second;
  assert(bi->maxVal == 2);
  ret = fdd_ithvar(bi->baseRhs, 1);
  ret = bdd_exist(ret, fdd_ithset(sizeInfo));
  return ret;
}
bdd ProgramBddContext::unsetPost(std::string var) const
{
  bdd ret;
  ProgramBddContext::const_iterator iter = (this->find(var));
  if(iter == this->end())
    LOG(FATAL) << "attempted unsetPost () on \"" << var << "\". i don't recognize this name\n";
  const bddinfo_t bi = iter->second;
  assert(bi->maxVal == 2);
  ret = fdd_ithvar(bi->baseRhs, 0);
  ret = bdd_exist(ret, fdd_ithset(sizeInfo));
  return ret;
}
bdd ProgramBddContext::setPost(std::string var, unsigned val) const
{
  bdd ret;
  ProgramBddContext::const_iterator iter = (this->find(var));
  if(iter == this->end())
    LOG(FATAL) << "attempted setPost () on \"" << var << "\". i don't recognize this name\n";
  const bddinfo_t bi = iter->second;
  if(val >= bi->maxVal)
    LOG(FATAL) << "setPost: val too large\n";
  ret = fdd_ithvar(bi->baseLhs, val);
  ret = bdd_exist(ret, fdd_ithset(sizeInfo));
  return ret;
}

bdd ProgramBddContext::bddAnd() const
{
  bdd ret =
    (fdd_ithvar(regAInfo->baseRhs,1) &
     fdd_ithvar(regBInfo->baseRhs,1) &
     fdd_ithvar(regAInfo->baseExtra,1))
    |
    (fdd_ithvar(regAInfo->baseRhs,1) &
     fdd_ithvar(regBInfo->baseRhs,0) &
     fdd_ithvar(regAInfo->baseExtra,0))
    |
    (fdd_ithvar(regAInfo->baseRhs,0) &
     fdd_ithvar(regBInfo->baseRhs,1) &
     fdd_ithvar(regAInfo->baseExtra,0))
    |
    (fdd_ithvar(regAInfo->baseRhs,0) &
     fdd_ithvar(regBInfo->baseRhs,0) &
     fdd_ithvar(regAInfo->baseExtra,0));
  ret = ret & fdd_ithvar(sizeInfo, 2);
  return ret;
}

bdd ProgramBddContext::bddOr() const
{
  bdd ret =
    (fdd_ithvar(regAInfo->baseRhs,1) &
     fdd_ithvar(regBInfo->baseRhs,1) &
     fdd_ithvar(regAInfo->baseExtra,1))
    |
    (fdd_ithvar(regAInfo->baseRhs,1) &
     fdd_ithvar(regBInfo->baseRhs,0) &
     fdd_ithvar(regAInfo->baseExtra,1))
    |
    (fdd_ithvar(regAInfo->baseRhs,0) &
     fdd_ithvar(regBInfo->baseRhs,1) &
     fdd_ithvar(regAInfo->baseExtra,1))
    |
    (fdd_ithvar(regAInfo->baseRhs,0) &
     fdd_ithvar(regBInfo->baseRhs,0) &
     fdd_ithvar(regAInfo->baseExtra,0));
  ret = ret & fdd_ithvar(sizeInfo, 2);
  return ret;
}

bdd ProgramBddContext::bddNot() const
{
  bdd ret =
    (fdd_ithvar(regAInfo->baseRhs,1) &
     fdd_ithvar(regAInfo->baseExtra,0))
    |
    (fdd_ithvar(regAInfo->baseRhs,0) &
     fdd_ithvar(regAInfo->baseExtra,1));
  ret = ret & fdd_ithvar(sizeInfo, 2);
  return ret;
}

bdd ProgramBddContext::bddPlus(unsigned in1Size, unsigned in2Size) const
{
  //std::cerr << "bddPlus(" << in1Size << ", " << in2Size << "):\n";
  if(in1Size != in2Size){
    LOG(WARNING) <<
      "[ProgramBddContext::bddPlus] Different sizes of registers in operation."
      << " Longer register / constant will be clipped.\n";
    if(in1Size < in2Size)
      in2Size = in1Size;
    else
      in1Size = in2Size;
  }
  unsigned outSize = in1Size;
  int num_bits = ::details::log2_ceiling(outSize - 1);

  // Not really fast and slow, but fast and slow to construct.
  bdd fast_adder, slow_adder;

  // Build fast adder then maybe build slow adder
  if (::details::is_power_of_two(outSize)) {
    //std::cerr << "  Building slow adder because " << outSize << " is"
    //          << (details::is_power_of_two(outSize) ? " " : " not ")
    //          << "a power of two\n";
    assert(in1Size <= (unsigned)fdd_domainsize(regAInfo->baseRhs));
    fast_adder = ::details::make_adder(regAInfo->baseRhs, regBInfo->baseRhs, regAInfo->baseExtra, num_bits);
    fast_adder = fast_adder & fdd_ithvar(sizeInfo, outSize);
  }

#ifndef BINREL_ALWAYS_BUILD_SLOW_ADDER
#error "You must define BINREL_ALWAYS_BUILD_SLOW_ADDER to 0 or 1"
#endif

  if (BINREL_ALWAYS_BUILD_SLOW_ADDER
      || (BINREL_ALLOW_BUILD_SLOW_ADDER &&
          !::details::is_power_of_two(outSize)))
  {
    //std::cerr << "  Building slow adder because"
    //          << (BINREL_ALWAYS_BUILD_SLOW_ADDER ? " we always do;" : "")
    //          << (details::is_power_of_two(outSize) ? "" : "outsize is not a power of two")
    //          << "\n";

    slow_adder = bddfalse;
    for(unsigned i=0; i<in1Size; ++i){
      for(unsigned j=0; j<in2Size; ++j){
        int k = (i + j) % outSize;
        slow_adder = slow_adder  |
          (fdd_ithvar(regAInfo->baseRhs,i) &
           fdd_ithvar(regBInfo->baseRhs,j) &
           fdd_ithvar(regAInfo->baseExtra,k));
      }
    }
    slow_adder = slow_adder & fdd_ithvar(sizeInfo, outSize);
  }

  // If we built both, check they are the same
  if (fast_adder.id() != 0
      && slow_adder.id() != 0
      && fast_adder != slow_adder)
  {
    std::cerr << "WARNING: the two methods of creating a plus BDD differ\n";
    std::cerr << "   in1size: " << in1Size << "\n";
    std::cerr << "   in2size: " << in2Size << "\n";
    bdd_fnprintdot((char*)"slow_plus.dot", slow_adder);
    bdd_fnprintdot((char*)"fast_plus.dot", fast_adder);
    assert(fast_adder == slow_adder);
  }

  if (fast_adder.id() != 0) {
    assert(BINREL_ALWAYS_BUILD_SLOW_ADDER || slow_adder.id() == 0);
    return fast_adder;
  }
  else {
    assert(slow_adder.id() != 0);
    return slow_adder;
  }
}

#ifdef BINREL_I_WANT_MINUS_TIMES_AND_DIV_EVEN_THOUGH_THEY_CAN_BE_EXPONENTIALLY_SLOW
bdd ProgramBddContext::bddMinus(unsigned in1Size, unsigned in2Size) const
{
  if(in1Size != in2Size){
    LOG(WARNING) <<
      "[ProgramBddContext::bddPlus] Different sizes of registers in operation."
      << " Longer register / constant will be clipped.\n";
    if(in1Size < in2Size)
      in2Size = in1Size;
    else
      in1Size = in2Size;
  }
  unsigned outSize = in1Size;
  bdd ret = bddfalse;
  for(unsigned i=0; i<in1Size; ++i){
    for(unsigned j=0; j<in2Size; ++j){
      int k = (i - j + outSize) % outSize;
      ret = ret  |
        (fdd_ithvar(regAInfo->baseRhs,i) &
         fdd_ithvar(regBInfo->baseRhs,j) &
         fdd_ithvar(regAInfo->baseExtra,k));
    }
  }
  ret = ret & fdd_ithvar(sizeInfo, outSize);
  return ret;
}

bdd ProgramBddContext::bddTimes(unsigned in1Size, unsigned in2Size) const
{
  if(in1Size != in2Size){
    LOG(WARNING) <<
      "[ProgramBddContext::bddPlus] Different sizes of registers in operation."
      << " Longer register / constant will be clipped.\n";
    if(in1Size < in2Size)
      in2Size = in1Size;
    else
      in1Size = in2Size;
  }
  unsigned outSize = in1Size;
  bdd ret = bddfalse;
  for(unsigned i=0; i<in1Size; ++i){
    for(unsigned j=0; j<in2Size; ++j){
      int k = (i * j) % outSize;
      ret = ret  |
        (fdd_ithvar(regAInfo->baseRhs,i) &
         fdd_ithvar(regBInfo->baseRhs,j) &
         fdd_ithvar(regAInfo->baseExtra,k));
    }
  }
  ret = ret & fdd_ithvar(sizeInfo, outSize);
  return ret;
}

bdd ProgramBddContext::bddDiv(unsigned in1Size, unsigned in2Size) const
{
  if(in1Size != in2Size){
    LOG(WARNING) <<
      "[ProgramBddContext::bddPlus] Different sizes of registers in operation."
      << " Longer register / constant will be clipped.\n";
    if(in1Size < in2Size)
      in2Size = in1Size;
    else
      in1Size = in2Size;
  }
  unsigned outSize = in1Size;
  bdd ret = bddfalse;
  for(unsigned i=0; i<in1Size; ++i){
    for(unsigned j=0; j<in2Size; ++j){
      int k;
      if(j == 0)
        //arbitrary
        k = 0;
      else
        k = (int) i / j;
      ret = ret |
        (fdd_ithvar(regAInfo->baseRhs,i) &
         fdd_ithvar(regBInfo->baseRhs,j) &
         fdd_ithvar(regAInfo->baseExtra,k));
    }
  }
  ret = ret & fdd_ithvar(sizeInfo, outSize);
  return ret;
}
#endif

unsigned ProgramBddContext::getRegSize(bdd forThis) const
{
  //Inefficient!!!
  for(unsigned i = 0; i <= maxSize; ++i){
    bdd tmp = fdd_ithvar(sizeInfo, i);
    if((tmp & forThis) != bddfalse)
      return i;
  }
  LOG(FATAL) << "[ProgramBddContext::getRegSize] bdd does not have a recognizable regSize\n";
  return 0;
}

bdd ProgramBddContext::Assign(std::string var, bdd expr) const
{
  bddinfo_t bi;
  if(this->find(var) == this->end()){
    LOG(WARNING) << "[ProgramBddContext::Assign] Unknown Variable: " << var;
    //This is a safe result. We assume that anything can be assigned to anything!
    return bddtrue;
  }else{
    bi = this->find(var)->second;
  }

  unsigned rvalsize = getRegSize(expr);
  //if(rvalsize > bi->maxVal){
  //  LOG(WARNING) << "[ProgramBddContext::Assign] The register size of rhs is greater than what the variable can hold";
  //  return bddfalse;
  //}

  // If rhs max size is smaller, only copy the relevant bits.
  unsigned copysize = bi->maxVal;
  copysize = (rvalsize < copysize) ? rvalsize : copysize;

  //redundant?
  bddPair *regARhs2Extra = bdd_newpair();
  fdd_setpair(
      regARhs2Extra,
      regAInfo->baseRhs,
      regAInfo->baseExtra
      );
  expr = bdd_replace(expr,regARhs2Extra);
  bdd_freepair(regARhs2Extra);
  //up to here.

  bdd regA2var = bddfalse;
  for(unsigned i = 0; i < copysize; ++i)
    regA2var = regA2var |
      (fdd_ithvar(regAInfo->baseExtra,i) & fdd_ithvar(bi->baseRhs,i));
  expr = bdd_relprod(expr,regA2var,fdd_ithset(regAInfo->baseExtra));

  // All transformers are slight perturbations of identity.
  bdd c = baseId;
  c = bdd_exist(c, fdd_ithset(bi->baseLhs));
  c = bdd_exist(c, fdd_ithset(bi->baseRhs));
  return bdd_exist(expr & c, fdd_ithset(sizeInfo));
}

bdd ProgramBddContext::Assume(bdd expr1, bdd expr2) const
{

  bddPair *regARhs2Extra = bdd_newpair();
  fdd_setpair(
      regARhs2Extra,
      regAInfo->baseRhs,
      regAInfo->baseExtra
      );
  expr1 = bdd_replace(expr1, regARhs2Extra);
  bdd_freepair(regARhs2Extra);

  bddPair *regARhs2BExtra = bdd_newpair();
  fdd_setpair(
      regARhs2BExtra,
      regAInfo->baseRhs,
      regBInfo->baseExtra
      );
  expr2 = bdd_replace(expr2, regARhs2BExtra);
  bdd_freepair(regARhs2BExtra);

  expr2 = bdd_replace(expr2, baseLhs2Rhs.get());

  bdd equate = baseId &
    fdd_equals(
        regAInfo->baseExtra,
        regBInfo->baseExtra
        );

  // expr1 and expr2 can have different regSize.
  // In that case, only the smaller number of bits are constrainted.
  expr1 = bdd_exist(expr1, fdd_ithset(sizeInfo));
  expr2 = bdd_exist(expr2, fdd_ithset(sizeInfo));

  bdd ret = expr1 & expr2 & equate;

  ret = bdd_exist(ret, fdd_ithset(regAInfo->baseExtra));
  ret = bdd_exist(ret, fdd_ithset(regBInfo->baseExtra));

  return ret;
}

bdd ProgramBddContext::tGetRandomTransformer(bool isTensored, unsigned seed) const
{
  if(seed != 0)
    srand(seed);
  bdd ret = bddfalse;
  int numRounds = rand() % 10 + 1;
  for(int c=0; c < numRounds; ++c){
    bdd inbdd = rand()%2?bddfalse:bddtrue;
    for(BddContext::const_iterator iter = this->begin();
        iter != this->end();
        ++iter){
      int siz = iter->second->maxVal;
      int n;
      if(!isTensored){
        n = rand() % 4;
        if(n==0)
          inbdd = inbdd & fdd_ithvar(iter->second->baseLhs,rand()%siz);
        if(n==1)
          inbdd = inbdd | fdd_ithvar(iter->second->baseLhs,rand()%siz);
        n = rand() % 4;
        if(n==0)
          inbdd = inbdd & fdd_ithvar(iter->second->baseRhs,rand()%siz);
        if(n==1)
          inbdd = inbdd | fdd_ithvar(iter->second->baseRhs,rand()%siz);
      }else{
        n = rand() % 4;
        if(n==0)
          inbdd = inbdd & fdd_ithvar(iter->second->tensor1Lhs,rand()%siz);
        if(n==1)
          inbdd = inbdd | fdd_ithvar(iter->second->tensor1Lhs,rand()%siz);
        n = rand() % 4;
        if(n==0)
          inbdd = inbdd & fdd_ithvar(iter->second->tensor1Rhs,rand()%siz);
        if(n==1)
          inbdd = inbdd | fdd_ithvar(iter->second->tensor1Rhs,rand()%siz);
        n = rand() % 4;
        if(n==0)
          inbdd = inbdd & fdd_ithvar(iter->second->tensor2Lhs,rand()%siz);
        if(n==1)
          inbdd = inbdd | fdd_ithvar(iter->second->tensor2Lhs,rand()%siz);
        n = rand() % 4;
        if(n==0)
          inbdd = inbdd & fdd_ithvar(iter->second->tensor2Rhs,rand()%siz);
        if(n==1)
          inbdd = inbdd | fdd_ithvar(iter->second->tensor2Rhs,rand()%siz);
      }
    }
    ret = rand() % 2 ? ret & inbdd : ret | inbdd;
  }
  return ret;
}

void ProgramBddContext::printHistory(std::ostream& o) const
{
  boost::archive::text_oarchive oa(o);
  oa << creationHistory;
}

ProgramBddContext * ProgramBddContext::buildFromHistory(std::istream& in)
{
  CreationHistory ch;
  boost::archive::text_iarchive ia(in);
  ia >> ch;

  ProgramBddContext * con = new ProgramBddContext();
  for(CreationHistory::const_iterator ci = ch.begin(); ci != ch.end(); ++ci){
    CreationCall cc = *ci;
    switch(cc.cf){
      case SET_INT_VECT:
        con->setIntVars(cc.arg);
        break;
      case SET_INT_MAP:
        con->setIntVars(cc.arg[0]);
        break;
      case ADD_INT:
        {
          map< string, int >::iterator m = cc.arg[0].begin();
          con->addIntVar(m->first, m->second);
        }
        break;
      case ADD_BOOL:
      default:
        assert(0);
    }
  }
  return con;
}

template<class Archive>
void ProgramBddContext::CreationCall::serialize(Archive &ar, const unsigned int)
{
  //ignore file_version
  ar & cf;
  ar & arg;
}

ostream& ProgramBddContext::CreationCall::print(ostream& o) const
{
  switch(cf){
    case SET_INT_VECT:
      o << "SET_INT_VECT";
      break;
    case SET_INT_MAP:
      o << "SET_INT_MAP";
      break;
    case ADD_INT:
      o << "ADD_INT";
      break;
    case ADD_BOOL:
      o << "ADD_BOOL";
      break;
    default:
      o << "UNKNOWN";
  }
  o << ": ";
  for(vector< map< string, int > >::const_iterator cvi = arg.begin(); cvi != arg.end(); ++cvi){
    o << "{";
    bool first = true;
    for(map< string, int >::const_iterator cmi = cvi->begin(); cmi != cvi->end(); ++cmi){
      if(!first)
        o << ", ";
      o << cmi->first << ": " << cmi->second;
      first = false;
    }
    o << "}";
  }
  o << endl;
  return o;
}
// Yo, Emacs!
// Local Variables:
//   c-file-style: "ellemtel"
//   c-basic-offset: 2
// End:


/* tsne_python.cc
   Jeremy Barnes, 20 January 2010
   Copyright (c) 2010 Jeremy Barnes.  All rights reserved.

   Python wrapper for TSNE.
*/

#include <Python.h>
#include <numpy/arrayobject.h>
#include "tsne.h"
#include <iostream>
#include "jml/arch/exception.h"
#include "jml/utils/unnamed_bool.h"
#include "jml/arch/demangle.h"
#include <cxxabi.h>
#include <typeinfo>
#include <boost/bind.hpp>

using namespace std;
using namespace ML;

// For the following: /usr/include/python2.6/numpy/__multiarray_api.h:958: error: ‘int _import_array()’ defined but not used
int (*fn) () = &_import_array;

template<class Object>
struct PyRef {
    PyRef(Object * obj)
        : obj(obj)
    {
    }

    template<class Other>
    PyRef(Other * obj)
        : obj(reinterpret_cast<Object *>(obj))
    {
    }

    Object * obj;

    JML_IMPLEMENT_OPERATOR_BOOL(obj);

    template<class ObjectOut>
    operator ObjectOut * () const { return reinterpret_cast<ObjectOut *>(obj); }

    Object * release()
    {
        Object * result = obj;
        obj = 0;
        return result;
    }

    template<class ObjectOut>
    ObjectOut * release()
    {
        ObjectOut * result = *this;
        obj = 0;
        return result;
    }

    ~PyRef()
    {
        if (obj) Py_DECREF(obj);
    }
};

struct Interrupt_Exception : public Exception {
    Interrupt_Exception()
        : Exception("Keyboard Interrupt")
    {
    }
};

typedef PyRef<PyObject> PyObjectRef;
typedef PyRef<PyArrayObject> PyArrayRef;

PyObject * to_python_exception(const std::type_info * exc_type,
                               const char * what = 0)
{
    if (*exc_type == typeid(Interrupt_Exception)) {
        PyErr_SetString(PyExc_KeyboardInterrupt, "JML processing interrupted");
        return NULL;
    }

    std::string message;
    if (what)
        message = format("JML Exception of type %s caught: %s",
                         demangle(exc_type->name()).c_str(),
                         what);
    else
        message = format("JML Exception of type %s caught",
                         demangle(exc_type->name()).c_str());

    PyErr_SetString(PyExc_RuntimeError, message.c_str());

    return NULL;
}

PyObject * to_python_exception(const std::exception & exc)
{
    return to_python_exception(&typeid(exc), exc.what());
}

PyObject * to_python_exception()
{
    const std::type_info * exc_type = abi::__cxa_current_exception_type();
    if (!exc_type) {
        cerr << "exception type was null" << endl;
        abort();
    }

    return to_python_exception(exc_type);
}

enum BlockOp {
    UNBLOCK,
    BLOCK
};

volatile int num_signals_received = 0;

void handle_sigint(int action)
{
    PyErr_SetInterrupt();
    ++num_signals_received;
}

struct PyThreads {
    PyThreads(BlockOp op = UNBLOCK)
        : _save(0), old_sigint(0)
    {
        if (op == UNBLOCK)
            unblock();
    }

    ~PyThreads()
    {
        if (_save)
            block();
    }

    PyThreadState *_save;
    PyOS_sighandler_t old_sigint;
    int signals_before;

    void unblock()
    {
        if (_save) {
            cerr << "threads were already unblocked; pairing error"
                 << endl;
            abort();
        }

        old_sigint = PyOS_setsig(SIGINT, handle_sigint);
        signals_before = num_signals_received;
        Py_UNBLOCK_THREADS;
    }

    void block()
    {
        if (!_save) {
            cerr << "threads were already unblocked; pairing error"
                 << endl;
            abort();
        }

        Py_BLOCK_THREADS;
        PyOS_setsig(SIGINT, old_sigint);
    }

    bool interrupted()
    {
        return num_signals_received > signals_before;
    }
};

static PyObject *
tsne_vectors_to_distances(PyObject *self, PyObject *args)
{
    PyObject * in_array;

    int args_ok = PyArg_ParseTuple(args, "O!", &PyArray_Type, &in_array);
    if (!args_ok)
        return NULL;

    /* Convert the argument to a float array. */
    PyArrayRef input_as_float32
        = PyArray_FromAny(in_array,
                          PyArray_DescrFromType(NPY_FLOAT32),
                          2, 2,
                          NPY_C_CONTIGUOUS | NPY_FORCECAST | NPY_ALIGNED,
                          0);
    if (!input_as_float32)
        return NULL;

    // TODO: inc reference?

    int n = PyArray_DIM(input_as_float32, 0);
    int d = PyArray_DIM(input_as_float32, 1);

    npy_intp npy_shape[2] = { n, n };

    /* Allocate an object (in memory) for the result */
    PyArrayRef result_array
        = PyArray_SimpleNew(2 /* num dims */,
                            npy_shape,
                            NPY_FLOAT);
    if (!result_array)
        return NULL;

    // TODO: inc reference?

    try {
        PyThreads threads(UNBLOCK);

        /* Copy into a boost multi array (TODO: avoid this copy) */
        boost::multi_array<float, 2> array(boost::extents[n][d]);

        const float * data_in = (const float *)PyArray_DATA(input_as_float32);

        std::copy(data_in, data_in + (n * d), array.data());

        input_as_float32.release();

        boost::multi_array<float, 2> result
            = ML::vectors_to_distances(array);

        if (result.shape()[0] != n || result.shape()[1] != n)
            throw Exception("wrong shapes");

        float * data_out = (float *)PyArray_DATA(result_array);

        std::copy(result.data(), result.data() + n * n, data_out);
    } catch (const std::exception & exc) {
        return to_python_exception(exc);
    } catch (...) {
        return to_python_exception();
    }

    return result_array.release<PyObject>();
}

static PyObject *
tsne_distances_to_probabilities(PyObject *self, PyObject *args)
{
    PyObject * in_array;
    double tolerance;
    double perplexity;

    int args_ok = PyArg_ParseTuple(args, "O!dd", &PyArray_Type, &in_array,
                                   &tolerance, &perplexity);
    if (!args_ok)
        return NULL;

    /* Convert the argument to a float array. */
    PyArrayRef input_as_float32
        = PyArray_FromAny(in_array,
                          PyArray_DescrFromType(NPY_FLOAT32),
                          2, 2,
                          NPY_C_CONTIGUOUS | NPY_FORCECAST | NPY_ALIGNED,
                          0);
    if (!input_as_float32)
        return NULL;

    int n = PyArray_DIM(input_as_float32, 0);
    int n2 = PyArray_DIM(input_as_float32, 1);

    if (n != n2) {
        PyErr_SetString(PyExc_RuntimeError, "array must be square");
        return NULL;
    }

    /* Allocate an object (in memory) for the result */
    npy_intp npy_shape[2] = { n, n };
    PyArrayRef result_array
        = PyArray_SimpleNew(2 /* num dims */,
                            npy_shape,
                            NPY_FLOAT);
    if (!result_array)
        return NULL;

    try {
        PyThreads threads(UNBLOCK);

        /* Copy into a boost multi array (TODO: avoid this copy) */
        boost::multi_array<float, 2> array(boost::extents[n][n]);

        const float * data_in = (const float *)PyArray_DATA(input_as_float32);

        std::copy(data_in, data_in + (n * n), array.data());

        input_as_float32.release();

        boost::multi_array<float, 2> result
            = distances_to_probabilities(array,
                                         tolerance,
                                         perplexity);

        if (result.shape()[0] != n || result.shape()[1] != n)
            throw Exception("wrong shapes");

        float * data_out = (float *)PyArray_DATA(result_array);

        std::copy(result.data(), result.data() + n * n, data_out);
    } catch (const std::exception & exc) {
        return to_python_exception(exc);
    } catch (...) {
        return to_python_exception();
    }

    return result_array.release<PyObject>();
}

bool tsne_callback(int signals_before, int, float, const char *)
{
    return signals_before == num_signals_received;
}

static PyObject *
tsne_tsne(PyObject *self, PyObject *args, PyObject * kwds)
{
    PyObject * in_array;
    TSNE_Params params;

    int num_dims = 2;

    static const char * const kwlist[] =
        { "array", "num_dims", "max_iter", "initial_momentum", "final_momentum",
          "eta", "min_gain", "min_prob", NULL };

    if (!PyArg_ParseTupleAndKeywords(args, kwds,
                                     "O!|iiddddd", (char **)kwlist,
                                     &PyArray_Type, &in_array,
                                     &num_dims,
                                     &params.max_iter,
                                     &params.initial_momentum,
                                     &params.final_momentum,
                                     &params.eta,
                                     &params.min_gain,
                                     &params.min_prob))
        return NULL;

    /* Convert the input array to a float array. */
    PyArrayRef input_as_float32
        = PyArray_FromAny(in_array,
                          PyArray_DescrFromType(NPY_FLOAT32),
                          2, 2,
                          NPY_C_CONTIGUOUS | NPY_FORCECAST | NPY_ALIGNED,
                          0);
    if (!input_as_float32)
        return NULL;

    int n = PyArray_DIM(input_as_float32, 0);
    int n2 = PyArray_DIM(input_as_float32, 1);

    if (n != n2) {
        PyErr_SetString(PyExc_RuntimeError, "array must be square");
        return NULL;
    }

    /* Allocate an object (in memory) for the result */
    npy_intp npy_shape[2] = { n, num_dims };
    PyArrayRef result_array
        = PyArray_SimpleNew(2 /* num dims */,
                            npy_shape,
                            NPY_FLOAT);
    if (!result_array)
        return NULL;

    try {
        PyThreads threads(UNBLOCK);

        /* Copy into a boost multi array (TODO: avoid this copy) */
        boost::multi_array<float, 2> array(boost::extents[n][n]);

        const float * data_in = (const float *)PyArray_DATA(input_as_float32);

        std::copy(data_in, data_in + (n * n), array.data());

        input_as_float32.release();

        boost::multi_array<float, 2> result
            = tsne(array, num_dims, params,
                   boost::bind(tsne_callback,
                               threads.signals_before,
                               _1, _2, _3));

        if (threads.interrupted())
            throw Interrupt_Exception();

        if (result.shape()[0] != n || result.shape()[1] != num_dims)
            throw Exception("wrong shapes");

        float * data_out = (float *)PyArray_DATA(result_array);

        std::copy(result.data(), result.data() + n * num_dims, data_out);
    } catch (const std::exception & exc) {
        return to_python_exception(exc);
    } catch (...) {
        return to_python_exception();
    }

    return result_array.release<PyObject>();
}

static PyMethodDef TsneMethods[] = {
    {"vectors_to_distances",  tsne_vectors_to_distances, METH_VARARGS,
     "Convert an array of vectors in a coordinate space to a symmetric square"
     "matrix of distances between them."},
    {"distances_to_probabilities",  tsne_distances_to_probabilities, METH_VARARGS,
     "Convert an array of distances between points to an array of joint "
     "probabilities by modelling as gaussians."},
    {"tsne",  (PyCFunction)tsne_tsne, METH_VARARGS | METH_KEYWORDS,
     "reduce the (n x d) matrix to a (n x num_dims) matrix using t-SNE."},
    {NULL, NULL, 0, NULL}        /* Sentinel */
};

extern "C" {

PyMODINIT_FUNC
init_tsne(void)
{
    import_array();

    PyObject *m;

    m = Py_InitModule("_tsne", TsneMethods);
    if (m == NULL)
        return;

#if 0
    TsneError = PyErr_NewException("tsne.error", NULL, NULL);
    Py_INCREF(TsneError);
    PyModule_AddObject(m, "error", TsneError);
#endif
}

} // extern "C"


/*
 * Copyright (c) 2011, Mattia Penati <mattia.penati@gmail.com>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification,
 * are permitted provided that the following conditions are met:
 *
 *     * Redistributions of source code must retain the above copyright notice,
 *       this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright notice,
 *       this list of conditions and the following disclaimer in the documentation
 *       and/or other materials provided with the distribution.
 *     * Neither the name of the Politecnico di Milano nor the names of its
 *       contributors may be used to endorse or promote products derived from
 *       this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#ifndef AMA_TENSOR_IEXP_IEXP_CALCULATOR_HPP
#define AMA_TENSOR_IEXP_IEXP_CALCULATOR_HPP 1

#include <ama/common/size_t.hpp>
#include <ama/tensor/iexp/indices.hpp>
#include <boost/mpl/advance.hpp>
#include <boost/mpl/begin_end.hpp>
#include <boost/mpl/comparison.hpp>
#include <boost/mpl/contains.hpp>
#include <boost/mpl/count.hpp>
#include <boost/mpl/empty.hpp>
#include <boost/mpl/fold.hpp>
#include <boost/mpl/if.hpp>
#include <boost/mpl/logical.hpp>
#include <boost/mpl/max.hpp>
#include <boost/mpl/placeholders.hpp>
#include <boost/mpl/push_back.hpp>
#include <boost/mpl/size_t.hpp>
#include <boost/mpl/vector/vector0.hpp>

/* forward declaration */
namespace ama
{
  template <typename T, size_t D, size_t CT, size_t CO> class tensor;

  namespace tensor_
  {
    template <typename TENSOR, typename CTLIST, typename COLIST> class iexp_constant;
    template <typename TENSOR, typename CTLIST, typename COLIST> class iexp_mutable;
    template <typename TENSOR, typename CTLIST, typename COLIST> class iexp_temporary;
  }
}

namespace ama
{
  namespace tensor_
  {

    namespace mpl = ::boost::mpl;

    /* check if the list contains repeated indices */
    template <typename ILIST>
    struct has_repeated_indices:
        mpl::fold<
              ILIST
            , mpl::false_
            , mpl::or_<
                  mpl::_1
                , mpl::greater<
                        mpl::count<ILIST, mpl::_2>
                      , mpl::size_t<1>
                      >
                >
            >::type { };

    /* check if all indices are repeated */
    template <typename ILIST>
    struct all_repeated_indices:
        mpl::fold<
              ILIST
            , mpl::true_
            , mpl::and_<
                  mpl::_1
                , mpl::greater<
                        mpl::count<ILIST, mpl::_2>
                      , mpl::size_t<1>
                      >
                >
            >::type { };

    /* compute the return type of an iexp expression */
    template <typename TENSOR, typename ILIST, typename CONST>
    struct iexp_calculator:
        mpl::if_<
              has_repeated_indices<ILIST>
            , typename mpl::if_<
                    all_repeated_indices<ILIST>
                  , typename TENSOR::value_type
                  , iexp_temporary<
                          typename iexp_reduce<TENSOR, ILIST>::tensor_type
                        , typename iexp_reduce<TENSOR, ILIST>::controvariant
                        , typename iexp_reduce<TENSOR, ILIST>::covariant
                        >
                  >::type
            , typename mpl::if_<
                    CONST
                  , iexp_constant<
                          TENSOR
                        , typename controvariant<TENSOR,ILIST>::type
                        , typename covariant<TENSOR,ILIST>::type
                        >
                  , iexp_mutable<
                          TENSOR
                        , typename controvariant<TENSOR,ILIST>::type
                        , typename covariant<TENSOR,ILIST>::type
                        >
                  >::type
            >
    {
      typedef typename mpl::fold<
            ILIST
          , mpl::size_t<0>
          , mpl::max<
                  mpl::_1
                , mpl::count<ILIST, mpl::_1>
                >
          >::type max_;

      /* check if the indeces are repeated at most twice */
      BOOST_MPL_ASSERT_MSG(
            (mpl::less_equal< max_, mpl::size_t<2> >::type::value)
          , AN_INDEX_COULD_BE_REPEATED_AT_MOST_TWICE
          , (TENSOR, ILIST));

      typedef typename controvariant<TENSOR,ILIST>::type controvariant_;
      typedef typename covariant<TENSOR,ILIST>::type covariant_;

      typedef typename mpl::not_< has_repeated_indices<controvariant_> >::type bct_;
      typedef typename mpl::not_< has_repeated_indices<covariant_> > bco_;

      /* check for reduced indeces */
      BOOST_MPL_ASSERT_MSG(
            (mpl::and_<bct_,bco_>::type::value)
          , YOU_CANNOT_REDUCE_TWO_INDICES_OF_SAME_TYPE
          , (TENSOR, ILIST));
    };

  }
}

#endif /* AMA_TENSOR_IEXP_IEXP_CALCULATOR_HPP */