/*
Part of the Fluid Corpus Manipulation Project (http://www.flucoma.org/)
Copyright 2017-2019 University of Huddersfield.
Licensed under the BSD-3 License.
See license.md file in the project root for full license information.
This project has received funding from the European Research Council (ERC)
under the European Union’s Horizon 2020 research and innovation programme
(grant agreement No 725899).
*/

#pragma once

#include "../util/FluidEigenMappings.hpp"
#include "../../data/FluidDataSet.hpp"
#include "../../data/FluidIndex.hpp"
#include "../../data/FluidTensor.hpp"
#include "../../data/TensorTypes.hpp"
#include <Eigen/Core>
#include <set>
#include <string>

namespace fluid {
namespace algorithm {

class DataSetQuery
{

public:
  using string = std::string;

  using DataSet = FluidDataSet<string, double, 1>;

  struct Condition
  {
    index  column;
    index  comparison;
    double value;

    bool test(RealVectorView point)
    {
      using namespace std;
      switch (comparison)
      {
      case 0: return point(column) == value;
      case 1: return point(column) != value;
      case 2: return point(column) < value;
      case 3: return point(column) <= value;
      case 4: return point(column) > value;
      case 5: return point(column) >= value;
      }
      return false;
    }
  };

  DataSetQuery()
  {
    mComparisons = std::vector<std::string>{"==", "!=", "<", "<=", ">", ">="};
  }

  void addColumn(index col) { mColumns.emplace(col); }

  void addRange(index from, index count)
  {
    for (index i = from; i < from + count; i++) { mColumns.emplace(i); }
  }

  index numColumns() { return asSigned(mColumns.size()); }

  bool hasAndConditions() { return (mAndConditions.size() > 0); }

  index maxColumn() { return mColumns.empty() ? 0 : *mColumns.rbegin(); }

  bool addCondition(index column, string comparison, double value,
                    bool conjunction)
  {
    auto pos = std::find(mComparisons.begin(), mComparisons.end(), comparison);
    if (pos == mComparisons.end()) return false;
    index i = std::distance(mComparisons.begin(), pos);
    if (conjunction)
      mAndConditions.emplace_back(Condition{column, i, value});
    else
      mOrConditions.emplace_back(Condition{column, i, value});
    return true;
  }

  void process(const DataSet& input, DataSet& current, DataSet& output)
  {
    auto data = input.getData();
    auto ids = input.getIds();
    mTmpPoint = RealVector(asUnsigned(current.pointSize()) + mColumns.size());
    index limit = mLimit == 0 ? input.size() : mLimit;
    index count = 0;
    for (index i = 0; i < input.size() && count < limit; i++)
    {
      bool matchesAllAnd = true;
      auto point = data.row(i);
      for (index j = 0; j < asSigned(mAndConditions.size()); j++)
        if (!mAndConditions[asUnsigned(j)].test(point)) matchesAllAnd = false;
      if (matchesAllAnd)
      {
        addRow(ids(i), point, current, output);
        count++;
        continue;
      }
      else
      {
        bool matchesAnyOr = false;
        for (index k = 0; k < asSigned(mOrConditions.size()); k++)
          if (mOrConditions[asUnsigned(k)].test(point)) matchesAnyOr = true;
        if (matchesAnyOr)
        {
          addRow(ids(i), point, current, output);
          count++;
        }
      }
    }
  }

  void print() const {}

  void limit(index rows) { mLimit = rows; }
  void clear()
  {
    mColumns.clear();
    mAndConditions.clear();
    mOrConditions.clear();
    mLimit = 0;
  }

private:
  void addRow(string id, RealVectorView point, const DataSet& current,
              DataSet& out)
  {
    mTmpPoint.fill(0);
    index currentSize = current.pointSize();
    bool  shouldAdd = true;
    if (currentSize > 0)
    {
      RealVectorView currentData = mTmpPoint(Slice(0, currentSize));
      shouldAdd = current.get(id, currentData);
    }
    if (shouldAdd)
    {
      for (auto c : mColumns) mTmpPoint(currentSize++) = point(c);
      out.add(id, mTmpPoint);
    }
  }

  index                    mLimit{0};
  std::set<index>          mColumns;
  RealVector               mTmpPoint;
  std::vector<std::string> mComparisons;
  std::vector<Condition>   mAndConditions;
  std::vector<Condition>   mOrConditions;
};
} // namespace algorithm
} // namespace fluid


/*
 *  filters.cpp
 *  cufflinks
 *
 *  Created by Cole Trapnell on 10/27/09.
 *  Copyright 2009 Cole Trapnell. All rights reserved.
 *
 */

#include "filters.h"
#include <algorithm>
#include <numeric>
#include <boost/graph/adjacency_list.hpp>
#include <boost/graph/depth_first_search.hpp>
#include <boost/graph/visitors.hpp>
#include <boost/graph/graph_traits.hpp>
#include <boost/graph/connected_components.hpp>

using namespace std;
using namespace boost;

void filter_introns(int bundle_length,
					int bundle_left,
					vector<Scaffold>& hits,
					double fraction,
					bool filter_on_intron_overlap,
					bool filter_with_intron_doc)
{
	vector<float> depth_of_coverage(bundle_length,0);
	vector<double> scaff_doc;
	map<pair<int,int>, float> intron_doc;
	vector<Scaffold> filtered_hits;
	vector<bool> toss(hits.size(), false);

	double bundle_avg_doc = compute_doc(bundle_left,
										hits,
										depth_of_coverage,
										intron_doc,
										false);

	double bundle_avg_thresh = bundle_avg_doc * fraction;

	if (filter_with_intron_doc && !intron_doc.empty())
	{
		bundle_avg_doc = major_isoform_intron_doc(intron_doc);
		bundle_avg_thresh = fraction * bundle_avg_doc;
		verbose_msg("\tFiltering bundle introns, avg (intron) doc = %lf, thresh = %f\n", bundle_avg_doc, bundle_avg_thresh);
	}
	else
	{
		verbose_msg("\tFiltering bundle introns, avg bundle doc = %lf, thresh = %f\n", bundle_avg_doc, bundle_avg_thresh);
	}

	for(map<pair<int, int>, float>::const_iterator itr = intron_doc.begin();
		itr != intron_doc.end();
		++itr)
	{
		for (size_t j = 0; j < hits.size(); ++j)
		{
			//fprintf(stderr, "considering read [%d-%d] with min doc = %lf contained in intron with doc = %lf\n", hits[j].left(), hits[j].right(), doc, idoc);
			const vector<AugmentedCuffOp>& ops = hits[j].augmented_ops();

			for (size_t i = 0; i < ops.size(); ++i)
			{
				if (ops[i].opcode == CUFF_INTRON)
				{
					map<pair<int, int>, float>::const_iterator itr;
					itr = intron_doc.find(make_pair(ops[i].g_left(), ops[i].g_right()));

					double doc = itr->second;
					if (doc < bundle_avg_thresh)
					{
						toss[j] = true;
						verbose_msg("\t Filtering intron %d - %d: %f thresh %f\n", itr->first.first, itr->first.second, doc, bundle_avg_thresh);
						continue;
					}

					if (!filter_on_intron_overlap)
						continue;

					for (map<pair<int,int>, float>::const_iterator itr2 = intron_doc.begin();
						 itr2 != intron_doc.end();
						 ++itr2)
					{
						if (itr == itr2 ||
							!overlap_in_genome(itr->first.first,
											   itr->first.second,
											   itr2->first.first,
											   itr2->first.second))
							continue;

						double thresh = itr2->second * fraction;
						if (doc < thresh)
						{
							verbose_msg("\t Filtering intron (due to overlap) %d - %d: %f thresh %f\n", itr->first.first, itr->first.second, doc, bundle_avg_thresh);
							toss[j] = true;
						}
					}
				}
			}
		}
	}

	for (size_t j = 0; j < hits.size(); ++j)
	{
		if (!toss[j])
		{
			filtered_hits.push_back(hits[j]);
//#if verbose_msg
//			if (hits[j].has_intron())
//			{
//				fprintf(stderr, "KEEPING intron scaff [%d-%d]\n", hits[j].left(), hits[j].right());
//			}
//#endif
		}
		else
		{
			if (hits[j].has_intron())
			{

				verbose_msg("\tFiltering intron scaff [%d-%d]\n", hits[j].left(), hits[j].right());
			}
		}
	}

	verbose_msg("\tIntron filtering pass finished: excluded %d fragments\n", (int)hits.size() - (int)filtered_hits.size());
	hits = filtered_hits;
}

double background_rate(const vector<float> depth_of_coverage,
                       int left,
                       int right)
{
    vector<float> tmp;

    size_t r_bound = (size_t)min(right, (int) depth_of_coverage.size());
    size_t l_bound = (size_t)max(left, 0);

    tmp.insert(tmp.end(),
               depth_of_coverage.begin() + l_bound,
               depth_of_coverage.begin() + r_bound);

    if (tmp.empty())
        return 0;

    vector<float>::iterator new_end =  remove(tmp.begin(), tmp.end(), 0);
    tmp.erase(new_end, tmp.end());
    sort(tmp.begin(), tmp.end());

    size_t median = (size_t)floor(tmp.size() / 2);
    double median_doc = tmp[median];
    return median_doc;
}

void pre_mrna_filter(int bundle_length,
					 int bundle_left,
					 vector<Scaffold>& hits)
{
	vector<float> depth_of_coverage(bundle_length,0);
	vector<double> scaff_doc;
	map<pair<int,int>, float> intron_doc;
	vector<Scaffold> filtered_hits;
	vector<bool> toss(hits.size(), false);
	vector<float> through_introns; //for each location, how many introns pass through

	vector<int> scaff_intron_status;
	// Make sure the avg only uses stuff we're sure isn't pre-mrna fragments
    double bundle_avg_doc = compute_doc(bundle_left,
										hits,
										depth_of_coverage,
										intron_doc,
										true,
										&through_introns,
										&scaff_intron_status);
    verbose_msg("Pre-mRNA flt: bundle average doc = %lf\n", bundle_avg_doc);
    /*
     //2nd call not needed, the vectors won't change, only the return value
     compute_doc(bundle_left,
     hits,
     depth_of_coverage,
     intron_doc,
     false);
     */
	record_doc_for_scaffolds(bundle_left,
							 hits,
							 depth_of_coverage,
							 intron_doc,
							 scaff_doc);

	for(map<pair<int, int>, float >::const_iterator itr = intron_doc.begin();
		itr != intron_doc.end();
		++itr)
	{
		int i_left = itr->first.first;
		int i_right = itr->first.second;
		int i_doc   = itr->second;
        double intron_background = background_rate(depth_of_coverage,
                                                   i_left - bundle_left,
                                                   i_right - bundle_left);
        double cumul_cov = 0;
        for (int i = 0; i < i_right - i_left; ++i)
        {
            size_t pos = (i_left - bundle_left) + i;
            cumul_cov += depth_of_coverage[pos];
        }
        cumul_cov /= i_right - i_left;
        verbose_msg("Pre-mRNA flt: intron %d-%d : background: %lf, inner coverage: %lf, junction coverage: %f\n",
                    i_left, i_right, intron_background, cumul_cov, i_doc);
        if (cumul_cov / bundle_avg_doc >= pre_mrna_fraction)
        {
            //fprintf(stderr, "\tskipping\n");
            continue;
        }

        ////double thresh = (1.0/pre_mrna_fraction) * intron_background;
        double thresh = pre_mrna_fraction * intron_background;

        for (size_t j = 0; j < hits.size(); ++j)
        {
            if (hits[j].left()>i_right) break;
            if (hits[j].is_ref())
                continue;
            if (toss[j])
                continue;
            //find maximum intron support in the hit region

            int len = 0;
            double doc = 0.0;
            size_t curr_op = 0;
            const vector<AugmentedCuffOp>& ops = hits[j].augmented_ops();
            while (curr_op != ops.size())
            {
                const AugmentedCuffOp&  op = ops[curr_op];
                if (op.opcode == CUFF_MATCH)
                {
                    int op_len = 0;
                    double op_doc = 0.0;
                    int left_off = op.g_left();
                    if (left_off + op.genomic_length > i_left && left_off < i_right)
                    {
                        if (left_off > i_left)
                        {
                            if (left_off + op.genomic_length <= i_right + 1)
                            {
                                op_len += op.genomic_length;
                                int L = left_off - bundle_left;
                                int R = L + op.genomic_length;
                                op_doc += accumulate(depth_of_coverage.begin() + L, depth_of_coverage.begin() + R, 0.0);
                            }
                            else
                            {
                                op_len += i_right - left_off;
                                int L = left_off - bundle_left;
                                int R = L + (i_right - left_off);
                                op_doc += accumulate(depth_of_coverage.begin() + L, depth_of_coverage.begin() + R, 0.0);
                            }
                        }
                        else
                        {
                            if (left_off + op.genomic_length <= i_right + 1)
                            {
                                op_len += (left_off + op.genomic_length - i_left);
                                int L = left_off - bundle_left;
                                int R = L + (left_off + op.genomic_length - i_left);
                                op_doc += accumulate(depth_of_coverage.begin() + L, depth_of_coverage.begin() + R, 0.0);
                            }
                            else
                            {
                                op_len = i_right - i_left;
                                int L = left_off - bundle_left;
                                int R = L + (i_right - i_left);
                                op_doc = accumulate(depth_of_coverage.begin() + L, depth_of_coverage.begin() + R, 0.0);
                            }
                        }
                    }

                    len += op_len;
                    doc += op_doc;
                }

                if (op.g_left() >= i_right)
                    break;
                ++curr_op;
            }

            if (len)
            {
                double hit_doc_in_region = doc / len;
                if (hit_doc_in_region < thresh)
                {
                    toss[j] = true;
                    if (hits[j].has_intron())
                    {
                        //    fprintf(stderr, "\t$$$ Filtering intron scaff [%d-%d]\n", hits[j].left(), hits[j].right());

                        verbose_msg("\t@@@ Filtering intron scaff [%d-%d] (scaff_doc=%lf, doc_in_region=%lf)\n",
                                    hits[j].left(), hits[j].right(), scaff_doc[j], hit_doc_in_region);
                    }
                }
            }
		} //for each scaffold
	} //for each intron
	for (size_t j = 0; j < hits.size(); ++j)
	{
		if (!toss[j])
		{
			filtered_hits.push_back(hits[j]);
		}
		/*else
         {
         if (hits[j].has_intron())
         {

         verbose_msg( "\t@@@ Filtering intron scaff [%d-%d]\n", hits[j].left(), hits[j].right());
         }
         }
         */
	}

	if (cuff_verbose && hits.size()>filtered_hits.size())
        verbose_msg("\tPre-mRNA flt tossed %lu fragments\n", hits.size() - filtered_hits.size());

	hits = filtered_hits;
}

void filter_hits(int bundle_length,
				 int bundle_left,
				 vector<Scaffold>& hits)
{

	pre_mrna_filter(bundle_length, bundle_left, hits);

	vector<float> depth_of_coverage(bundle_length+1,0);
	vector<double> scaff_doc;
	map<pair<int,int>, float> intron_doc;
	vector<Scaffold> filtered_hits;
	vector<bool> toss(hits.size(), false);

	// Make sure the avg only uses stuff we're sure isn't pre-mrna fragments
	double bundle_avg_doc = compute_doc(bundle_left,
										hits,
										depth_of_coverage,
										intron_doc,
										true);

	// recompute the real DoCs
	/* not needed, vectors are not changed
	compute_doc(bundle_left,
				hits,
				depth_of_coverage,
				intron_doc,
				false);
	*/

	record_min_doc_for_scaffolds(bundle_left,
								 hits,
								 depth_of_coverage,
								 intron_doc,
								 scaff_doc);

	//double bundle_avg_thresh = min_isoform_fraction * bundle_avg_doc;

	if (!intron_doc.empty())
	{
		double intron_avg_doc = major_isoform_intron_doc(intron_doc);
		double intron_multiplier = intron_avg_doc / bundle_avg_doc;

		// we don't want this to be more than 1.0 ...
		intron_multiplier = min(intron_avg_doc, 1.0);
		//bundle_avg_thresh = min_isoform_fraction * bundle_avg_doc;

		set<pair<int, int> > tossed_introns;
		for(map<pair<int, int>, float>::const_iterator itr = intron_doc.begin();
			itr != intron_doc.end();
			++itr)
		{
			for (size_t j = 0; j < hits.size(); ++j)
			{
				if (hits[j].is_ref())
                {
					continue;
                }
				int i_left = itr->first.first;
				int i_right = itr->first.second;
				int j_match_len = hits[j].match_length(i_left, i_right);
				if (j_match_len > 0)
				{
					double idoc = itr->second;
					double doc = scaff_doc[j];

					if (!hits[j].has_intron() &&
						doc < pre_mrna_fraction * (idoc * intron_multiplier))
                    {
						toss[j] = true;
                    }

					const vector<AugmentedCuffOp>& ops = hits[j].augmented_ops();

					unsigned int num_mismatches = 0;
					assert (hits[j].mate_hits().size() == 1);
					const MateHit& hit = **(hits[j].mate_hits().begin());
					num_mismatches = hit.edit_dist();

					double percent_mismatches = num_mismatches / (double)hits[j].length();

					bool intron_pokin_read = false;

					const AugmentedCuffOp& first = ops.front();
					// intron                    =========================
					// hit                     ******************
					if (first.g_left() < i_left && first.g_right() > i_left  && first.g_right() < i_right)
					{
						intron_pokin_read = true;
					}

					// intron          =========================
					// hit                      ******************
					if (first.g_left() < i_right && first.g_right() > i_right  && first.g_left() > i_left)
					{
						intron_pokin_read = true;
					}

					const AugmentedCuffOp& last = ops.back();
					// intron          =========================
					// hit                     ******************
					if (last.g_left() < i_left && last.g_right() > i_left  && last.g_right() < i_right)
					{
						intron_pokin_read = true;
					}

					// intron          =========================
					// hit                            ******************
					if (last.g_left() < i_right && last.g_right() > i_right  && last.g_left() > i_left)
					{
						intron_pokin_read = true;
					}

					if (intron_pokin_read)
					{
						double fraction;
//						if (!hits[j].has_intron())
//						{
//							fraction = (3 * pre_mrna_fraction) + percent_mismatches;
//						}
//						else
						{
							fraction = pre_mrna_fraction + percent_mismatches;
						}
						double thresh = fraction * (intron_avg_doc * intron_multiplier);
						if (doc < thresh)
						{
							toss[j] = true;
//							if (hits[j].has_intron())
//							{
//								fprintf(stderr, "\t^^^Filtering intron scaff [%d-%d]\n", hits[j].left(), hits[j].right());
//							}
						}
					}
				}
			}
		}
	}

	for (size_t j = 0; j < hits.size(); ++j)
	{
		if (!toss[j])
		{
			filtered_hits.push_back(hits[j]);
//#if verbose_msg
//			if (hits[j].has_intron())
//			{
//
//				fprintf(stderr, "KEEPING intron scaff [%d-%d]\n", hits[j].left(), hits[j].right());
//			}
//#endif
		}
		else
		{
			if (hits[j].has_intron())
			{

				verbose_msg("\t!!!Filtering intron scaff [%d-%d]\n", hits[j].left(), hits[j].right());
			}
		}
	}

//#if verbose_msg
//	fprintf(stderr, "\tInitial filter pass complete\n");
//#endif

	hits = filtered_hits;

	scaff_doc.clear();
	filtered_hits.clear();

	toss = vector<bool>(hits.size(), false);

	map<pair<int, int>, float> dummy;
	bundle_avg_doc = compute_doc(bundle_left,
								 hits,
								 depth_of_coverage,
								 dummy,
								 false);

//#if verbose_msg
//	fprintf(stderr, "\tUpdated avg bundle doc = %lf\n", bundle_avg_doc);
//#endif

	record_doc_for_scaffolds(bundle_left,
							 hits,
							 depth_of_coverage,
							 intron_doc,
							 scaff_doc);

//#if verbose_msg
//    double bundle_thresh = pre_mrna_fraction * bundle_avg_doc;
//	fprintf(stderr, "\tthreshold is = %lf\n", bundle_thresh);
//#endif

	if (!intron_doc.empty())
	{
//		filter_introns(bundle_length,
//					   bundle_left,
//					   hits,
//					   min_isoform_fraction,
//					   true,
//					   true);
		if (bundle_avg_doc > 3000)
		{
			filter_introns(bundle_length,
						   bundle_left,
						   hits,
						   min_isoform_fraction,
						   true,
						   false);
		}
	}

	for (size_t j = 0; j < hits.size(); ++j)
	{
		if (!toss[j])
		{
			filtered_hits.push_back(hits[j]);
//#if verbose_msg
//			if (hits[j].has_intron())
//			{
//
//				fprintf(stderr, "KEEPING intron scaff [%d-%d]\n", hits[j].left(), hits[j].right());
//			}
//#endif
		}
		else
		{
			if (hits[j].has_intron())
			{
				verbose_msg("\t***Filtering intron scaff [%d-%d]\n", hits[j].left(), hits[j].right());
			}
		}
	}

	//fprintf(stderr, "\tTossed %d hits as noise\n", (int)hits.size() - filtered_hits.size());

	hits = filtered_hits;
}

void filter_junk_isoforms(vector<boost::shared_ptr<Abundance> >& transcripts,
						  vector<double>& abundances,
                          const vector<boost::shared_ptr<Abundance> >& mapped_transcripts,
                          double locus_mass)
{
	//	vector<double>::iterator max_ab = std::max_element(abundances.begin(),
	//													   abundances.end());
	double max_fwd_ab = -1.0;
	double max_rev_ab = -1.0;

	for (size_t t = 0; t < transcripts.size(); ++t)
	{
		boost::shared_ptr<Scaffold> scaff = transcripts[t]->transfrag();
		if (scaff->strand() == CUFF_FWD || scaff->strand() == CUFF_STRAND_UNKNOWN)
		{
			if (abundances[t] > max_fwd_ab)
				max_fwd_ab = abundances[t];
		}
		if (scaff->strand() == CUFF_REV || scaff->strand() == CUFF_STRAND_UNKNOWN)
		{
			if (abundances[t] > max_rev_ab)
				max_rev_ab = abundances[t];
		}
	}

	// Try to categorize the crap transcripts for suppression
	vector<bool> pre_mrna_junk(transcripts.size(), false); //intra-intron, much lower abundance than container
	vector<bool> chaff(transcripts.size(), false); // only a single MateHit, impossible to reliably quantitate
	vector<bool> repeats(transcripts.size(), false); // too many low-quality hits
	vector<bool> too_rare(transcripts.size(), false); // too rare to be reliably quantitated, could be error

	//cerr << "Chucked : ";
	for (size_t t = 0; t < transcripts.size(); ++t)
	{
		boost::shared_ptr<Scaffold> scaff = transcripts[t]->transfrag();

		if (!(scaff->is_ref()) && allow_junk_filtering)
		{
			const vector<const MateHit*> hits = scaff->mate_hits();

			const vector<AugmentedCuffOp>& ops = scaff->augmented_ops();

			if (ops.size() == 1 && ops[0].opcode == CUFF_MATCH)
			{
				for (size_t j = 0; j < transcripts.size(); ++j)
				{
					const vector<AugmentedCuffOp>& j_ops = scaff->augmented_ops();
					for (size_t L = 0; L < j_ops.size(); L++)
					{
						if (AugmentedCuffOp::overlap_in_genome(ops[0], j_ops[L]) &&
							j_ops[L].opcode == CUFF_INTRON)
						{
							pre_mrna_junk[t] = true;
						}
					}
				}
			}

            if (library_type != "transfrags")
            {
                double multi_hits = 0.0;
                //static const double low_qual_err_prob = high_phred_err_prob; // hits with error_prob() above this are low quality;
                //static const double low_qual_thresh = 0.75; // hits with more than this fraction of low qual hits are repeats
                for (vector<const MateHit*>::const_iterator itr = hits.begin();
                     itr != hits.end();
                     ++itr)
                {
                    if ((*itr)->is_multi())
                        multi_hits += 1.0;
                }

                double low_qual_frac = multi_hits / (double)hits.size();
                if (low_qual_frac > max_multiread_fraction)
                    repeats[t] = true;
            }
            if (scaff->strand() == CUFF_FWD &&
                (abundances[t] / max_fwd_ab) < min_isoform_fraction)
                too_rare[t] = true;
            if ((scaff->strand() == CUFF_REV ||  scaff->strand() == CUFF_STRAND_UNKNOWN) &&
                (abundances[t] / max_rev_ab) < min_isoform_fraction)
                too_rare[t] = true;

            const vector<double>* cond_probs = (mapped_transcripts[t]->cond_probs());
            if (cond_probs)
            {
                assert (library_type != "transfrags");
                double supporting_hits = abundances[t] * locus_mass;
                if (supporting_hits < min_frags_per_transfrag)
                    chaff[t] = true;
            }
		}
	}

	vector<boost::shared_ptr<Abundance> > non_junk_transcripts;
	vector<double> non_junk_abundances;
	for (size_t t = 0; t < transcripts.size(); ++t)
	{
		if (!repeats[t] && !pre_mrna_junk[t] && !too_rare[t] && !chaff[t])
		{
			non_junk_transcripts.push_back(transcripts[t]);
			non_junk_abundances.push_back(abundances[t]);
		}
        else
        {
            verbose_msg( "Filtering isoform %d-%d\n", transcripts[t]->transfrag()->left(), transcripts[t]->transfrag()->right());
        }
	}

	transcripts = non_junk_transcripts;
	abundances = non_junk_abundances;
}

// Designed to strip out remaining pre-mrna genes, assembled repeats, and
// fragments from isoforms too short to be reliably quantitated.
void filter_junk_genes(vector<Gene>& genes)
{
	vector<Gene> good_genes;
	vector<Isoform> all_isoforms;
	for (size_t i = 0; i < genes.size(); ++i)
	{
		all_isoforms.insert(all_isoforms.end(),
							genes[i].isoforms().begin(),
							genes[i].isoforms().end());
	}

	for (size_t i = 0; i < genes.size(); ++i)
	{
		const Gene& g = genes[i];

		if(g.has_ref_trans())
		{
			good_genes.push_back(g);
			continue;
		}

		bool good_gene = true;
		for (size_t j = 0; j < all_isoforms.size(); ++j)
		{
			vector<pair<int, int> > introns = all_isoforms[j].scaffold().gaps();

            //assert (!allow_junk_filtering || all_isoforms[j].scaffold().mate_hits().size() >= min_frags_per_transfrag);
			for (size_t k = 0; k < introns.size(); ++k)
			{
				if (g.left() > introns[k].first && g.right() < introns[k].second &&
					g.FPKM() / all_isoforms[j].FPKM() < pre_mrna_fraction)
				{
					good_gene = false;
				}
			}
		}
        if (allow_junk_filtering)
        {
            if (g.FPKM() == 0)
            {
                good_gene = false;
            }
        }
		if (good_gene)
        {
			good_genes.push_back(g);
        }
        else
        {
            verbose_msg("Filtering transfrags from gene %d-%d\n", g.left(), g.right());
        }
	}

	genes = good_genes;

}

void clip_by_3_prime_dropoff(vector<Scaffold>& scaffolds)
{
    vector<pair<double, Scaffold*> > three_prime_ends;

    if (library_type != "transfrags")
    {
        BOOST_FOREACH (Scaffold& scaff, scaffolds)
        {
            if (!(scaff.strand() == CUFF_FWD || scaff.strand() == CUFF_REV))
                continue;

            int scaff_len = scaff.length();
            vector<double> coverage(scaff_len, 0.0);

            double total = 0;
            BOOST_FOREACH(const MateHit* hit, scaff.mate_hits())
            {
                int start, end, frag_len;
                if (!scaff.map_frag(*hit, start, end, frag_len)) continue;

                if (scaff.strand() == CUFF_REV)
                {
                    start = scaff_len - 1 - start;
                    end = scaff_len - 1 - end;
                    swap(start, end);
                }

                for(int i = start; i <= end; ++i)
                {
                    coverage[i] += hit->mass();
                    total += hit->mass();
                }
            }
            double avg_cov = total/scaff_len;
    //        if (avg_cov < trim_3_avgcov_thresh)
    //            continue;

            const AugmentedCuffOp* exon_3 = NULL;
            int mult;
            int offset;

            if (scaff.strand() == CUFF_REV)
            {
                mult = 1;
                offset = 0;
                exon_3 = &scaff.augmented_ops().front(); //last exon at 3' end
            }
            else if (scaff.strand() == CUFF_FWD)
            {
                mult = -1;
                offset = scaff_len - 1;
                exon_3 = &scaff.augmented_ops().back();//last exon at 3' end
            }
            else
            {
                continue;
            }

            int to_remove;
            double min_cost = numeric_limits<double>::max();
            double mean_to_keep = 0.0;
            double mean_to_trim = 0.0;
            double tmp_mean_to_trim = 0.0;
            double tmp_mean_to_keep = 0.0;
            double tmp_mean_3prime = 0.0;
            for (int i = 0; i < exon_3->genomic_length; i++)
            {
                tmp_mean_3prime += coverage[offset + mult*i];
            }
            tmp_mean_3prime /= exon_3->genomic_length;

            double base_cost = 0.0;
            for (int i = 0; i < exon_3->genomic_length; i++)
            {
                double d = (coverage[offset + mult*i] - tmp_mean_3prime);
                d *= d;
                base_cost += d;
            }
            base_cost /= exon_3->genomic_length;

            size_t min_cost_x = -1;
            for (to_remove = 1; to_remove < exon_3->genomic_length - 1; to_remove++)
            {
                tmp_mean_to_trim = 0.0;
                tmp_mean_to_keep = 0.0;
                for (int i = 0; i < exon_3->genomic_length; i++)
                {
                    if (i <= to_remove)
                    {
                        tmp_mean_to_trim += coverage[offset + mult*i];
                    }
                    else
                    {
                        tmp_mean_to_keep += coverage[offset + mult*i];
                    }
                }

                tmp_mean_to_trim /= to_remove;
                tmp_mean_to_keep /= (exon_3->genomic_length - to_remove);

                double tmp_mean_trim_cost = 0.0;
                double tmp_mean_keep_cost = 0.0;
                for (int i = 0; i < exon_3->genomic_length; i++)
                {
                    if (i <= to_remove)
                    {
                        double d = (coverage[offset + mult*i] - tmp_mean_to_trim);
                        d *= d;
                        tmp_mean_trim_cost += d;
                    }
                    else
                    {
                        double d = (coverage[offset + mult*i] - tmp_mean_to_keep);
                        d *= d;
                        tmp_mean_keep_cost += d;
                    }
                }

                tmp_mean_trim_cost /= to_remove;
                tmp_mean_keep_cost /= (exon_3->genomic_length - to_remove);

                double new_cost = tmp_mean_trim_cost + tmp_mean_keep_cost;

                if (new_cost < min_cost && trim_3_dropoff_frac * tmp_mean_to_keep > tmp_mean_to_trim && new_cost < base_cost && to_remove > scaff_len * 0.05)
                {
                    min_cost = tmp_mean_trim_cost + tmp_mean_keep_cost;
                    min_cost_x = to_remove;
                    mean_to_keep = tmp_mean_to_keep;
                    mean_to_trim = tmp_mean_to_trim;
                }
            }

            // If trimming reduces the overall mean squared error of the coverage
            // do it
            if (avg_cov >= trim_3_avgcov_thresh && min_cost_x < exon_3->genomic_length)
            {
                scaff.trim_3(min_cost_x);
            }

            // store the mean squared error for this exon
            tmp_mean_3prime = 0.0;
            for (int i = 0; i < exon_3->genomic_length; i++)
            {
                tmp_mean_3prime += coverage[offset + mult*i];
            }
            tmp_mean_3prime /= exon_3->genomic_length;

            base_cost = 0.0;
            for (int i = 0; i < exon_3->genomic_length; i++)
            {
                double d = (coverage[offset + mult*i] - tmp_mean_3prime);
                d *= d;
                base_cost += d;
            }
            base_cost /= exon_3->genomic_length;
            three_prime_ends.push_back(make_pair(base_cost, &scaff));
        }
    }
    else
    {
        BOOST_FOREACH (Scaffold& scaff, scaffolds)
        {
            if (!(scaff.strand() == CUFF_FWD || scaff.strand() == CUFF_REV))
                continue;

            int scaff_len = scaff.length();
            vector<double> coverage(scaff_len, 0.0);

            double total = 0;
            BOOST_FOREACH(const MateHit* hit, scaff.mate_hits())
            {
                int start, end, frag_len;
                if (!scaff.map_frag(*hit, start, end, frag_len)) continue;

                if (scaff.strand() == CUFF_REV)
                {
                    start = scaff_len - 1 - start;
                    end = scaff_len - 1 - end;
                    swap(start, end);
                }

                for(int i = start; i <= end; ++i)
                {
                    coverage[i] += hit->mass();
                    total += hit->mass();
                }
            }

            const AugmentedCuffOp* exon_3 = NULL;
            int mult;
            int offset;

            if (scaff.strand() == CUFF_REV)
            {
                mult = 1;
                offset = 0;
                exon_3 = &scaff.augmented_ops().front();
            }
            else if (scaff.strand() == CUFF_FWD)
            {
                mult = -1;
                offset = scaff_len - 1;
                exon_3 = &scaff.augmented_ops().back();
            }
            else
            {
                continue;
            }

            three_prime_ends.push_back(make_pair(scaff.fpkm(), &scaff));
        }

    }

    adjacency_list <vecS, vecS, undirectedS> G;

	for (size_t i = 0; i < three_prime_ends.size(); ++i)
	{
		add_vertex(G);
	}

	for (size_t i = 0; i < three_prime_ends.size(); ++i)
	{
		Scaffold* scaff_i = three_prime_ends[i].second;
		//assert (scaff_i);

        const AugmentedCuffOp* scaff_i_exon_3 = NULL;

        if (scaff_i->strand() == CUFF_REV)
        {
            scaff_i_exon_3 = &(scaff_i->augmented_ops().front());
        }
        else if (scaff_i->strand() == CUFF_FWD)
        {
            scaff_i_exon_3 = &(scaff_i->augmented_ops().back());
        }

		for (size_t j = i + 1; j < three_prime_ends.size(); ++j)
		{
			Scaffold* scaff_j = three_prime_ends[j].second;

            if (scaff_i->strand() != scaff_j->strand())
                continue;

            const AugmentedCuffOp* scaff_j_exon_3 = NULL;

            if (scaff_j->strand() == CUFF_REV)
            {
                scaff_j_exon_3 = &(scaff_j->augmented_ops().front());
            }
            else if (scaff_j->strand() == CUFF_FWD)
            {
                scaff_j_exon_3 = &(scaff_j->augmented_ops().back());
            }

			if (AugmentedCuffOp::overlap_in_genome(*scaff_j_exon_3, *scaff_i_exon_3) &&
                AugmentedCuffOp::compatible(*scaff_j_exon_3, *scaff_i_exon_3, 0))
				add_edge(i, j, G);
		}
	}

	std::vector<int> component(num_vertices(G));
	connected_components(G, &component[0]);

	vector<vector<bool> > clusters(three_prime_ends.size(),
								   vector<bool>(three_prime_ends.size(), false));

	//vector<vector<size_t> > cluster_indices(three_prime_ends.size());

    vector<vector<pair<double, Scaffold*> > > grouped_scaffolds(three_prime_ends.size());
	for (size_t i = 0; i < three_prime_ends.size(); ++i)
	{
		clusters[component[i]][i] = true;
		grouped_scaffolds[component[i]].push_back(three_prime_ends[i]);
	}

    for (size_t i = 0; i < grouped_scaffolds.size(); ++i)
    {
        vector<pair<double, Scaffold*> >& group = grouped_scaffolds[i];
        sort(group.begin(), group.end());
        if (group.empty())
            continue;

        Scaffold* group_leader = NULL;
        int trim_point = -1;

        const AugmentedCuffOp* group_exon_3 = NULL;
        vector<pair<double, Scaffold*> >::iterator l_itr = group.begin();
        while (l_itr != group.end())
        {
            Scaffold* possible_leader = l_itr->second;
            bool ok_clip_leader = true;
            vector<pair<double, Scaffold*> >::iterator g_itr = group.begin();
            const AugmentedCuffOp* l_exon_3 = NULL;
            CuffStrand s = possible_leader->strand();

            if (s != CUFF_STRAND_UNKNOWN)
            {
                if (s == CUFF_REV)
                    l_exon_3 = &(possible_leader->augmented_ops().front());
                else
                    l_exon_3 = &(possible_leader->augmented_ops().back());
                for (; g_itr != group.end(); ++g_itr)
                {
                    const AugmentedCuffOp* g_exon_3 = NULL;
                    if (s == CUFF_REV)
                    {
                        //  bad:
                        //              leader
                        //    follower
                        g_exon_3 = &(g_itr->second->augmented_ops().front());
                        if (g_exon_3->g_right() <= l_exon_3->g_left())
                            ok_clip_leader = false;

                        // for meta-assembly libraries, don't ever allow clipping, just extension
                        // bad:
                        //             leader
                        //         follower
                        if (library_type == "transfrags" &&
                            g_exon_3->g_left() < l_exon_3->g_left())
                            ok_clip_leader = false;
                    }
                    else
                    {
                        //  bad:
                        //          follower
                        //  leader
                        g_exon_3 = &(g_itr->second->augmented_ops().back());
                        if (g_exon_3->g_left() >= l_exon_3->g_right())
                            ok_clip_leader = false;

                        // for meta-assembly libraries, don't ever allow clipping, just extension
                        // bad:
                        //             leader
                        //                follower
                        if (library_type == "transfrags" &&
                            g_exon_3->g_right() > l_exon_3->g_right())
                            ok_clip_leader = false;
                    }
                }
            }
            else
            {
                ok_clip_leader = false;
            }

            if (ok_clip_leader)
            {
                if (s == CUFF_REV)
                {
                    if (trim_point == -1)
                        trim_point = l_exon_3->g_left();
                    else if (l_exon_3->g_left() < trim_point)
                        ok_clip_leader = false;
                }
                else
                {
                    if (trim_point == -1)
                        trim_point = l_exon_3->g_right();
                    else if (l_exon_3->g_right() > trim_point)
                        ok_clip_leader = false;
                }
            }

            if (ok_clip_leader)
            {
                group_leader = possible_leader;
                group_exon_3 = l_exon_3;
                break;
            }
            ++l_itr;
        }

        if (!group_leader || !group_exon_3)
            continue;

        for (size_t j = 0; j < group.size(); ++j)
        {
            const AugmentedCuffOp* exon_3 = NULL;
            int end_diff = 0;
            if (group_leader->strand() == CUFF_REV)
            {
                exon_3 = &(group[j].second->augmented_ops().front());
                end_diff = group_exon_3->g_left() - exon_3->g_left();
            }
            else
            {
                exon_3 = &(group[j].second->augmented_ops().back());
                end_diff = exon_3->g_right() - group_exon_3->g_right();
            }

            if (end_diff > 0)
            {
                // leader
                //     follower
                group[j].second->trim_3(end_diff);
            }
            else if (end_diff < 0)
            {
                //        leader
                //   follower
                group[j].second->extend_3(-end_diff);
            }
        }
    }

	return;

}


#pragma once

#include <string>
#include <armadillo> // http://arma.sourceforge.net/docs.html
#include <vector>
#include <thread>   /* std::this_thread::sleep_for */

class EvolutionaryAlgorithm{
    public:
        EvolutionaryAlgorithm();
        ~EvolutionaryAlgorithm();
        void saveBestIndvParamsCSV();
        void initializePop(int);
        void normalizeFitness(double*);
        void evaluatePop();
        void crossover(int,arma::rowvec,arma::rowvec);
        void elitism();
        void tournament();
        void roulette();
        void mutate(int);
        void selectionAndMutation();
        void createCSV(std::string);
        std::string formatParamsString(int);
        void saveGenerationData(int);
        void increaseMutation();
        void predationOfOne();
        void partIncrease();
        void oneRemainingPopReset();
        void endEABatch();
        bool eventHappens(int);
        void checkEvents();
        void startEventTriggerList();
        void evoAlg(std::string);
        void fillInitialsWithBests(int);
        void semiFinalsTournment(int);
        void finalTournment();
        void scriptLoop();
        void setFitness(std::vector<std::pair<int,int>>);
    private:

        EAGameControler &m_eaGameControler;

        pthread_t scriptThread;
        pthread_mutex_t mutex;
        pthread_mutex_t mutex2;

        arma::rowvec bestIndividual;

        double indvFitness;
        const std::string parametersList[28] = {
            "ON_ATTACK_MULTIPLIER",
            "POP_INCREASE_TAX",
            "POP_PER_CONSTRUCTION",
            "INIT_POP_VALUE",
            "INIT_STAT_VALUE",
            "ON_ATTACK_DECAY_TAX",
            "NORMAL_DECAY_TAX",
            "ATTACK_FREQUENCY",
            "INIT_RESOURCES_VALUE",
            "TAX_REDUCT",
            "DECAY_DELAY_MICRO",
            "MIN_LOSS_0",
            "MIN_LOSS_1",
            "MIN_LOSS_2",
            "MIN_LOSS_3",
            "MAX_LOSS_0",
            "MAX_LOSS_1",
            "MAX_LOSS_2",
            "MAX_LOSS_3",
            "TOT_ROBOTS_INI",
            "FREE_ROBOTS_INI",
            "PROD_COST_INI",
            "TIME_STEP",
            "INIT_TIME_STEP",
            "MAX_TIME_STEPS",
            "MIN_REWARD",
            "REWARD_RANGE",
            "FAILURE_TAX"
        };
        bool continueEA;
        bool eaFinished;
        const double MIN_MULT = 0.5;
        const double MAX_MULT = 1.5;
        const double MUTATE_POSITIVE_MULT = 1.0;
        const double MUTATE_NEGATIVE_MULT = -1.0;

        #define NB_PARAMETERS 28   // number of parameters that will be adjusted
        #define POPULATION_SIZE 20
        #define TOURNMENT_RATE 10

        const long MAX_MUTATION_RATE = 10;
        const double MUTATION_INCREASE_RATIO = 1.1; // mutation increase 10% at a time
        const double INITIAL_MUTATION = 0.05;

        const long BEGIN_DECAY_THRESHOLD = 50; // after how many generations without improvement the mutation starts to decay
        const long APPLY_DECAY_MODULE = 15; // decay will be applied when the generation index % APPLY_DECAY_MODULE = 0

        const long APPLY_PREDATION_INTERVAL = 10; // interval (in generations) to apply predation

        const long MAX_PARAM_VALUE = 1; // max value a parameter can reach
        enum tournments{ SEMI_FINALS, FINALS, INITIALS};

        arma::mat::fixed<TOURNMENT_RATE, NB_PARAMETERS> bestTournmentIndv[2] = {
            arma::mat::fixed<TOURNMENT_RATE, NB_PARAMETERS>(),
            arma::mat::fixed<TOURNMENT_RATE, NB_PARAMETERS>()
        };

        int eventTriggerModule[6] = {1, 10, 5, 10, 2, 10};

        enum selectionMethods { TOURNAMENT, ELITISM, ROULETTE };
        selectionMethods partsSelectionMethods[20] = {
            TOURNAMENT,TOURNAMENT,TOURNAMENT,TOURNAMENT,TOURNAMENT,TOURNAMENT,TOURNAMENT,TOURNAMENT,TOURNAMENT,TOURNAMENT,
            ELITISM,ELITISM,ELITISM,ELITISM,ELITISM,ELITISM,ELITISM,ELITISM,ELITISM,ELITISM
        };

        int partPos = 0;
        selectionMethods selectionMethod = partsSelectionMethods[partPos];

        arma::mat::fixed<POPULATION_SIZE, NB_PARAMETERS> population; // arma::mat : armadillo matrix (Mat<double>)

        // Each LINE (ROW) is the individual's parameters
        double fitness[POPULATION_SIZE];
        double bestFitness = 0, worstFitness = 2000000000;
        int bestFitIndex = -1, worstFitIndex = -1;
        double mutationRate = INITIAL_MUTATION;
        int nbNoImprovementGens = 0; // number of generations in a row without any improvement (best indiviNdual remains the same)
        int generationIndex = 0;
        bool continueEA;

        // Choose if population evaluation will be according to max fitness value or min fitness
        enum evalTypes{ BY_MIN, BY_MAX };
        evalTypes EVAL_TYPE = BY_MIN;

        bool continueBatch;
        void (EvolutionaryAlgorithm::*selectionType[3])() = {
            &EvolutionaryAlgorithm::tournament,
            &EvolutionaryAlgorithm::elitism,
            &EvolutionaryAlgorithm::roulette
        };

        void (EvolutionaryAlgorithm::*eventTypes[5])() = {
            &EvolutionaryAlgorithm::increaseMutation,
            &EvolutionaryAlgorithm::predationOfOne,
            &EvolutionaryAlgorithm::partIncrease,
            &EvolutionaryAlgorithm::oneRemainingPopReset,
            &EvolutionaryAlgorithm::endEABatch
        };
    int remainingFitnessToCalc;

};
void *runScript(void*);


#include "ns_ter_impl.h"

#include <algorithm>
#include <cstdio>
#include <cassert>
#include <iostream>
#include <limits>
#include <unordered_map>
#include <set>
#include <boost/functional/hash.hpp>

static const int ter_short_circuit_long_sentences = -1;

using namespace std;

namespace NewScorer {

struct COSTS {
  static const float substitution;
  static const float deletion;
  static const float insertion;
  static const float shift;
};
const float COSTS::substitution = 1.0f;
const float COSTS::deletion = 1.0f;
const float COSTS::insertion = 1.0f;
const float COSTS::shift = 1.0f;

static const int MAX_SHIFT_SIZE = 10;
static const int MAX_SHIFT_DIST = 50;

TERScorerImpl::TERScorerImpl(const vector<WordID>& ref) : ref_(ref) {
  for (unsigned i = 0; i < ref.size(); ++i)
    rwexists_.insert(ref[i]);
}

float TERScorerImpl::Calculate(
    const vector<WordID>& hyp,
    int* subs, int* ins, int* dels, int* shifts) const {
  return CalculateAllShifts(hyp, subs, ins, dels, shifts);
}

float TERScorerImpl::MinimumEditDistance(
    const vector<WordID>& hyp,
    const vector<WordID>& ref,
    vector<TransType>* path) {
  vector<vector<TransType> > bmat(hyp.size() + 1, vector<TransType>(ref.size() + 1, MATCH));
  vector<vector<float> > cmat(hyp.size() + 1, vector<float>(ref.size() + 1, 0));
  for (int i = 0; i <= hyp.size(); ++i)
    cmat[i][0] = i;
  for (int j = 0; j <= ref.size(); ++j)
    cmat[0][j] = j;
  for (int i = 1; i <= hyp.size(); ++i) {
    const WordID& hw = hyp[i-1];
    for (int j = 1; j <= ref.size(); ++j) {
      const WordID& rw = ref[j-1];
      float& cur_c = cmat[i][j];
      TransType& cur_b = bmat[i][j];

      if (rw == hw) {
        cur_c = cmat[i-1][j-1];
        cur_b = MATCH;
      } else {
        cur_c = cmat[i-1][j-1] + COSTS::substitution;
        cur_b = SUBSTITUTION;
      }
      float cwoi = cmat[i-1][j];
      if (cur_c > cwoi + COSTS::insertion) {
        cur_c = cwoi + COSTS::insertion;
        cur_b = INSERTION;
      }
      float cwod = cmat[i][j-1];
      if (cur_c > cwod + COSTS::deletion) {
        cur_c = cwod + COSTS::deletion;
        cur_b = DELETION;
      }
    }
  }

  // trace back along the best path and record the transition types
  path->clear();
  int i = hyp.size();
  int j = ref.size();
  while (i > 0 || j > 0) {
    if (j == 0) {
      --i;
      path->push_back(INSERTION);
    } else if (i == 0) {
      --j;
      path->push_back(DELETION);
    } else {
      TransType t = bmat[i][j];
      path->push_back(t);
      switch (t) {
        case SUBSTITUTION:
        case MATCH:
          --i; --j; break;
        case INSERTION:
          --i; break;
        case DELETION:
          --j; break;
      }
    }
  }
  std::reverse(path->begin(), path->end());
  return cmat[hyp.size()][ref.size()];
}

void TERScorerImpl::BuildWordMatches(
    const vector<WordID>& hyp, NgramToIntsMap* nmap) const {
  nmap->clear();
  set<WordID> exists_both;
  for (int i = 0; i < hyp.size(); ++i)
    if (rwexists_.find(hyp[i]) != rwexists_.end())
      exists_both.insert(hyp[i]);
  for (int start=0; start<ref_.size(); ++start) {
    if (exists_both.find(ref_[start]) == exists_both.end()) continue;
    vector<WordID> cp;
    int mlen = min(MAX_SHIFT_SIZE, static_cast<int>(ref_.size() - start));
    for (int len=0; len<mlen; ++len) {
      if (len && exists_both.find(ref_[start + len]) == exists_both.end()) break;
      cp.push_back(ref_[start + len]);
      (*nmap)[cp].insert(start);
    }
  }
}

void TERScorerImpl::PerformShift(
    const vector<WordID>& in,
    int start, int end, int moveto, vector<WordID>* out) {
  // cerr << "ps: " << start << " " << end << " " << moveto << endl;
  out->clear();
  if (moveto == -1) {
    for (int i = start; i <= end; ++i)
      out->push_back(in[i]);
    for (int i = 0; i < start; ++i)
      out->push_back(in[i]);
    for (int i = end+1; i < in.size(); ++i)
      out->push_back(in[i]);
  } else if (moveto < start) {
    for (int i = 0; i <= moveto; ++i)
      out->push_back(in[i]);
    for (int i = start; i <= end; ++i)
      out->push_back(in[i]);
    for (int i = moveto+1; i < start; ++i)
      out->push_back(in[i]);
    for (int i = end+1; i < in.size(); ++i)
      out->push_back(in[i]);
  } else if (moveto > end) {
    for (int i = 0; i < start; ++i)
      out->push_back(in[i]);
    for (int i = end+1; i <= moveto; ++i)
      out->push_back(in[i]);
    for (int i = start; i <= end; ++i)
      out->push_back(in[i]);
    for (int i = moveto+1; i < in.size(); ++i)
      out->push_back(in[i]);
  } else {
    for (int i = 0; i < start; ++i)
      out->push_back(in[i]);
    for (int i = end+1; (i < in.size()) && (i <= end + (moveto - start)); ++i)
      out->push_back(in[i]);
    for (int i = start; i <= end; ++i)
      out->push_back(in[i]);
    for (int i = (end + (moveto - start))+1; i < in.size(); ++i)
      out->push_back(in[i]);
  }
  if (out->size() != in.size()) {
    cerr << "ps: " << start << " " << end << " " << moveto << endl;
    // cerr << "in=" << TD::GetString(in) << endl;
    // cerr << "out=" << TD::GetString(*out) << endl;
  }
  assert(out->size() == in.size());
  // cerr << "ps: " << TD::GetString(*out) << endl;
}

void TERScorerImpl::GetAllPossibleShifts(
    const vector<WordID>& hyp,
    const vector<int>& ralign,
    const vector<bool>& herr,
    const vector<bool>& rerr,
    const int min_size,
    vector<vector<Shift> >* shifts) const {
  for (int start = 0; start < hyp.size(); ++start) {
    vector<WordID> cp(1, hyp[start]);
    NgramToIntsMap::iterator niter = nmap_.find(cp);
    if (niter == nmap_.end()) continue;
    bool ok = false;
    int moveto;
    for (set<int>::iterator i = niter->second.begin(); i != niter->second.end(); ++i) {
      moveto = *i;
      int rm = ralign[moveto];
      ok = (start != rm &&
            (rm - start) < MAX_SHIFT_DIST &&
            (start - rm - 1) < MAX_SHIFT_DIST);
      if (ok) break;
    }
    if (!ok) continue;
    cp.clear();
    for (int end = start + min_size - 1;
         ok && end < hyp.size() && end < (start + MAX_SHIFT_SIZE); ++end) {
      cp.push_back(hyp[end]);
      vector<Shift>& sshifts = (*shifts)[end - start];
      ok = false;
      NgramToIntsMap::iterator niter = nmap_.find(cp);
      if (niter == nmap_.end()) break;
      bool any_herr = false;
      for (int i = start; i <= end && !any_herr; ++i)
        any_herr = herr[i];
      if (!any_herr) {
        ok = true;
        continue;
      }
      for (set<int>::iterator mi = niter->second.begin();
           mi != niter->second.end(); ++mi) {
        int moveto = *mi;
        int rm = ralign[moveto];
        if (! ((rm != start) &&
               ((rm < start) || (rm > end)) &&
               (rm - start <= MAX_SHIFT_DIST) &&
               ((start - rm - 1) <= MAX_SHIFT_DIST))) continue;
        ok = true;
        bool any_rerr = false;
        for (int i = 0; (i <= end - start) && (!any_rerr); ++i)
          any_rerr = rerr[moveto+i];
        if (!any_rerr) continue;
        for (int roff = 0; roff <= (end - start); ++roff) {
          int rmr = ralign[moveto+roff];
          if ((start != rmr) && ((roff == 0) || (rmr != ralign[moveto])))
            sshifts.push_back(Shift(start, end, moveto + roff));
        }
      }
    }
  }
}

bool TERScorerImpl::CalculateBestShift(
    const vector<WordID>& cur,
    const vector<WordID>& /*hyp*/,
    float curerr,
    const vector<TransType>& path,
    vector<WordID>* new_hyp,
    float* newerr,
    vector<TransType>* new_path) const {
  vector<bool> herr, rerr;
  vector<int> ralign;
  int hpos = -1;
  for (int i = 0; i < path.size(); ++i) {
    switch (path[i]) {
      case MATCH:
        ++hpos;
        herr.push_back(false);
        rerr.push_back(false);
        ralign.push_back(hpos);
        break;
      case SUBSTITUTION:
        ++hpos;
        herr.push_back(true);
        rerr.push_back(true);
        ralign.push_back(hpos);
        break;
      case INSERTION:
        ++hpos;
        herr.push_back(true);
        break;
      case DELETION:
        rerr.push_back(true);
        ralign.push_back(hpos);
        break;
    }
  }
#if 0
  cerr << "RALIGN: ";
  for (int i = 0; i < rerr.size(); ++i)
    cerr << ralign[i] << " ";
  cerr << endl;
  cerr << "RERR: ";
  for (int i = 0; i < rerr.size(); ++i)
    cerr << (bool)rerr[i] << " ";
  cerr << endl;
  cerr << "HERR: ";
  for (int i = 0; i < herr.size(); ++i)
    cerr << (bool)herr[i] << " ";
  cerr << endl;
#endif

  vector<vector<Shift> > shifts(MAX_SHIFT_SIZE + 1);
  GetAllPossibleShifts(cur, ralign, herr, rerr, 1, &shifts);
  float cur_best_shift_cost = 0;
  *newerr = curerr;
  vector<TransType> cur_best_path;
  vector<WordID> cur_best_hyp;

  bool res = false;
  for (int i = shifts.size() - 1; i >=0; --i) {
    float curfix = curerr - (cur_best_shift_cost + *newerr);
    float maxfix = 2.0f * (1 + i) - COSTS::shift;
    if ((curfix > maxfix) || ((cur_best_shift_cost == 0) && (curfix == maxfix))) break;
    for (int j = 0; j < shifts[i].size(); ++j) {
      const Shift& s = shifts[i][j];
      curfix = curerr - (cur_best_shift_cost + *newerr);
      maxfix = 2.0f * (1 + i) - COSTS::shift;  // TODO remove?
      if ((curfix > maxfix) || ((cur_best_shift_cost == 0) && (curfix == maxfix))) continue;
      vector<WordID> shifted(cur.size());
      PerformShift(cur, s.begin(), s.end(), ralign[s.moveto()], &shifted);
      vector<TransType> try_path;
      float try_cost = MinimumEditDistance(shifted, ref_, &try_path);
      float gain = (*newerr + cur_best_shift_cost) - (try_cost + COSTS::shift);
      if (gain > 0.0f || ((cur_best_shift_cost == 0.0f) && (gain == 0.0f))) {
        *newerr = try_cost;
        cur_best_shift_cost = COSTS::shift;
        new_path->swap(try_path);
        new_hyp->swap(shifted);
        res = true;
        // cerr << "Found better shift " << s.begin() << "..." << s.end() << " moveto " << s.moveto() << endl;
      }
    }
  }

  return res;
}

void TERScorerImpl::GetPathStats(
    const vector<TransType>& path, int* subs, int* ins, int* dels) {
  *subs = *ins = *dels = 0;
  for (int i = 0; i < path.size(); ++i) {
    switch (path[i]) {
      case SUBSTITUTION:
        ++(*subs);
      case MATCH:
        break;
      case INSERTION:
        ++(*ins); break;
      case DELETION:
        ++(*dels); break;
    }
  }
}

float TERScorerImpl::CalculateAllShifts(
    const vector<WordID>& hyp,
    int* subs, int* ins, int* dels, int* shifts) const {
  BuildWordMatches(hyp, &nmap_);
  vector<TransType> path;
  float med_cost = MinimumEditDistance(hyp, ref_, &path);
  float edits = 0;
  vector<WordID> cur = hyp;
  *shifts = 0;
  if (ter_short_circuit_long_sentences < 0 ||
      ref_.size() < ter_short_circuit_long_sentences) {
    while (true) {
      vector<WordID> new_hyp;
      vector<TransType> new_path;
      float new_med_cost;
      if (!CalculateBestShift(cur, hyp, med_cost, path, &new_hyp, &new_med_cost, &new_path))
        break;
      edits += COSTS::shift;
      ++(*shifts);
      med_cost = new_med_cost;
      path.swap(new_path);
      cur.swap(new_hyp);
    }
  }
  GetPathStats(path, subs, ins, dels);
  return med_cost + edits;
}

} // namespace NewScorer


/*
 * crystal.cc
 *
 *  Copyright (C) 2017 Diamond Light Source
 *
 *  Author: James Parkhurst
 *
 *  This code is distributed under the BSD license, a copy of which is
 *  included in the root directory of this package.
 */
#include <boost/python.hpp>
#include <boost/python/def.hpp>
#include <boost_adaptbx/optional_conversions.h>
#include <string>
#include <sstream>
#include <dxtbx/model/crystal.h>

namespace dxtbx { namespace model { namespace boost_python {

  using namespace boost::python;
  using boost_adaptbx::optional_conversions::to_and_from_python;

  static Crystal *make_crystal_default(const vec3<double> &real_space_a,
                                       const vec3<double> &real_space_b,
                                       const vec3<double> &real_space_c,
                                       const cctbx::sgtbx::space_group &space_group) {
    Crystal *crystal =
      new Crystal(real_space_a, real_space_b, real_space_c, space_group);

    return crystal;
  }

  static Crystal *make_crystal_with_symbol(const vec3<double> &real_space_a,
                                           const vec3<double> &real_space_b,
                                           const vec3<double> &real_space_c,
                                           const std::string &space_group_symbol) {
    Crystal *crystal = new Crystal(
      real_space_a,
      real_space_b,
      real_space_c,
      cctbx::sgtbx::space_group(cctbx::sgtbx::space_group_symbols(space_group_symbol)));

    return crystal;
  }

  static Crystal *make_crystal_with_A(const mat3<double> &A,
                                      const cctbx::sgtbx::space_group &space_group,
                                      const bool &reciprocal) {
    Crystal *crystal = new Crystal(A, space_group, reciprocal);

    return crystal;
  }

  static Crystal *make_crystal_with_A_symbol(const mat3<double> &A,
                                             const std::string &space_group_symbol,
                                             const bool &reciprocal) {
    Crystal *crystal = new Crystal(
      A,
      cctbx::sgtbx::space_group(cctbx::sgtbx::space_group_symbols(space_group_symbol)),
      reciprocal);

    return crystal;
  }

  static MosaicCrystalKabsch2010 *make_kabsch2010_mosaic_crystal_default(
    const vec3<double> &real_space_a,
    const vec3<double> &real_space_b,
    const vec3<double> &real_space_c,
    const cctbx::sgtbx::space_group &space_group) {
    MosaicCrystalKabsch2010 *crystal = new MosaicCrystalKabsch2010(
      real_space_a, real_space_b, real_space_c, space_group);

    return crystal;
  }

  static MosaicCrystalKabsch2010 *make_kabsch2010_mosaic_crystal_with_symbol(
    const vec3<double> &real_space_a,
    const vec3<double> &real_space_b,
    const vec3<double> &real_space_c,
    const std::string &space_group_symbol) {
    MosaicCrystalKabsch2010 *crystal = new MosaicCrystalKabsch2010(
      real_space_a,
      real_space_b,
      real_space_c,
      cctbx::sgtbx::space_group(cctbx::sgtbx::space_group_symbols(space_group_symbol)));

    return crystal;
  }

  static MosaicCrystalSauter2014 *make_sauter2014_mosaic_crystal_default(
    const vec3<double> &real_space_a,
    const vec3<double> &real_space_b,
    const vec3<double> &real_space_c,
    const cctbx::sgtbx::space_group &space_group) {
    MosaicCrystalSauter2014 *crystal = new MosaicCrystalSauter2014(
      real_space_a, real_space_b, real_space_c, space_group);

    return crystal;
  }

  static MosaicCrystalSauter2014 *make_sauter2014_mosaic_crystal_with_symbol(
    const vec3<double> &real_space_a,
    const vec3<double> &real_space_b,
    const vec3<double> &real_space_c,
    const std::string &space_group_symbol) {
    MosaicCrystalSauter2014 *crystal = new MosaicCrystalSauter2014(
      real_space_a,
      real_space_b,
      real_space_c,
      cctbx::sgtbx::space_group(cctbx::sgtbx::space_group_symbols(space_group_symbol)));

    return crystal;
  }

  static void Crystal_set_A_at_scan_points_from_tuple(CrystalBase &self,
                                                      boost::python::tuple l) {
    scitbx::af::shared<mat3<double> > A_list;
    for (std::size_t i = 0; i < boost::python::len(l); ++i) {
      mat3<double> A = boost::python::extract<mat3<double> >(l[i]);
      A_list.push_back(A);
    }
    self.set_A_at_scan_points(A_list.const_ref());
  }

  static void Crystal_set_A_at_scan_points_from_list(CrystalBase &self,
                                                     boost::python::list l) {
    scitbx::af::shared<mat3<double> > A_list;
    for (std::size_t i = 0; i < boost::python::len(l); ++i) {
      mat3<double> A = boost::python::extract<mat3<double> >(l[i]);
      A_list.push_back(A);
    }
    self.set_A_at_scan_points(A_list.const_ref());
  }

  static void Crystal_set_B_covariance_from_tuple(CrystalBase &self,
                                                  boost::python::object obj) {
    scitbx::af::versa<double, scitbx::af::c_grid<2> > B_cov(
      scitbx::af::c_grid<2>(9, 9));
    DXTBX_ASSERT(boost::python::len(obj) == 9 * 9);
    for (std::size_t i = 0; i < boost::python::len(obj); ++i) {
      B_cov[i] = boost::python::extract<double>(obj[i]);
    }
    self.set_B_covariance(B_cov.const_ref());
  }

  struct CrystalPickleSuite : boost::python::pickle_suite {
    static boost::python::tuple getinitargs(const Crystal &obj) {
      scitbx::af::shared<vec3<double> > real_space_v = obj.get_real_space_vectors();
      return boost::python::make_tuple(
        real_space_v[0], real_space_v[1], real_space_v[2], obj.get_space_group());
    }

    static boost::python::tuple getstate(boost::python::object obj) {
      const Crystal &crystal = boost::python::extract<const Crystal &>(obj)();
      return boost::python::make_tuple(obj.attr("__dict__"),
                                       crystal.get_A_at_scan_points(),
                                       crystal.get_B_covariance(),
                                       crystal.get_B_covariance_at_scan_points(),
                                       crystal.get_recalculated_unit_cell(),
                                       crystal.get_recalculated_cell_parameter_sd(),
                                       crystal.get_recalculated_cell_volume_sd());
    }

    static void setstate(boost::python::object obj, boost::python::tuple state) {
      Crystal &crystal = boost::python::extract<Crystal &>(obj)();
      DXTBX_ASSERT(boost::python::len(state) == 7);

      // restore the object's __dict__
      boost::python::dict d =
        boost::python::extract<boost::python::dict>(obj.attr("__dict__"))();
      d.update(state[0]);

      // restore the internal state of the C++ object
      scitbx::af::const_ref<mat3<double> > A_scanpoints =
        boost::python::extract<scitbx::af::const_ref<mat3<double> > >(state[1]);
      scitbx::af::const_ref<double, scitbx::af::c_grid<2> > cov_B =
        boost::python::extract<scitbx::af::const_ref<double, scitbx::af::c_grid<2> > >(
          state[2]);
      scitbx::af::const_ref<double, scitbx::af::c_grid<3> > cov_B_scanpoints =
        boost::python::extract<scitbx::af::const_ref<double, scitbx::af::c_grid<3> > >(
          state[3]);
      boost::optional<cctbx::uctbx::unit_cell> recalculated_unit_cell =
        boost::python::extract<boost::optional<cctbx::uctbx::unit_cell> >(state[4]);
      scitbx::af::small<double, 6> recalculated_cell_parameter_sd =
        boost::python::extract<scitbx::af::small<double, 6> >(state[5]);
      double recalculated_cell_volume_sd = boost::python::extract<double>(state[6]);
      crystal.set_A_at_scan_points(A_scanpoints);
      crystal.set_B_covariance(cov_B);
      crystal.set_B_covariance_at_scan_points(cov_B_scanpoints);
      if (recalculated_unit_cell) {
        crystal.set_recalculated_unit_cell(*recalculated_unit_cell);
      }
      crystal.set_recalculated_cell_parameter_sd(recalculated_cell_parameter_sd);
      crystal.set_recalculated_cell_volume_sd(recalculated_cell_volume_sd);
    }

    static bool getstate_manages_dict() {
      return true;
    }
  };

  struct MosaicCrystalKabsch2010PickleSuite : CrystalPickleSuite {
    static boost::python::tuple getinitargs(const MosaicCrystalKabsch2010 &obj) {
      scitbx::af::shared<vec3<double> > real_space_v = obj.get_real_space_vectors();
      return boost::python::make_tuple(
        real_space_v[0], real_space_v[1], real_space_v[2], obj.get_space_group());
    }

    static boost::python::tuple getstate(boost::python::object obj) {
      const MosaicCrystalKabsch2010 &crystal =
        boost::python::extract<const MosaicCrystalKabsch2010 &>(obj)();
      return boost::python::make_tuple(obj.attr("__dict__"),
                                       crystal.get_A_at_scan_points(),
                                       crystal.get_B_covariance(),
                                       crystal.get_B_covariance_at_scan_points(),
                                       crystal.get_recalculated_unit_cell(),
                                       crystal.get_recalculated_cell_parameter_sd(),
                                       crystal.get_recalculated_cell_volume_sd(),
                                       crystal.get_mosaicity());
    }

    static void setstate(boost::python::object obj, boost::python::tuple state) {
      MosaicCrystalKabsch2010 &crystal =
        boost::python::extract<MosaicCrystalKabsch2010 &>(obj)();
      DXTBX_ASSERT(boost::python::len(state) == 8);

      // restore the object's __dict__
      boost::python::dict d =
        boost::python::extract<boost::python::dict>(obj.attr("__dict__"))();
      d.update(state[0]);

      // restore the internal state of the C++ object
      scitbx::af::const_ref<mat3<double> > A_scanpoints =
        boost::python::extract<scitbx::af::const_ref<mat3<double> > >(state[1]);
      scitbx::af::const_ref<double, scitbx::af::c_grid<2> > cov_B =
        boost::python::extract<scitbx::af::const_ref<double, scitbx::af::c_grid<2> > >(
          state[2]);
      scitbx::af::const_ref<double, scitbx::af::c_grid<3> > cov_B_scanpoints =
        boost::python::extract<scitbx::af::const_ref<double, scitbx::af::c_grid<3> > >(
          state[3]);
      boost::optional<cctbx::uctbx::unit_cell> recalculated_unit_cell =
        boost::python::extract<boost::optional<cctbx::uctbx::unit_cell> >(state[4]);
      scitbx::af::small<double, 6> recalculated_cell_parameter_sd =
        boost::python::extract<scitbx::af::small<double, 6> >(state[5]);
      double recalculated_cell_volume_sd = boost::python::extract<double>(state[6]);
      double mosaicity = boost::python::extract<double>(state[7]);
      crystal.set_A_at_scan_points(A_scanpoints);
      crystal.set_B_covariance(cov_B);
      crystal.set_B_covariance_at_scan_points(cov_B_scanpoints);
      if (recalculated_unit_cell) {
        crystal.set_recalculated_unit_cell(*recalculated_unit_cell);
      }
      crystal.set_recalculated_cell_parameter_sd(recalculated_cell_parameter_sd);
      crystal.set_recalculated_cell_volume_sd(recalculated_cell_volume_sd);
      crystal.set_mosaicity(mosaicity);
    }
  };

  struct MosaicCrystalSauter2014PickleSuite : CrystalPickleSuite {
    static boost::python::tuple getinitargs(const MosaicCrystalSauter2014 &obj) {
      scitbx::af::shared<vec3<double> > real_space_v = obj.get_real_space_vectors();
      return boost::python::make_tuple(
        real_space_v[0], real_space_v[1], real_space_v[2], obj.get_space_group());
    }

    static boost::python::tuple getstate(boost::python::object obj) {
      const MosaicCrystalSauter2014 &crystal =
        boost::python::extract<const MosaicCrystalSauter2014 &>(obj)();
      return boost::python::make_tuple(obj.attr("__dict__"),
                                       crystal.get_A_at_scan_points(),
                                       crystal.get_B_covariance(),
                                       crystal.get_B_covariance_at_scan_points(),
                                       crystal.get_recalculated_unit_cell(),
                                       crystal.get_recalculated_cell_parameter_sd(),
                                       crystal.get_recalculated_cell_volume_sd(),
                                       crystal.get_half_mosaicity_deg(),
                                       crystal.get_domain_size_ang());
    }

    static void setstate(boost::python::object obj, boost::python::tuple state) {
      MosaicCrystalSauter2014 &crystal =
        boost::python::extract<MosaicCrystalSauter2014 &>(obj)();
      DXTBX_ASSERT(boost::python::len(state) == 9);

      // restore the object's __dict__
      boost::python::dict d =
        boost::python::extract<boost::python::dict>(obj.attr("__dict__"))();
      d.update(state[0]);

      // restore the internal state of the C++ object
      scitbx::af::const_ref<mat3<double> > A_scanpoints =
        boost::python::extract<scitbx::af::const_ref<mat3<double> > >(state[1]);
      scitbx::af::const_ref<double, scitbx::af::c_grid<2> > cov_B =
        boost::python::extract<scitbx::af::const_ref<double, scitbx::af::c_grid<2> > >(
          state[2]);
      scitbx::af::const_ref<double, scitbx::af::c_grid<3> > cov_B_scanpoints =
        boost::python::extract<scitbx::af::const_ref<double, scitbx::af::c_grid<3> > >(
          state[3]);
      boost::optional<cctbx::uctbx::unit_cell> recalculated_unit_cell =
        boost::python::extract<boost::optional<cctbx::uctbx::unit_cell> >(state[4]);
      scitbx::af::small<double, 6> recalculated_cell_parameter_sd =
        boost::python::extract<scitbx::af::small<double, 6> >(state[5]);
      double recalculated_cell_volume_sd = boost::python::extract<double>(state[6]);
      double half_mosaicity_deg = boost::python::extract<double>(state[7]);
      double domain_size_ang = boost::python::extract<double>(state[8]);
      crystal.set_A_at_scan_points(A_scanpoints);
      crystal.set_B_covariance(cov_B);
      crystal.set_B_covariance_at_scan_points(cov_B_scanpoints);
      if (recalculated_unit_cell) {
        crystal.set_recalculated_unit_cell(*recalculated_unit_cell);
      }
      crystal.set_recalculated_cell_parameter_sd(recalculated_cell_parameter_sd);
      crystal.set_recalculated_cell_volume_sd(recalculated_cell_volume_sd);
      crystal.set_half_mosaicity_deg(half_mosaicity_deg);
      crystal.set_domain_size_ang(domain_size_ang);
    }
  };

  inline void CrystalBase_set_unit_cell_real_space_vectors(
    CrystalBase &self,
    scitbx::vec3<double> &real_space_a,
    scitbx::vec3<double> &real_space_b,
    scitbx::vec3<double> &real_space_c) {
    self.set_unit_cell(real_space_a, real_space_b, real_space_c);
  }

  inline void CrystalBase_set_unit_cell(CrystalBase &self,
                                        cctbx::uctbx::unit_cell &unit_cell) {
    self.set_unit_cell(unit_cell);
  }

  inline void CrystalBase_set_recalculated_unit_cell(
    CrystalBase &self,
    cctbx::uctbx::unit_cell &unit_cell) {
    self.set_recalculated_unit_cell(unit_cell);
  }

  inline void CrystalBase_set_recalculated_cell_parameter_sd(
    CrystalBase &self,
    const scitbx::af::small<double, 6> &unit_cell_sd) {
    self.set_recalculated_cell_parameter_sd(unit_cell_sd);
  }

  void export_crystal() {
    // Expose the optional values
    to_and_from_python<boost::optional<cctbx::uctbx::unit_cell> >();

    class_<CrystalBase, boost::noncopyable>("CrystalBase", no_init)
      .def("set_unit_cell", CrystalBase_set_unit_cell_real_space_vectors)
      .def("set_unit_cell", CrystalBase_set_unit_cell)
      .def("update_B", &CrystalBase::update_B)
      .def("set_U", &CrystalBase::set_U)
      .def("get_U", &CrystalBase::get_U)
      .def("set_B", &CrystalBase::set_B)
      .def("get_B", &CrystalBase::get_B)
      .def("set_A", &CrystalBase::set_A)
      .def("get_A", &CrystalBase::get_A)
      .def("get_unit_cell", &CrystalBase::get_unit_cell)
      .def("get_real_space_vectors", &CrystalBase::get_real_space_vectors)
      .def("set_space_group", &CrystalBase::set_space_group)
      .def("get_space_group", &CrystalBase::get_space_group)
      .add_property("num_scan_points", &CrystalBase::get_num_scan_points)
      .def("get_num_scan_points", &CrystalBase::get_num_scan_points)
      .def("set_A_at_scan_points", &CrystalBase::set_A_at_scan_points)
      .def("set_A_at_scan_points", &Crystal_set_A_at_scan_points_from_tuple)
      .def("set_A_at_scan_points", &Crystal_set_A_at_scan_points_from_list)
      .def("get_A_at_scan_point", &CrystalBase::get_A_at_scan_point)
      .def("get_B_at_scan_point", &CrystalBase::get_B_at_scan_point)
      .def("get_U_at_scan_point", &CrystalBase::get_U_at_scan_point)
      .def("get_unit_cell_at_scan_point", &CrystalBase::get_unit_cell_at_scan_point)
      .def("reset_scan_points", &CrystalBase::reset_scan_points)
      .def("change_basis", &CrystalBase::change_basis)
      .def("update", &CrystalBase::update)
      .def("rotate_around_origin",
           &CrystalBase::rotate_around_origin,
           (arg("axis"), arg("angle"), arg("deg") = true))
      .def("is_similar_to",
           &CrystalBase::is_similar_to,
           (arg("other"),
            arg("angle_tolerance") = 0.01,
            arg("uc_rel_length_tolerance") = 0.01,
            arg("uc_abs_angle_tolerance") = 1.0))
      .def("get_B_covariance", &CrystalBase::get_B_covariance)
      .def("set_B_covariance", &CrystalBase::set_B_covariance)
      .def("set_B_covariance", &Crystal_set_B_covariance_from_tuple)
      .def("set_B_covariance_at_scan_points",
           &CrystalBase::set_B_covariance_at_scan_points)
      .def("get_B_covariance_at_scan_point",
           &CrystalBase::get_B_covariance_at_scan_point)
      .def("get_B_covariance_at_scan_points",
           &CrystalBase::get_B_covariance_at_scan_points)
      .def("get_cell_parameter_sd", &CrystalBase::get_cell_parameter_sd)
      .def("get_cell_volume_sd", &CrystalBase::get_cell_volume_sd)
      .def("get_recalculated_cell_volume_sd",
           &CrystalBase::get_recalculated_cell_volume_sd)
      .def("set_recalculated_cell_volume_sd",
           &CrystalBase::set_recalculated_cell_volume_sd)
      .def("get_cell_parameter_sd_at_scan_point",
           &CrystalBase::get_cell_parameter_sd_at_scan_point)
      .def("reset_unit_cell_errors", &CrystalBase::reset_unit_cell_errors)
      .def("set_recalculated_unit_cell", &CrystalBase_set_recalculated_unit_cell)
      .def("get_recalculated_unit_cell", &CrystalBase::get_recalculated_unit_cell)
      .def("set_recalculated_cell_parameter_sd",
           &CrystalBase_set_recalculated_cell_parameter_sd)
      .def("get_recalculated_cell_parameter_sd",
           &CrystalBase::get_recalculated_cell_parameter_sd)
      .def("__eq__", &CrystalBase::operator==)
      .def("__ne__", &CrystalBase::operator!=);

    class_<Crystal, bases<CrystalBase> >("Crystal", no_init)
      .def(init<const Crystal &>())
      .def("__init__",
           make_constructor(&make_crystal_default,
                            default_call_policies(),
                            (arg("real_space_a"),
                             arg("real_space_b"),
                             arg("real_space_c"),
                             arg("space_group"))))
      .def("__init__",
           make_constructor(&make_crystal_with_symbol,
                            default_call_policies(),
                            (arg("real_space_a"),
                             arg("real_space_b"),
                             arg("real_space_c"),
                             arg("space_group_symbol"))))
      .def("__init__",
           make_constructor(&make_crystal_with_A,
                            default_call_policies(),
                            (arg("A"), arg("space_group"), arg("reciprocal") = true)))
      .def("__init__",
           make_constructor(
             &make_crystal_with_A_symbol,
             default_call_policies(),
             (arg("A"), arg("space_group_symbol"), arg("reciprocal") = true)))
      .def_pickle(CrystalPickleSuite());

    // Create member-function pointers to specific is_similar_to overloads
    // - each of these crystals has a custom implementation along with the
    //   inherited interface, and we want to expose these explicitly
    bool (MosaicCrystalKabsch2010::*kabsch_is_similar_to)(
      const CrystalBase &, double, double, double, double) const =
      &MosaicCrystalKabsch2010::is_similar_to;
    bool (MosaicCrystalSauter2014::*sauter_is_similar_to)(
      const CrystalBase &, double, double, double, double, double) const =
      &MosaicCrystalSauter2014::is_similar_to;

    class_<MosaicCrystalKabsch2010, bases<Crystal> >("MosaicCrystalKabsch2010", no_init)
      .def(init<const MosaicCrystalKabsch2010 &>())
      .def(init<const Crystal &>())
      .def("__init__",
           make_constructor(&make_kabsch2010_mosaic_crystal_default,
                            default_call_policies(),
                            (arg("real_space_a"),
                             arg("real_space_b"),
                             arg("real_space_c"),
                             arg("space_group"))))
      .def("__init__",
           make_constructor(&make_kabsch2010_mosaic_crystal_with_symbol,
                            default_call_policies(),
                            (arg("real_space_a"),
                             arg("real_space_b"),
                             arg("real_space_c"),
                             arg("space_group_symbol"))))
      .def("is_similar_to",
           kabsch_is_similar_to,
           (arg("other"),
            arg("angle_tolerance") = 0.01,
            arg("uc_rel_length_tolerance") = 0.01,
            arg("uc_abs_angle_tolerance") = 1.0,
            arg("mosaicity_tolerance") = 0.8))
      .def(
        "get_mosaicity", &MosaicCrystalKabsch2010::get_mosaicity, (arg("deg") = true))
      .def("set_mosaicity",
           &MosaicCrystalKabsch2010::set_mosaicity,
           (arg("mosaicity"), arg("deg") = true))
      .def_pickle(MosaicCrystalKabsch2010PickleSuite());

    class_<MosaicCrystalSauter2014, bases<Crystal> >("MosaicCrystalSauter2014", no_init)
      .def(init<const MosaicCrystalSauter2014 &>())
      .def(init<const Crystal &>())
      .def("__init__",
           make_constructor(&make_sauter2014_mosaic_crystal_default,
                            default_call_policies(),
                            (arg("real_space_a"),
                             arg("real_space_b"),
                             arg("real_space_c"),
                             arg("space_group"))))
      .def("__init__",
           make_constructor(&make_sauter2014_mosaic_crystal_with_symbol,
                            default_call_policies(),
                            (arg("real_space_a"),
                             arg("real_space_b"),
                             arg("real_space_c"),
                             arg("space_group_symbol"))))
      .def("is_similar_to",
           sauter_is_similar_to,
           (arg("other"),
            arg("angle_tolerance") = 0.01,
            arg("uc_rel_length_tolerance") = 0.01,
            arg("uc_abs_angle_tolerance") = 1.0,
            arg("half_mosaicity_tolerance") = 0.4,
            arg("domain_size_tolerance") = 1.0))
      .def("get_half_mosaicity_deg", &MosaicCrystalSauter2014::get_half_mosaicity_deg)
      .def("set_half_mosaicity_deg",
           &MosaicCrystalSauter2014::set_half_mosaicity_deg,
           (arg("half_mosaicity_deg")))
      .def("get_domain_size_ang", &MosaicCrystalSauter2014::get_domain_size_ang)
      .def("set_domain_size_ang",
           &MosaicCrystalSauter2014::set_domain_size_ang,
           (arg("domain_size_ang")))
      .def_pickle(MosaicCrystalSauter2014PickleSuite());

    register_ptr_to_python<boost::shared_ptr<CrystalBase> >();
    register_ptr_to_python<boost::shared_ptr<Crystal> >();
    register_ptr_to_python<boost::shared_ptr<MosaicCrystalKabsch2010> >();
    register_ptr_to_python<boost::shared_ptr<MosaicCrystalSauter2014> >();
  }

}}}  // namespace dxtbx::model::boost_python


#include <scitbx/array_family/boost_python/flex_fwd.h>

#include <boost/python/module.hpp>
#include <boost/python/class.hpp>
#include <boost/python/def.hpp>
#include <boost/python/args.hpp>
#include <boost/python/list.hpp>
#include <boost/python/return_arg.hpp>
#include <boost/python/return_value_policy.hpp>
#include <boost/python/return_by_value.hpp>

#include <scitbx/mat3.h>
#include <scitbx/sym_mat3.h>
#include <scitbx/constants.h>
#include <scitbx/array_family/ref.h>
#include <scitbx/array_family/tiny.h>
#include <scitbx/array_family/accessors/c_grid_padded_periodic.h>
#include <scitbx/boost_python/container_conversions.h>
#include <boost/shared_array.hpp>
#include <boost/shared_ptr.hpp>

#include <gltbx/error.h>
#include <gltbx/util.h>

#if defined(GLTBX_HAVE_GL2PS)
#include <gl2ps.h>
#endif

namespace gltbx { namespace util {

  namespace af = scitbx::af;

  template<typename T>
  struct gl_enum_type_of
  {};

  template<>
  struct gl_enum_type_of<GLfloat>
  {
    static const GLenum type = GL_FLOAT;
  };

  template<>
  struct gl_enum_type_of<GLdouble>
  {
    static const GLenum type = GL_DOUBLE;
  };

  inline
  void
  translate(scitbx::vec3<double> const& t)
  {
    glTranslated(t[0], t[1], t[2]);
  }

  template <typename ElementType>
  boost::python::list
  as_python_list(ElementType* elements, unsigned size)
  {
    boost::python::list result;
    for(unsigned i=0;i<size;i++) result.append(elements[i]);
    return result;
  }

  template <unsigned Size>
  struct gl_vector_as_python_list
  {
    static
    boost::python::list
    int_(GLenum pname)
    {
      GLint vector[Size];
      glGetIntegerv(pname, vector);
      return as_python_list(vector, Size);
    }

    static
    boost::python::list
    double_(GLenum pname)
    {
      GLdouble vector[Size];
      glGetDoublev(pname, vector);
      return as_python_list(vector, Size);
    }
  };

  inline
  boost::python::list
  get_gl_modelview_matrix()
  {
    return gl_vector_as_python_list<16>::double_(GL_MODELVIEW_MATRIX);
  }

  inline
  boost::python::list
  get_gl_projection_matrix()
  {
    return gl_vector_as_python_list<16>::double_(GL_PROJECTION_MATRIX);
  }

  inline
  boost::python::list
  get_gl_viewport()
  {
    return gl_vector_as_python_list<4>::int_(GL_VIEWPORT);
  }

  inline
  scitbx::mat3<double>
  extract_rotation_from_gl_matrix(GLdouble* m)
  {
    return scitbx::mat3<double>(
      m[0], m[4], m[8],
      m[1], m[5], m[9],
      m[2], m[6], m[10]);
  }

  inline
  scitbx::vec3<double>
  extract_translation_from_gl_matrix(GLdouble* m)
  {
    return scitbx::vec3<double>(m[12], m[13], m[14]);
  }

  inline
  scitbx::mat3<double>
  extract_rotation_from_gl_modelview_matrix()
  {
    GLdouble mvm[16];
    glGetDoublev(GL_MODELVIEW_MATRIX, mvm);
    return extract_rotation_from_gl_matrix(mvm);
  }

  inline
  scitbx::vec3<double>
  extract_translation_from_gl_modelview_matrix()
  {
    GLdouble mvm[16];
    glGetDoublev(GL_MODELVIEW_MATRIX, mvm);
    return extract_translation_from_gl_matrix(mvm);
  }

  inline
  scitbx::vec3<double>
  object_as_eye_coordinates(
    scitbx::vec3<double> const& object_coordinates)
  {
    GLdouble mvm[16];
    glGetDoublev(GL_MODELVIEW_MATRIX, mvm);
    return extract_rotation_from_gl_matrix(mvm) * object_coordinates
         + extract_translation_from_gl_matrix(mvm);
  }

  inline
  void
  translate_object(double eye_x, double eye_y, double eye_z)
  {
    GLdouble mvm[16];
    glGetDoublev(GL_MODELVIEW_MATRIX, mvm);
    glMatrixMode(GL_MODELVIEW);
    glLoadIdentity();
    glTranslated(eye_x, eye_y, eye_z);
    glMultMatrixd(mvm);
  }

  inline
  void
  translate_object(scitbx::vec3<double> const& eye_vector)
  {
    translate_object(eye_vector[0], eye_vector[1], eye_vector[2]);
  }

  inline
  void
  translate_object(
    double s,
    double x,
    double y,
    double mousex,
    double mousey)
  {
    translate_object(s * (x - mousex), s * (mousey - y), 0.0);
  }

  inline
  void
  rotate_object_about_eye_x_and_y(
    double s,
    double xcenter,
    double ycenter,
    double zcenter,
    double x,
    double y,
    double mousex,
    double mousey)
  {
    GLdouble mvm[16];
    glGetDoublev(GL_MODELVIEW_MATRIX, mvm);
    scitbx::vec3<double> eye_center =
        extract_rotation_from_gl_matrix(mvm)
          * scitbx::vec3<double>(xcenter, ycenter, zcenter)
      + extract_translation_from_gl_matrix(mvm);
    glMatrixMode(GL_MODELVIEW);
    glLoadIdentity();
    translate(eye_center);
    glRotated((s * (y - mousey)), 1.0, 0.0, 0.0);
    glRotated((s * (x - mousex)), 0.0, 1.0, 0.0);
    translate(-eye_center);
    glMultMatrixd(mvm);
  }

  inline
  void
  rotate_object_about_eye_vector(
    double xcenter,
    double ycenter,
    double zcenter,
    double xvector,
    double yvector,
    double zvector,
    double angle)
  {
    GLdouble mvm[16];
    glGetDoublev(GL_MODELVIEW_MATRIX, mvm);
    scitbx::vec3<double> eye_center =
        extract_rotation_from_gl_matrix(mvm)
          * scitbx::vec3<double>(xcenter, ycenter, zcenter)
      + extract_translation_from_gl_matrix(mvm);
    glMatrixMode(GL_MODELVIEW);
    glLoadIdentity();
    translate(eye_center);
    glRotated(angle, xvector, yvector, zvector);
    translate(-eye_center);
    glMultMatrixd(mvm);
  }

  inline
  void
  TranslateScene(
    double s,
    double x,
    double y,
    double mousex,
    double mousey)
  {
    GLdouble mat[16];
    glGetDoublev(GL_MODELVIEW_MATRIX, mat);
    glMatrixMode(GL_MODELVIEW);
    glLoadIdentity();
    glTranslated((s * (x - mousex)), (s * (mousey - y)), 0.0);
    glMultMatrixd(mat);
  }

  inline
  void
  RotateScene(
    double s,
    double xcenter,
    double ycenter,
    double zcenter,
    double x,
    double y,
    double mousex,
    double mousey)
  {
    GLdouble mat[16];
    glGetDoublev(GL_MODELVIEW_MATRIX, mat);
    glMatrixMode(GL_MODELVIEW);
    glLoadIdentity();
    glTranslated(xcenter, ycenter, zcenter);
    glRotated((s * (y - mousey)), 1.0, 0.0, 0.0);
    glRotated((s * (x - mousex)), 0.0, 1.0, 0.0);
    glTranslated(-xcenter, -ycenter, -zcenter);
    glMultMatrixd(mat);
  }

  inline
  void
  RotateAboutVector(
    double xcenter,
    double ycenter,
    double zcenter,
    double xvector,
    double yvector,
    double zvector,
    double angle)
  {
    glMatrixMode(GL_MODELVIEW);
    GLdouble mat[16];
    glGetDoublev(GL_MODELVIEW_MATRIX, mat);
    glLoadIdentity();
    glTranslated(xcenter, ycenter, zcenter);
    glRotated(angle, xvector, yvector, zvector);
    glTranslated(-xcenter, -ycenter, -zcenter);
    glMultMatrixd(mat);
  }

  /// Array of vertices with the associated normals
  /** Implementated with OpenGL vertex arrays. Since OpenGL is a state machine,
  it means it is foolish to construct two such objects and interwin calls to
  draw_xxx
  */
  template<typename IndexType, typename CoordinatesType>
  class vertex_array
  {
    public:
      typedef af::const_ref< scitbx::vec3<CoordinatesType> > input_type;
      typedef af::tiny<IndexType, 3> triangle;

      /// Construct an array from the given vertices and normals
      vertex_array(input_type const& vertices,
                   input_type const& normals)
        : vertices_(vertices), normals_(normals)
      {
        SCITBX_ASSERT(vertices.size() == normals.size())
                     (vertices.size())(normals.size());
        glVertexPointer(
          3, gl_enum_type_of<CoordinatesType>::type, 0, vertices.begin());
        glNormalPointer(
          gl_enum_type_of<CoordinatesType>::type, 0, normals.begin());
      }

      /** Draw the triangles: triangles[i] is a triplet of indices to look
      into the array vertices passed to the constructor */
      void draw_triangles(af::const_ref<triangle> const& triangles) {
        for(std::size_t i=0; i < triangles.size(); i++) {
          glBegin(GL_TRIANGLES);
          for(int j=0; j<3; j++) glArrayElement(triangles[i][j]);
          glEnd();
        }
      }
    private:
      input_type vertices_;
      input_type normals_;
  };

  void IsoMesh (
    af::const_ref< scitbx::vec3< double > > const& vertices,
    af::const_ref< af::tiny<int, 3> > const& triangles)
  {
    for (std::size_t i = 0; i < triangles.size(); i++) {
      glBegin(GL_TRIANGLES);
      for (int j=0; j<3; j++) {
        scitbx::vec3< double > const& vertex = vertices[triangles[i][j]];
        glVertex3f(vertex[0], vertex[1], vertex[2]);
      }
      glEnd();
    }
  }

  //! Based on freeglut-2.4.0/src/freeglut_geometry.c
  /* Compute lookup table of cos and sin values forming a cirle
   *
   * Notes:
   *    It is the responsibility of the caller to free these tables
   *    The size of the table is (n+1) to form a connected loop
   *    The last entry is exactly the same as the first
   *    The sign of n can be flipped to get the reverse loop
   */
  struct CircleTable
  {
    boost::shared_array<double> memory;
    double* s;
    double* c;

    CircleTable(int n)
    {
      /* Table size, the sign of n flips the circle direction */
      unsigned size = std::abs(n);
      memory = boost::shared_array<double>(new double[2*(size+1)]);
      s = memory.get();
      c = s + (size+1);

      /* Determine the angle between samples */
      double angle = scitbx::constants::two_pi / (n == 0 ? 1 : n);

      /* Compute cos and sin around the circle */
      s[0] = 0;
      c[0] = 1;
      for (unsigned i=1; i<size; i++) {
        s[i] = std::sin(angle*i);
        c[i] = std::cos(angle*i);
      }

      /* Last sample is duplicate of the first */
      s[size] = 0;
      c[size] = 1;
    }
  };

  //! Based on freeglut-2.4.0/src/freeglut_geometry.c
  void
  SolidSphere(double radius, int slices, int stacks)
  {
    CircleTable ct1(-slices);
    CircleTable ct2(stacks*2);

    /* The top stack is covered with a triangle fan */
    double z0 = 1;
    double z1 = ct2.c[stacks > 0 ? 1 : 0];
    double r0 = 0;
    double r1 = ct2.s[stacks > 0 ? 1 : 0];
    glBegin(GL_TRIANGLE_FAN);
    glNormal3d(0,0,1);
    glVertex3d(0,0,radius);
    for (int j=slices; j>=0; j--) {
      glNormal3d(ct1.c[j]*r1,        ct1.s[j]*r1,        z1       );
      glVertex3d(ct1.c[j]*r1*radius, ct1.s[j]*r1*radius, z1*radius);
    }
    glEnd();

    /* Cover each stack with a quad strip, except the top and bottom stacks */
    for(int i=1; i<stacks-1; i++) {
      z0 = z1;
      z1 = ct2.c[i+1];
      r0 = r1;
      r1 = ct2.s[i+1];
      glBegin(GL_QUAD_STRIP);
      for(int j=0; j<=slices; j++) {
        glNormal3d(ct1.c[j]*r1,        ct1.s[j]*r1,        z1       );
        glVertex3d(ct1.c[j]*r1*radius, ct1.s[j]*r1*radius, z1*radius);
        glNormal3d(ct1.c[j]*r0,        ct1.s[j]*r0,        z0       );
        glVertex3d(ct1.c[j]*r0*radius, ct1.s[j]*r0*radius, z0*radius);
      }
      glEnd();
    }

    /* The bottom stack is covered with a triangle fan */
    z0 = z1;
    r0 = r1;
    glBegin(GL_TRIANGLE_FAN);
    glNormal3d(0,0,-1);
    glVertex3d(0,0,-radius);
    for (int j=0; j<=slices; j++) {
      glNormal3d(ct1.c[j]*r0,        ct1.s[j]*r0,        z0       );
      glVertex3d(ct1.c[j]*r0*radius, ct1.s[j]*r0*radius, z0*radius);
    }
    glEnd();
  }

  //! Based on freeglut-2.4.0/src/freeglut_geometry.c
  void
  WireSphere(double radius, int slices, int stacks)
  {
    CircleTable ct1(-slices);
    CircleTable ct2(stacks*2);

    /* Draw a line loop for each stack */
    for (int i=1; i<stacks; i++) {
      double z = ct2.c[i];
      double r = ct2.s[i];
      glBegin(GL_LINE_LOOP);
      for(int j=0; j<=slices; j++) {
        double x = ct1.c[j];
        double y = ct1.s[j];
        glNormal3d(x,y,z);
        glVertex3d(x*r*radius, y*r*radius, z*radius);
      }
      glEnd();
    }

    /* Draw a line loop for each slice */
    for (int i=0; i<slices; i++) {
      glBegin(GL_LINE_STRIP);
      for(int j=0; j<=stacks; j++) {
        double x = ct1.c[i] * ct2.s[j];
        double y = ct1.s[i] * ct2.s[j];
        double z = ct2.c[j];
        glNormal3d(x,y,z);
        glVertex3d(x*radius, y*radius, z*radius);
      }
      glEnd();
    }
  }

  template<typename IndexType, typename CoordinatesType>
  struct vertex_array_wrapper
  {
    typedef vertex_array<IndexType, CoordinatesType> wt;
    typedef typename wt::input_type const& inp_t;

    static void wrap(char const *name) {
      using namespace boost::python;
      class_<wt>(name, no_init)
        .def(init<inp_t, inp_t>((
             arg("vertices"),
             arg("normals"))))
        .def("draw_triangles", &wt::draw_triangles, (arg("triangles")))
      ;
    }
  };

  struct matrix_wrapper
  {
    /// OpenGL matrix (column-major as per the standard)
    af::tiny<GLfloat, 16> m;

    /// Construct an uninitalised matrix
    /* Use glTranslate, glRotate, etc and then the member function get to
       initialise it
     */
    matrix_wrapper() {}

    /// Construct with the given matrix a
    /** a is assumed to be stored row-major (the convention through the scitbx)
     */
    matrix_wrapper(scitbx::mat3<double> const &a) {
      m[0] = a[0]; m[4] = a[1]; m[8]  = a[2]; m[12] = 0;
      m[1] = a[3]; m[5] = a[4]; m[9]  = a[5]; m[13] = 0;
      m[2] = a[6]; m[6] = a[7]; m[10] = a[8]; m[14] = 0;
      m[3] =   0 ; m[7] =  0  ; m[11] =  0  ; m[15] = 1;
    }

    /// Construct with the given symmetric matrix a
    /** a is assumed to be stored in packed U format
     */
    matrix_wrapper(scitbx::sym_mat3<double> const &a) {
      m[0] = a[0]; m[4] = a[1]; m[8]  = a[2]; m[12] = 0;
      m[1] = a[1]; m[5] = a[3]; m[9]  = a[4]; m[13] = 0;
      m[2] = a[2]; m[6] = a[4]; m[10] = a[5]; m[14] = 0;
      m[3] =   0 ; m[7] =  0  ; m[11] =  0  ; m[15] = 1;
    }

    matrix_wrapper &get() {
      GLint matrix_mode;
      glGetIntegerv(GL_MATRIX_MODE, &matrix_mode);
      GLenum matrix_kind =   GL_MODELVIEW  ? GL_MODELVIEW_MATRIX
                           : GL_PROJECTION ? GL_PROJECTION_MATRIX
                           :                 GL_NONE;
      GLTBX_ASSERT(matrix_kind != GL_NONE);
      glGetFloatv(matrix_kind, m.begin());
      return *this;
    }

    void load() {  glLoadMatrixf(m.begin()); }

    void multiply() { glMultMatrixf(m.begin()); }

    static void wrap() {
      using namespace boost::python;
      using namespace scitbx::boost_python::container_conversions;
      tuple_mapping_fixed_size<af::tiny<float, 16> >();
      return_value_policy<return_by_value> rbv;
      class_<matrix_wrapper>("matrix", no_init)
        .def(init<>())
        .def(init<scitbx::mat3<double> const &>())
        .def(init<scitbx::sym_mat3<double> const &>())
        .add_property("m", make_getter(&matrix_wrapper::m, rbv))
        .def("get", &matrix_wrapper::get, return_self<>())
        .def("load", &matrix_wrapper::load)
        .def("multiply", &matrix_wrapper::multiply)
        ;
    }
  };

  bool
  gl2ps_interface(
    char const* file_name,
    bool draw_background,
    boost::python::object const& callback)
  {
#if !defined(GLTBX_HAVE_GL2PS)
    if (   file_name == 0
        && !draw_background
        && callback.is_none()) return false;
    throw std::runtime_error("gl2ps is not available.");
#else
    if (file_name == 0) return true;
    boost::shared_ptr<FILE> stream(fopen(file_name, "wb"), fclose);
    GLTBX_ASSERT(stream.get() != 0);
    GLint options = GL2PS_USE_CURRENT_VIEWPORT;
    if (draw_background) {
      options |= GL2PS_DRAW_BACKGROUND;
    }
    int buffersize = 1024 * 1024;
    int state = GL2PS_OVERFLOW;
    while (state == GL2PS_OVERFLOW) {
      buffersize *= 2;
      gl2psBeginPage(
        /* title */ "test",
        /* producer */ "gltbx.util.gl2ps_interface",
        /* viewport */ NULL,
        /* format */ GL2PS_PDF,
        /* sort */ GL2PS_SIMPLE_SORT,
        options,
        /* colormode */ GL_RGBA,
        /* colorsize */ 0,
        /* colormap */ NULL,
        /* nr */ 0,
        /* ng */ 0,
        /* nb */ 0,
        buffersize,
        stream.get(),
        file_name);
      if (!callback.is_none()) {
        try {
          callback();
        }
        catch (...) {
          gl2psEndPage();
          throw;
        }
      }
      state = gl2psEndPage();
    }
    return true;
#endif
  }

  void
  init_module()
  {
    using namespace boost::python;
    def("handle_error", handle_error);
    def("get_gl_modelview_matrix", get_gl_modelview_matrix);
    def("get_gl_projection_matrix", get_gl_projection_matrix);
    def("get_gl_viewport", get_gl_viewport);
    def("extract_rotation_from_gl_modelview_matrix",
      extract_rotation_from_gl_modelview_matrix);
    def("extract_translation_from_gl_modelview_matrix",
      extract_translation_from_gl_modelview_matrix);
    def("object_as_eye_coordinates", object_as_eye_coordinates, (
      arg("object_coordinates")));
    def("translate_object",
      (void(*)(double, double, double)) translate_object, (
        arg("eye_x"),
        arg("eye_y"),
        arg("eye_z")));
    def("translate_object",
      (void(*)(scitbx::vec3<double> const&)) translate_object, (
        arg("eye_vector")));
    def("translate_object",
      (void(*)(double, double, double, double, double)) translate_object, (
        arg("s"),
        arg("x"),
        arg("y"),
        arg("mousex"),
        arg("mousey")));
    def("rotate_object_about_eye_x_and_y", rotate_object_about_eye_x_and_y, (
      arg("s"),
      arg("xcenter"),
      arg("ycenter"),
      arg("zcenter"),
      arg("x"),
      arg("y"),
      arg("mousex"),
      arg("mousey")));
    def("rotate_object_about_eye_vector", rotate_object_about_eye_vector, (
      arg("xcenter"),
      arg("ycenter"),
      arg("zcenter"),
      arg("xvector"),
      arg("yvector"),
      arg("zvector"),
      arg("angle")));
    def("TranslateScene", TranslateScene, (
      arg("s"),
      arg("x"),
      arg("y"),
      arg("mousex"),
      arg("mousey")));
    def("RotateScene", RotateScene, (
      arg("s"),
      arg("xcenter"),
      arg("ycenter"),
      arg("zcenter"),
      arg("x"),
      arg("y"),
      arg("mousex"),
      arg("mousey")));
    def("RotateAboutVector", RotateAboutVector, (
      arg("xcenter"),
      arg("ycenter"),
      arg("zcenter"),
      arg("xvector"),
      arg("yvector"),
      arg("zvector"),
      arg("angle")));
    def("SolidSphere", SolidSphere, (
      arg("radius"),
      arg("slices"),
      arg("stacks")));
    def("WireSphere", WireSphere, (
      arg("radius"),
      arg("slices"),
      arg("stacks")));
    def("IsoMesh", IsoMesh, (
      arg("vertices"),
      arg("triangles")));
    {
      // compatibility with scitbx/iso_surface/iso_surface_ext.cpp
      typedef af::c_grid_padded_periodic<3>::index_value_type ivt;
      vertex_array_wrapper<ivt, GLdouble>::wrap("vertex_array");
    }
    matrix_wrapper::wrap();
    def("gl2ps_interface", gl2ps_interface, (
      arg("file_name"),
      arg("draw_background"),
      arg("callback")));
  }

}} // namespace gltbx::util

BOOST_PYTHON_MODULE(gltbx_util_ext)
{
  gltbx::util::init_module();
}