/* $Id: Paraclu.cpp,v 1.29 2016/08/02 07:21:36 severin Exp $ */

/***

NAME - EEDB::Tools::Paraclu

SYNOPSIS

DESCRIPTION

  Port of the paraclu program into a ZENBU SPStream module
  Copyright 2011 Martin C. Frith

  Find clusters in 1-dimensional data, using the method described in
  MC Frith et al. Genome Res. 2008 18(1):1-12.

  http://www.cbrc.jp/paraclu/

  The original command line code uses the input
    4 columns: chromosome, strand, coordinate, value.
    For example: chr20 + 60026 2

CONTACT

Jessica Severin <severin@gsc.riken.jp>

LICENSE

 * Software License Agreement (BSD License)
 * EdgeExpressDB [eeDB] system
 * copyright (c) 2007-2013 Jessica Severin RIKEN OSC
 * All rights reserved.
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *     * Neither the name of Jessica Severin RIKEN OSC nor the
 *       names of its contributors may be used to endorse or promote products
 *       derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ''AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

APPENDIX

The rest of the documentation details each of the object methods. Internal methods are usually preceded with a _

***/

#include <stdio.h>
#include <stdlib.h>
#include <iostream>
#include <string>
#include <stdarg.h>

#include <algorithm>  // max, min
#include <cstdlib>  // EXIT_SUCCESS, EXIT_FAILURE
#include <fstream>
#include <iomanip>
#include <iostream>
#include <sstream>
#include <stdexcept>
#include <vector>

#include <rapidxml.hpp>  //rapidxml must be include before boost
#include <boost/algorithm/string.hpp>
#include <EEDB/Experiment.h>
#include <EEDB/Feature.h>
#include <EEDB/Expression.h>
#include <EEDB/Symbol.h>
#include <EEDB/SPStream.h>
#include <EEDB/Tools/Paraclu.h>

using namespace std;

const char*  EEDB::Tools::Paraclu::class_name = "EEDB::Tools::Paraclu";

//function prototypes
void _tools_paraclu_delete_func(MQDB::DBObject *obj) {
  delete (EEDB::Tools::Paraclu*)obj;
}
void _tools_paraclu_xml_func(MQDB::DBObject *obj, string &xml_buffer) {
  ((EEDB::Tools::Paraclu*)obj)->_xml(xml_buffer);
}
string _tools_paraclu_display_desc_func(MQDB::DBObject *obj) {
  return ((EEDB::Tools::Paraclu*)obj)->_display_desc();
}

const double infinity = 1e100;
void check_over_memory();  //from TrackCacheBuilder

////////////////////////////////////////////////////////////////////////////
//
// Cluster object
//
////////////////////////////////////////////////////////////////////////////

EEDB::Tools::Paraclu::Cluster::Cluster() {
  //constructor
  beg = NULL;
  end = NULL;
  totalValue = 0.0;
  minDensity = infinity;
  maxDensity = infinity;
  stability = 0.0;
}

EEDB::Tools::Paraclu::Cluster::~Cluster() {
  // destructor
  beg = NULL;
  end = NULL;
  totalValue = 0.0;
  minDensity = infinity;
  maxDensity = infinity;
  stability = 0.0;
}

void  EEDB::Tools::Paraclu::Cluster::show() {
  long cluster_length = end->feature->chrom_start() - beg->feature->chrom_start() + 1;

  std::cerr << beg->feature->chrom_name() << "\t" << beg->feature->strand() << "\t"
  << std::setprecision(20)
  << beg->feature->chrom_start() << "\t" << end->feature->chrom_start() << "\t"
  << cluster_length << "\t" << totalValue << "\t"
  << std::setprecision(3)
  << minDensity << "\t" << maxDensity ;
  std::cerr << "\t" << stability;
//  if(stability >= _min_stability) { std::cerr << "\tSTABLE"; }
//  std::cerr << "\n";
}

////////////////////////////////////////////////////////////////////////////
//
// creation methods
//
////////////////////////////////////////////////////////////////////////////

EEDB::Tools::Paraclu::Paraclu() {
  init();
}

EEDB::Tools::Paraclu::~Paraclu() {
}

void EEDB::Tools::Paraclu::init() {
  MQDB::DBObject::init();
  _classname                 = EEDB::Tools::Paraclu::class_name;
  _funcptr_delete            = _tools_paraclu_delete_func;
  _funcptr_xml               = _tools_paraclu_xml_func;
  _funcptr_simple_xml        = _tools_paraclu_xml_func;
  _funcptr_display_desc      = _tools_paraclu_display_desc_func;

  _min_cutoff_value    = 10.0;
  _max_cluster_length  = 100;
  _min_stability       = 1;
  _min_density         = 0;
  _selection_mode      = STABILITY_CUT;

  _stream_chrom        = NULL;
  _fl_buffer_start     = NULL;
  _fl_buffer_end       = NULL;
  _best_cluster        = NULL;
  _cluster_number      = 1;

  //attributes
  _feature_source = new EEDB::FeatureSource();
  _feature_source->name("Paraclu");
  _feature_source->category("cluster");

  _output_buffer = NULL;
}

void  EEDB::Tools::Paraclu::output_buffer(EEDB::Tools::ResortBuffer* buffer) {
  _output_buffer = buffer;
}

void  EEDB::Tools::Paraclu::feature_source(EEDB::FeatureSource* source) {
  if(_feature_source) { _feature_source->release(); _feature_source = NULL; }
  if(source) { source->retain(); }
  _feature_source = source;
}

////////////////////////////////////////////////////////////////////////////
//
//  creation from XML section
//
////////////////////////////////////////////////////////////////////////////

string EEDB::Tools::Paraclu::_display_desc() {
  string str = "Paraclu";
  return str;
}

void EEDB::Tools::Paraclu::_xml(string &xml_buffer) {
  char buffer[256];
  snprintf(buffer, 256, "<min_cutoff>%f</min_cutoff>", _min_cutoff_value);
  xml_buffer.append(buffer);

  snprintf(buffer, 256, "<stability>%f</stability>", _min_stability);
  xml_buffer.append(buffer);

  snprintf(buffer, 256, "<max_cluster_length>%ld</max_cluster_length>", _max_cluster_length);
  xml_buffer.append(buffer);

  if(_min_density>0) {
    snprintf(buffer, 256, "<min_density>%f</min_density>", _min_density);
    xml_buffer.append(buffer);
  }

  switch(_selection_mode) {
    case FULL_HIERARCHY:  xml_buffer.append("<selection_mode>full_hierarchy</selection_mode>"); break;
    case BEST_STABLE:     xml_buffer.append("<selection_mode>best_stable</selection_mode>"); break;
    case STABILITY_CUT:   xml_buffer.append("<selection_mode>stability_cut</selection_mode>"); break;
    case SMALL_STABLE:    xml_buffer.append("<selection_mode>small_stable</selection_mode>"); break;
  }

}

EEDB::Tools::Paraclu::Paraclu(void *xml_node) {
  //constructor using a rapidxml <spstream> description
  init();
  if(xml_node==NULL) { return; }

  rapidxml::xml_node<>      *root_node = (rapidxml::xml_node<>*)xml_node;
  rapidxml::xml_node<>      *node;

  if((node = root_node->first_node("min_cutoff")) != NULL) {
    _min_cutoff_value = strtod(node->value(), NULL);
  }

  if((node = root_node->first_node("stability")) != NULL) {
    _min_stability = strtod(node->value(), NULL);
    if(_min_stability < 1.0) { _min_stability = 1.0; }
  }

  if((node = root_node->first_node("max_cluster_length")) != NULL) {
    _max_cluster_length = strtol(node->value(), NULL, 10);
  }

  if((node = root_node->first_node("min_density")) != NULL) {
    _min_density = strtod(node->value(), NULL);
    if(_min_density < 0) { _min_density = 0; }
  }

  _selection_mode = STABILITY_CUT;
  if(((node = root_node->first_node("full_hierarchy")) != NULL) and (string(node->value()) == "true")) {
    _selection_mode = FULL_HIERARCHY;
  }
  if(((node = root_node->first_node("most_stable")) != NULL) and (string(node->value()) == "true")) {
    _selection_mode = BEST_STABLE;
  }
  if((node = root_node->first_node("selection_mode")) != NULL) {
    if(string(node->value()) == "full_hierarchy") { _selection_mode = FULL_HIERARCHY; }
    if(string(node->value()) == "best_stable")    { _selection_mode = BEST_STABLE; }
    if(string(node->value()) == "stability_cut")  { _selection_mode = STABILITY_CUT; }
    if(string(node->value()) == "small_stable")  { _selection_mode = SMALL_STABLE; }
  }

}

////////////////////////////////////////////////////////////

void  EEDB::Tools::Paraclu::process_feature(EEDB::Feature* feature) {
  //final post processing of subfeat prior to returning
  if(feature==NULL) { return; }
  if(!_output_buffer) { return; }

  // check if we are finished with clusters in the _output_buffer
  //_output_buffer->transfer_completed(feature->chrom_start());

  //fill in the featurelink buffer to have enough room to cluster properly
  if(!_stream_chrom) {
    _stream_chrom = feature->chrom();
    _stream_chrom->retain();
  }

  FeatureLink *fl = new EEDB::FeatureLink();
  fl->feature = feature;
  feature->retain();
  //fprintf(stderr, "add feature %s to links\n", feature->chrom_location().c_str());

  if(_fl_buffer_start == NULL) {
    //printf("reset buffer start..end\n");
    _fl_buffer_start = fl;
    _fl_buffer_end   = fl;
  } else {
    // add new feature onto end of feature link buffer
    _fl_buffer_end->next = fl;
    fl->prev             = _fl_buffer_end;
    _fl_buffer_end       = fl;
  }

  if(_fl_buffer_end->feature->chrom_start() - _fl_buffer_start->feature->chrom_start()+1 > _max_cluster_length * 9) {
    //_show_buffer(_fl_buffer_start, _fl_buffer_end);
    //long length = _fl_buffer_end->feature->chrom_start() - _fl_buffer_start->feature->chrom_start() + 1;

    switch(_selection_mode) {
      case FULL_HIERARCHY:
      case STABILITY_CUT:
        //fprintf(stderr, "process buffer %ld .. %ld [ %ld ]\n", _fl_buffer_start->feature->chrom_start(), _fl_buffer_end->feature->chrom_start(), length);
        _process_cluster_range_full(_fl_buffer_start, _fl_buffer_end, -infinity);
        break;
      case BEST_STABLE:
        _process_cluster_range_best(_fl_buffer_start, _fl_buffer_end);
        break;
      case SMALL_STABLE:
        int rtn = _process_cluster_range_smallest(_fl_buffer_start, _fl_buffer_end, -infinity);
        if(rtn == 0) {
          //fprintf(stderr, "entire buffer is too weak now\n");
          _discard_region(_fl_buffer_start, _fl_buffer_end);
        }
        break;
    }
    check_over_memory();
  }
}

void  EEDB::Tools::Paraclu::finish_processing() {
  if(!_output_buffer) { return; }
  FeatureLink *last_fl_start = NULL;

  while(_fl_buffer_start != NULL) {
    //_show_buffer(_fl_buffer_start, _fl_buffer_end);
    //long length = _fl_buffer_end->feature->chrom_start() - _fl_buffer_start->feature->chrom_start() + 1;
    last_fl_start = _fl_buffer_start;

    switch(_selection_mode) {
      case FULL_HIERARCHY:
      case STABILITY_CUT:
        //fprintf(stderr, "finalize buffer %ld .. %ld [ %ld ]\n", _fl_buffer_start->feature->chrom_start(), _fl_buffer_end->feature->chrom_start(), length);
        _process_cluster_range_full(_fl_buffer_start, _fl_buffer_end, -infinity);
        break;
      case BEST_STABLE:
        _process_cluster_range_best(_fl_buffer_start, _fl_buffer_end);
        break;
      case SMALL_STABLE:
        int rtn = _process_cluster_range_smallest(_fl_buffer_start, _fl_buffer_end, -infinity);
        if(rtn == 0) {
          //fprintf(stderr, "entire buffer is too weak now\n");
          _discard_region(_fl_buffer_start, _fl_buffer_end);
        }
        break;
    }
    if(_fl_buffer_start == last_fl_start) {
      //fprintf(stderr, "hmm buffer didn't change - problem\n");
      _discard_region(_fl_buffer_start, _fl_buffer_end);
    }
  }

  // input stream is empty so move all _subfeature_buffer into _completed_subfeatures
  //_output_buffer->transfer_all_to_completed();
}

void  EEDB::Tools::Paraclu::reset() {
  //reset for next region query
  if(_output_buffer) { _output_buffer->reset(); }

  //reset parameters to defaults
  _min_cutoff_value    = 10.0;
  _max_cluster_length  = 100;
  _min_stability       = 1;
  _min_density         = 0;
  _selection_mode      = STABILITY_CUT;

  //clear from previous run
  if(_stream_chrom) { _stream_chrom->release(); }
  _discard_region(_fl_buffer_start, _fl_buffer_end);
  if(_best_cluster) { delete _best_cluster; }

  _stream_chrom        = NULL;
  _fl_buffer_start     = NULL;
  _fl_buffer_end       = NULL;
  _best_cluster        = NULL;
  _cluster_number      = 1;
}

//--------------------------------------------------------------

// Copyright 2011 Martin C. Frith

// Find clusters in 1-dimensional data, using the method described in
// MC Frith et al. Genome Res. 2008 18(1):1-12.

// The input has 4 columns: chromosome, strand, coordinate, value.
// For example: chr20 + 60026 2

// Ported into the ZENBU Feature and SPStream system: Jessica Severin copyright 2012

double  EEDB::Tools::Paraclu::_calc_total_value(FeatureLink *beg, FeatureLink *end) {
  if(beg==NULL) { return 0; }
  FeatureLink  *flink = beg;
  double       totalValue = 0;
  while(flink) {
    totalValue += flink->feature->significance();
    if(flink == end) { break; }
    flink = flink->next;
  }
  return totalValue;
}

double  EEDB::Tools::Paraclu::_weakestPrefix(FeatureLink *beg, FeatureLink *end) {
  if(beg==end) { return 0.0; }
  unsigned origin   = beg->feature->chrom_start();
  unsigned stop_pos = end->feature->chrom_start();
  //printf("_weakestPrefix\n");

  _minPrefix = NULL;  //maybe not needed
  _minPrefixDensity = infinity;

  FeatureLink  *flink = beg;
  double       totalValue = 0;
  unsigned     pos=0;

  //density is calculated as
  while(flink) {
    pos = flink->feature->chrom_start();
    //loop on identical starts in case input was not pre-clustered at 1bp resolution
    while(flink && (flink->feature->chrom_start() == pos)) {
      totalValue += flink->feature->significance();
      flink = flink->next;
    }
    if(!flink) { break; }
    if(flink->feature->chrom_start() > stop_pos) { break; }

    double density = totalValue / (flink->feature->chrom_start() - origin);
    if(density < _minPrefixDensity) {
      _minPrefix = flink;
      _minPrefixDensity = density;
    }
  }
  //printf(" total=%f  minDensity = %f  pos = %s\n", totalValue, _minPrefixDensity, _minPrefix->feature->chrom_location().c_str());
  return totalValue;
}

double  EEDB::Tools::Paraclu::_weakestSuffix(FeatureLink *beg, FeatureLink *end) {
  if(beg==end) { return 0.0; }
  unsigned beg_pos = beg->feature->chrom_start();
  unsigned end_pos = end->feature->chrom_start();
  //printf("_weakestSuffix\n");

  _minSuffix = NULL;
  _minSuffixDensity = infinity;

  FeatureLink  *flink = end;
  double       totalValue = 0;
  unsigned     pos = 0;

  while(flink) {
    pos = flink->feature->chrom_start();
    //printf("  pos = %d\n", pos);
    //loop on identical chrom_starts in case input was not pre-clustered at 1bp resolution
    while(flink && (flink->feature->chrom_start() == pos)) {
      totalValue += flink->feature->significance();
      //printf("   add to density %s\n", flink->feature->chrom_location().c_str());
      flink = flink->prev;
    }
    if(!flink) { break; }
    if(flink->feature->chrom_start() < beg_pos) { break; }

    double density = totalValue / (end_pos - flink->feature->chrom_start());
    //printf("   density = %f\n", density);
    if(density < _minSuffixDensity) {
      _minSuffix = flink->next;
      _minSuffixDensity = density;
      //printf("     !NEW minSuffix %s [ %f ]\n", _minSuffix->feature->chrom_location().c_str(), density);
    }
  }
  //printf(" total=%f  minDensity = %f  pos = %s\n", totalValue, _minSuffixDensity, _minSuffix->feature->chrom_location().c_str());
  return totalValue;
}

void EEDB::Tools::Paraclu::_show_buffer(FeatureLink *beg, FeatureLink *end) {
  printf("=== buffer\n");
  FeatureLink *fl = beg;
  while(1) {
    printf("  %s\n", fl->feature->chrom_location().c_str());
    if(fl == end) { break; }
    fl = fl->next;
  }
}

void  EEDB::Tools::Paraclu::_discard_region(FeatureLink *beg, FeatureLink *end) {
  if((beg==NULL) || (end==NULL)) { return; }

  //long cluster_length = end->feature->chrom_start() - beg->feature->chrom_start() + 1;
  //fprintf(stderr, "_discard_region %ld .. %ld [ %ld ]\n", beg->feature->chrom_start(), end->feature->chrom_start(), cluster_length);

  //cut off this section from the feature link buffer
  FeatureLink *b2 = beg->prev;
  FeatureLink *e2 = end->next;
  if(b2) { b2->next = e2; }
  if(e2) { e2->prev = b2; }
  if(_fl_buffer_start == beg) {
    _fl_buffer_start = e2;
    //printf("new _fl_buffer_start\n");
  }
  if(_fl_buffer_end == end)   {
    _fl_buffer_end = b2;
    //printf("new _fl_buffer_end\n");
  }

  beg->prev = NULL;
  end->next = NULL;

  EEDB::FeatureLink *flink = beg;
  while(flink!=NULL) {
    //printf("  remove link %s\n", flink->feature->chrom_location().c_str());
    flink->feature->release();
    flink->feature = NULL;
    FeatureLink *fl2 = flink;
    flink = flink->next;
    delete fl2;
  }
}

/*
void  EEDB::Tools::Paraclu::_processOneStream(std::istream &stream) {
  std::string newSeqname;
  char newStrand;
  unsigned p;
  double v;
  while (stream >> newSeqname >> newStrand >> p >> v) {
    if (newSeqname == seqname && newStrand == strand) {
      if (p > sites.back().chrom_start())
        sites.push_back(Site(p, v));
      else if (p == sites.back().chrom_start())
        sites.back().value += v;
      else
        throw std::runtime_error("unsorted input");
    } else {
      _writeClusters(sites.begin(), sites.end(), -infinity);
      sites.clear();
      seqname = newSeqname;
      strand = newStrand;
      sites.push_back(Site(p, v));
    }
  }
  if (!stream.eof()) throw std::runtime_error("bad input");
  _writeClusters(sites.begin(), sites.end(), -infinity);
}
*/

/////////////////////////////////////////////////////////////////////////////////////
//
// original ported version. close to direct interpretation of the original ParaClu algorithm
// but flipped recursion logic so that parent decides based on children and self
// also working with the sliding input buffer and the ZENBU datamodel objects
//
/////////////////////////////////////////////////////////////////////////////////////

void  EEDB::Tools::Paraclu::_process_cluster_range_full(FeatureLink *beg, FeatureLink *end, double parentDensity) {
  //OK this is a closer attempt at martin's original algorithm, full hierarchy
  //recursive algorithm which subdivides sites
  //original paraclu called the parameter minDensity, but parentDensity is better name

  if((beg==NULL) || (end==NULL)) { return; }
  //if(beg==end) {  return; }

  long   cluster_length = end->feature->chrom_start() - beg->feature->chrom_start() + 1;
  double totalValue     = _calc_total_value(beg, end);
  double density        = totalValue / cluster_length;

  //fprintf(stderr, "_process_cluster_range_full  %ld .. %ld  [ %ld ] [total=%f density=%f]\n", beg->feature->chrom_start(), end->feature->chrom_start(), cluster_length, totalValue, density);

  if((totalValue < _min_cutoff_value) || (density < _min_density)) {
    //weak region so can fall out
    if(cluster_length > _max_cluster_length) {
      //if >max_length then we are walking down the left-side, so can discard and fall out
      //fprintf(stderr, "discard large-weak region %ld .. %ld  [ %ld ] [total=%f density=%f]\n", beg->feature->chrom_start(), end->feature->chrom_start(), cluster_length, totalValue, density);
      _discard_region(beg, end);
    }
    return;
  }

  if(beg==end) {
    double stability = density/parentDensity; //my stability
    //fprintf(stderr, "_process_cluster_range  beg == end density[%f] parentDensity[%f] stability[%f] minstab[%f]\n", density, parentDensity, stability, _min_stability);
    if((_selection_mode==FULL_HIERARCHY) || ((_selection_mode==STABILITY_CUT) && (stability >= _min_stability))) {
      //this is part of the hierarchy
      //fprintf(stderr, "single 1bp stable cluster [%ld] & STABLE [%f]\n", cluster_length, stability);
      EEDB::Feature *feature = _generate_cluster(beg, end);
      if(feature) {
        feature->metadataset()->add_tag_data("density", d_to_string(density));
        feature->metadataset()->add_tag_data("maxDensity", d_to_string(density));  //1bp so no children
        feature->metadataset()->add_tag_data("parentDensity", d_to_string(parentDensity));
        feature->metadataset()->add_tag_data("stability", d_to_string(stability));
        _output_buffer->insert_feature(feature);
      }
    }
    return;
  }

  //
  // find best mid-cut-point
  //
  _weakestPrefix(beg, end);
  _weakestSuffix(beg, end);

  double maxDensity;
  FeatureLink *mid = NULL;
  if(_minPrefixDensity < _minSuffixDensity) {
    maxDensity = _minPrefixDensity;
    mid = _minPrefix;
    //fprintf(stderr, "choose prefix %f %d\n", _minPrefixDensity, _minPrefix->feature->chrom_start());
  } else {
    maxDensity = _minSuffixDensity;
    mid = _minSuffix;
    //fprintf(stderr, "choose suffix %f %d\n", _minSuffixDensity, _minSuffix->feature->chrom_start());
  }

  //if no mid point pop out
  if(mid==NULL) {
    //did not find a cut so bounce out
    //fprintf(stderr, "Paraclu PROBLEM mid is null\n");
    return;
  }
  //printf("  maxDensity %f\n", maxDensity);
  //printf("  parentDensity %f\n", parentDensity);
  //printf("  mid link %s\n", mid->feature->chrom_location().c_str());

  double child_stability = maxDensity / parentDensity;

  if((cluster_length <= _max_cluster_length) && (maxDensity>parentDensity)) {

    //this is part of the hierarchy
    /*
     std::cerr << beg->feature->chrom_name() << "\t" << beg->feature->strand() << "\t"
     << std::setprecision(20)
     << beg->feature->chrom_start() << "\t" << end->feature->chrom_start() << "\t"
     << cluster_length << "\t" << totalValue << "\t"
     << std::setprecision(3)
     << parentDensity << "\t" << maxDensity ;
     std::cerr << "\t" << stability;
     if(stability >= _min_stability) { std::cerr << "\tSTABLE"; }
     std::cerr << "\n";
     */

    if((_selection_mode==FULL_HIERARCHY) || ((_selection_mode==STABILITY_CUT) && (child_stability >= _min_stability))) {
      EEDB::Feature *feature = _generate_cluster(beg, end);
      if(feature) {
        feature->metadataset()->add_tag_data("density", d_to_string(density));
        feature->metadataset()->add_tag_data("maxDensity", d_to_string(maxDensity));
        feature->metadataset()->add_tag_data("parentDensity", d_to_string(parentDensity));
        feature->metadataset()->add_tag_data("child_stability", d_to_string(child_stability));
        _output_buffer->insert_feature(feature);
        //fprintf(stderr, "make cluster %ld .. %ld  [ %ld ] [total=%f density=%f]\n", beg->feature->chrom_start(), end->feature->chrom_start(), cluster_length, totalValue, density);
      }
    }

    if((_selection_mode==STABILITY_CUT) && (child_stability >= _min_stability)) {
      // no need to dig deeper, martin's paraclu-cut picks biggest cluster in hierarchy
      // Remove clusters that are contained in larger clusters.  This assumes
      // that the input is sorted by start position (ascending) then end
      // position (descending).
      //fprintf(stderr, "stop stability_cut\n");
      return;
    }

  }
  double newMinDensity = max(parentDensity, maxDensity);
  //double newMinDensity = parentDensity;
  //if(cluster_length <= _max_cluster_length) { newMinDensity = max(parentDensity, maxDensity); }

  FeatureLink *mid1 = mid->prev;
  long   left_length  = mid1->feature->chrom_start() - beg->feature->chrom_start() + 1;

  //fprintf(stderr, "left\n");
  _process_cluster_range_full(beg, mid1, newMinDensity);

  //only dig down right side when inside left most cluster <= max_cluster_length
  if(cluster_length <= _max_cluster_length) {
    //fprintf(stderr, "right\n");
    _process_cluster_range_full(mid, end, newMinDensity);
  }

  //special transition, from above max_cluster into left-most cluster
  //once finished here we poping completely out so discard this left-most region
  if((left_length <= _max_cluster_length) && (cluster_length > _max_cluster_length)) {
    //fprintf(stderr, "discard left-side %ld .. %ld  [ %ld ] [total=%f density=%f]\n", beg->feature->chrom_start(), end->feature->chrom_start(), cluster_length, totalValue, density);
    _discard_region(beg, mid1);
  }

}

/////////////////////////////////////////////////////////////////////////////////////
//
// variation on full-heirarchy cluster-selection where it picks the
// "smallest stable" cluster. chooses from bottom up of recusrison.
//
/////////////////////////////////////////////////////////////////////////////////////

int  EEDB::Tools::Paraclu::_process_cluster_range_smallest(FeatureLink *beg, FeatureLink *end, double parentDensity) {
  //recursive algorithm which subdivides sites
  // return 0 = false weak region < min_cutoff_value
  // return 1 = true made cluster, pop out
  // return 2 = discarded some weak reagion in buffer, pop out
  //printf("_process_cluster_range\n");
  if((beg==NULL) || (end==NULL)) { return 0; }
  //if(beg==end) {  return 0; }

  long cluster_length = end->feature->chrom_start() - beg->feature->chrom_start() + 1;
  double totalValue   = _calc_total_value(beg, end);
  double density      = totalValue / cluster_length;

  //fprintf(stderr, "_process_cluster_range  %ld .. %ld  [ %ld ] [total=%f density=%f]\n", beg->feature->chrom_start(), end->feature->chrom_start(), cluster_length, totalValue, density);

  if(cluster_length > _max_cluster_length) {
    //use self density if cluster region is greater than max_cluster_length
    //don't track parent density above _max_cluster_length since it is meaningless is the ZENBU buffering design
    parentDensity = density;
  }
  double newParentDensity = max(parentDensity, density); //most dense parent for children, either me or my densest parent
  double stability = density/parentDensity; //my stability

  if(cluster_length <= _max_cluster_length) {
    //fprintf(stderr, "_process_cluster_range  %ld .. %ld  [ %ld ] [total=%f stability=%f]\n", beg->feature->chrom_start(), end->feature->chrom_start(), cluster_length, totalValue, stability);
  }

  if(totalValue < _min_cutoff_value) { //weak region so can fall out
    return 0;
  }

  if(beg==end) {
    //fprintf(stderr, "_process_cluster_range  beg == end density[%f] parentDensity[%f] stability[%f] minstab[%f]\n", density, parentDensity, stability, _min_stability);
    if(stability >= _min_stability) {
      //this is part of the hierarchy
      //fprintf(stderr, "this region is small [%ld] & STABLE [%f]\n", cluster_length, stability);
      EEDB::Feature *feature = _generate_cluster(beg, end);
      if(feature) {
        feature->metadataset()->add_tag_data("density", d_to_string(density));
        feature->metadataset()->add_tag_data("maxDensity", d_to_string(density));  //1bp so no children
        feature->metadataset()->add_tag_data("parentDensity", d_to_string(parentDensity));
        feature->metadataset()->add_tag_data("stability", d_to_string(stability));
        _output_buffer->insert_feature(feature);
        return 1;
      }
    }
    //all other cases is a fail
    return 0;
  }

  //
  // find best mid-cut-point
  //
  _weakestPrefix(beg, end);
  _weakestSuffix(beg, end);

  double maxDensity;
  FeatureLink *mid = NULL;
  if(_minPrefixDensity < _minSuffixDensity) {
    maxDensity = _minPrefixDensity;
    mid = _minPrefix;
    //fprintf(stderr, "choose prefix %f %d\n", _minPrefixDensity, _minPrefix->feature->chrom_start());
  } else {
    maxDensity = _minSuffixDensity;
    mid = _minSuffix;
    //fprintf(stderr, "choose suffix %f %d\n", _minSuffixDensity, _minSuffix->feature->chrom_start());
  }

  if(mid==NULL) {
    //did not find a cut so bounce out
    //fprintf(stderr, "Paraclu PROBLEM mid is null\n");
    return 0;
  }
  //printf("  maxDensity %f\n", maxDensity);
  //printf("  parentDensity %f\n", parentDensity);
  //printf("  mid link %s\n", mid->feature->chrom_location().c_str());

  int rtn =0;
  int rtn1=0;
  int rtn2=0;

  //
  //subdivide code
  //

  //printf("  subdivide\n");
  //check left child first, if it makes a cluster return out
  //printf("\nleft child  %s .. %s  [ %ld ]\n", beg->feature->chrom_location().c_str(), end->feature->chrom_location().c_str(), cluster_length);

  FeatureLink *mid1 = mid->prev;
  long   left_length  = mid1->feature->chrom_start() - beg->feature->chrom_start() + 1;
  //double left_total   = _calc_total_value(beg, mid1);
  //double left_density = left_total / left_length;

  long   right_length  = end->feature->chrom_start() - mid->feature->chrom_start() + 1;
  //double right_total   = _calc_total_value(mid, end);
  //double right_density = right_total/ right_length;

  //fprintf(stderr, "left\n");
  rtn1 = _process_cluster_range_smallest(beg, mid1, newParentDensity);
  rtn = rtn1;

  if((left_length <= _max_cluster_length) && (cluster_length > _max_cluster_length)) {
    //special transition, check if child can be recovered based on signal
    if(rtn1==0) {
      double left_total   = _calc_total_value(beg, mid1);
      double left_density = left_total / left_length;
      if(left_total >= _min_cutoff_value) {
        //children failed to find stable sub clusters, but there is enough signal here to make a cluster
        EEDB::Feature *feature = _generate_cluster(beg, mid1);
        if(feature) {
          feature->metadataset()->add_tag_data("density", d_to_string(left_density));
          feature->metadataset()->add_tag_data("maxDensity", d_to_string(maxDensity));
          feature->metadataset()->add_tag_data("parentDensity", d_to_string(newParentDensity));
          feature->metadataset()->add_tag_data("stability", d_to_string(left_density/newParentDensity));
          string name = feature->primary_name() + "-wholeleft";
          feature->primary_name(name);
          _output_buffer->insert_feature(feature);
          rtn = 1;
        }
      }
    }
    if(rtn==0) { rtn=2; }
    //fprintf(stderr, "discard left-side now\n");
    _discard_region(beg, mid1);
    return rtn;
  }

  if(cluster_length > _max_cluster_length) {
    if(rtn==0) {
      //do partial trimming of left side to slide buffer if still nothing happened below
      double left_total = _calc_total_value(beg, mid1);
      if(left_total < _min_cutoff_value) {
        //fprintf(stderr, "discard left-side now\n");
        _discard_region(beg, mid1);
        return 2;
      }
    }
    return rtn;
  }

  //
  // next check right child only if we are < max_cluster_length
  //

  //fprintf(stderr, "right child  %ld .. %ld .. %ld  [ %ld ]\n", beg->feature->chrom_start(), mid->feature->chrom_start(), end->feature->chrom_start(), cluster_length);
  //cerr <<"right\n";
  //fprintf(stderr, "right\n");
  rtn2 = _process_cluster_range_smallest(mid, end, newParentDensity);
  if(rtn2==1) { rtn = rtn2; }
  if(rtn2==2) {
    fprintf(stderr, "Paraclu PROBLEM right side returned a 2 which should never happen\n");
  }

  if(rtn1==0 && rtn2==1) {
    //right made clusters, check if left  has enough signal to be cluster
    double left_total   = _calc_total_value(beg, mid1);
    double left_density = left_total / left_length;
    if(left_total >= _min_cutoff_value) {
      EEDB::Feature *feature = _generate_cluster(beg, mid1);
      if(feature) {
        //fprintf(stderr, "LEFT-fill because right was stable\n");
        feature->metadataset()->add_tag_data("density", d_to_string(left_density));
        feature->metadataset()->add_tag_data("maxDensity", d_to_string(maxDensity));
        feature->metadataset()->add_tag_data("parentDensity", d_to_string(newParentDensity));
        feature->metadataset()->add_tag_data("stability", d_to_string(left_density/newParentDensity));
        //fprintf(stderr, "cluster %s\n", feature->chrom_location().c_str());
        string name = feature->primary_name() + "-leftfill";
        feature->primary_name(name);
        _output_buffer->insert_feature(feature);
        rtn = 1;
      }
    }
  }

  if(rtn1==1 && rtn2==0) {
    //left made clusters, check if right  has enough signal to be cluster
    double right_total   = _calc_total_value(mid, end);
    double right_density = right_total / right_length;
    if(right_total >= _min_cutoff_value) {
      //fprintf(stderr, "RIGHT-fill because left made cluster\n");
      EEDB::Feature *feature = _generate_cluster(mid, end);
      if(feature) {
        feature->metadataset()->add_tag_data("density", d_to_string(right_density));
        feature->metadataset()->add_tag_data("maxDensity", d_to_string(maxDensity));
        feature->metadataset()->add_tag_data("parentDensity", d_to_string(newParentDensity));
        feature->metadataset()->add_tag_data("stability", d_to_string(right_density/newParentDensity));
        //fprintf(stderr, "cluster %s\n", feature->chrom_location().c_str());
        string name = feature->primary_name() + "-rightfill";
        feature->primary_name(name);
        _output_buffer->insert_feature(feature);
        rtn=1;
      }
    }
  }

  if((cluster_length <= _max_cluster_length) && (stability >= _min_stability)) {
    //this is part of the hierarchy
    //fprintf(stderr, "this region is small [%ld] & STABLE [%f]\n", cluster_length, stability);
    /*
    std::cerr << beg->feature->chrom_name() << "\t" << beg->feature->strand() << "\t"
    << std::setprecision(20)
    << beg->feature->chrom_start() << "\t" << end->feature->chrom_start() << "\t"
    << cluster_length << "\t" << totalValue << "\t"
    << std::setprecision(3)
    << parentDensity << "\t" << maxDensity ;
    std::cerr << "\t" << stability;
    if(stability >= _min_stability) { std::cerr << "\tSTABLE"; }
    std::cerr << "\n";
    */

    //if children failed to make clusters, BUT I am stable then I can make a cluster
    if(rtn==0) {
      EEDB::Feature *feature = _generate_cluster(beg, end);
      if(feature) {
        feature->metadataset()->add_tag_data("density", d_to_string(density));
        feature->metadataset()->add_tag_data("maxDensity", d_to_string(maxDensity));
        feature->metadataset()->add_tag_data("parentDensity", d_to_string(parentDensity));
        feature->metadataset()->add_tag_data("stability", d_to_string(stability));
        _output_buffer->insert_feature(feature);
        return 1;
      }
    }
  }

  return rtn;
}

EEDB::Feature*  EEDB::Tools::Paraclu::_generate_cluster(FeatureLink *beg, FeatureLink *end) {
  //this range has been deemed a valid cluster so convert into a Feature for output

  //fprintf(stderr, "_generate_cluster %ld .. %ld\n", beg->feature->chrom_start(), end->feature->chrom_start());
  EEDB::Feature *feature = EEDB::Feature::realloc();
  feature->feature_source(_feature_source);
  feature->chrom(_stream_chrom);
  feature->chrom_start(beg->feature->chrom_start());
  feature->chrom_end(end->feature->chrom_start());
  feature->strand(beg->feature->strand());

  EEDB::FeatureLink *flink = beg;
  while(flink) {
    feature->collate_expression(flink->feature, EEDB::CL_SUM);
    if(flink==end) { break; }
    flink = flink->next;
  }

  feature->calc_significance(CL_SUM);
  if(feature->significance() < _min_cutoff_value) {
    //fprintf(stderr, "trying to make cluster, but it is too weak so give up\n");
    feature->release();
    return NULL;
  }

  char buffer[256];
  snprintf(buffer, 256, "paraclu_%ld_%s", _cluster_number++, feature->chrom_location().c_str());
  feature->primary_name(buffer);

  return feature;
}

///////////////////////////////////////////////////////////////////////////////////////////
//
// re-interpretation of paraclu, should behave the same but new "cut/select" idea
//   works only on left-side recursion and adds concept of "best_cluster" to the
//   depth-first search. no need for min_stability cutoff
//
///////////////////////////////////////////////////////////////////////////////////////////

void  EEDB::Tools::Paraclu::_process_cluster_range_best(FeatureLink *beg, FeatureLink *end) {

  if(_best_cluster) {
    delete _best_cluster;
    _best_cluster = NULL;
  }
  _best_cluster = new EEDB::Tools::Paraclu::Cluster;

  _recurse_cluster_range(beg, end, -infinity);

  //if((_best_cluster->totalValue >= _min_cutoff_value) && (_best_cluster->stability >= _min_stability)) {
  //if((cluster_length <= _max_cluster_length) && (stability ) && (maxDensity > minDensity)) {
  if(_best_cluster->totalValue >= _min_cutoff_value) {
    EEDB::Feature *feature = _generate_cluster(_best_cluster);
    //printf("cluster %s\n", feature->chrom_location().c_str());
    _output_buffer->insert_feature(feature);
  } else {
    _recover_singletons(_best_cluster->beg, _best_cluster->end);
  }
  _discard_region(beg, _best_cluster->end);
}

void  EEDB::Tools::Paraclu::_recurse_cluster_range(FeatureLink *beg, FeatureLink *end, double minDensity) {
  //recursive algorithm which subdivides sites
  //printf("_recurse_cluster_range\n");
  if((beg==NULL) || (end==NULL)) { return; }

  if(beg==end) {
    if(_best_cluster->stability == 0) {
      //singleton, no best above it
      _best_cluster->beg = beg;
      _best_cluster->end = end;
      _best_cluster->totalValue = beg->feature->significance();
      _best_cluster->stability = 1.0;
      _best_cluster->minDensity = minDensity;
      _best_cluster->maxDensity = minDensity;
      //_best_cluster->show();
    }
    return;
  }

  long cluster_length = end->feature->chrom_start() - beg->feature->chrom_start() + 1;
  //fprintf(stderr, "_recurse_cluster_range  %s .. %s  [ %ld ]\n", beg->feature->chrom_location().c_str(), end->feature->chrom_location().c_str(), cluster_length);

  double maxDensity=0.0, totalValue=0.0;
  FeatureLink *mid = NULL;

  totalValue = _weakestPrefix(beg, end);
  if (totalValue < _min_cutoff_value) {
    //weak region
    if(_best_cluster->stability == 0) {
      _best_cluster->beg = beg;
      _best_cluster->end = end;
      _best_cluster->totalValue = totalValue;
      //_best_cluster->show();
      //printf(" WEAK < min_cutoff\n");
    }
    return;
  }
  totalValue = _weakestSuffix(beg, end);

  if(_minPrefixDensity < _minSuffixDensity) {
    maxDensity = _minPrefixDensity;
    mid = _minPrefix;
    //fprintf(stderr, "choose prefix %f %d\n", _minPrefixDensity, _minPrefix->feature->chrom_start());
  } else {
    maxDensity = _minSuffixDensity;
    mid = _minSuffix;
    //fprintf(stderr, "choose suffix %f %d\n", _minSuffixDensity, _minSuffix->feature->chrom_start());
  }

  //double maxDensity = min(_minPrefixDensity, _minSuffixDensity);
  double stability = maxDensity/minDensity;
  //printf("  maxDensity %f\n", maxDensity);
  //printf("  minDensity %f\n", minDensity);
  //printf("  stability %f\n", stability);

  /*
  if (maxDensity > minDensity) {
    std::cerr << beg->feature->chrom_name() << "\t" << beg->feature->strand() << "\t"
    << std::setprecision(20)
    << beg->feature->chrom_start() << "\t" << end->feature->chrom_start() << "\t"
    << cluster_length << "\t" << totalValue << "\t"
    << std::setprecision(3)
    << minDensity << "\t" << maxDensity ;
    std::cerr << "\t" << stability;
    if(stability >= _min_stability) { std::cerr << "\tSTABLE"; }
    std::cerr << "\n";
  }
  */

  if((cluster_length <= _max_cluster_length) && (maxDensity > minDensity)) {
  //if((cluster_length <= _max_cluster_length) && (stability >= _min_stability) && (maxDensity > minDensity)) {
    //printf("  this region is small [%ld] & STABLE [%f], but check children first\n", cluster_length, stability);
    if(_best_cluster->stability < stability) {
      //printf("new best cluster\n");
      _best_cluster->beg = beg;
      _best_cluster->end = end;
      _best_cluster->minDensity = minDensity;
      _best_cluster->maxDensity = maxDensity;
      _best_cluster->totalValue = totalValue;
      _best_cluster->stability = stability;
    }
  }

  //FeatureLink *mid = (_minPrefixDensity < _minSuffixDensity) ? _minPrefix : _minSuffix;
  double newMinDensity = max(minDensity, maxDensity);
  //printf("  mid link %s\n", mid->feature->chrom_location().c_str());

  //if <max_length and stable -> create cluster

  if(mid && (maxDensity < infinity)) {
    //printf("  subdivide\n");
    //check left child first, if it makes a cluster return out
    //printf("\nleft child  %s .. %s  [ %ld ]\n", beg->feature->chrom_location().c_str(), end->feature->chrom_location().c_str(), cluster_length);
    _recurse_cluster_range(beg, mid->prev, newMinDensity);
  }
  //printf("returned to parent  %s .. %s  [ %ld ]\n", beg->feature->chrom_location().c_str(), end->feature->chrom_location().c_str(), cluster_length);
}

EEDB::Feature*  EEDB::Tools::Paraclu::_generate_cluster(Cluster *cl) {
  //this range has been deemed a valid cluster
  //convert into a feature for output and readjust feature links to remove these features

  //fprintf(stderr, "_generate_cluster %ld .. %ld\n", cl->beg->feature->chrom_start(), cl->end->feature->chrom_start());
  EEDB::Feature *feature = EEDB::Feature::realloc();
  feature->feature_source(_feature_source);
  feature->chrom(_stream_chrom);
  feature->chrom_start(cl->beg->feature->chrom_start());
  feature->chrom_end(cl->end->feature->chrom_start());
  feature->strand(cl->beg->feature->strand());
  feature->primary_name("paraclu_" + feature->chrom_location());

  EEDB::FeatureLink *flink = cl->beg;
  while(flink) {
    feature->collate_expression(flink->feature, EEDB::CL_SUM);
    if(flink==cl->end) { break; }
    flink = flink->next;
  }
  feature->calc_significance(CL_SUM);

  return feature;
}

void  EEDB::Tools::Paraclu::_recover_singletons(FeatureLink *beg, FeatureLink *end) {
  //if I am about to throw away a region I need to recover any strong CTSS singletons
  //TODO: figure out where I need to call this
  EEDB::FeatureLink *flink = beg;
  while(flink) {
    if(flink->feature->significance() > _min_cutoff_value) {
      flink->feature->retain();
      _output_buffer->insert_feature(flink->feature);
    }
    if(flink==end) { break; }
    flink = flink->next;
  }
}


/*
 *            Copyright 2009-2017 The VOTCA Development Team
 *                       (http://www.votca.org)
 *
 *      Licensed under the Apache License, Version 2.0 (the "License")
 *
 * You may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *              http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

// Overload of uBLAS prod function with MKL/GSL implementations

#include <boost/filesystem.hpp>
#include <boost/format.hpp>
#include <votca/ctp/logger.h>
#include <votca/tools/constants.h>
#include <votca/tools/linalg.h>
#include <votca/xtp/gwbse.h>
#include <votca/xtp/numerical_integrations.h>
#include <votca/xtp/qmpackagefactory.h>

using boost::format;
using namespace boost::filesystem;

namespace votca {
namespace xtp {
namespace ub = boost::numeric::ublas;

// +++++++++++++++++++++++++++++ //
// GWBSE MEMBER FUNCTIONS        //
// +++++++++++++++++++++++++++++ //

void GWBSE::CleanUp() {}

void GWBSE::Initialize(Property *options) {

#if (GWBSE_DOUBLE)
  CTP_LOG(ctp::logDEBUG, *_pLog) << " Compiled with full double support"
                                 << flush;
#else
  CTP_LOG(ctp::logDEBUG, *_pLog)
      << " Compiled with float/double mixture (standard)" << flush;
#endif

  string key = Identify();

  // getting level ranges
  _ranges = options->ifExistsReturnElseReturnDefault<string>(key + ".ranges",
                                                             "default");

  // now check validity, and get rpa, qp, and bse level ranges accordingly

  if (_ranges == "factor") {
    // get factors
    _rpamaxfactor = options->get(key + ".rpamax").as<double>();
    _qpminfactor = options->get(key + ".qpmin").as<double>();
    _qpmaxfactor = options->get(key + ".qpmax").as<double>();
    _bseminfactor = options->get(key + ".bsemin").as<double>();
    _bsemaxfactor = options->get(key + ".bsemax").as<double>();
  } else if (_ranges == "explicit") {
    // get explicit numbers
    _rpamax = options->get(key + ".rpamax").as<unsigned int>();
    _qpmin = options->get(key + ".qpmin").as<unsigned int>();
    _qpmax = options->get(key + ".qpmax").as<unsigned int>();
    _bse_vmin = options->get(key + ".bsemin").as<unsigned int>();
    _bse_cmax = options->get(key + ".bsemax").as<unsigned int>();
  } else if (_ranges == "" || _ranges == "default") {
    _ranges = "default";
  } else if (_ranges == "full") {
    _ranges = "full";
  } else {
    cerr << "\nSpecified range option " << _ranges << " invalid. ";
    throw std::runtime_error(
        "\nValid options are: default,factor,explicit,full");
  }

  _ignore_corelevels = options->ifExistsReturnElseReturnDefault<bool>(
      key + ".ignore_corelevels", false);

  _bse_nmax =
      options->ifExistsReturnElseReturnDefault<int>(key + ".exctotal", 25);
  _bse_nprint =
      options->ifExistsReturnElseReturnDefault<int>(key + ".print", 25);
  _fragA = options->ifExistsReturnElseReturnDefault<int>(key + ".fragment", -1);

  string BSEtype =
      options->ifExistsReturnElseReturnDefault<string>(key + ".BSEtype", "TDA");

  if (BSEtype == "full") {
    _do_full_BSE = true;
    CTP_LOG(ctp::logDEBUG, *_pLog) << " BSE type: full" << flush;
  } else {
    _do_full_BSE = false;
    CTP_LOG(ctp::logDEBUG, *_pLog) << " BSE type: TDA" << flush;
  }

  _openmp_threads =
      options->ifExistsReturnElseReturnDefault<int>(key + ".openmp", 0);

  if (options->exists(key + ".vxc")) {
    _doVxc =
        options->ifExistsReturnElseThrowRuntimeError<bool>(key + ".vxc.dovxc");
    if (_doVxc) {
      _functional = options->ifExistsReturnElseThrowRuntimeError<string>(
          key + ".vxc.functional");
      _grid = options->ifExistsReturnElseReturnDefault<string>(
          key + ".vxc.grid", "medium");
    }
  }

  _gwbasis_name =
      options->ifExistsReturnElseThrowRuntimeError<string>(key + ".gwbasis");
  _dftbasis_name =
      options->ifExistsReturnElseThrowRuntimeError<string>(key + ".dftbasis");

  _shift = options->ifExistsReturnElseThrowRuntimeError<double>(key + ".shift");
  _g_sc_limit = options->ifExistsReturnElseReturnDefault<double>(
      key + ".g_sc_limit",
      0.00001);  // convergence criteria for qp iteration [Hartree]]
  _g_sc_max_iterations = options->ifExistsReturnElseReturnDefault<int>(
      key + ".g_sc_max_iterations",
      40);  // convergence criteria for qp iteration [Hartree]]

  _gw_sc_max_iterations = options->ifExistsReturnElseReturnDefault<int>(
      key + ".gw_sc_max_iterations",
      20);  // convergence criteria for qp iteration [Hartree]]

  _gw_sc_limit = options->ifExistsReturnElseReturnDefault<double>(
      key + ".gw_sc_limit", 0.00001);  // convergence criteria for shift it
  _iterate_gw = false;
  string _shift_type =
      options->ifExistsReturnElseThrowRuntimeError<string>(key + ".shift_type");
  if (_shift_type != "fixed") {
    _iterate_gw = true;
  }
  CTP_LOG(ctp::logDEBUG, *_pLog) << " Shift: " << _shift_type << flush;
  CTP_LOG(ctp::logDEBUG, *_pLog) << " g_sc_limit [Hartree]: " << _g_sc_limit
                                 << flush;
  if (_iterate_gw) {
    CTP_LOG(ctp::logDEBUG, *_pLog) << " gw_sc_limit [Hartree]: " << _gw_sc_limit
                                   << flush;
    // if(_g_sc_max_iterations<20){
    //_g_sc_max_iterations=20;
    // CTP_LOG(ctp::logDEBUG, *_pLog) << " Setting G iterations to 20, as it
    // speeds up the GW iterations" << flush;
    // }
  }
  _min_print_weight = options->ifExistsReturnElseReturnDefault<double>(
      key + ".bse_print_weight",
      0.2);  // print exciton WF composition weight larger that thin minimum

  // setting some defaults
  _do_qp_diag = false;
  _do_bse_singlets = false;
  _do_bse_triplets = false;
  // possible tasks
  // diagQP, singlets, triplets, all, ibse
  string _tasks_string =
      options->ifExistsReturnElseThrowRuntimeError<string>(key + ".tasks");
  if (_tasks_string.find("all") != std::string::npos) {
    _do_qp_diag = true;
    _do_bse_singlets = true;
    _do_bse_triplets = true;
  }
  if (_tasks_string.find("qpdiag") != std::string::npos) _do_qp_diag = true;
  if (_tasks_string.find("singlets") != std::string::npos)
    _do_bse_singlets = true;
  if (_tasks_string.find("triplets") != std::string::npos)
    _do_bse_triplets = true;
  _store_eh_interaction = false;
  _do_bse_diag = true;
  // special construction for ibse mode
  if (_tasks_string.find("igwbse") != std::string::npos) {
    _do_qp_diag = false;   // no qp diagonalization
    _do_bse_diag = false;  // no diagonalization of BSE Hamiltonian
    _store_eh_interaction = true;
  }

  // possible storage
  // qpPert, qpdiag_energies, qp_diag_coefficients, bse_singlet_energies,
  // bse_triplet_energies, bse_singlet_coefficients, bse_triplet_coefficients
  _store_qp_pert = true;

  _store_qp_diag = false;
  _store_bse_triplets = false;
  _store_bse_singlets = false;
  string _store_string =
      options->ifExistsReturnElseThrowRuntimeError<string>(key + ".store");
  if ((_store_string.find("all") != std::string::npos) ||
      (_store_string.find("") != std::string::npos)) {
    // store according to tasks choice
    if (_do_qp_diag) _store_qp_diag = true;
    if (_do_bse_singlets && _do_bse_diag) _store_bse_singlets = true;
    if (_do_bse_triplets && _do_bse_diag) _store_bse_triplets = true;
  }
  if (_store_string.find("qpdiag") != std::string::npos) _store_qp_diag = true;
  if (_store_string.find("singlets") != std::string::npos)
    _store_bse_singlets = true;
  if (_store_string.find("triplets") != std::string::npos)
    _store_bse_triplets = true;
  if (_store_string.find("ehint") != std::string::npos)
    _store_eh_interaction = true;

  CTP_LOG(ctp::logDEBUG, *_pLog) << " Tasks: " << flush;
  if (_do_qp_diag) {
    CTP_LOG(ctp::logDEBUG, *_pLog) << " qpdiag " << flush;
  }
  if (_do_bse_singlets) {
    CTP_LOG(ctp::logDEBUG, *_pLog) << " singlets " << flush;
  }
  if (_do_bse_triplets) {
    CTP_LOG(ctp::logDEBUG, *_pLog) << " triplets " << flush;
  }
  CTP_LOG(ctp::logDEBUG, *_pLog) << " Store: " << flush;
  if (_store_qp_diag) {
    CTP_LOG(ctp::logDEBUG, *_pLog) << " qpdiag " << flush;
  }
  if (_store_bse_singlets) {
    CTP_LOG(ctp::logDEBUG, *_pLog) << " singlets " << flush;
  }
  if (_store_bse_triplets) {
    CTP_LOG(ctp::logDEBUG, *_pLog) << " triplets " << flush;
  }
  if (_store_eh_interaction) {
    CTP_LOG(ctp::logDEBUG, *_pLog) << " ehint " << flush;
  }

  return;
}

void GWBSE::addoutput(Property *_summary) {

  const double hrt2ev = tools::conv::hrt2ev;
  Property *_gwbse_summary = &_summary->add("GWBSE", "");
  _gwbse_summary->setAttribute("units", "eV");
  _gwbse_summary->setAttribute("DeltaHLGap",
                               (format("%1$+1.6f ") % (_shift * hrt2ev)).str());

  _gwbse_summary->setAttribute(
      "DFTEnergy", (format("%1$+1.6f ") % _orbitals->getQMEnergy()).str());
  int printlimit = _bse_nprint;  // I use this to determine how much is printed,
                                 // I do not want another option to pipe through

  Property *_dft_summary = &_gwbse_summary->add("dft", "");
  _dft_summary->setAttribute("HOMO", _homo);
  _dft_summary->setAttribute("LUMO", _homo + 1);
  int begin = _homo - printlimit;
  int end = _homo + printlimit + 1;
  if (begin < 0) {
    begin = 0;
    end = 2 * _homo + 1;
  }
  for (int state = begin; state < end; state++) {

    Property *_level_summary = &_dft_summary->add("level", "");
    _level_summary->setAttribute("number", state);
    _level_summary->add("dft_energy",
                        (format("%1$+1.6f ") %
                         ((_orbitals->MOEnergies())(_qpmin + state) * hrt2ev))
                            .str());
    _level_summary->add(
        "gw_energy",
        (format("%1$+1.6f ") % (_qp_energies(_qpmin + state) * hrt2ev)).str());

    if (_do_qp_diag) {
      // cout << "_do_qp_diag" <<_do_qp_diag<<endl;
      _level_summary->add(
          "qp_energy",
          (format("%1$+1.6f ") % (_qp_diag_energies(_qpmin + state) * hrt2ev))
              .str());
    }
  }

  if (_do_bse_singlets) {
    Property *_singlet_summary = &_gwbse_summary->add("singlets", "");
    for (int state = 0; state < printlimit; ++state) {
      Property *_level_summary = &_singlet_summary->add("level", "");
      _level_summary->setAttribute("number", state + 1);
      _level_summary->add("omega", (format("%1$+1.6f ") %
                                    (_bse_singlet_energies(state) * hrt2ev))
                                       .str());
      if (_orbitals->hasTransitionDipoles()) {

        const tools::vec &dipoles = (_orbitals->TransitionDipoles())[state];
        double f = 2 * dipoles * dipoles * _bse_singlet_energies(state) / 3.0;

        _level_summary->add("f", (format("%1$+1.6f ") % f).str());
        Property *_dipol_summary = &_level_summary->add(
            "Trdipole", (format("%1$+1.4f %2$+1.4f %3$+1.4f") % dipoles.getX() %
                         dipoles.getY() % dipoles.getZ())
                            .str());
        _dipol_summary->setAttribute("unit", "e*bohr");
        _dipol_summary->setAttribute("gauge", "length");
      }
    }
  }
  if (_do_bse_triplets) {
    Property *_triplet_summary = &_gwbse_summary->add("triplets", "");
    for (int state = 0; state < printlimit; ++state) {

      Property *_level_summary = &_triplet_summary->add("level", "");
      _level_summary->setAttribute("number", state + 1);
      _level_summary->add("omega", (format("%1$+1.6f ") %
                                    (_bse_triplet_energies(state) * hrt2ev))
                                       .str());
    }
  }
  return;
}

/*
 *    Many-body Green's fuctions theory implementation
 *
 *  data required from orbitals file
 *  - atomic coordinates
 *  - DFT molecular orbitals (energies and coeffcients)
 *  - DFT exchange-correlation potential matrix in atomic orbitals
 *  - number of electrons, number of levels
 *

 */

bool GWBSE::Evaluate() {

// set the parallelization
#ifdef _OPENMP
  if (_openmp_threads > 0) {
    omp_set_num_threads(_openmp_threads);
    CTP_LOG(ctp::logDEBUG, *_pLog) << ctp::TimeStamp() << " Using "
                                   << omp_get_max_threads() << " threads"
                                   << flush;
  }
#endif
  /* check which QC program was used for the DFT run
   * -> implicit info about MO coefficient storage order
   */
  string _dft_package = _orbitals->getQMpackage();
  CTP_LOG(ctp::logDEBUG, *_pLog) << ctp::TimeStamp()
                                 << " DFT data was created by " << _dft_package
                                 << flush;

  std::vector<ctp::QMAtom *> _atoms;
  for (const auto &atom : _orbitals->QMAtoms()) {
    if (!atom->from_environment) {
      _atoms.push_back(atom);
    }
  }
  // load DFT basis set (element-wise information) from xml file
  BasisSet dftbs;

  if (_dftbasis_name != _orbitals->getDFTbasis()) {
    throw std::runtime_error(
        "Name of the Basisset from .orb file: " + _orbitals->getDFTbasis() +
        " and from GWBSE optionfile " + _dftbasis_name + " do not agree.");
  }

  dftbs.LoadBasisSet(_dftbasis_name);
  _orbitals->setDFTbasis(_dftbasis_name);
  CTP_LOG(ctp::logDEBUG, *_pLog) << ctp::TimeStamp() << " Loaded DFT Basis Set "
                                 << _dftbasis_name << flush;

  // fill DFT AO basis by going through all atoms

  _dftbasis.AOBasisFill(&dftbs, _atoms, _fragA);
  CTP_LOG(ctp::logDEBUG, *_pLog) << ctp::TimeStamp()
                                 << " Filled DFT Basis of size "
                                 << _dftbasis.AOBasisSize() << flush;
  if (_dftbasis._AOBasisFragB > 0) {
    CTP_LOG(ctp::logDEBUG, *_pLog) << ctp::TimeStamp() << " FragmentA size "
                                   << _dftbasis._AOBasisFragA << flush;
    CTP_LOG(ctp::logDEBUG, *_pLog) << ctp::TimeStamp() << " FragmentB size "
                                   << _dftbasis._AOBasisFragB << flush;
  }

  if (_do_full_BSE)
    _orbitals->setBSEtype("full");
  else {
    _orbitals->setBSEtype("TDA");
  }
  /* Preparation of calculation parameters:
   *  - number of electrons -> index of HOMO
   *  - number of levels
   *  - highest level considered in RPA
   *  - lowest and highest level considered in GWA
   *  - lowest and highest level considered in BSE
   *  - number of excitations calculates in BSE
   */

  // convert _rpamax if needed
  _homo = _orbitals->getNumberOfElectrons() - 1;  // indexed from 0

  unsigned int _ignored_corelevels = 0;
  if (_ignore_corelevels) {
    std::string _ecpsave = _orbitals->getECP();
    _orbitals->setECP("ecp");
    int _valence_levels =
        _orbitals->FragmentNuclearCharges(_atoms.size())(0) / 2;
    _orbitals->setECP(_ecpsave);
    _ignored_corelevels = _orbitals->getNumberOfElectrons() - _valence_levels;
    CTP_LOG(ctp::logDEBUG, *_pLog) << ctp::TimeStamp() << " Can ignore "
                                   << _ignored_corelevels << " core levels "
                                   << flush;
  }

  _rpamin = 0;  // lowest index occ min(gwa%mmin, screening%nsum_low) ! always 1
  if (_ranges == "default" || _ranges == "full") {
    _rpamax = _orbitals->getNumberOfLevels() - 1;  // total number of levels
  } else if (_ranges == "factor") {
    _rpamax = _rpamaxfactor * _orbitals->getNumberOfLevels() -
              1;  // total number of levels
  }
  if (_rpamax > _orbitals->getNumberOfLevels() - 1) {
    _rpamax = _orbitals->getNumberOfLevels() - 1;
  }
  // convert _qpmin and _qpmax if needed
  if (_ranges == "default") {
    _qpmin = 0;              // indexed from 0
    _qpmax = 2 * _homo + 1;  // indexed from 0
  } else if (_ranges == "factor") {
    if (_orbitals->getNumberOfElectrons() -
            int(_qpminfactor * _orbitals->getNumberOfElectrons()) - 1 <
        0) {
      _qpmin = 0;
    } else {
      _qpmin = _orbitals->getNumberOfElectrons() -
               int(_qpminfactor * _orbitals->getNumberOfElectrons()) - 1;
    }
    _qpmax = _orbitals->getNumberOfElectrons() +
             int(_qpmaxfactor * _orbitals->getNumberOfElectrons()) - 1;
  } else if (_ranges == "explicit") {
    _qpmin -= 1;
    _qpmax -= 1;
  } else if (_ranges == "full") {
    _qpmin = 0;
    _qpmax = _orbitals->getNumberOfLevels() - 1;
  }
  if (_qpmax > unsigned(_orbitals->getNumberOfLevels() - 1)) {
    _qpmax = _orbitals->getNumberOfLevels() - 1;
  }

  // autoignore core levels in QP
  if (_ignore_corelevels && (_qpmin < _ignored_corelevels)) {
    _qpmin = _ignored_corelevels;
  }

  // set BSE band range indices
  // anything else would be stupid!
  _bse_vmax = _homo;
  _bse_cmin = _homo + 1;

  if (_ranges == "default") {
    _bse_vmin = 0;              // indexed from 0
    _bse_cmax = 2 * _homo + 1;  // indexed from 0
  } else if (_ranges == "factor") {
    _bse_vmin = _orbitals->getNumberOfElectrons() -
                int(_bseminfactor * _orbitals->getNumberOfElectrons()) - 1;
    if (_orbitals->getNumberOfElectrons() -
            int(_bseminfactor * _orbitals->getNumberOfElectrons()) - 1 <
        0) {
      _bse_vmin = 0;
    }
    _bse_cmax = _orbitals->getNumberOfElectrons() +
                int(_bsemaxfactor * _orbitals->getNumberOfElectrons()) - 1;
  } else if (_ranges == "explicit") {
    _bse_vmin -= 1;
    _bse_cmax -= 1;
  } else if (_ranges == "full") {
    _bse_vmin = 0;
    _bse_cmax = _orbitals->getNumberOfLevels() - 1;
  }
  if (_bse_cmax > unsigned(_orbitals->getNumberOfLevels() - 1)) {
    _bse_cmax = _orbitals->getNumberOfLevels() - 1;
  }

  // autoignore core levels in BSE
  if (_ignore_corelevels && (_bse_vmin < _ignored_corelevels)) {
    _bse_vmin = _ignored_corelevels;
  }

  _bse_vtotal = _bse_vmax - _bse_vmin + 1;
  _bse_ctotal = _bse_cmax - _bse_cmin + 1;
  _bse_size = _bse_vtotal * _bse_ctotal;

  // indexing info BSE vector index to occupied/virtual orbital
  for (unsigned _v = 0; _v < _bse_vtotal; _v++) {
    for (unsigned _c = 0; _c < _bse_ctotal; _c++) {
      _index2v.push_back(_bse_vmin + _v);
      _index2c.push_back(_bse_cmin + _c);
    }
  }

  // some QP - BSE consistency checks are required
  if (_bse_vmin < _qpmin) _qpmin = _bse_vmin;
  if (_bse_cmax > _qpmax) _qpmax = _bse_cmax;

  _qptotal = _qpmax - _qpmin + 1;
  if (_bse_nmax > int(_bse_size) || _bse_nmax < 0) _bse_nmax = int(_bse_size);
  if (_bse_nprint > _bse_nmax) _bse_nprint = _bse_nmax;

  // store information in _orbitals for later use
  _orbitals->setRPAindices(_rpamin, _rpamax);
  _orbitals->setGWAindices(_qpmin, _qpmax);
  _orbitals->setBSEindices(_bse_vmin, _bse_vmax, _bse_cmin, _bse_cmax,
                           _bse_nmax);

  // information for hybrid DFT

  _ScaHFX = -1;

  CTP_LOG(ctp::logDEBUG, *_pLog) << ctp::TimeStamp() << " Set RPA level range ["
                                 << _rpamin + 1 << ":" << _rpamax + 1 << "]"
                                 << flush;
  CTP_LOG(ctp::logDEBUG, *_pLog) << ctp::TimeStamp() << " Set QP  level range ["
                                 << _qpmin + 1 << ":" << _qpmax + 1 << "]"
                                 << flush;
  CTP_LOG(ctp::logDEBUG, *_pLog)
      << ctp::TimeStamp() << " Set BSE level range occ[" << _bse_vmin + 1 << ":"
      << _bse_vmax + 1 << "]  virt[" << _bse_cmin + 1 << ":" << _bse_cmax + 1
      << "]" << flush;
  CTP_LOG(ctp::logDEBUG, *_pLog)
      << ctp::TimeStamp() << " BSE Hamiltonian has size " << _bse_size << "x"
      << _bse_size << flush;

  // process the DFT data
  // a) form the expectation value of the XC functional in MOs

  _ScaHFX = _orbitals->getScaHFX();

  ub::matrix<double> _vxc_ao;
  if (_orbitals->hasAOVxc()) {
    if (_doVxc) {
      CTP_LOG(ctp::logDEBUG, *_pLog)
          << ctp::TimeStamp()
          << " There is already a Vxc matrix loaded from DFT, did you maybe "
             "run a DFT code with outputVxc?\n I will take the external "
             "implementation"
          << flush;
      _doVxc = false;
    }
    CTP_LOG(ctp::logDEBUG, *_pLog) << ctp::TimeStamp()
                                   << " Loaded external Vxc matrix" << flush;
    _vxc_ao = _orbitals->AOVxc();
  } else if (_doVxc) {

    NumericalIntegration _numint;
    _numint.setXCfunctional(_functional);
    double ScaHFX_temp = _numint.getExactExchange(_functional);
    if (ScaHFX_temp != _ScaHFX) {
      throw std::runtime_error(
          (boost::format("GWBSE exact exchange a=%s differs from qmpackage "
                         "exact exchange a=%s, probably your functionals are "
                         "inconsistent") %
           ScaHFX_temp % _ScaHFX)
              .str());
    }
    _numint.GridSetup(_grid, &dftbs, _atoms, &_dftbasis);
    CTP_LOG(ctp::logDEBUG, *_pLog)
        << ctp::TimeStamp()
        << " Setup grid for integration with gridsize: " << _grid << " with "
        << _numint.getGridSize() << " points, divided into "
        << _numint.getBoxesSize() << " boxes" << flush;

    CTP_LOG(ctp::logDEBUG, *_pLog)
        << ctp::TimeStamp() << " Converted DFT orbital coefficient order from "
        << _dft_package << " to XTP" << flush;
    CTP_LOG(ctp::logDEBUG, *_pLog)
        << ctp::TimeStamp() << " Integrating Vxc in VOTCA with functional "
        << _functional << flush;
    ub::matrix<double> DMAT = _orbitals->DensityMatrixGroundState();

    _vxc_ao = _numint.IntegrateVXC(DMAT);
    CTP_LOG(ctp::logDEBUG, *_pLog) << ctp::TimeStamp()
                                   << " Calculated Vxc in VOTCA" << flush;

  } else {
    throw std::runtime_error(
        "So your DFT data contains no Vxc, if you want to proceed use the "
        "dovxc option.");
  }

  CTP_LOG(ctp::logDEBUG, *_pLog) << ctp::TimeStamp()
                                 << " Set hybrid exchange factor: " << _ScaHFX
                                 << flush;

  // now get expectation values but only for those in _qpmin:_qpmax range
  ub::matrix<double> _mos =
      ub::project(_dft_orbitals, ub::range(_qpmin, _qpmax + 1),
                  ub::range(0, _dftbasis.AOBasisSize()));

  ub::matrix<double> _temp = ub::prod(_vxc_ao, ub::trans(_mos));
  _vxc = ub::prod(_mos, _temp);
  CTP_LOG(ctp::logDEBUG, *_pLog)
      << ctp::TimeStamp()
      << " Calculated exchange-correlation expectation values " << flush;
  _vxc_ao.resize(0, 0);

  /// ------- actual calculation begins here -------

  // load auxiliary GW basis set (element-wise information) from xml file
  BasisSet gwbs;
  gwbs.LoadBasisSet(_gwbasis_name);
  CTP_LOG(ctp::logDEBUG, *_pLog) << ctp::TimeStamp() << " Loaded GW Basis Set "
                                 << _gwbasis_name << flush;

  // fill auxiliary GW AO basis by going through all atoms
  AOBasis gwbasis;
  gwbasis.AOBasisFill(&gwbs, _atoms);
  _orbitals->setGWbasis(_gwbasis_name);
  CTP_LOG(ctp::logDEBUG, *_pLog) << ctp::TimeStamp()
                                 << " Filled GW Basis of size "
                                 << gwbasis.AOBasisSize() << flush;

  /*
   * for the representation of 2-point functions with the help of the
   * auxiliary GW basis, its AO overlap matrix is required.
   * cf. M. Rohlfing, PhD thesis, ch. 3
   */
  AOOverlap _gwoverlap;
  // Fill overlap
  _gwoverlap.Fill(gwbasis);

  CTP_LOG(ctp::logDEBUG, *_pLog) << ctp::TimeStamp()
                                 << " Filled GW Overlap matrix of dimension: "
                                 << _gwoverlap.Matrix().size1() << flush;

  /*
   *  for the calculation of Coulomb and exchange term in the self
   *  energy and electron-hole interaction, the Coulomb interaction
   *  is represented using the auxiliary GW basis set.
   *  Here, we need to prepare the Coulomb matrix expressed in
   *  the AOs of the GW basis
   */

  // get Coulomb matrix as AOCoulomb
  AOCoulomb _gwcoulomb;

  // Fill Coulomb matrix
  _gwcoulomb.Fill(gwbasis);
  CTP_LOG(ctp::logDEBUG, *_pLog) << ctp::TimeStamp()
                                 << " Filled GW Coulomb matrix of dimension: "
                                 << _gwcoulomb.Matrix().size1() << flush;

  // PPM is symmetric, so we need to get the sqrt of the Coulomb matrix

  ub::matrix<double> _gwoverlap_cholesky = _gwoverlap.Matrix();
  linalg_cholesky_decompose(_gwoverlap_cholesky);

// remove L^T from Cholesky
#pragma omp parallel for
  for (unsigned i = 0; i < _gwoverlap_cholesky.size1(); i++) {
    for (unsigned j = i + 1; j < _gwoverlap_cholesky.size1(); j++) {
      _gwoverlap_cholesky(i, j) = 0.0;
    }
  }

  ub::matrix<double> _gwoverlap_cholesky_inverse;  // will also be needed in PPM
                                                   // itself
  int removed =
      linalg_invert_svd(_gwoverlap_cholesky, _gwoverlap_cholesky_inverse, 1e7);
  CTP_LOG(ctp::logDEBUG, *_pLog)
      << ctp::TimeStamp() << " Removed " << removed
      << " functions from gwoverlap to avoid near linear dependencies" << flush;

  int removed_functions = _gwcoulomb.Symmetrize(_gwoverlap_cholesky);
  CTP_LOG(ctp::logDEBUG, *_pLog)
      << ctp::TimeStamp() << " Prepared GW Coulomb matrix for symmetric PPM"
      << flush;
  CTP_LOG(ctp::logDEBUG, *_pLog)
      << ctp::TimeStamp() << " Removed " << removed_functions
      << " functions from gwcoulomb to avoid near linear dependencies" << flush;
  /* calculate 3-center integrals,  convoluted with DFT eigenvectors
   *
   *  M_mn(beta) = \int{ \psi^DFT_m(r) \phi^GW_beta(r) \psi^DFT_n d3r  }
   *             = \sum_{alpha,gamma} { c_m,alpha c_n,gamma \int
   * {\phi^DFT_alpha(r) \phi^GW_beta(r) \phi^DFT_gamma(r) d3r}  }
   *
   *  cf. M. Rohlfing, PhD thesis, ch. 3.2
   *
   */

  // --- prepare a vector (gwdacay) of matrices (orbitals, orbitals) as
  // container => M_mn
  // prepare 3-center integral object

  TCMatrix _Mmn;
  _Mmn.Initialize(gwbasis.AOBasisSize(), _rpamin, _qpmax, _rpamin, _rpamax);
  _Mmn.Fill(gwbasis, _dftbasis, _dft_orbitals);
  CTP_LOG(ctp::logDEBUG, *_pLog)
      << ctp::TimeStamp()
      << " Calculated Mmn_beta (3-center-repulsion x orbitals)  " << flush;

  // make _Mmn symmetric
  _Mmn.Symmetrize(_gwcoulomb.Matrix());
  CTP_LOG(ctp::logDEBUG, *_pLog)
      << ctp::TimeStamp() << " Symmetrize Mmn_beta for self-energy  " << flush;

  // for use in RPA, make a copy of _Mmn with dimensions
  // (1:HOMO)(gwabasissize,LUMO:nmax)
  TCMatrix _Mmn_RPA;
  _Mmn_RPA.Initialize(gwbasis.AOBasisSize(), _rpamin, _homo, _homo + 1,
                      _rpamax);
  RPA_prepare_threecenters(_Mmn_RPA, _Mmn);
  CTP_LOG(ctp::logDEBUG, *_pLog) << ctp::TimeStamp()
                                 << " Prepared Mmn_beta for RPA  " << flush;

  // fix the frequencies for PPM
  _screening_freq = ub::zero_matrix<double>(2, 2);  // two frequencies
  // first one
  _screening_freq(0, 0) = 0.0;  // real part
  _screening_freq(0, 1) = 0.0;  // imaginary part
  // second one
  _screening_freq(1, 0) = 0.0;  // real part
  _screening_freq(1, 1) = 0.5;  // imaginary part  //hartree

  // one entry to epsilon for each frequency
  _epsilon.resize(_screening_freq.size1());

  /* for automatic iteration of _shift, we need to
   * - make a copy of _Mmn
   * - calculate eps
   * - construct ppm
   * - threecenters for sigma
   * - sigma_x
   * - sigma_c
   * - test for convergence
   *
   */

  // initialize _qp_energies;
  // shift unoccupied levels by the shift
  _qp_energies = ub::zero_vector<double>(_orbitals->getNumberOfLevels());
  for (size_t i = 0; i < _qp_energies.size(); ++i) {
    _qp_energies(i) = _orbitals->MOEnergies()(i);
    if (i > _homo) {
      _qp_energies(i) += _shift;
    }
  }

  _sigma_c.resize(_qptotal);
  _sigma_x.resize(_qptotal);

  TCMatrix _Mmn_backup;
  if (_iterate_gw) {

    // make copy of _Mmn, memory++

    _Mmn_backup.Initialize(gwbasis.AOBasisSize(), _rpamin, _qpmax, _rpamin,
                           _rpamax);
    int _mnsize = _Mmn_backup.get_mtot();
    for (int _i = 0; _i < _mnsize; _i++) {
      _Mmn_backup[_i] = _Mmn[_i];
    }
    CTP_LOG(ctp::logDEBUG, *_pLog) << ctp::TimeStamp()
                                   << " Made backup of _Mmn  " << flush;
  } else {
    _gw_sc_max_iterations = 1;
  }

  const ub::vector<double> &_dft_energies = _orbitals->MOEnergies();
  for (unsigned gw_iteration = 0; gw_iteration < _gw_sc_max_iterations;
       ++gw_iteration) {

    ub::vector<double> _qp_old_rpa = _qp_energies;
    if (_iterate_gw) {
      CTP_LOG(ctp::logDEBUG, *_pLog) << ctp::TimeStamp() << " GW Iteraton "
                                     << gw_iteration + 1 << " of "
                                     << _gw_sc_max_iterations << flush;
    }

    // for symmetric PPM, we can initialize _epsilon with the overlap matrix!
    for (unsigned _i_freq = 0; _i_freq < _screening_freq.size1(); _i_freq++) {
      _epsilon[_i_freq] = _gwoverlap.Matrix();
    }

    // determine epsilon from RPA
    RPA_calculate_epsilon(_Mmn_RPA);
    CTP_LOG(ctp::logDEBUG, *_pLog) << ctp::TimeStamp()
                                   << " Calculated epsilon via RPA  " << flush;

    // construct PPM parameters
    PPM_construct_parameters(_gwoverlap_cholesky_inverse);
    CTP_LOG(ctp::logDEBUG, *_pLog) << ctp::TimeStamp()
                                   << " Constructed PPM parameters  " << flush;

    // prepare threecenters for Sigma
    sigma_prepare_threecenters(_Mmn);
    CTP_LOG(ctp::logDEBUG, *_pLog)
        << ctp::TimeStamp() << " Prepared threecenters for sigma  " << flush;

    sigma_diag(_Mmn);
    CTP_LOG(ctp::logDEBUG, *_pLog)
        << ctp::TimeStamp() << " Calculated diagonal part of Sigma  " << flush;
    // iterative refinement of qp energies

    double _DFTgap = _dft_energies(_homo + 1) - _dft_energies(_homo);
    double _QPgap = _qp_energies(_homo + 1) - _qp_energies(_homo);
    _shift = _QPgap - _DFTgap;

    // qp energies outside the update range are simply shifted.
    for (unsigned i = _qpmax + 1; i < _dft_energies.size(); ++i) {
      _qp_energies(i) = _dft_energies(i) + _shift;
    }

    if (_iterate_gw) {
      bool _gw_converged = true;
      ub::vector<double> diff = _qp_old_rpa - _qp_energies;
      unsigned int _l_not_converged = 0;
      double E_max = 0;
      for (unsigned l = 0; l < diff.size(); l++) {
        if (std::abs(diff(l)) > std::abs(E_max)) {
          _l_not_converged = l;
          E_max = diff(l);
        }
        if (std::abs(diff(l)) > _gw_sc_limit) {
          _gw_converged = false;
        }
      }
      double alpha = 0.0;
      _qp_energies = alpha * _qp_old_rpa + (1 - alpha) * _qp_energies;
      if (tools::globals::verbose) {
        CTP_LOG(ctp::logDEBUG, *_pLog)
            << ctp::TimeStamp() << " GW_Iteration: " << gw_iteration + 1
            << " shift=" << _shift << " E_diff max=" << E_max
            << " StateNo:" << _l_not_converged << flush;
      }

      if (_gw_converged) {
        CTP_LOG(ctp::logDEBUG, *_pLog)
            << ctp::TimeStamp() << " Converged after " << gw_iteration + 1
            << " GW iterations" << flush;
        break;
      } else if (gw_iteration == _gw_sc_max_iterations - 1) {
        // continue regardless for now, but drop WARNING
        CTP_LOG(ctp::logDEBUG, *_pLog)
            << ctp::TimeStamp() << " WARNING! GWA spectrum not converged after "
            << _gw_sc_max_iterations << " iterations." << flush;
        CTP_LOG(ctp::logDEBUG, *_pLog)
            << ctp::TimeStamp() << "          GWA level " << _l_not_converged
            << " energy changed by " << diff(_l_not_converged) << flush;
        CTP_LOG(ctp::logDEBUG, *_pLog)
            << ctp::TimeStamp()
            << "          Run continues. Inspect results carefully!" << flush;
        break;
      }

      int _mnsize = _Mmn_backup.get_mtot();
      for (int _i = 0; _i < _mnsize; _i++) {
        _Mmn[_i] = _Mmn_backup[_i];
      }
    }
  }

  sigma_offdiag(_Mmn);
  CTP_LOG(ctp::logDEBUG, *_pLog)
      << ctp::TimeStamp() << " Calculated offdiagonal part of Sigma  " << flush;
  _gwoverlap.Matrix().resize(0, 0);
  _gwoverlap_cholesky_inverse.resize(0, 0);
  _Mmn_RPA.Cleanup();
  if (_iterate_gw) {
    _Mmn_backup.Cleanup();
    CTP_LOG(ctp::logDEBUG, *_pLog)
        << ctp::TimeStamp() << " Cleaned up Overlap, MmnRPA and Mmn_backup "
        << flush;
  } else {
    CTP_LOG(ctp::logDEBUG, *_pLog) << ctp::TimeStamp()
                                   << " Cleaned up Overlap and MmnRPA" << flush;
  }
  // free no longer required three-center matrices in _Mmn
  // max required is _bse_cmax (could be smaller than _qpmax)
  _Mmn.Prune(gwbasis.AOBasisSize(), _bse_vmin, _bse_cmax);

  // Output of quasiparticle energies after all is done:

  CTP_LOG(ctp::logINFO, *_pLog)
      << (format(
              "  ====== Perturbative quasiparticle energies (Hartree) ====== "))
             .str()
      << flush;
  CTP_LOG(ctp::logINFO, *_pLog)
      << (format("   DeltaHLGap = %1$+1.6f Hartree") % _shift).str() << flush;

  for (unsigned _i = 0; _i < _qptotal; _i++) {
    if ((_i + _qpmin) == _homo) {
      CTP_LOG(ctp::logINFO, *_pLog)
          << (format("  HOMO  = %1$4d DFT = %2$+1.4f VXC = %3$+1.4f S-X = "
                     "%4$+1.4f S-C = %5$+1.4f GWA = %6$+1.4f") %
              (_i + _qpmin + 1) % _dft_energies(_i + _qpmin) % _vxc(_i, _i) %
              _sigma_x(_i, _i) % _sigma_c(_i, _i) % _qp_energies(_i + _qpmin))
                 .str()
          << flush;
    } else if ((_i + _qpmin) == _homo + 1) {
      CTP_LOG(ctp::logINFO, *_pLog)
          << (format("  LUMO  = %1$4d DFT = %2$+1.4f VXC = %3$+1.4f S-X = "
                     "%4$+1.4f S-C = %5$+1.4f GWA = %6$+1.4f") %
              (_i + _qpmin + 1) % _dft_energies(_i + _qpmin) % _vxc(_i, _i) %
              _sigma_x(_i, _i) % _sigma_c(_i, _i) % _qp_energies(_i + _qpmin))
                 .str()
          << flush;

    } else {
      CTP_LOG(ctp::logINFO, *_pLog)
          << (format("  Level = %1$4d DFT = %2$+1.4f VXC = %3$+1.4f S-X = "
                     "%4$+1.4f S-C = %5$+1.4f GWA = %6$+1.4f") %
              (_i + _qpmin + 1) % _dft_energies(_i + _qpmin) % _vxc(_i, _i) %
              _sigma_x(_i, _i) % _sigma_c(_i, _i) % _qp_energies(_i + _qpmin))
                 .str()
          << flush;
    }
  }

  // store perturbative QP energy data in orbitals object (DFT, S_x,S_c, V_xc,
  // E_qp)
  if (_store_qp_pert) {
    ub::matrix<double> &_qp_energies_store = _orbitals->QPpertEnergies();
    _qp_energies_store.resize(_qptotal, 5);
    for (unsigned _i = 0; _i < _qptotal; _i++) {
      _qp_energies_store(_i, 0) = _dft_energies(_i + _qpmin);
      _qp_energies_store(_i, 1) = _sigma_x(_i, _i);
      _qp_energies_store(_i, 2) = _sigma_c(_i, _i);
      _qp_energies_store(_i, 3) = _vxc(_i, _i);
      _qp_energies_store(_i, 4) = _qp_energies(_i + _qpmin);
    }
  }

  // constructing full quasiparticle Hamiltonian and diagonalize, if requested
  if (_do_qp_diag || _do_bse_singlets || _do_bse_triplets) {
    FullQPHamiltonian();
    if (_do_qp_diag) {
      CTP_LOG(ctp::logDEBUG, *_pLog)
          << ctp::TimeStamp() << " Full quasiparticle Hamiltonian  " << flush;
      CTP_LOG(ctp::logINFO, *_pLog)
          << (format("  ====== Diagonalized quasiparticle energies (Hartree) "
                     "====== "))
                 .str()
          << flush;
      for (unsigned _i = 0; _i < _qptotal; _i++) {
        if ((_i + _qpmin) == _homo) {
          CTP_LOG(ctp::logINFO, *_pLog)
              << (format("  HOMO  = %1$4d PQP = %2$+1.4f DQP = %3$+1.4f ") %
                  (_i + _qpmin + 1) % _qp_energies(_i + _qpmin) %
                  _qp_diag_energies(_i))
                     .str()
              << flush;
        } else if ((_i + _qpmin) == _homo + 1) {
          CTP_LOG(ctp::logINFO, *_pLog)
              << (format("  LUMO  = %1$4d PQP = %2$+1.4f DQP = %3$+1.4f ") %
                  (_i + _qpmin + 1) % _qp_energies(_i + _qpmin) %
                  _qp_diag_energies(_i))
                     .str()
              << flush;

        } else {
          CTP_LOG(ctp::logINFO, *_pLog)
              << (format("  Level = %1$4d PQP = %2$+1.4f DQP = %3$+1.4f ") %
                  (_i + _qpmin + 1) % _qp_energies(_i + _qpmin) %
                  _qp_diag_energies(_i))
                     .str()
              << flush;
        }
      }

      // free memory

      if (!_store_qp_diag) {
        _qp_diag_coefficients.resize(0, 0);
        _qp_diag_energies.resize(0);
      }
    }  // _do_qp_diag
  }    // constructing full quasiparticle Hamiltonian

  // proceed only if BSE requested
  if (_do_bse_singlets || _do_bse_triplets) {

    // calculate direct part of eh interaction, needed for singlets and triplets
    BSE_d_setup(_Mmn);
    // add qp part to Eh_d
    BSE_Add_qp2H(_eh_d);
    // Only if verbose do we use this really
    if (votca::tools::globals::verbose) {
      BSE_qp_setup();
    }

    CTP_LOG(ctp::logDEBUG, *_pLog)
        << ctp::TimeStamp() << " Direct part of e-h interaction " << flush;

    if (_do_full_BSE) {
      BSE_d2_setup(_Mmn);
      CTP_LOG(ctp::logDEBUG, *_pLog) << ctp::TimeStamp()
                                     << " Direct part of e-h interaction RARC "
                                     << flush;
    }

    if (_do_bse_triplets && _do_bse_diag) {
      BSE_solve_triplets();
      CTP_LOG(ctp::logDEBUG, *_pLog) << ctp::TimeStamp()
                                     << " Solved BSE for triplets " << flush;
      // analyze and report results
      BSE_analyze_triplets();

    }  // do_triplets

    // constructing electron-hole interaction for BSE
    if (_do_bse_singlets) {
      // calculate exchange part of eh interaction, only needed for singlets
      BSE_x_setup(_Mmn);
      CTP_LOG(ctp::logDEBUG, *_pLog)
          << ctp::TimeStamp() << " Exchange part of e-h interaction " << flush;
    }

    if (_do_bse_singlets && _do_bse_diag) {

      if (_do_full_BSE) {
        BSE_solve_singlets_BTDA();
        CTP_LOG(ctp::logDEBUG, *_pLog)
            << ctp::TimeStamp() << " Solved full BSE for singlets " << flush;
        BSE_analyze_singlets_BTDA();
      } else {
        BSE_solve_singlets();
        CTP_LOG(ctp::logDEBUG, *_pLog) << ctp::TimeStamp()
                                       << " Solved BSE for singlets " << flush;
        BSE_analyze_singlets();
      }

      if (!_store_eh_interaction) {
        _eh_d.resize(0, 0);
        _eh_x.resize(0, 0);
      }
    }
  }
  CTP_LOG(ctp::logDEBUG, *_pLog) << ctp::TimeStamp()
                                 << " GWBSE calculation finished " << flush;
  return true;
}
}
};


//=======================================================================
// Copyright 1997, 1998, 1999, 2000 University of Notre Dame.
// Copyright 2009 Trustees of Indiana University.
// Authors: Andrew Lumsdaine, Lie-Quan Lee, Jeremy G. Siek, Michael Hansen
//
// Distributed under the Boost Software License, Version 1.0. (See
// accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
//=======================================================================

#ifndef BOOST_GRAPH_DIJKSTRA_NO_COLOR_MAP_HPP
#define BOOST_GRAPH_DIJKSTRA_NO_COLOR_MAP_HPP

#include <boost/pending/indirect_cmp.hpp>
#include <boost/graph/relax.hpp>
#include <boost/pending/relaxed_heap.hpp>
#include <boost/graph/detail/d_ary_heap.hpp>
#include <boost/graph/dijkstra_shortest_paths.hpp>
#include <boost/graph/iteration_macros.hpp>
#include <boost/array.hpp>

namespace boost {
/*
namespace detail{
    //template <class Graph, class IndexMap, class Value>
    //struct my_vertex_property_map_generator_helper {};
    template <class Graph, class IndexMap, class Value>
    struct my_vertex_property_map_generator_helper{
        typedef boost::iterator_property_map<Value*, IndexMap> type;
        static type build(const Graph& g, const IndexMap& index, boost::array<Value>& array_holder) {
            // array_holder.reset(new Value[num_vertices(g)]);
            std::fill(array_holder.get(), array_holder.get() + num_vertices(g), Value());
            return make_iterator_property_map(array_holder.get(), index);
        }
    };
}*/

// template <class Graph, class IndexMap, class Value>
// static boost::iterator_property_map<Value*, IndexMap> build(const Graph& g, const IndexMap& index, boost::array<Value>& array_holder){
//     array_holder.reset(new Value[num_vertices(g)]);
//     std::fill(array_holder.get(), array_holder.get() + num_vertices(g), Value());
//     // Vertex index to fetch the pointer address at array holder.
//     // It maps the indexmap (or index vector) to the values stored in the iteratior, which is the array holder.
//     return make_iterator_property_map(array_holder.get(), index);
// };

/**
 *  This is an optimized driving distance function based on dijkstra_shortest_paths_no_color_map_no_init
 *  in BGL.
 *
 *  Input:
 *      graph,...,distance_zero: graph definitions
 *      delta: upper bound distance for early stopping
 *      Note: the predecessor map must be initialized with the vertex at each position
 *      and the distance map initialized with infinity
 *
 *  Returns:
 *      predecessor_map: similar to Dijkstra
 *      distance_map: similar to Dijkstra
 *      examined_vertex_map: examined vertexes, that will be used
 *      to clean the predecessor_map and distance_map in repeated queries.
 *
 */
template <typename Graph,
          typename PredecessorMap, typename DistanceMap,
          typename WeightMap, typename VertexIndexMap,
          typename DistanceCompare, typename DistanceWeightCombine,
          typename DistanceInfinity, typename DistanceZero>
void dijkstra_shortest_paths_upperbound
(const Graph& graph,
 typename graph_traits<Graph>::vertex_descriptor start_vertex,
 PredecessorMap predecessor_map,
 DistanceMap distance_map,
 WeightMap weight_map,
 VertexIndexMap index_map,
 DistanceCompare distance_compare,
 DistanceWeightCombine distance_weight_combine,
 DistanceInfinity distance_infinity,
 DistanceZero distance_zero, double distance_goal,
 std::vector<typename Graph::vertex_descriptor> &nodes_within_goal,
 std::vector<typename Graph::vertex_descriptor> &examined_vertices)
{
    typedef typename graph_traits<Graph>::vertex_descriptor Vertex;
    typedef typename property_traits<DistanceMap>::value_type Distance;

    typedef indirect_cmp<DistanceMap, DistanceCompare> DistanceIndirectCompare;
    DistanceIndirectCompare
    distance_indirect_compare(distance_map, distance_compare);

    // Default - use d-ary heap (d = 4)
    // typedef
    // detail::vertex_property_map_generator<Graph, VertexIndexMap, std::size_t>
    // IndexInHeapMapHelper;
    // typedef typename IndexInHeapMapHelper::type IndexInHeapMap;
    typedef boost::iterator_property_map<std::size_t *,VertexIndexMap> IndexInHeapMap;
    typedef
    d_ary_heap_indirect<Vertex, 4, IndexInHeapMap, DistanceMap, DistanceCompare>
    VertexQueue;
    // 80%
    // boost::scoped_array<std::size_t> index_in_heap_map_holder;
    // // IndexInHeapMap is actually a boost::iterator_property_map<Value*, IndexMap>
    // IndexInHeapMap index_in_heap =
    //     IndexInHeapMapHelper::build(graph, index_map,
    //                                 index_in_heap_map_holder);
    // typedef boost::array<std::size_t, std::size_t num_vertices(g)> array;
    // *dheap_on_stack = new boost::array(std::size_t,num_vertices(g));
    std::size_t * dheap_on_stack = new std::size_t[num_vertices(graph)];
    boost::iterator_property_map<std::size_t *,VertexIndexMap> index_in_heap_map =
        make_iterator_property_map(dheap_on_stack, index_map);

    // The first two arguments in the template of VertexQueue are already defined
    VertexQueue vertex_queue(distance_map, index_in_heap_map, distance_compare);

    double m_distance_goal; //Delta

    // Add vertex to the queue
    vertex_queue.push(start_vertex);

    // Starting vertex will always be the first discovered vertex
    // visitor.discover_vertex(start_vertex, graph);

    while (!vertex_queue.empty()) {
        Vertex min_vertex = vertex_queue.top();
        vertex_queue.pop();
        // visitor.examine_vertex(min_vertex, graph);
        nodes_within_goal.push_back(min_vertex);
        if (distance_map[min_vertex] > distance_goal) {
            nodes_within_goal.pop_back();
            delete[] dheap_on_stack;
            return;
        }

        // Check if any other vertices can be reached
        Distance min_vertex_distance = get(distance_map, min_vertex);

        if (!distance_compare(min_vertex_distance, distance_infinity)) {
            // This is the minimum vertex, so all other vertices are unreachable
            delete[] dheap_on_stack;
            return;
        }

        // Examine neighbors of min_vertex
        BGL_FORALL_OUTEDGES_T(min_vertex, current_edge, graph, Graph) {
            // visitor.examine_edge(current_edge, graph);

            // Check if the edge has a negative weight
            if (distance_compare(get(weight_map, current_edge), distance_zero)) {
                boost::throw_exception(negative_edge());
            }

            // Extract the neighboring vertex and get its distance
            Vertex neighbor_vertex = target(current_edge, graph);
            Distance neighbor_vertex_distance = get(distance_map, neighbor_vertex);
            bool is_neighbor_undiscovered =
                !distance_compare(neighbor_vertex_distance, distance_infinity);

            // Attempt to relax the edge
            bool was_edge_relaxed = relax(current_edge, graph, weight_map,
                                          predecessor_map, distance_map,
                                          distance_weight_combine, distance_compare);

            if (was_edge_relaxed) {
                // visitor.edge_relaxed(current_edge, graph);
                examined_vertices.push_back(neighbor_vertex);
                if (is_neighbor_undiscovered) {
                    // visitor.discover_vertex(neighbor_vertex, graph);
                    vertex_queue.push(neighbor_vertex);
                } else {
                    vertex_queue.update(neighbor_vertex);
                }
            } else {
                // visitor.edge_not_relaxed(current_edge, graph);
            }

        } // end out edge iteration

        // visitor.finish_vertex(min_vertex, graph);
    } // end while queue not empty
    delete[] dheap_on_stack;
} // dijkstra_shortest_paths_upperbound

} // namespace boost

#endif // BOOST_GRAPH_DIJKSTRA_NO_COLOR_MAP_HPP