/*
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation; either version 3 of the License, or
  * (at your option) any later version.
  *
  * Written (W) 2013 Nicolo' Navarin
  */

#include "DDK_FeatureGenerator.h"
 //#include <shogun/features/Labels.h>
#include <climits>
#include <deque>
//#include <boost/container/stable_vector.hpp>
//#include <boost/random/uniform_int.hpp>
//#include <boost/random/random_number_generator.hpp>
//#include <boost/random.hpp>
//#include <boost/generator_iterator.hpp>

//#include <boost/random/variate_generator.hpp>
//#include <boost/generator_iterator.hpp>

// #include <shogun/mathematics/Math.h>

 //TODO eliminare
 double lossy_e=0.001;

 DDkernel_FeatureGenerator::DDkernel_FeatureGenerator()
 : learn_rate(0.1), max_iter(1000), NSPDK_FeatureGenerator("DDK")
 {
 }
/*
 DDkernel_FeatureGenerator::DDkernel_FeatureGenerator(CGraphFeatures* traindat, CLabels* trainlab)
 :  learn_rate(.1), max_iter(1000), NSPDK_FeatureGenerator("DDK")
 {
     //TODO inizializzazione
   //  set_features(traindat);
   //  set_labels(trainlab);
 }*/

 DDkernel_FeatureGenerator::~DDkernel_FeatureGenerator()
 {
	 NSPDK_FeatureGenerator("ddk");
 }
 DDkernel_FeatureGenerator::DDkernel_FeatureGenerator(const std::string& id) :
 		NSPDK_FeatureGenerator(id) {
		new_flag(&mTreeLambda, "mTreeLambda", "(double)\n tree_lambda parameter for DDK kernel");
 }
 void DDkernel_FeatureGenerator::OutputParameters(ostream& out) const {
 	out << "Radius: " << mRadius << endl;
 	out << "Radius2: " << mRadiusTwo << endl;

 	out << "Distance: " << mDistance << endl;
 	out << "Match_Type: " << mMatchType << endl;
 	out << "Hash_Bit_Size: " << mHashBitSize << endl;
 	out << "Hash_Bit_mask: " << mHashBitMask << endl;
 	out << "Min_Kernel: " << mMinKernel << endl;
 	out << "Normalization: " << mNormalization << endl;
 	out << "Vertex Degree Threshold: " << mVertexDegreeThreshold << endl;
 	out << "Debug_Verbosity: " << mDebugVerbosity << endl;
 	out << "Tree_lambda: " << mTreeLambda << endl;

 }

 void DDkernel_FeatureGenerator::generate_feature_vector(const GraphClass& aG, SVector& x, const vector<unsigned>& aFirstEndpointList) {
 }
	 /*
	 // OutputParameters(cout);
	 vector<GraphClass> graphs;
	 graphs.push_back(aG);
	 CGraphFeatures* data = new CGraphFeatures(graphs);
	// set_lambda(mTreeLambda);
	 //TODO dag_h is now set as the radius parameter r
	// cout<<"dag_h="<<mRadius<<endl;
	 svectorFeatures(data,x,mRadius+1 ,10000000);
	// data->;
	 graphs.clear();
	 delete data;

 }*/

 void DDkernel_FeatureGenerator::generate_vertex_feature_vector(const GraphClass& aG, vector<SVector>& x_list, const vector<unsigned>& aFirstEndpointList) {

 }

 void DDkernel_FeatureGenerator::GenerateAbstractVertexFeatures(unsigned aSrcID, const GraphClass& aG, unsigned aRadius, unsigned aDistance, SVector& x) {

 }

 //---------------------------------------------------------------------------------

//TODO output on svector
 /*
 void DDkernel_FeatureGenerator::svectorFeatures(CGraphFeatures* data,SVector& out_vector, int dag_h, int max_features = 10000000){
      //  vector<mymapForHash*> HashMaps = vector<mymapForHash*>(lhs->get_num_vectors());
     // for di tutti i grafi cosi creo i bigDAG.

 long time = 1.0 / lossy_e;

 vector<double> BER_total;

 OnlineFeatures onlinefeatures(max_features);
      //  OnlineFeatures onlinefeatures_old;

         int64_t numero = 0;

         int training =   data->get_num_vectors();
         string a;
         ofstream output(a.c_str());

         for (int idx_a=0;idx_a<training;idx_a++){

         Graph* a = new Graph() ;
       int32_t size_a = 0;
       ((CGraphFeatures*) data)->get_feature_vector(a,idx_a);

    //   	 try{
       OnlineHashBig2DAGLossy* test = onlinefeatures.generateFeatures(a, dag_h,idx_a,data->get_num_vectors());

       OnlineHashBig2DAGOrdered ordered_map;
    //TODO add features to a OnlineHashBig2DAGOrdered map
       // cout<<"test: "<<test->size();
        BOOST_FOREACH(OnlineHashBig2DAGLossy::value_type local_row, *test) {
           ordered_map[local_row.second.get<2>()]=local_row.second;
       }

    //   cout<<"ordered_map: "<<ordered_map.size();
        //normalize the feature vector

  //      }
        SVector z;

    unsigned int last = 0;
         BOOST_FOREACH(OnlineHashBig2DAGOrdered::value_type i_row,  ordered_map){
           //  cout<<"lambda "<<lambda<<endl;
				z.set((unsigned int)i_row.second.get<2>() % (unsigned int)pow(2,mHashBitSize), pow((double)mTreeLambda, (double)i_row.second.get<0>() / 2.0)* (double)i_row.second.get<3>());

                 ASSERT((unsigned int)i_row.second.get<2>() >= last);
                 last = (unsigned int)i_row.second.get<2>();
         }
      //   if (mNormalization){
        // 				z.normalize();}

		out_vector.add(z);
		//if (mNormalization){
		//		out_vector.normalize();}

     delete a;
            test->clear();
            ordered_map.clear();
         delete test;

     //  	 }catch(shogun::ShogunException e){
     //  		 cout<<e.get_exception_string();

      // 	 }

       }
     //   cout<<endl;
        output.close();

     //  cout<<"FEATURES has been printed"<<endl;

     }

bool GraphSortCriterion (const tuple<double,Graph,long> p1, const tuple<double,Graph,long> p2)
{
    /// a graph is less than another graph

    return p1.get<0>()<p2.get<0>();
}

bool FeaturesSortCriterion (const tuple<double,OnlineHashBig2DAG,long> p1, const tuple<double,OnlineHashBig2DAG,long> p2)
{
    // a graph is less than another graph

    return p1.get<0>()<p2.get<0>();
}

*/

void DDkernel_FeatureGeneratorNew::generate_feature_vector(const GraphClass& aG, SVector& x, const vector<unsigned>& aFirstEndpointList) {
//	 OnlineFeaturesEDeN of;
//	cout<<"Lambda="<<mTreeLambda<<endl;
//	cout<<"Radius="<<mRadius<<endl;

	 x = generate_feature_vector_core(aG, mRadius);
	 if (mNormalization){
	 				x.normalize();
	 }

}

SVector&  DDkernel_FeatureGeneratorNew::generate_feature_vector_core(const GraphClass& aG , int dag_h){
	//cout<<mTreeLambda<<endl;
	//cout<<"start generation of feature vector.."<<endl;
			SVector* x = new SVector();

		//	cout<<"n of nodes "<<aG.VertexSize()<<endl;
		    for (unsigned i=0; i< aG.VertexSize(); i++){
		    	map<pair<unsigned, unsigned>, int> oSrcDestMaptoDistance;
		    	map<unsigned, vector<unsigned> > MapNodetoParents;
		    	map<unsigned, vector<unsigned> > MapNodetoChildren;

		    	vector<unsigned> TopologicalSort;
		    	int maxLevel = 0;
		    	//aG. // codice calcolo features here
		      	aG.SingleVertexBoundedBreadthFirstVisitTree(i,dag_h, oSrcDestMaptoDistance,  MapNodetoParents,MapNodetoChildren, TopologicalSort, maxLevel);
		//    cout<<"MaxLevel: "<<maxLevel<<endl;
		      	// 	cout<<"Breadth first visit ended.."<<endl;
		      	// I have the DAG for the vertex i. Need topological sort.
		      	//map every node to him productions
		    	map<unsigned, vector<unsigned> > MapNodeToProductionsID;
		    	map<unsigned, vector<int> > MapNodetoFrequencies;

		    	map<unsigned, int > MapProductionIDtoSize;

		  //  	cout<<"topological sort size "<<TopologicalSort.size()<<endl;

		      	//maybe I meed the maximum depth (distance)
		      	for (int ii=TopologicalSort.size()-1; ii>=0; ii--){
		      		// cout<<ii<<endl;
		      		//calculate child size
			    	vector<unsigned> vertex_adjacency_list = MapNodetoChildren[TopologicalSort[ii]];
		      		int max_child_heigth=0;
		        	for (unsigned j = 0; j < vertex_adjacency_list.size(); ++j) {
		    			int child_vertex_id= vertex_adjacency_list[j];
		    			int child_heigth=MapNodeToProductionsID[child_vertex_id].size();
		    			if(child_heigth > max_child_heigth){
		    				max_child_heigth = child_heigth;
		    			}
		        	}

		      		for (int depth=0; depth<= max_child_heigth; ++depth){ //era maxLevel
		      			unsigned hash_subgraph_code = 1;
		      		//depth 0; only label
		      			if(depth==0){
		      		//		cout<<"depth=0"<<endl;
		      				unsigned enc= Radius0RootedGraphCanonicalFormEncoding(TopologicalSort[ii], aG);
		      		MapNodeToProductionsID[TopologicalSort[ii]].push_back(enc);
		      		// scorro i figli e creo etichetta tramite funzione di hash
		      		// depth > 0; the length of the children ID list
		      		//imposta valore feature a 1 (TODO lambda)
		      		//							level(o-mRadius)
			    	int frequency = 0;
			    	if(max_child_heigth==0){
		      		//leaf node
			    	frequency=  maxLevel - oSrcDestMaptoDistance[make_pair(i,TopologicalSort[ii])];
			    	}
			    	SVector z;
		      		z.set(enc, (double)(frequency+1.0) * sqrt(mTreeLambda));
			    //	cout<<"Feature: "<<aG.GetVertexLabelConcatenated(TopologicalSort[ii])<<" freq:"<<(double)(frequency+1.0) * sqrt(mTreeLambda)<<" code: "<<enc<<endl;
			    	x->add(z);
			    	MapNodetoFrequencies[TopologicalSort[ii]].push_back(frequency);
		      		MapProductionIDtoSize[enc]= 1; //sqrt(mTreeLambda);
		      			}
		      			else{
		      	//			cout<<"depth="<<depth<<endl;

		      			int size = 0;
		      		string encoding;
		      		encoding = aG.GetVertexLabelConcatenated(TopologicalSort[ii]);
			    	vector<unsigned> vertex_adjacency_list = MapNodetoChildren[TopologicalSort[ii]];
		    		vector<unsigned>  vertex_label_id_list;
		    		int min_freq_children =INT_MAX;
			    	for (unsigned j = 0; j < vertex_adjacency_list.size(); ++j) {

			    		unsigned child_vertex_id = vertex_adjacency_list[j];
			    		int size_map =  MapNodeToProductionsID[child_vertex_id].size();//min (depth-1) e dim
			    		unsigned child_hash = MapNodeToProductionsID[child_vertex_id][min(size_map,depth)-1];// id hash del figlio;
			    		int freq_child = MapNodetoFrequencies[child_vertex_id][min(size_map,depth)-1];
			    		if (freq_child < min_freq_children){
			    			min_freq_children = freq_child;
			    		}
			    		vertex_label_id_list.push_back(child_hash);
			    		size += MapProductionIDtoSize[child_hash];
			    	}

			    	// vertex_label_id_list contiene gli hash delle prod dei figli. Va ordinato e usato x generare l hash attuale
			    	sort(vertex_label_id_list.begin(), vertex_label_id_list.end());

			    	if (vertex_label_id_list.size() > 0){
			    		std::stringstream ss;
			    		ss <<":"<<vertex_label_id_list[0];
			    		encoding += ss.str() ;
			    	}//originale senza .
			    	for (unsigned i = 1; i < vertex_label_id_list.size(); i++){
			    		std::stringstream ss;
			    		ss << vertex_label_id_list[i];
			    		encoding += "." + ss.str() ;
			    	}
			    //DEBUG	cout<<"encoding "<<encoding<<endl;
			    	//calculate frequency

			    	hash_subgraph_code = HashFunc(encoding);
			    	MapNodeToProductionsID[TopologicalSort[ii]].push_back(hash_subgraph_code);
			    	size += 1; // current node
			    	MapProductionIDtoSize[hash_subgraph_code]= size;
			    	int frequency = min_freq_children;
			    	MapNodetoFrequencies[TopologicalSort[ii]].push_back(frequency);

			    	SVector z;
			   // 	cout<<"Feature: "<<encoding<<" freq:"<<(double)(frequency+1.0) * sqrt(pow(mTreeLambda,size))<<" code: "<<hash_subgraph_code<<endl;
			    	z.set(hash_subgraph_code,(double)(frequency+1.0) * sqrt(pow(mTreeLambda,size)));
			    	x->add(z);

			    //	cout<<"weight "<<sqrt(pow(mTreeLambda,size))<<endl;

		      			}

		      		}
		      	}

		    }
		    return *x;

	 }

DDkernel_FeatureGeneratorNew::DDkernel_FeatureGeneratorNew(const std::string& id) :
		DDkernel_FeatureGenerator(id) {
}

NSDDkernel_FeatureGenerator::NSDDkernel_FeatureGenerator(const std::string& id) :
		DDkernel_FeatureGeneratorNew(id) {
	mRadiusTwo = 0;
			new_flag(&mRadiusTwo, "mRadiusTwo", "(double)\n second radius parameter for NSDDK kernel");

}

void NSDDkernel_FeatureGenerator::generate_feature_vector(const GraphClass& aG, SVector& x, const vector<unsigned>& aFirstEndpointList){
//cout<<mRadiusTwo<<endl;
	 x = generate_feature_vector_core(aG, mRadius);
	//x = generate_feature_vector_core_DISTANCE(aG, mRadius); //TEST distanze

	// if (mNormalization){
	 //				x.normalize();
	// }
}

SVector&   NSDDkernel_FeatureGenerator::generate_feature_vector_core(const GraphClass& aG , int dag_h){
   // test rNSPDK 1/2 d=1/2
	GraphClass ag2 = aG;
	SVector* out = new SVector();
//	cout<<"Generate feature vector core"<<endl;
	for (unsigned i=0; i< aG.VertexSize(); i++){
	//	cout<<"i:"<<i<<endl;

		//SVector DD= generate_vertex_feature_vector(i,ag2 ,dag_h);
		vector<SVector> features; //(dag_h+1);
		SVector DD= generate_vertex_feature_vector_DIVIDED(i,ag2 ,mRadius,features);
	//	cout<<"got DD vector"<<endl;
		//cout<<features.size();
	//	out->add(DD);
		//NSPDK
	//	cout<<"start generation of NSPKD feature vector.. mDistance"<<mDistance<<endl;

		//NSPDK_FeatureGenerator::GenerateVertexFeatures(i,aG,NSPDK);
	//	NSPDK_FeatureGenerator::generate_feature_vector(aG,NSPDK);
		vector<unsigned> first_endpoint_list= vector<unsigned>();
		first_endpoint_list.push_back(i);
			//GetFirstEndpoints(aG, first_endpoint_list);

		int horizon=max(mDistance, mRadiusTwo); //   max(mDistance, mRadius);
			aG.ComputePairwiseDistanceInformation(horizon, first_endpoint_list);
			if (aG.Check() == false)
				throw logic_error("ERROR10: the graph data structure is not sound and it has not passed the checkup procedure"); //check graph data structure soundness

			InitFeatureCache(aG, mRadiusTwo ); //mRadius
			for (unsigned r = 0; r <= mRadius ; r++) { //max(1.0,mRadius / 2.0)
				for (unsigned r2 = 0; r2 <= mRadiusTwo ; r2++) {
				for (unsigned d = 0; d <= mDistance; d++) { // era 0
					SVector z;
					for (unsigned i = 0; i < first_endpoint_list.size(); i++) {
						unsigned src_id = first_endpoint_list[i];
						if (aG.GetVertexViewPoint(src_id) && aG.GetVertexKernelPoint(src_id) && aG.GetVertexAlive(src_id)) { //proceed to extract features only if the *src* vertex is a kernel point and is alive
							SVector zv;

							vector<unsigned> endpoint_list(4);
							endpoint_list[0] = r2;
							endpoint_list[1] = d;

						//	unsigned src_code = GenerateVertexNeighbourhoodHashCode(aSrcID, aG, aRadius);
				//	TODO	for(unsigned src_code in DD)

							vector<unsigned> dest_id_list = aG.GetFixedDistanceVertexIDList(i, d);
							for (unsigned dest_j = 0; dest_j < dest_id_list.size(); dest_j++) {
								unsigned dest_id = dest_id_list[dest_j];
								unsigned dest_code = 0;
								if (aG.GetVertexKernelPoint(dest_id) && aG.GetVertexAlive(dest_id)) { //proceed to extract features only if the *dest* vertex is a kernel point and is alive
									dest_code = GenerateVertexNeighbourhoodHashCode(dest_id, aG, r2); //
									// first is the DDK feature, second NSPDK feature
						//  for(const SVector::Pair *p = DD; p->i>=0; p++) {
						 for(const SVector::Pair *p = features[r]; p->i>=0; p++) {

									endpoint_list[2] = p->i;
										//TODO scorrere le feature di DD
										endpoint_list[3] = dest_code;
									unsigned code = HashFunc(endpoint_list, mHashBitMask);

									SVector z;
									z.set(code, p->v);
									zv.add(z);
								}
								}
							}

							z.add(zv);
						}
						}

					if (mNormalization)
						z.normalize();
				out->add(z);
				} //d
			} //r
			} //r2

	//	cout<<"features size DD:"<<DD.sparse_size()<<" out:"<<out->sparse_size()<<endl;

}
	return *out;
}

SVector&   NSDDkernel_FeatureGenerator::generate_feature_vector_core_DISTANCE(const GraphClass& aG , int dag_h){
   // test rNSPDK 1/2 d=1/2
	GraphClass ag2 = aG;
	SVector* out = new SVector();
//	cout<<"Generate feature vector core"<<endl;
	for (unsigned i=0; i< aG.VertexSize(); i++){
	//	cout<<"i:"<<i<<endl;

		//SVector DD= generate_vertex_feature_vector(i,ag2 ,dag_h);
		vector< vector< SVector> > features; //(dag_h+1);
		SVector DD= generate_vertex_feature_vector_DIVIDED_DISTANCE(i,ag2 ,dag_h,features);
		//cout<<features.size();
	//	out->add(DD);
		//NSPDK
	//	cout<<"start generation of NSPKD feature vector.. mDistance"<<mDistance<<endl;

		//NSPDK_FeatureGenerator::GenerateVertexFeatures(i,aG,NSPDK);
	//	NSPDK_FeatureGenerator::generate_feature_vector(aG,NSPDK);
		vector<unsigned> first_endpoint_list= vector<unsigned>();
		first_endpoint_list.push_back(i);
			//GetFirstEndpoints(aG, first_endpoint_list);

		int horizon=max(mDistance, mRadiusTwo); //   max(mDistance, mRadius);
			aG.ComputePairwiseDistanceInformation(horizon, first_endpoint_list);
			if (aG.Check() == false)
				throw logic_error("ERROR10: the graph data structure is not sound and it has not passed the checkup procedure"); //check graph data structure soundness

			InitFeatureCache(aG, mRadiusTwo ); //mRadius
			for (unsigned r = 0; r <= mRadius ; r++) { //max(1.0,mRadius / 2.0)
				for (unsigned r2 = 0; r2 <= mRadiusTwo ; r2++) {
				for (unsigned d = 0; d <= mDistance; d++) { // era 0
					SVector z;
					for (unsigned i = 0; i < first_endpoint_list.size(); i++) {
						unsigned src_id = first_endpoint_list[i];
						if (aG.GetVertexViewPoint(src_id) && aG.GetVertexKernelPoint(src_id) && aG.GetVertexAlive(src_id)) { //proceed to extract features only if the *src* vertex is a kernel point and is alive
							SVector zv;

							vector<unsigned> endpoint_list(5);
							endpoint_list[0] = r2;
							endpoint_list[1] = d;

						//	unsigned src_code = GenerateVertexNeighbourhoodHashCode(aSrcID, aG, aRadius);
				//	TODO	for(unsigned src_code in DD)

							vector<unsigned> dest_id_list = aG.GetFixedDistanceVertexIDList(i, d);
							for (unsigned dest_j = 0; dest_j < dest_id_list.size(); dest_j++) {
								unsigned dest_id = dest_id_list[dest_j];
								unsigned dest_code = 0;
								if (aG.GetVertexKernelPoint(dest_id) && aG.GetVertexAlive(dest_id)) { //proceed to extract features only if the *dest* vertex is a kernel point and is alive
									dest_code = GenerateVertexNeighbourhoodHashCode(dest_id, aG, r2); //
									// first is the DDK feature, second NSPDK feature
						//  for(const SVector::Pair *p = DD; p->i>=0; p++) {
									for(int distance=0; distance<features[r].size(); distance++){ //dag_h+1
										endpoint_list[4]=distance; //from root in DD
						 for(const SVector::Pair *p = features[r][distance]; p->i>=0; p++) {

									endpoint_list[2] = p->i;
										//TODO scorrere le feature di DD
										endpoint_list[3] = dest_code;
										//TEST output
								//		cout<<"R "<<endpoint_list[0]<<" d "<<endpoint_list[1]<<" DD "<<endpoint_list[2]<<" NSPDK "<<endpoint_list[3]<<" dRoot "<<endpoint_list[4]<<endl;
									unsigned code = HashFunc(endpoint_list, mHashBitMask);

									SVector z;
									z.set(code, p->v);
									zv.add(z);
								}
								}
								}
							}

							z.add(zv);
						}
						}

					if (mNormalization)
						z.normalize();
				out->add(z);
				} //d
			} //r
			}//r2

	//	cout<<"features size DD:"<<DD.sparse_size()<<" out:"<<out->sparse_size()<<endl;

}
	return *out;
}

SVector&   NSDDkernel_FeatureGenerator::generate_vertex_feature_vector(unsigned i, const GraphClass& aG , int dag_h){
	//cout<<"start generation of feature vector.."<<endl;
			SVector* x = new SVector();

		//	cout<<"n of nodes "<<aG.VertexSize()<<endl;
		    assert(i< aG.VertexSize());
		    	map<pair<unsigned, unsigned>, int> oSrcDestMaptoDistance;
		    	map<unsigned, vector<unsigned> > MapNodetoParents;
		    	map<unsigned, vector<unsigned> > MapNodetoChildren;

		    	vector<unsigned> TopologicalSort;
		    	int maxLevel = 0;
		    	//aG. // codice calcolo features here
		      	aG.SingleVertexBoundedBreadthFirstVisitTree(i,dag_h, oSrcDestMaptoDistance,  MapNodetoParents,MapNodetoChildren, TopologicalSort, maxLevel);
		//    cout<<"MaxLevel: "<<maxLevel<<endl;
		      	// 	cout<<"Breadth first visit ended.."<<endl;
		      	// I have the DAG for the vertex i. Need topological sort.
		      	//map every node to him productions
		    	map<unsigned, vector<unsigned> > MapNodeToProductionsID;
		    	map<unsigned, vector<int> > MapNodetoFrequencies;

		    	map<unsigned, int > MapProductionIDtoSize;

		  //  	cout<<"topological sort size "<<TopologicalSort.size()<<endl;

		      	//maybe I meed the maximum depth (distance)
		      	for (int ii=TopologicalSort.size()-1; ii>=0; ii--){
		      		// cout<<ii<<endl;
		      		//calculate child size
			    	vector<unsigned> vertex_adjacency_list = MapNodetoChildren[TopologicalSort[ii]];
		      		int max_child_heigth=0;
		        	for (unsigned j = 0; j < vertex_adjacency_list.size(); ++j) {
		    			int child_vertex_id= vertex_adjacency_list[j];
		    			int child_heigth=MapNodeToProductionsID[child_vertex_id].size();
		    			if(child_heigth > max_child_heigth){
		    				max_child_heigth = child_heigth;
		    			}
		        	}

		      		for (int depth=0; depth<= max_child_heigth; ++depth){ //era maxLevel
		      			unsigned hash_subgraph_code = 1;
		      		//depth 0; only label
		      			if(depth==0){
		      		//		cout<<"depth=0"<<endl;
		      				unsigned enc= Radius0RootedGraphCanonicalFormEncoding(TopologicalSort[ii], aG);
		      		MapNodeToProductionsID[TopologicalSort[ii]].push_back(enc);
		      		// scorro i figli e creo etichetta tramite funzione di hash
		      		// depth > 0; the length of the children ID list
		      		//imposta valore feature a 1 (TODO lambda)
		      		//							level(o-mRadius)
			    	int frequency = 0;
			    	if(max_child_heigth==0){
		      		//leaf node
			    	frequency=  maxLevel - oSrcDestMaptoDistance[make_pair(i,TopologicalSort[ii])];
			    	}
			    	SVector z;
		      		z.set(enc, (double)(frequency+1.0) * sqrt(mTreeLambda));
			    //	cout<<"Feature: "<<aG.GetVertexLabelConcatenated(TopologicalSort[ii])<<" freq:"<<(double)(frequency+1.0) * sqrt(mTreeLambda)<<" code: "<<enc<<endl;
			    	x->add(z);
			    	MapNodetoFrequencies[TopologicalSort[ii]].push_back(frequency);
		      		MapProductionIDtoSize[enc]= 1; //sqrt(mTreeLambda);
		      			}
		      			else{
		      	//			cout<<"depth="<<depth<<endl;

		      			int size = 0;
		      		string encoding;
		      		encoding = aG.GetVertexLabelConcatenated(TopologicalSort[ii]);
			    	vector<unsigned> vertex_adjacency_list = MapNodetoChildren[TopologicalSort[ii]];
		    		vector<unsigned>  vertex_label_id_list;
		    		int min_freq_children =INT_MAX;
			    	for (unsigned j = 0; j < vertex_adjacency_list.size(); ++j) {

			    		unsigned child_vertex_id = vertex_adjacency_list[j];
			    		int size_map =  MapNodeToProductionsID[child_vertex_id].size();//min (depth-1) e dim
			    		unsigned child_hash = MapNodeToProductionsID[child_vertex_id][min(size_map,depth)-1];// id hash del figlio;
			    		int freq_child = MapNodetoFrequencies[child_vertex_id][min(size_map,depth)-1];
			    		if (freq_child < min_freq_children){
			    			min_freq_children = freq_child;
			    		}
			    		vertex_label_id_list.push_back(child_hash);
			    		size += MapProductionIDtoSize[child_hash];
			    	}

			    	// vertex_label_id_list contiene gli hash delle prod dei figli. Va ordinato e usato x generare l hash attuale
			    	sort(vertex_label_id_list.begin(), vertex_label_id_list.end());

			    	if (vertex_label_id_list.size() > 0){
			    		std::stringstream ss;
			    		ss <<":"<<vertex_label_id_list[0];
			    		encoding += ss.str() ;
			    	}//originale senza .
			    	for (unsigned i = 1; i < vertex_label_id_list.size(); i++){
			    		std::stringstream ss;
			    		ss << vertex_label_id_list[i];
			    		encoding += "." + ss.str() ;
			    	}
			    //DEBUG	cout<<"encoding "<<encoding<<endl;
			    	//calculate frequency

			    	hash_subgraph_code = HashFunc(encoding);
			    	MapNodeToProductionsID[TopologicalSort[ii]].push_back(hash_subgraph_code);
			    	size += 1; // current node
			    	MapProductionIDtoSize[hash_subgraph_code]= size;
			    	int frequency = min_freq_children;
			    	MapNodetoFrequencies[TopologicalSort[ii]].push_back(frequency);

			    	SVector z;
			   // 	cout<<"Feature: "<<encoding<<" freq:"<<(double)(frequency+1.0) * sqrt(pow(mTreeLambda,size))<<" code: "<<hash_subgraph_code<<endl;
			    	z.set(hash_subgraph_code,(double)(frequency+1.0) * sqrt(pow(mTreeLambda,size)));
			    	x->add(z);

			    //	cout<<"weight "<<sqrt(pow(mTreeLambda,size))<<endl;

		      			}

		      		}
		      	}

		    return *x;

	 }

SVector&   NSDDkernel_FeatureGenerator::generate_vertex_feature_vector_DIVIDED(unsigned i, const GraphClass& aG , int dag_h, vector<SVector> & features_grouped_by_depth){
	//cout<<"start generation of feature vector.."<<endl;
			SVector* x = new SVector();
			features_grouped_by_depth= vector<SVector>(dag_h+1);
		/*	for(int i=0; i<dag_h;i++){
				features_grouped_by_depth.push_back(SVector(0));

			}*/
		//	cout<<"n of nodes "<<aG.VertexSize()<<endl;
		    assert(i< aG.VertexSize());
		    	map<pair<unsigned, unsigned>, int> oSrcDestMaptoDistance;
		    	map<unsigned, vector<unsigned> > MapNodetoParents;
		    	map<unsigned, vector<unsigned> > MapNodetoChildren;

		    	vector<unsigned> TopologicalSort;
		    	int maxLevel = 0;
		    	//aG. // codice calcolo features here
		      	aG.SingleVertexBoundedBreadthFirstVisitTree(i,dag_h, oSrcDestMaptoDistance,  MapNodetoParents,MapNodetoChildren, TopologicalSort, maxLevel);
		//    cout<<"MaxLevel: "<<maxLevel<<endl;
		      	// 	cout<<"Breadth first visit ended.."<<endl;
		      	// I have the DAG for the vertex i. Need topological sort.
		      	//map every node to him productions
		    	map<unsigned, vector<unsigned> > MapNodeToProductionsID;
		    	map<unsigned, vector<int> > MapNodetoFrequencies;

		    	map<unsigned, int > MapProductionIDtoSize;

		  //  	cout<<"topological sort size "<<TopologicalSort.size()<<endl;

		      	//maybe I meed the maximum depth (distance)
		      	for (int ii=TopologicalSort.size()-1; ii>=0; ii--){
		      		// cout<<ii<<endl;
		      		//calculate child size
			    	vector<unsigned> vertex_adjacency_list = MapNodetoChildren[TopologicalSort[ii]];
		      		int max_child_heigth=0;
		        	for (unsigned j = 0; j < vertex_adjacency_list.size(); ++j) {
		    			int child_vertex_id= vertex_adjacency_list[j];
		    			int child_heigth=MapNodeToProductionsID[child_vertex_id].size();
		    			if(child_heigth > max_child_heigth){
		    				max_child_heigth = child_heigth;
		    			}
		        	}

		      		for (int depth=0; depth<= max_child_heigth; ++depth){ //era maxLevel
		      			unsigned hash_subgraph_code = 1;
		      		//depth 0; only label
		      			if(depth==0){
		      		//		cout<<"depth=0"<<endl;
		      				unsigned enc= Radius0RootedGraphCanonicalFormEncoding(TopologicalSort[ii], aG);
		      		MapNodeToProductionsID[TopologicalSort[ii]].push_back(enc);
		      		// scorro i figli e creo etichetta tramite funzione di hash
		      		// depth > 0; the length of the children ID list
		      		//imposta valore feature a 1 (TODO lambda)
		      		//							level(o-mRadius)
			    	int frequency = 0;
			    	if(max_child_heigth==0){
		      		//leaf node
			    	frequency=  maxLevel - oSrcDestMaptoDistance[make_pair(i,TopologicalSort[ii])];
			    	}
			    	SVector z;
		      		z.set(enc, (double)(frequency+1.0) * sqrt(mTreeLambda));
			    //	cout<<"Feature: "<<aG.GetVertexLabelConcatenated(TopologicalSort[ii])<<" freq:"<<(double)(frequency+1.0) * sqrt(mTreeLambda)<<" code: "<<enc<<endl;
			    	x->add(z);
			    	features_grouped_by_depth[depth].add(z);
			    	MapNodetoFrequencies[TopologicalSort[ii]].push_back(frequency);
		      		MapProductionIDtoSize[enc]= 1; //sqrt(mTreeLambda);
		      			}
		      			else{
		      	//			cout<<"depth="<<depth<<endl;

		      			int size = 0;
		      		string encoding;
		      		encoding = aG.GetVertexLabelConcatenated(TopologicalSort[ii]);
			    	vector<unsigned> vertex_adjacency_list = MapNodetoChildren[TopologicalSort[ii]];
		    		vector<unsigned>  vertex_label_id_list;
		    		int min_freq_children =INT_MAX;
			    	for (unsigned j = 0; j < vertex_adjacency_list.size(); ++j) {

			    		unsigned child_vertex_id = vertex_adjacency_list[j];
			    		int size_map =  MapNodeToProductionsID[child_vertex_id].size();//min (depth-1) e dim
			    		unsigned child_hash = MapNodeToProductionsID[child_vertex_id][min(size_map,depth)-1];// id hash del figlio;
			    		int freq_child = MapNodetoFrequencies[child_vertex_id][min(size_map,depth)-1];
			    		if (freq_child < min_freq_children){
			    			min_freq_children = freq_child;
			    		}
			    		vertex_label_id_list.push_back(child_hash);
			    		size += MapProductionIDtoSize[child_hash];
			    	}

			    	// vertex_label_id_list contiene gli hash delle prod dei figli. Va ordinato e usato x generare l hash attuale
			    	sort(vertex_label_id_list.begin(), vertex_label_id_list.end());

			    	if (vertex_label_id_list.size() > 0){
			    		std::stringstream ss;
			    		ss <<":"<<vertex_label_id_list[0];
			    		encoding += ss.str() ;
			    	}//originale senza .
			    	for (unsigned i = 1; i < vertex_label_id_list.size(); i++){
			    		std::stringstream ss;
			    		ss << vertex_label_id_list[i];
			    		encoding += "." + ss.str() ;
			    	}
			    //DEBUG	cout<<"encoding "<<encoding<<endl;
			    	//calculate frequency

			    	hash_subgraph_code = HashFunc(encoding);
			    	MapNodeToProductionsID[TopologicalSort[ii]].push_back(hash_subgraph_code);
			    	size += 1; // current node
			    	MapProductionIDtoSize[hash_subgraph_code]= size;
			    	int frequency = min_freq_children;
			    	MapNodetoFrequencies[TopologicalSort[ii]].push_back(frequency);

			    	SVector z;
			   // 	cout<<"Feature: "<<encoding<<" freq:"<<(double)(frequency+1.0) * sqrt(pow(mTreeLambda,size))<<" code: "<<hash_subgraph_code<<endl;
			    	z.set(hash_subgraph_code,(double)(frequency+1.0) * sqrt(pow(mTreeLambda,size)));
			    	x->add(z);
			    	features_grouped_by_depth[depth].add(z);
			    //	cout<<"weight "<<sqrt(pow(mTreeLambda,size))<<endl;

		      			}

		      		}
		      	}

		    return *x;

	 }

SVector&   NSDDkernel_FeatureGenerator::generate_vertex_feature_vector_DIVIDED_DISTANCE(unsigned i, const GraphClass& aG , int dag_h, vector<vector<SVector> > & features_grouped_by_depth){
	//cout<<"start generation of feature vector.."<<endl;
			SVector* x = new SVector();
			features_grouped_by_depth= vector<vector<SVector> >(dag_h+1);

		//	cout<<"n of nodes "<<aG.VertexSize()<<endl;
		    assert(i< aG.VertexSize());
		    	map<pair<unsigned, unsigned>, int> oSrcDestMaptoDistance;
		    	map<unsigned, vector<unsigned> > MapNodetoParents;
		    	map<unsigned, vector<unsigned> > MapNodetoChildren;

		    	vector<unsigned> TopologicalSort;
		    	int maxLevel = 0;
		    	//aG. // codice calcolo features here
		      	aG.SingleVertexBoundedBreadthFirstVisitTree(i,dag_h, oSrcDestMaptoDistance,  MapNodetoParents,MapNodetoChildren, TopologicalSort, maxLevel);
		//    cout<<"MaxLevel: "<<maxLevel<<endl;
				for(int t=0; t<maxLevel+1;t++){
					features_grouped_by_depth[t] =(vector<SVector>(dag_h+1));

				}

		      	// 	cout<<"Breadth first visit ended.."<<endl;
		      	// I have the DAG for the vertex i. Need topological sort.
		      	//map every node to him productions
		    	map<unsigned, vector<unsigned> > MapNodeToProductionsID;
		    	map<unsigned, vector<int> > MapNodetoFrequencies;

		    	map<unsigned, int > MapProductionIDtoSize;

		  //  	cout<<"topological sort size "<<TopologicalSort.size()<<endl;

		      	//maybe I meed the maximum depth (distance)
		      	for (int ii=TopologicalSort.size()-1; ii>=0; ii--){
		      		// cout<<ii<<endl;
		      		//calculate child size
			    	vector<unsigned> vertex_adjacency_list = MapNodetoChildren[TopologicalSort[ii]];
		      		int max_child_heigth=0;
		        	for (unsigned j = 0; j < vertex_adjacency_list.size(); ++j) {
		    			int child_vertex_id= vertex_adjacency_list[j];
		    			int child_heigth=MapNodeToProductionsID[child_vertex_id].size();
		    			if(child_heigth > max_child_heigth){
		    				max_child_heigth = child_heigth;
		    			}
		        	}

		      		for (int depth=0; depth<= max_child_heigth; ++depth){ //era maxLevel
		      			unsigned hash_subgraph_code = 1;
		      		//depth 0; only label
		      			if(depth==0){
		      		//		cout<<"depth=0"<<endl;
		      				unsigned enc= Radius0RootedGraphCanonicalFormEncoding(TopologicalSort[ii], aG);
		      		MapNodeToProductionsID[TopologicalSort[ii]].push_back(enc);
		      		// scorro i figli e creo etichetta tramite funzione di hash
		      		// depth > 0; the length of the children ID list
		      		//imposta valore feature a 1 (TODO lambda)
		      		//							level(o-mRadius)
			    	int frequency = 0;
			    	if(max_child_heigth==0){
		      		//leaf node
			    	frequency=  maxLevel - oSrcDestMaptoDistance[make_pair(i,TopologicalSort[ii])];
			    	}
			    	SVector z;
		      		z.set(enc, (double)(frequency+1.0) * sqrt(mTreeLambda));
			    //	cout<<"Feature: "<<aG.GetVertexLabelConcatenated(TopologicalSort[ii])<<" freq:"<<(double)(frequency+1.0) * sqrt(mTreeLambda)<<" code: "<<enc<<endl;
			    	x->add(z);
			    	features_grouped_by_depth[depth][oSrcDestMaptoDistance[make_pair(i,TopologicalSort[ii])]].add(z);
			    	MapNodetoFrequencies[TopologicalSort[ii]].push_back(frequency);
		      		MapProductionIDtoSize[enc]= 1; //sqrt(mTreeLambda);
		      			}
		      			else{
		      	//			cout<<"depth="<<depth<<endl;

		      			int size = 0;
		      		string encoding;
		      		encoding = aG.GetVertexLabelConcatenated(TopologicalSort[ii]);
			    	vector<unsigned> vertex_adjacency_list = MapNodetoChildren[TopologicalSort[ii]];
		    		vector<unsigned>  vertex_label_id_list;
		    		int min_freq_children =INT_MAX;
			    	for (unsigned j = 0; j < vertex_adjacency_list.size(); ++j) {

			    		unsigned child_vertex_id = vertex_adjacency_list[j];
			    		int size_map =  MapNodeToProductionsID[child_vertex_id].size();//min (depth-1) e dim
			    		unsigned child_hash = MapNodeToProductionsID[child_vertex_id][min(size_map,depth)-1];// id hash del figlio;
			    		int freq_child = MapNodetoFrequencies[child_vertex_id][min(size_map,depth)-1];
			    		if (freq_child < min_freq_children){
			    			min_freq_children = freq_child;
			    		}
			    		vertex_label_id_list.push_back(child_hash);
			    		size += MapProductionIDtoSize[child_hash];
			    	}

			    	// vertex_label_id_list contiene gli hash delle prod dei figli. Va ordinato e usato x generare l hash attuale
			    	sort(vertex_label_id_list.begin(), vertex_label_id_list.end());

			    	if (vertex_label_id_list.size() > 0){
			    		std::stringstream ss;
			    		ss <<":"<<vertex_label_id_list[0];
			    		encoding += ss.str() ;
			    	}//originale senza .
			    	for (unsigned id = 1; id < vertex_label_id_list.size(); id++){
			    		std::stringstream ss;
			    		ss << vertex_label_id_list[id];
			    		encoding += "." + ss.str() ;
			    	}
			    //DEBUG	cout<<"encoding "<<encoding<<endl;
			    	//calculate frequency

			    	hash_subgraph_code = HashFunc(encoding);
			    	MapNodeToProductionsID[TopologicalSort[ii]].push_back(hash_subgraph_code);
			    	size += 1; // current node
			    	MapProductionIDtoSize[hash_subgraph_code]= size;
			    	int frequency = min_freq_children;
			    	MapNodetoFrequencies[TopologicalSort[ii]].push_back(frequency);

			    	SVector z;
			   // 	cout<<"Feature: "<<encoding<<" freq:"<<(double)(frequency+1.0) * sqrt(pow(mTreeLambda,size))<<" code: "<<hash_subgraph_code<<endl;
			    	z.set(hash_subgraph_code,(double)(frequency+1.0) * sqrt(pow(mTreeLambda,size)));
			    	x->add(z);
			    	features_grouped_by_depth[depth][oSrcDestMaptoDistance[make_pair(i,TopologicalSort[ii])]].add(z); //TODO distance from root
			    //	cout<<"weight "<<sqrt(pow(mTreeLambda,size))<<endl;

		      			}

		      		}
		      	}

		    return *x;

	 }

//Kernel su dati con astrazione (nodi di relazione)
ANSDDkernel_FeatureGenerator::ANSDDkernel_FeatureGenerator(const std::string& id) :
		NSDDkernel_FeatureGenerator(id) {
		//new_flag(&mTreeLambda, "mTreeLambda", "(double)\n tree_lambda parameter for DDK kernel");
}

void ANSDDkernel_FeatureGenerator::generate_feature_vector(const GraphClass& aG, SVector& x, const vector<unsigned>& aFirstEndpointList) {
	//cout<<"generate_vertex_feature_vector.... OK for abstract"<<endl;
	vector<unsigned> first_endpoint_list = aFirstEndpointList;
	GetFirstEndpoints(aG, first_endpoint_list);
	if (first_endpoint_list.size() == 0)
		throw std::logic_error("ERROR6: Something went wrong: cannot generate features over an empty set of first endpoints!");
	int horizon = max(mDistance, mRadius);
	aG.ComputePairwiseDistanceInformation(horizon, first_endpoint_list);
	if (aG.Check() == false)
		throw logic_error("ERROR11: the graph data structure is not sound and it has not passed the checkup procedure"); //check graph data structure soundness

	InitFeatureCache(aG, mRadiusTwo);
	for (unsigned i = 0; i < first_endpoint_list.size(); i++) {
		SVector z;
		unsigned src_id = first_endpoint_list[i];
		//InitFeatureCache(aG, mRadiusTwo);

		if (aG.GetVertexViewPoint(src_id) && aG.GetVertexKernelPoint(src_id) && aG.GetVertexAlive(src_id)) { //proceed to extract features only if the *src* vertex is a kernel point and is alive
			//for (unsigned r = 0; r <= mRadius; r++) {
			//	for (unsigned d = 0; d <= mDistance; d++) {
				//	GenerateVertexFeatures(src_id, aG, r, d, z);
			GenerateVertexFeatures(src_id, aG, mRadius, mDistance, z);
				//}
			//}
		} else if (aG.GetVertexAbstraction(src_id) && aG.GetVertexAlive(src_id)) {
			//for (unsigned r = 0; r <= mRadius; r++) {
			//	for (unsigned d = 0; d <= mDistance; d++) {
				//	GenerateAbstractVertexFeatures(src_id, aG, r, d, z);
			GenerateAbstractVertexFeatures(src_id, aG, mRadius, mDistance, z);

			//	}
			//}
		}
		if (mMinKernel)
			ConvertSparseVectorToMinFeatureVector(z);
		//x_list.push_back(z);
		x.add(z);
	}
	if (mDebugVerbosity > 0) {

		OutputFeatureMap(cout);
		aG.Output(cout);
	}
}
void ANSDDkernel_FeatureGenerator::GenerateVertexFeatures(unsigned aRootVertexIndex, const GraphClass& aG, unsigned aRadius, unsigned aDistance, SVector& x) {
	x= generateVertexFeatures_DISTANCE(aRootVertexIndex, aG ,aRadius);
}

SVector&   ANSDDkernel_FeatureGenerator::generateVertexFeatures_DISTANCE(unsigned i, const GraphClass& aG , int dag_h){
   // test rNSPDK 1/2 d=1/2
	GraphClass ag2 = aG;
	SVector* out = new SVector();
//	cout<<"Generate feature vector core"<<endl;
	//	cout<<"i:"<<i<<endl;

		//SVector DD= generate_vertex_feature_vector(i,ag2 ,dag_h);
		vector< vector< SVector> > features; //(dag_h+1);
		SVector DD= generate_vertex_feature_vector_DIVIDED_DISTANCE(i,ag2 ,dag_h,features);
		//cout<<features.size();
	//	out->add(DD);
		//NSPDK
	//	cout<<"start generation of NSPKD feature vector.. mDistance"<<mDistance<<endl;

		//NSPDK_FeatureGenerator::GenerateVertexFeatures(i,aG,NSPDK);
	//	NSPDK_FeatureGenerator::generate_feature_vector(aG,NSPDK);
		vector<unsigned> first_endpoint_list= vector<unsigned>();
		first_endpoint_list.push_back(i);
			//GetFirstEndpoints(aG, first_endpoint_list);

		int horizon=max(mDistance, mRadiusTwo); //   max(mDistance, mRadius);
			aG.ComputePairwiseDistanceInformation(horizon, first_endpoint_list);
			if (aG.Check() == false)
				throw logic_error("ERROR10: the graph data structure is not sound and it has not passed the checkup procedure"); //check graph data structure soundness

			InitFeatureCache(aG, mRadiusTwo );
			for (unsigned r = 0; r <= mRadius ; r++) { //max(1.0,mRadius / 2.0)
				for (unsigned d = 0; d <= mDistance; d++) { // era 0
					for (unsigned r2 = 0; r2 <= mRadiusTwo ; r2++) {
					SVector z;
					for (unsigned i = 0; i < first_endpoint_list.size(); i++) {
						unsigned src_id = first_endpoint_list[i];
						if (aG.GetVertexViewPoint(src_id) && aG.GetVertexKernelPoint(src_id) && aG.GetVertexAlive(src_id)) { //proceed to extract features only if the *src* vertex is a kernel point and is alive
							SVector zv;

							vector<unsigned> endpoint_list(5);
							endpoint_list[0] = r2;
							endpoint_list[1] = d;

						//	unsigned src_code = GenerateVertexNeighbourhoodHashCode(aSrcID, aG, aRadius);
				//	TODO	for(unsigned src_code in DD)

							vector<unsigned> dest_id_list = aG.GetFixedDistanceVertexIDList(i, d);
							for (unsigned dest_j = 0; dest_j < dest_id_list.size(); dest_j++) {
								unsigned dest_id = dest_id_list[dest_j];
								unsigned dest_code = 0;
								if (aG.GetVertexKernelPoint(dest_id) && aG.GetVertexAlive(dest_id)) { //proceed to extract features only if the *dest* vertex is a kernel point and is alive
									dest_code = GenerateVertexNeighbourhoodHashCode(dest_id, aG, r2); //
									// first is the DDK feature, second NSPDK feature
						//  for(const SVector::Pair *p = DD; p->i>=0; p++) {
									for(int distance=0; distance<features[r].size(); distance++){
										endpoint_list[4]=distance; //from root in DD
						 for(const SVector::Pair *p = features[r][distance]; p->i>=0; p++) {

									endpoint_list[2] = p->i;
										//TODO scorrere le feature di DD
										endpoint_list[3] = dest_code;
										//TEST output
								//		cout<<"R "<<endpoint_list[0]<<" d "<<endpoint_list[1]<<" DD "<<endpoint_list[2]<<" NSPDK "<<endpoint_list[3]<<" dRoot "<<endpoint_list[4]<<endl;
									unsigned code = HashFunc(endpoint_list, mHashBitMask);

									SVector z;
									z.set(code, p->v);
									zv.add(z);
								}
								}

								}
							}

							z.add(zv);
						}
						}

					if (mNormalization)
						z.normalize();
				out->add(z);
				} //d
			} //r
			} //r2

	//	cout<<"features size DD:"<<DD.sparse_size()<<" out:"<<out->sparse_size()<<endl;
		features.clear();

	return *out;
}

void ANSDDkernel_FeatureGenerator::GenerateAbstractVertexFeatures(unsigned aSrcID, const GraphClass& aG, unsigned aRadius, unsigned aDistance, SVector& x) {
	//cout<<"ABSTRACT FEATURES"<<endl;
	vector<unsigned> endpoint_list(5);
	for (unsigned r = 0; r <= mRadius ; r++) { //max(1.0,mRadius / 2.0)
					for (unsigned d = 0; d <= mDistance; d++) { // era 0
						for (unsigned r2 = 0; r2 <= mRadiusTwo ; r2++) { //max(1.0,mRadius / 2.0)

	endpoint_list[0] = r;
	endpoint_list[1] = d;

	//ensure that the src vertex is of the abstraction type
	if (aG.GetVertexAbstraction(aSrcID) == false)
		throw std::logic_error("ERROR2: Something went wrong: expecting an abstraction vertex, but abstraction test fails.");
	//extract all adjacent vertices and partition them into part_of (lower level) and abstraction_of (upper level)
	vector<unsigned> part_of_list;
	vector<unsigned> abstraction_of_list;

	vector<unsigned> vertex_adjacency_list = aG.GetVertexAdjacentList(aSrcID);
	vector<unsigned> edge_adjacency_list = aG.GetEdgeAdjacentList(aSrcID);
	if (vertex_adjacency_list.size() != edge_adjacency_list.size())
		throw std::logic_error("ERROR3: Something went wrong: expecting vertex adjacency list to be the same size as the edge adjacency list.");
	if (vertex_adjacency_list.size() == 0)
		throw std::logic_error("ERROR3b: Something went wrong: expecting a non empty vertex adjacency list for each abstract vertex.");
	for (unsigned i = 0; i < vertex_adjacency_list.size(); ++i) {
		unsigned child_vertex_id = vertex_adjacency_list[i];
		unsigned child_edge_id = edge_adjacency_list[i];
		if (aG.GetEdgePartOf(child_edge_id))
			part_of_list.push_back(child_vertex_id);
		if (aG.GetEdgeAbstractionOf(child_edge_id))
			abstraction_of_list.push_back(child_vertex_id);
	}
	//----------------------------

	//----------------------------------------
						//starting from each upper level vertex find all vertices at distance aDistance
	if (abstraction_of_list.size() == 0)
		throw std::logic_error("ERROR5: Something went wrong: expecting a non empty abstraction_of list.");
	set<unsigned> abstraction_of_set;
	for (unsigned i = 0; i < abstraction_of_list.size(); ++i) {
		unsigned v = abstraction_of_list[i];
		vector<unsigned> dest_id_list = aG.GetFixedDistanceVertexIDList(v, d);
		//-----------?????????????????????????????
	//	abstraction_of_set.insert(abstraction_of_list.begin(), abstraction_of_list.end());
		abstraction_of_set.insert(dest_id_list.begin(), dest_id_list.end());

	}
	//starting from each lower level vertex find all vertices at distance aDistance
	if (part_of_list.size() == 0)
		throw std::logic_error("ERROR4: Something went wrong: expecting a non empty part_of list.");
	set<unsigned> part_of_set;
	for (unsigned i = 0; i < part_of_list.size(); ++i) {
		unsigned v = part_of_list[i];
		vector<unsigned> dest_id_list = aG.GetFixedDistanceVertexIDList(v, d);
		//-----------?????????????????????????????
		part_of_set.insert(dest_id_list.begin(), dest_id_list.end());
	}
	//make all possible pairs of distant lower level vertices with distant upper level vertices
	for (set<unsigned>::iterator it = part_of_set.begin(); it != part_of_set.end(); ++it) {
		unsigned part_of_id = *it;

		//TODO modifica con mio kernel
		vector< vector< SVector> > features; //(dag_h+1);
		SVector DD= generate_vertex_feature_vector_DIVIDED_DISTANCE(part_of_id, aG ,aRadius,features);
		//SVector DD= generate_vertex_feature_vector_DIVIDED(part_of_id, aG ,aRadius,features);

		//For every feature in DD, by height and distance from the root
		for(int distance=0; distance<features[r].size(); distance++){
			endpoint_list[4]=distance; //from root in DD
			for(const SVector::Pair *p = features[r][distance]; p->i>=0; p++) {

			endpoint_list[2] = p->i;
			//TODO scorrere le feature di DD
				//TEST output
			//		cout<<"R "<<endpoint_list[0]<<" d "<<endpoint_list[1]<<" DD "<<endpoint_list[2]<<" NSPDK "<<endpoint_list[3]<<" dRoot "<<endpoint_list[4]<<endl;

			//
		for (set<unsigned>::iterator jt = abstraction_of_set.begin(); jt != abstraction_of_set.end(); ++jt) {
			unsigned abstraction_of_id = *jt;
			//build features with one neighborhood graph signature from the lower level and one from the upper
			unsigned abstraction_of_code = GenerateVertexNeighbourhoodHashCode(abstraction_of_id, aG, r2);
			endpoint_list[3] = abstraction_of_code;
			unsigned code = HashFunc(endpoint_list, mHashBitMask);
			if (mDebugVerbosity > 0)
				mDebugInfo.StoreFeatureCodeToFeatureInfo(code, endpoint_list);
			SVector z;
			z.set(code, 1);
			x.add(z);
		}
	}
		}
	}

	//i have to repeat for both sides for ddk
	//make all possible pairs of distant lower level vertices with distant upper level vertices
	for (set<unsigned>::iterator it = part_of_set.begin(); it != part_of_set.end(); ++it) {
		unsigned part_of_id = *it;

		//TODO modifica con mio kernel
		vector< vector< SVector> > features; //(dag_h+1);
		SVector DD= generate_vertex_feature_vector_DIVIDED_DISTANCE(part_of_id, aG ,aRadius,features);

			//For every feature in DD, by height and distance from the root
			for(int distance=0; distance<features[r].size(); distance++){
				endpoint_list[4]=distance; //from root in DD
				for(const SVector::Pair *p = features[r][distance]; p->i>=0; p++) {

				endpoint_list[2] = p->i;

		unsigned part_of_code = GenerateVertexNeighbourhoodHashCode(part_of_id, aG, aRadius);
		endpoint_list[2] = part_of_code;
		for (set<unsigned>::iterator jt = abstraction_of_set.begin(); jt != abstraction_of_set.end(); ++jt) {
			unsigned abstraction_of_id = *jt;
			//build features with one neighborhood graph signature from the lower level and one from the upper
			unsigned abstraction_of_code = GenerateVertexNeighbourhoodHashCode(abstraction_of_id, aG, aRadius);
			endpoint_list[3] = abstraction_of_code;
			unsigned code = HashFunc(endpoint_list, mHashBitMask);
			if (mDebugVerbosity > 0)
				mDebugInfo.StoreFeatureCodeToFeatureInfo(code, endpoint_list);
			SVector z;
			z.set(code, 1);
			x.add(z);
		}
	}
	}

}
					}
	}
}
}


// Copyright (c) 2018 yshurik
//
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
//
// The use in another cyptocurrency project the code is licensed under
// Jelurida Public License (JPL). See https://www.jelurida.com/resources/jpl

#include "pegdata.h"

#include <map>
#include <set>
#include <cstdint>
#include <utility>
#include <algorithm>
#include <type_traits>

#include <boost/multiprecision/cpp_int.hpp>

#include "util.h"

#include <zconf.h>
#include <zlib.h>

using namespace std;
using namespace boost;

CFractions::CFractions()
    :nFlags(VALUE)
    ,f(new int64_t[PEG_SIZE])
{
    f[0] = 0; // fast init first item
}
CFractions::CFractions(int64_t value, uint32_t flags)
    :nFlags(flags)
    ,f(new int64_t[PEG_SIZE])
{
    if (flags & VALUE)
        f[0] = value; // fast init first item
    else if (flags & STD) {
        f[0] = value;
        nFlags = VALUE;
        ToStd();
        nFlags = flags;
    }
    else {
        assert(0);
    }
}
CFractions::CFractions(const CFractions & o)
    :nFlags(o.nFlags)
    ,nLockTime(o.nLockTime)
    ,sReturnAddr(o.sReturnAddr)
    ,f(new int64_t[PEG_SIZE])
{
    for(int i=0; i< PEG_SIZE; i++) {
        f[i] = o.f[i];
    }
}

CFractions& CFractions::operator=(const CFractions& o)
{
    nFlags = o.nFlags;
    nLockTime = o.nLockTime;
    sReturnAddr = o.sReturnAddr;
    for(int i=0; i< PEG_SIZE; i++) {
        f[i] = o.f[i];
    }
    return *this;
}

void CFractions::ToDeltas(int64_t* deltas) const
{
    int64_t fp = 0;
    for(int i=0; i<PEG_SIZE; i++) {
        if (i==0) {
            fp = deltas[0] = f[0];
            continue;
        }
        deltas[i] = f[i]-fp*(PEG_RATE-1)/PEG_RATE;
        fp = f[i];
    }
}

void CFractions::FromDeltas(const int64_t* deltas)
{
    int64_t fp = 0;
    for(int i=0; i<PEG_SIZE; i++) {
        if (i==0) {
            fp = f[0] = deltas[0];
            continue;
        }
        f[i] = deltas[i]+fp*(PEG_RATE-1)/PEG_RATE;
        fp = f[i];
    }
}

bool CFractions::Pack(CDataStream& out, unsigned long* report_len, bool compress) const
{
    if (nFlags & VALUE) {
        if (report_len) *report_len = sizeof(int64_t);
        out << nVersion;
        out << uint32_t(nFlags | SER_VALUE);
        out << nLockTime;
        out << sReturnAddr;
        out << f[0];
    } else if (compress) {
        int64_t deltas[PEG_SIZE];
        ToDeltas(deltas);

        int zlevel = 9;
        unsigned char zout[2*PEG_SIZE*sizeof(int64_t)];
        unsigned long n = PEG_SIZE*sizeof(int64_t);
        unsigned long zlen = PEG_SIZE*2*sizeof(int64_t);
        auto src = reinterpret_cast<const unsigned char *>(deltas);
        int res = ::compress2(zout, &zlen, src, n, zlevel);
        if (res == Z_OK) {
            if (report_len) *report_len = zlen;
            out << nVersion;
            out << uint32_t(nFlags | SER_ZDELTA);
            out << nLockTime;
            out << sReturnAddr;
            auto ser = reinterpret_cast<const char *>(zout);
            out << zlen;
            out.write(ser, zlen);
        }
        else {
            if (report_len) *report_len = PEG_SIZE*sizeof(int64_t);
            out << nVersion;
            out << uint32_t(nFlags | SER_RAW);
            out << nLockTime;
            out << sReturnAddr;
            auto ser = reinterpret_cast<const char *>(f.get());
            out.write(ser, PEG_SIZE*sizeof(int64_t));
        }
    } else {
        if (report_len) *report_len = PEG_SIZE*sizeof(int64_t);
        out << nVersion;
        out << uint32_t(nFlags | SER_RAW);
        out << nLockTime;
        out << sReturnAddr;
        auto ser = reinterpret_cast<const char *>(f.get());
        out.write(ser, PEG_SIZE*sizeof(int64_t));
    }
    return true;
}

bool CFractions::Unpack(CDataStream& inp)
{
    uint32_t nSerFlags = 0;
    inp >> nVersion;
    inp >> nSerFlags;
    inp >> nLockTime;
    inp >> sReturnAddr;

    if (nSerFlags & SER_VALUE) {
        nFlags = nSerFlags | VALUE;
        inp >> f[0];
    }
    else if (nSerFlags & SER_ZDELTA) {
        unsigned long zlen = 0;
        inp >> zlen;

        if (zlen>(2*PEG_SIZE*sizeof(int64_t))) {
            // data are broken, no read
            return false;
        }

        unsigned char zinp[2*PEG_SIZE*sizeof(int64_t)];
        unsigned long n = PEG_SIZE*sizeof(int64_t);
        auto ser = reinterpret_cast<char *>(zinp);
        inp.read(ser, zlen);

        int64_t deltas[PEG_SIZE];
        auto src = reinterpret_cast<const unsigned char *>(ser);
        auto dst = reinterpret_cast<unsigned char *>(deltas);
        int res = ::uncompress(dst, &n, src, zlen);
        if (res != Z_OK) {
            // data are broken, can not uncompress
            return false;
        }
        FromDeltas(deltas);
        nFlags = nSerFlags | STD;
    }
    else if (nSerFlags & SER_RAW) {
        auto ser = reinterpret_cast<char *>(f.get());
        inp.read(ser, PEG_SIZE*sizeof(int64_t));
        nFlags = nSerFlags | STD;
    }
    nFlags &= SER_MASK;

    return true;
}

CFractions CFractions::Std() const
{
    if ((nFlags & VALUE) ==0)
        return *this;

    CFractions fstd;
    fstd.sReturnAddr = sReturnAddr;
    fstd.nLockTime = nLockTime;
    fstd.nFlags = nFlags;
    fstd.nFlags &= ~uint32_t(VALUE);
    fstd.nFlags |= STD;

    int64_t v = f[0];
    for(int i=0;i<PEG_SIZE;i++) {
        if (i == PEG_SIZE-1) {
            fstd.f[i] = v;
            break;
        }
        int64_t frac = v/PEG_RATE;
        fstd.f[i] = frac;
        v -= frac;
    }
    return fstd;
}

bool CFractions::IsPositive() const
{
    if (nFlags & VALUE)
        return true;

    for(int i=0;i<PEG_SIZE;i++) {
        if (f[i] <0)
            return false;
    }
    return true;
}

bool CFractions::IsNegative() const
{
    if (nFlags & VALUE)
        return false;

    for(int i=0;i<PEG_SIZE;i++) {
        if (f[i] >0)
            return false;
    }
    return true;
}

int64_t CFractions::Total() const
{
    int64_t nValue =0;
    if (nFlags & VALUE)
        return f[0];

    for(int i=0;i<PEG_SIZE;i++) {
        nValue += f[i];
    }
    return nValue;
}

int64_t CFractions::Low(int supply) const
{
    int64_t nValue =0;
    if (nFlags & VALUE)
        return Std().Low(supply);

    for(int i=0;i<supply;i++) {
        nValue += f[i];
    }
    return nValue;
}

int64_t CFractions::High(int supply) const
{
    int64_t nValue =0;
    if (nFlags & VALUE)
        return Std().High(supply);

    for(int i=supply;i<PEG_SIZE;i++) {
        nValue += f[i];
    }
    return nValue;
}

int64_t CFractions::Low(const CPegLevel & peglevel) const
{
    int64_t nValue =0;
    if (nFlags & VALUE)
        return Std().Low(peglevel);

    int to = peglevel.nSupply + peglevel.nShift;
    if (to <0) return 0;
    if (to >= PEG_SIZE) return 0;

    if (peglevel.nShiftLastPart >0 &&
        peglevel.nShiftLastTotal >0) {
        // partial value to use
        int64_t v = f[to];
        int64_t vpart = ::RatioPart(v,
                                    peglevel.nShiftLastPart,
                                    peglevel.nShiftLastTotal);
        if (vpart < v) vpart++; // better rounding
        nValue += vpart;
    }

    for(int i=0;i<to;i++) {
        nValue += f[i];
    }

    return nValue;
}

int64_t CFractions::High(const CPegLevel & peglevel) const
{
    int64_t nValue =0;
    if (nFlags & VALUE)
        return Std().High(peglevel);

    int from = peglevel.nSupply + peglevel.nShift;
    if (from <0) return 0;
    if (from >= PEG_SIZE) return 0;

    if (peglevel.nShiftLastPart >0 &&
        peglevel.nShiftLastTotal >0) {
        // partial value to use
        int64_t v = f[from];
        int64_t vpart = ::RatioPart(v,
                                    peglevel.nShiftLastPart,
                                    peglevel.nShiftLastTotal);
        if (vpart < v) vpart++; // better rounding
        nValue += (v - vpart);
        from++;
    }

    for(int i=from;i<PEG_SIZE;i++) {
        nValue += f[i];
    }

    return nValue;
}

int64_t CFractions::NChange(const CPegLevel & peglevel) const
{
    CPegLevel peglevel_next = peglevel;
    peglevel_next.nSupply = peglevel_next.nSupplyNext;
    peglevel_next.nSupplyNext = peglevel_next.nSupplyNextNext;

    int64_t nValueSrc = High(peglevel);
    int64_t nValueDst = High(peglevel_next);
    return nValueDst - nValueSrc;
}

int64_t CFractions::NChange(int src_supply, int dst_supply) const
{
    int64_t nValueSrc = High(src_supply);
    int64_t nValueDst = High(dst_supply);
    return nValueDst - nValueSrc;
}

int16_t CFractions::HLI() const
{
    if (nFlags & VALUE)
        return Std().HLI();

    int64_t half = 0;
    int64_t total = Total();
    if (total ==0) {
        return 0;
    }
    for(int16_t i=0;i<PEG_SIZE;i++) {
        half += f[i];
        if (half > total/2) {
            return i;
        }
    }
    return 0;
}

void CFractions::ToStd()
{
    if ((nFlags & VALUE) == 0)
        return;

    nFlags &= ~uint32_t(VALUE);
    nFlags |= STD;

    int64_t v = f[0];
    for(int i=0;i<PEG_SIZE;i++) {
        if (i == PEG_SIZE-1) {
            f[i] = v;
            break;
        }
        int64_t frac = v/PEG_RATE;
        f[i] = frac;
        v -= frac;
    }
}

CFractions CFractions::Positive(int64_t* total) const
{
    if ((nFlags & STD) == 0) {
        return Std().Positive(total);
    }
    CFractions frPositive(0, CFractions::STD);
    for(int i=0; i<PEG_SIZE; i++) {
        if (f[i] <=0) continue;
        frPositive.f[i] = f[i];
        if (total) *total += f[i];
    }
    return frPositive;
}
CFractions CFractions::Negative(int64_t* total) const
{
    if ((nFlags & STD) == 0) {
        return Std().Negative(total);
    }
    CFractions frNegative(0, CFractions::STD);
    for(int i=0; i<PEG_SIZE; i++) {
        if (f[i] >=0) continue;
        frNegative.f[i] = f[i];
        if (total) *total += f[i];
    }
    return frNegative;
}

CFractions CFractions::LowPart(int supply, int64_t* total) const
{
    if ((nFlags & STD) == 0) {
        return Std().LowPart(supply, total);
    }
    CFractions frLowPart(0, CFractions::STD);
    for(int i=0; i<supply; i++) {
        if (total) *total += f[i];
        frLowPart.f[i] += f[i];
    }
    return frLowPart;
}
CFractions CFractions::HighPart(int supply, int64_t* total) const
{
    if ((nFlags & STD) == 0) {
        return Std().HighPart(supply, total);
    }
    CFractions frHighPart(0, CFractions::STD);
    for(int i=supply; i<PEG_SIZE; i++) {
        if (total) *total += f[i];
        frHighPart.f[i] += f[i];
    }
    return frHighPart;
}

CFractions CFractions::LowPart(const CPegLevel & peglevel, int64_t* total) const
{
    if ((nFlags & STD) == 0) {
        return Std().LowPart(peglevel, total);
    }

    CFractions frLowPart(0, CFractions::STD);

    int to = peglevel.nSupply + peglevel.nShift;
    if (to >=0 &&
            to <PEG_SIZE &&
            peglevel.nShiftLastPart >0 &&
            peglevel.nShiftLastTotal >0) {
        // partial value to use
        int64_t v = f[to];
        int64_t vpart = ::RatioPart(v,
                                    peglevel.nShiftLastPart,
                                    peglevel.nShiftLastTotal);
        if (vpart < v) vpart++;
        frLowPart.f[to] = vpart;
        if (total) *total += vpart;
    }

    for(int i=0; i<to; i++) {
        if (total) *total += f[i];
        frLowPart.f[i] = f[i];
    }
    return frLowPart;
}
CFractions CFractions::HighPart(const CPegLevel & peglevel, int64_t* total) const
{
    if ((nFlags & STD) == 0) {
        return Std().HighPart(peglevel, total);
    }

    CFractions frHighPart(0, CFractions::STD);

    int from = peglevel.nSupply + peglevel.nShift;
    if (from >=0 &&
            from <PEG_SIZE &&
            peglevel.nShiftLastPart >0 &&
            peglevel.nShiftLastTotal >0) {
        // partial value to use
        int64_t v = f[from];
        int64_t vpart = ::RatioPart(v,
                                    peglevel.nShiftLastPart,
                                    peglevel.nShiftLastTotal);
        if (vpart < v) vpart++;
        frHighPart.f[from] = (v - vpart);
        if (total) *total += (v - vpart);
        from++;
    }

    for(int i=from; i<PEG_SIZE; i++) {
        if (total) *total += f[i];
        frHighPart.f[i] = f[i];
    }
    return frHighPart;
}

CFractions CFractions::MidPart(const CPegLevel & peglevel_low,
                               const CPegLevel & peglevel_high) const
{
    int from = peglevel_low.nSupply + peglevel_low.nShift;
    int to = peglevel_high.nSupply + peglevel_high.nShift;

    if (from != to) {
        return HighPart(peglevel_low, nullptr).LowPart(peglevel_high, nullptr);
    }

    // same supply, different partial ratio
    CFractions mid(0, STD);
    int one = from;
    int64_t v = f[one];
    int64_t vone = f[one];

    if (peglevel_low.nShiftLastPart >0 &&
        peglevel_low.nShiftLastTotal >0) {
        int64_t vpart = ::RatioPart(vone,
                                    peglevel_low.nShiftLastPart,
                                    peglevel_low.nShiftLastTotal);
        v -= vpart;
    }

    if (peglevel_high.nShiftLastPart >0 &&
        peglevel_high.nShiftLastTotal >0) {
        int64_t vpart = ::RatioPart(vone,
                                    peglevel_high.nShiftLastPart,
                                    peglevel_high.nShiftLastTotal);
        if (vpart < vone) vpart++;
        v -= (vone - vpart);
    }

    mid.f[from] = v;
    return mid;
}

/** Take a part as ration part/total where part is also value (sum fraction)
 *  Returned fractions are also adjusted for part for rounding differences.
 *  #NOTE8
 */
CFractions CFractions::RatioPart(int64_t nPartValue) const {
    if ((nFlags & STD) == 0) {
        return Std().RatioPart(nPartValue);
    }
    int64_t nTotalValue = Total();
    int64_t nPartValueSum = 0;
    CFractions fPart(0, CFractions::STD);

    if (nPartValue == 0 && nTotalValue == 0)
        return fPart;
    if (nPartValue == 0)
        return fPart;
    if (nPartValue > nTotalValue)
        return Std();

    int adjust_from = PEG_SIZE;
    for(int i=0; i<PEG_SIZE; i++) {
        int64_t v = f[i];

        if (v != 0 && i < adjust_from) {
            adjust_from = i;
        }

        bool has_overflow = false;
        if (std::is_same<int64_t,long>()) {
            long m_test;
            has_overflow = __builtin_smull_overflow(v, nPartValue, &m_test);
        } else if (std::is_same<int64_t,long long>()) {
            long long m_test;
            has_overflow = __builtin_smulll_overflow(v, nPartValue, &m_test);
        } else {
            assert(0); // todo: compile error
        }

        if (has_overflow) {
            multiprecision::uint128_t v128(v);
            multiprecision::uint128_t part128(nPartValue);
            multiprecision::uint128_t f128 = (v128*part128)/nTotalValue;
            fPart.f[i] = f128.convert_to<int64_t>();
        }
        else {
            fPart.f[i] = (v*nPartValue)/nTotalValue;
        }

        nPartValueSum += fPart.f[i];
    }

    if (nPartValueSum == nPartValue)
        return fPart;
    if (nPartValueSum > nPartValue)
        return fPart; // todo:peg: validate if possible

    int idx = adjust_from;
    int64_t nAdjustValue = nPartValue - nPartValueSum;
    while(nAdjustValue >0) {
        // todo:peg: review all possible cases if rounding mismatch with adjust_from
        if (fPart.f[idx] < f[idx]) {
            nAdjustValue--;
            fPart.f[idx]++;
        }
        idx++;
        if (idx >= PEG_SIZE) {
            idx = adjust_from;
        }
    }

    return fPart;
}

/** Take a part as ration part/total where part is also value (sum fraction)
 *  Add taken part into destination fractions. Adjusted from adjust_from.
 *  Returns not completed amount (if source("this") has no enough)
 *  #NOTE8
 */
int64_t CFractions::MoveRatioPartTo(int64_t nValueToMove,
                                    CFractions& b)
{
    int64_t nTotalValue = Total();
    int64_t nPartValue = nValueToMove;

    if (nTotalValue == 0)
        return nValueToMove;
    if (nValueToMove == 0)
        return 0;

    if ((nFlags & STD) == 0)
        ToStd();
    if ((b.nFlags & STD) == 0)
        b.ToStd();

    if (nPartValue >= nTotalValue) {
        nPartValue = nTotalValue;
        b += *this; // move all
        for(int i=0; i<PEG_SIZE; i++) f[i] = 0; // taken all
        return nValueToMove - nPartValue;
    }

    int64_t nPartValueSum = 0;
    int adjust_from = PEG_SIZE;
    for(int i=0; i<PEG_SIZE; i++) {
        int64_t v = f[i];

        if (v != 0 && i < adjust_from) {
            adjust_from = i;
        }

        bool has_overflow = false;
        if (std::is_same<int64_t,long>()) {
            long m_test;
            has_overflow = __builtin_smull_overflow(v, nPartValue, &m_test);
        } else if (std::is_same<int64_t,long long>()) {
            long long m_test;
            has_overflow = __builtin_smulll_overflow(v, nPartValue, &m_test);
        } else {
            assert(0); // todo: compile error
        }

        int64_t vp = 0;

        if (has_overflow) {
            multiprecision::uint128_t v128(v);
            multiprecision::uint128_t part128(nPartValue);
            multiprecision::uint128_t f128 = (v128*part128)/nTotalValue;
            vp = f128.convert_to<int64_t>();
        }
        else {
            vp = (v*nPartValue)/nTotalValue;
        }

        nPartValueSum += vp;
        b.f[i] += vp;
        f[i] -= vp;
    }

    if (nPartValueSum == nPartValue)
        return 0;
    if (nPartValueSum > nPartValue)
        return 0; // todo:peg: validate if possible

    int idx = adjust_from;
    int64_t nAdjustValue = nPartValue - nPartValueSum;
    while(nAdjustValue >0) {
        if (f[idx] >0) {
            nAdjustValue--;
            b.f[idx]++;
            f[idx]--;
        }
        idx++;
        if (idx >= PEG_SIZE) {
            idx = adjust_from;
        }
    }

    return 0;
}

CFractions& CFractions::operator+=(const CFractions& b)
{
    if ((b.nFlags & STD) == 0) {
        return operator+=(b.Std());
    }
    if ((nFlags & STD) == 0) {
        ToStd();
    }
    for(int i=0; i<PEG_SIZE; i++) {
        f[i] += b.f[i];
    }
    return *this;
}

CFractions& CFractions::operator-=(const CFractions& b)
{
    if ((b.nFlags & STD) == 0) {
        return operator-=(b.Std());
    }
    if ((nFlags & STD) == 0) {
        ToStd();
    }
    for(int i=0; i<PEG_SIZE; i++) {
        f[i] -= b.f[i];
    }
    return *this;
}

CFractions CFractions::operator&(const CFractions& b) const
{
    CFractions a = *this;
    for(int i=0; i<PEG_SIZE; i++) {
        int64_t va = a.f[i];
        int64_t vb = b.f[i];
        if      (va >=0 && vb >=0) a.f[i] = std::min(va, vb);
        else if (va >=0 && vb < 0) a.f[i] = 0;
        else if (va < 0 && vb >=0) a.f[i] = 0;
        else if (va < 0 && vb < 0) a.f[i] = std::max(va, vb);
    }
    return a;
}

CFractions CFractions::operator-() const
{
    CFractions a = *this;
    for(int i=0; i<PEG_SIZE; i++) {
        int64_t va = a.f[i];
        a.f[i] = -va;
    }
    return a;
}

double CFractions::Distortion(const CFractions& b) const
{
    int64_t nTotalA = Total();
    int64_t nTotalB = b.Total();

    if (nTotalA == nTotalB) {

        if (nTotalA == 0) {
            return 0;
        }

        int64_t nDiff = 0;

        for(int i=0; i<PEG_SIZE; i++) {
            int64_t va = f[i];
            int64_t vb = b.f[i];
            if (va > vb) {
                nDiff += (va - vb);
            }
        }

        return double(nDiff) / double(nTotalA);
    }

    else if (nTotalA < nTotalB) {
        if (nTotalA == 0) {
            return nTotalB; // does not make sense to scale
        }

        CFractions b1 = b.RatioPart(nTotalA);
        return Distortion(b1);
    }

    else if (nTotalA > nTotalB) {
        if (nTotalB == 0) {
            return nTotalA; // does not make sense to scale
        }

        CFractions a1 = RatioPart(nTotalB);
        return a1.Distortion(b);
    }

    return 0;
}


/*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\
|  Phycas: Python software for phylogenetic analysis                          |
|  Copyright (C) 2006 Mark T. Holder, Paul O. Lewis and David L. Swofford     |
|                                                                             |
|  This program is free software; you can redistribute it and/or modify       |
|  it under the terms of the GNU General Public License as published by       |
|  the Free Software Foundation; either version 2 of the License, or          |
|  (at your option) any later version.                                        |
|                                                                             |
|  This program is distributed in the hope that it will be useful,            |
|  but WITHOUT ANY WARRANTY; without even the implied warranty of             |
|  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the              |
|  GNU General Public License for more details.                               |
|                                                                             |
|  You should have received a copy of the GNU General Public License along    |
|  with this program; if not, write to the Free Software Foundation, Inc.,    |
|  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.                |
\~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/

#if ! defined(GTR_MODEL_HPP)
#define GTR_MODEL_HPP

#include "states_patterns.hpp"
#include <boost/noncopyable.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/lambda/lambda.hpp>
#include <vector>
#include <numeric>
#include <algorithm>
#include "xlikelihood.hpp"
#include "mcmc_param.hpp"
#include "basic_cdf.hpp"
#include "q_matrix.hpp"
#include "model.hpp"
#include "multivariate_probability_distribution.hpp"

namespace phycas{

/*----------------------------------------------------------------------------------------------------------------------
|	Specialization of the base class Model that represents the General Time Reversible (GTR) model.
*/
class GTR: public Model
	{
	public:
                                        GTR();
                                        ~GTR()
                                            {
                                            //std::cerr << "GTR dying..." << std::endl;
                                            }

		virtual std::string             getModelName() const;
		virtual void                    createParameters(JointPriorManagerShPtr jpm, TreeShPtr t, MCMCUpdaterVect & edgelens, MCMCUpdaterVect & edgelen_hyperparams, MCMCUpdaterVect & parameters, int subset_pos);
		double                          calcUniformizationLambda() const;
        double                          calcLMat(double * * lMat) const;
        double                          calcUMat(double * * uMat) const;
		void                            calcPMat(double * * pMat, double edgeLength) const;

        void                            fixRelRates();
		void                            freeRelRates();

        std::vector<double>             getRelRates();
		void                            setRelRates(const std::vector<double> & rates);

        void                            calcRelRatesFromRatios();
		void                            setRelRateRatio(unsigned which, double value);

        void                            setStateFreqRatio(unsigned ratio_index, double value);

		void                            setRelRateUnnorm(unsigned param_index, double value);
		double                          getRelRateUnnorm(unsigned param_index);

		void                            setRelRateParamPrior(ProbDistShPtr d);
		ProbDistShPtr                   getRelRateParamPrior();

		void                            setRelRatePrior(MultivarProbDistShPtr d);
		MultivarProbDistShPtr           getRelRatePrior();

        void                            setNucleotideFreqs(double freqA, double freqC, double freqG, double freqT);
        void                            setAllFreqsEqual();
		virtual void                    setStateFreqUnnorm(unsigned param_index, double value);
        virtual void                    setStateFreqsUnnorm(const std::vector<double> & values);

        void                            setStateFreqPrior(MultivarProbDistShPtr d);
		MultivarProbDistShPtr           getStateFreqPrior();

        void                            setStateFreqParamPrior(ProbDistShPtr d);
		ProbDistShPtr                   getStateFreqParamPrior();

        virtual std::string             paramHeader() const;
		virtual std::string             paramReport(unsigned ndecimals, bool include_edgelen_hyperparams) const;
		double                          calcTRatio();

        virtual unsigned                getNumFreeParameters() const;
        virtual void                    appendPWKParamNames(std::vector<std::string> & names, std::string prefix = "") const;
        virtual void                    appendFreeParamNames(std::vector<std::string> & names, std::string prefix = "") const;
        virtual void                    appendParamNames(std::vector<std::string> & names, std::string prefix = "") const;
        virtual void                    appendUntransformedParamValues(std::vector<double> & values) const;
        virtual void                    appendTransformedParamValues(std::vector<double> & values) const;
        virtual bool                    setParamValueFromTransformed(std::string parameter_name, double transformed_value, TreeShPtr tree);
        virtual double                  calcLogDetJacobian() const;

	protected:

		MultivarProbDistShPtr           rel_rate_prior;		    /**< The prior distribution governing each relative rate (usually a gamma distribution with scale 1 and shape equal to the desired Dirichlet parameter) */
		ProbDistShPtr                   rel_rate_param_prior;	/**< The joint prior distribution governing all six relative rates */
		ProbDistShPtr                   freq_param_prior;	    /**< The prior distribution governing each frequency parameter (usually a gamma distribution with scale 1 and shape equal to the desired Dirichlet parameter; used if frequencies are updated separately by slice sampling) */
    	MultivarProbDistShPtr           freq_prior;	            /**< The prior distribution governing the vector of frequencies (used if frequencies are updated jointly by StateFreqMove) */
		bool                            rel_rates_fixed;	    /**< If true, the relative rate values will not change during MCMC updates */
		mutable MCMCUpdaterVect         rel_rate_params;	    /**< A vector containing copies of all six relative rate parameters (saved so that fixed/free status can be changed) */
		mutable QMatrix                 q_matrix;			    /**< A QMatrix object used to compute transition probabilities */

		string_vect_t                   relrate_name;			/**< Holds names of the 6 relative rates (e.g. rAC, rAG, rAT, rCG, rCT, rGT) used as headers in the param file */
		string_vect_t                   freq_name;				/**< Holds names of the 4 state frequencies (e.g. freqA, freqC, freqG, freqT) used as headers in the param file */

		// Below here are quantities that directly affect likelihood calculations and which should increment time_stamp when modified
		std::vector<double>             rel_rates;			    /**< A vector containing the six relative rates */
		std::vector<double>             rel_rate_ratios;	    /**< A vector for temporary storage of the five relative rate ratios: rAG/rAC, rAT/rAC, rCG/rAC, rCT/rAC, and rGT/rAC */
	};

typedef boost::shared_ptr<GTR> GTRShPtr;

} // namespace phycas

//#include "phycas/src/gtr_model.inl"

#endif


#include <iostream>
#include <string>
#include <vector>
#include "cnpy.h"
#include "float_conversion.h"
#include <fstream>
#include <chrono>
#include <thread>
#include <mutex>
#include <queue>
#include <algorithm>
#include <random>
#include <Eigen/Dense>
#include <sys/stat.h>

using namespace std;
using namespace Eigen;
using namespace std::string_literals;

constexpr size_t BLUE   =  0;
constexpr size_t GREEN  =  1;
constexpr size_t RED    =  2;
constexpr size_t RE1    =  3;
constexpr size_t RE2    =  4;
constexpr size_t RE3    =  5;
constexpr size_t NIR    =  6;
constexpr size_t SWIR1  =  7;
constexpr size_t SWIR2  =  8;
constexpr size_t CLOUDS =  9;
constexpr size_t SHAPE  = 10;

constexpr size_t NUM_THREADS = 8;
constexpr size_t BANDS = 6;

const array<float, 3> QUANTILES {0.2f, 0.5f, 0.8f};
const size_t QS = QUANTILES.size();

typedef array<float, BANDS> Point;
typedef vector<Point> PointCloud;
typedef pair<vector<float>, vector<PointCloud>> timeseries;

constexpr float START_DAY = 31 + 28; // Only use images starting in March
constexpr float MAX_DAY   = 365 - START_DAY - 31; // End with November

time_t basetime;

struct result
{
  string number;
  char type;
  vector<float> features;
};

array<thread, NUM_THREADS> threads;
array<string, NUM_THREADS> current;
array<vector<result>, NUM_THREADS> results;
mutex queue_lock;
queue<string> todo;

#define DATA(b) half_to_float(raw.data<uint16_t>()[ b * W * H * T + x * H * T + y * T + t ])

float clamp(float x, float min=-100, float max=100)
{
  if (x > max)
    return max;
  else if (x < min)
    return min;
  else if (isnormal(x))
    return x;
  else
    return 0;
}

timeseries parse(string filename)
{
  vector<float> timestamps;
  vector<PointCloud> all_points;

  cnpy::NpyArray raw = cnpy::npy_load(filename + ".npy"s);
  ifstream indexfile(filename + ".idx"s);
  if(indexfile.is_open())
  {
    string line;
    // Skip two lines
    getline(indexfile, line);
    getline(indexfile, line);
    // Skip the label
    getline(indexfile, line, '~');
    // Get entries
    while(getline(indexfile, line, ','))
    {
      struct tm tm;
      strptime(line.c_str(), "%Y-%m-%dT%H:%M:%S", &tm);
      float diff = difftime(mktime(&tm), basetime) / (60 * 60 * 24) - START_DAY;
      timestamps.push_back(diff);
    }
    indexfile.close();
  }

  size_t W = raw.shape[1];
  size_t H = raw.shape[2];
  size_t T = raw.shape[3];
  vector<float> used_timestamps;

  assert(timestamps.size() == T);

  for(size_t t = 0; t < T; ++t)
  {
    if(timestamps[t] <= 0)
      continue;
    PointCloud points;
    size_t glitchy_points = 0,
           cloudy_points = 0,
           total_points = 0;
    for(size_t x = 0; x < W; ++x) for(size_t y = 0; y < H; ++y)
    {
      if(DATA(SHAPE) > 0)
      {
        ++total_points;
        if(DATA(CLOUDS) > 0)
          ++cloudy_points;
        float red  = DATA(RED),
              grn  = DATA(GREEN),
              blu  = DATA(BLUE),
              nir  = DATA(NIR),
              swir1= DATA(SWIR1),
              swir2= DATA(SWIR2),
              re1  = DATA(RE1),
              re2  = DATA(RE2),
              re3  = DATA(RE3);
        if(red == 0 && grn == 0 && blu == 0)
          ++glitchy_points;
        float ndvi = (nir - red) / (nir + red); // Normalized Difference Vegetation Index
        float cai = (swir1 - swir2) / (swir1 + swir2); // Cellulose Absorption index
        float n = (2 * (nir * nir - red * red) + 1.5f * nir + 0.5 * red ) / (nir + red + 0.5f);
        float gemi = n * (1.0f - 0.25f * n) - (red - 0.125f) / (1 - red); // Global Environment Monitoring Index
        float gli = (2 * grn - red - blu) / (2 * grn + red + blu); // Green Leaf Index
        float cvi = (nir * red) / (grn * grn); // Chlorophyll vegetation index
        float ccci = ((nir - re1) / (nir + re1)) / ndvi; // Canopy Chlorophyll Content Index
        float dswi = (re3 - grn) / (swir1 + red); // Disease-Water Stress Index 5
        float nd790_670 = (re3 - red) / (re3 + red); // Normalized Difference 790 / 670
        float ndwi = (nir - swir2) / (nir + swir2); // Normalized Difference Water Index
        points.push_back({
            clamp(ndvi),
            // clamp(cai),
            clamp(gemi),
            clamp(gli),
            clamp(cvi),
            // clamp(ccci),
            clamp(dswi),
            // clamp(nd790_670),
            clamp(ndwi),
        });
      }
    }
    if(cloudy_points > 0.2 * total_points)
      continue;
    if(glitchy_points > 0.2 * total_points)
      continue;
    all_points.push_back(points);
    used_timestamps.push_back(timestamps[t] / MAX_DAY);
  }

  return make_pair(used_timestamps, all_points);
}

inline float hat(float t, float center, float width)
{
  return max(1 - abs(t-center) / width, 0.0f) / width;
}

const vector<float> get_features(timeseries data, float subsample=1.0f)
{
  vector<float> time = data.first;
  vector<PointCloud> point_series  = data.second;
  int count = point_series[0].size();

  vector<bool> used(count, true);
  if(subsample < 1.0f)
  {
    count = 0;
    random_device rd;
    mt19937 random(rd());
    uniform_real_distribution<> real(0.0, 1.0);
    for(auto &&b : used)
    {
      b = real(random) < subsample;
      count += b;
    }
  }

  vector<float> features;
  array<vector<float>, BANDS * QS> Q;

  for(PointCloud point_cloud : point_series)
  {
    for(size_t band = 0; band < BANDS; ++band)
    {
      vector<float> points;
      for(size_t i = 0; i < point_cloud.size(); ++i)
      {
        if(used[i] && !isnan(point_cloud[i][band]))
        {
          points.push_back(point_cloud[i][band]);
        }
      }
      if(points.size() > 0)
      {
        sort(points.begin(), points.end());
        for(size_t q = 0; q < QS; ++q)
          Q[band * QS + q].push_back(points[size_t(points.size() * QUANTILES[q])]);
      }
      else
      {
        for(float q : QUANTILES)
          Q[band * QS + q].push_back(0);
      }
    }
  }

  for(auto q : Q)
  {
    constexpr size_t DIM = 8;
    // vector<complex<float>> coefficients(NUM_FOURIER, 0);
    Eigen::Matrix<float, Eigen::Dynamic, DIM> model(q.size(), DIM);
    Eigen::Matrix<float, Eigen::Dynamic, 1> target(q.size(), 1);
    for(size_t k = 0; k < q.size(); ++k)
    {
      target(k, 0) = q[k];
      model(k, 0)  = hat(time[k], 0.0f  , 1.0f  ); // Left Edge
      model(k, 1)  = hat(time[k], 1.0f  , 1.0f  ); // Right Edge
      model(k, 2)  = hat(time[k], 0.5f  , 0.5f  ); // 1/2
      model(k, 3)  = hat(time[k], 0.25f , 0.25f ); // 1/4
      model(k, 4)  = hat(time[k], 0.75f , 0.25f ); // 3/4
      model(k, 5)  = hat(time[k], 0.125f, 0.125f); // 1/8
      model(k, 6)  = hat(time[k], 0.375f, 0.125f); // 3/8
      model(k, 7)  = hat(time[k], 0.625f, 0.125f); // 5/8
      // model(k, 8)  = hat(time[k], 0.875f, 0.125f); // 7/8
    }
    Eigen::Matrix<float, DIM, DIM> H = model.transpose() * model;
    H = H.completeOrthogonalDecomposition().pseudoInverse();
    Eigen::Matrix<float, DIM, 1> weight = H * model.transpose() * target;
    for(size_t k = 0; k < DIM; ++k)
    {
      features.push_back(weight(k, 0));
    }
  }
  return features;
}

inline bool file_exists (const std::string& name)
{
  struct stat buffer;
  return (stat (name.c_str(), &buffer) == 0);
}

void work(size_t thread_id)
{
  current[thread_id] = "Initializing";
  bool running = true;
  string number = "";
  while(running)
  {
    {
      lock_guard<mutex> lock(queue_lock);
      if(todo.empty())
      {
        break;
      } else {
        number = todo.front();
        current[thread_id] = number;
        todo.pop();
      }
    }

    string filename = "/home/konrad/dev/remote_sensing/ibiss_processed/cubes/" + number;
    if(!file_exists(filename + ".npy"s))
      continue;
    timeseries data = parse(filename);

    results[thread_id].push_back({ number, 'F', get_features(data, 0.5) });
  }
  current[thread_id] = "Done";
}

int main(int argc, char **argv)
{
  struct tm tm;
  strptime("2017-01-01T00:00:00", "%Y-%m-%dT%H:%M:%S", &tm);
  basetime = mktime(&tm);

  size_t count = 0;
  if(argc == 1)
  {
    string number = "";
    while(getline(cin, number))
    {
      todo.push(number);
      ++count;
    }
  }
  else
  {
    string number = "";
    ifstream input(argv[1]);
    if(!input.is_open())
      return 1;
    while(getline(input, number))
    {
      todo.push(number);
      ++count;
    }
    input.close();
  }

  cout << "Beginning feature extraction" << endl;
  auto features = vector<pair<string, vector<float>>>(count);
  size_t num = features.size();

  for(size_t i = 0; i < 100; ++i)
    cout << '_';
  cout << endl;

  for(size_t i = 0; i < NUM_THREADS; ++i)
  {
    threads[i] = thread(work, i);
  }

  size_t progress = 100;
  while(progress > 0)
  {
    if(progress * num > todo.size() * 100)
    {
      cout << '#' << flush;
      --progress;
    }
    else
    {
      this_thread::sleep_for(100ms);
    }
  }

  cout << endl;
  for(size_t i = 0; i < NUM_THREADS; ++i)
  {
    cout << "Thread (" << i << ") processed " << results[i].size() << " elements" << endl;
  }

  for(size_t i = 0; i < NUM_THREADS; ++i)
  {
    threads[i].join();
  }
  cout << "\nCollected all threads" << endl;

  cout << "\nWriting CSV..." << endl;

  ofstream out("/home/konrad/dev/remote_sensing/data/cfeatures.csv");

  for(size_t i = 0; i < 8*NUM_THREADS; ++i)
    cout << '_';
  cout << endl;

  for(size_t i = 0; i < NUM_THREADS; ++i)
  {
    for(auto p : results[i])
    {
      out << p.number << ',' << p.type;
      for(auto val : p.features)
        out << "," << val;
      out << '\n';
    }
    cout << "########" << flush;
    results[i].clear();
  }
  cout << endl;
  out.close();

  return 0;
}