#ifndef XOR_FAST_HASH_HPP
#define XOR_FAST_HASH_HPP

#include <boost/assert.hpp>

#include <algorithm>
#include <array>
#include <iterator>
#include <numeric>
#include <random>
#include <vector>

#include <cstdint>

namespace osrm
{
namespace util
{

/*
    This is an implementation of Tabulation hashing, which has suprising properties like
   universality.
    The space requirement is 2*2^16 = 256 kb of memory, which fits into L2 cache.
    Evaluation boils down to 10 or less assembly instruction on any recent X86 CPU:

    1: movq    table2(%rip), %rdx
    2: movl    %edi, %eax
    3: movzwl  %di, %edi
    4: shrl    $16, %eax
    5: movzwl  %ax, %eax
    6: movzbl  (%rdx,%rax), %eax
    7: movq    table1(%rip), %rdx
    8: xorb    (%rdx,%rdi), %al
    9: movzbl  %al, %eax
    10: ret

*/
template <std::size_t MaxNumElements = (1u << 16u)> class XORFastHash
{
    static_assert(MaxNumElements <= (1u << 16u), "only 65536 elements indexable with uint16_t");

    std::array<std::uint16_t, MaxNumElements> table1;
    std::array<std::uint16_t, MaxNumElements> table2;

  public:
    XORFastHash()
    {
        std::mt19937 generator; // impl. defined but deterministic default seed

        std::iota(begin(table1), end(table1), 0u);
        std::shuffle(begin(table1), end(table1), generator);

        std::iota(begin(table2), end(table2), 0u);
        std::shuffle(begin(table2), end(table2), generator);
    }

    inline std::uint16_t operator()(const std::uint32_t originalValue) const
    {
        std::uint16_t lsb = originalValue & 0xffffu;
        std::uint16_t msb = originalValue >> 16u;

        BOOST_ASSERT(lsb < table1.size());
        BOOST_ASSERT(msb < table2.size());

        return table1[lsb] ^ table2[msb];
    }
};
}
}

#endif // XOR_FAST_HASH_HPP


/***************************************************************************
 *   Copyright (C) 2007 by Reed A. Cartwright                              *
 *   reed@scit.us                                                          *
 *                                                                         *
 *   Permission is hereby granted, free of charge, to any person obtaining *
 *   a copy of this software and associated documentation files (the       *
 *   "Software"), to deal in the Software without restriction, including   *
 *   without limitation the rights to use, copy, modify, merge, publish,   *
 *   distribute, sublicense, and/or sell copies of the Software, and to    *
 *   permit persons to whom the Software is furnished to do so, subject to *
 *   the following conditions:                                             *
 *                                                                         *
 *   The above copyright notice and this permission notice shall be        *
 *   included in all copies or substantial portions of the Software.       *
 *                                                                         *
 *   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       *
 *   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    *
 *   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*
 *   IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR     *
 *   OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, *
 *   ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR *
 *   OTHER DEALINGS IN THE SOFTWARE.                                       *
 ***************************************************************************/

#ifdef HAVE_CONFIG_H
#	include "config.h"
#endif

#define _USE_MATH_DEFINES

#include <iostream>

#include <boost/math/special_functions/zeta.hpp>

#include "sample.h"
#include "sample_k2p.h"
#include "ccvector.h"
#include "series.h"
#include "invert_matrix.h"
#include "table.h"
#include "emdel.h"

using namespace std;

void sample_k2p_zeta::preallocate(size_t maxa, size_t maxd)
{
	p.resize(maxa+1,maxd+1,0.0);
}

void sample_k2p_zeta::presample_step(const params_type &params, const sequence &seq_a, const sequence &seq_d)
{
	size_t sz_max = std::max(sz_height, sz_width);
	size_t sz_anc = seq_a.size();
	size_t sz_dec = seq_d.size();
	sa = seq_a;
	sd = seq_d;

	const model_type &model = get_model();

	p(0,0) = prob_scale;

	for(size_t d = 1; d <= sz_dec; ++d) {
		p(0,d) = 0.0;
		for(size_t k = d; k > 0; --k)
			p(0,d) += p(0,d-k)*model.p_indel_size[k];
	}

	for(size_t a = 1; a <= sz_anc; ++a) {
		p(a,0) = 0.0;
		for(size_t k = a; k > 0; --k)
			p(a,0) += p(a-k,0)*model.p_indel_size[k];
	}

	for(size_t a=1;a<=sz_anc;++a) {
		for(size_t d=1;d<= sz_dec;++d) {
			double pt = p(a-1,d-1)*model.p_substitution[sa[a-1]][sd[d-1]];

			for(size_t k = d; k > 0; --k)
				pt += p(a,d-k)*model.p_indel_size[k];
			for(size_t k = a; k > 0; --k)
				pt += p(a-k,d)*model.p_indel_size[k];
			p(a,d) = pt;
		}
	}
}

void sample_k2p_zeta::sample_once(std::string &seq_a, std::string &seq_d)
{
	size_t a = sa.size();
	size_t d = sd.size();
	seq_a.clear();
	seq_d.clear();

	const model_type &model = get_model();

	while(a != 0 || d != 0) {
		double u = p(a,d)*myrand.uniform01();
		double t = (a > 0 && d > 0) ? p(a-1,d-1)*model.p_substitution[sa[a-1]][sd[d-1]] : 0.0;
		if(u < t) {
			// match
			a = a-1;
			d = d-1;
			seq_a.append(1, ccNuc[sa[a]]);
			seq_d.append(1, ccNuc[sd[d]]);
		} else {
			for(size_t k = 1; k <= a || k <= d; ++k) {
				if(k <= a) {
					t += p(a-k,d)*model.p_indel_size[k];
					if(u < t) {
						// gap in d
						seq_d.append(k, '-');
						while(k--) seq_a.append(1, ccNuc[sa[--a]]);
						break;
					}
				}
				if(k <= d)
				{
					t += p(a,d-k)*model.p_indel_size[k];
					if(u < t) {
						// gap in a
						seq_a.append(k, '-');
						while(k--) seq_d.append(1, ccNuc[sd[--d]]);
						break;
					}
				}
			}
		}
		//seq_a.append(1,',');
		//seq_d.append(1,',');
	}
	reverse(seq_a.begin(), seq_a.end());
	reverse(seq_d.begin(), seq_d.end());
}

void sample_k2p_geo::preallocate(size_t maxa, size_t maxd)
{
	p.resize(maxa+1,maxd+1,0.0);
}

void sample_k2p_geo::presample_step(const params_type &params, const sequence &seq_a, const sequence &seq_d)
{
	size_t sz_max = std::max(sz_height, sz_width);
	size_t sz_anc = seq_a.size();
	size_t sz_dec = seq_d.size();
	sa = seq_a;
	sd = seq_d;

	const model_type &model = get_model();

	double row_cache = 0.0;
	vector<double> col_cache(sz_dec+1, 0.0);

	p(0,0) = prob_scale;
	for(size_t d = 1; d <= sz_dec; ++d) {
		row_cache *= model.p_extend;
		row_cache += p(0,d-1)*model.p_open;
		p(0,d) = row_cache;
	}

	row_cache = 0.0;
	for(size_t a = 1; a <= sz_anc; ++a) {
		row_cache *= model.p_extend;
		row_cache += p(a-1,0)*model.p_open;
		p(a,0) = row_cache;
	}

	for(size_t a=1;a<=sz_anc;++a) {
		row_cache = 0.0;
		for(size_t d=1;d<= sz_dec;++d) {
			double pt = p(a-1,d-1)*model.p_substitution[sa[a-1]][sd[d-1]];
			row_cache *= model.p_extend;
			row_cache += p(a,d-1)*model.p_open;
			pt += row_cache;
			col_cache[d] *= model.p_extend;
			col_cache[d] += p(a-1,d)*model.p_open;
			pt += col_cache[d];
			p(a,d) = pt;
		}
	}
}

void sample_k2p_geo::sample_once(std::string &seq_a, std::string &seq_d)
{
	size_t a = sa.size();
	size_t d = sd.size();
	seq_a.clear();
	seq_d.clear();

	const model_type &model = get_model();

	while(a != 0 || d != 0) {
		double u = p(a,d)*myrand.uniform01();
		double t = (a > 0 && d > 0) ? p(a-1,d-1)*model.p_substitution[sa[a-1]][sd[d-1]] : 0.0;
		if(u < t) {
			// match
			a = a-1;
			d = d-1;
			seq_a.append(1, ccNuc[sa[a]]);
			seq_d.append(1, ccNuc[sd[d]]);
		} else {
			for(size_t k = 1; k <= a || k <= d; ++k) {
				if(k <= a) {
					t += p(a-k,d)*model.p_indel_size[k];
					if(u < t) {
						// gap in d
						seq_d.append(k, '-');
						while(k--) seq_a.append(1, ccNuc[sa[--a]]);
						break;
					}
				}
				if(k <= d)
				{
					t += p(a,d-k)*model.p_indel_size[k];
					if(u < t) {
						// gap in a
						seq_a.append(k, '-');
						while(k--) seq_d.append(1, ccNuc[sd[--d]]);
						break;
					}
				}
			}
		}
		//seq_a.append(1,',');
		//seq_d.append(1,',');
	}
	reverse(seq_a.begin(), seq_a.end());
	reverse(seq_d.begin(), seq_d.end());
}

// Draw from Zipf distribution, with parameter a > 1.0
// Devroye Luc (1986) Non-uniform random variate generation.
//     Springer-Verlag: Berlin. p551
inline unsigned int rand_zipf(double a)
{
	double b = pow(2.0, a-1.0);
	double x,t;
	do {
	 x = floor(pow(myrand.uniform01(), -1.0/(a-1.0)));
	 t = pow(1.0+1.0/x, a-1.0);
	} while( myrand.uniform01()*x*(t-1.0)*b >= t*(b-1.0));
	return (unsigned int)x;
}

const char g_nuc[] = "ACGT";
const char g_nuc2[] = "ACGT" "GTAC" "TGCA" "CATG";

void gen_sample_k2p_zeta::sample_once(std::string &seq_a, std::string &seq_d)
{
	seq_a.clear();
	seq_d.clear();

	const model_type &model = get_model();
	const double z = get_params()[model_k2p_zeta::pZ];
	while(1) {
		double p = myrand.uniform01();
		if(p < model.p_2h) {
			// M
			unsigned int x = myrand.uniform(4);
			seq_a.append(1, g_nuc[x]);
			p = myrand.uniform01();
			if(p < model.p_match) {
				seq_d.append(1, g_nuc2[x]);
			} else if(p < model.p_match+model.p_ts) {
				seq_d.append(1, g_nuc2[x+4]);
			} else if(p < 1.0 - 0.5*model.p_ts) {
				seq_d.append(1, g_nuc2[x+8]);
			} else {
				seq_d.append(1, g_nuc2[x+12]);
			}
		} else if(p <  model.p_2h+model.p_2g) {
			// U
			unsigned int x;
			do {
				x = rand_zipf(z);
			} while(x > nmax);
			seq_d.append(x, '-');
			while(x--) {
				seq_a.append(1, g_nuc[myrand.uniform(4)]);
			}
		} else if(p <  1.0-model.p_end) {
			// V
			unsigned int x;
			do {
				x = rand_zipf(z);
			} while(x > nmax);
			seq_a.append(x, '-');
			while(x--) {
				seq_d.append(1, g_nuc[myrand.uniform(4)]);
			}
		} else {
			// E
			break;
		}
	}
}

void gen_sample_k2p_geo::sample_once(std::string &seq_a, std::string &seq_d)
{
	seq_a.clear();
	seq_d.clear();

	const model_type &model = get_model();
	const double q = 1.0/get_params()[model_k2p_geo::pQ];
	while(1) {
		double p = myrand.uniform01();
		if(p < model.p_2h) {
			// M
			unsigned int x = myrand.uniform(4);
			seq_a.append(1, g_nuc[x]);
			p = myrand.uniform01();
			if(p < model.p_match) {
				seq_d.append(1, g_nuc2[x]);
			} else if(p < model.p_match+model.p_ts) {
				seq_d.append(1, g_nuc2[x+4]);
			} else if(p < 1.0 - 0.5*model.p_ts) {
				seq_d.append(1, g_nuc2[x+8]);
			} else {
				seq_d.append(1, g_nuc2[x+12]);
			}
		} else if(p <  model.p_2h+model.p_2g) {
			// U
			unsigned int x = myrand.geometric(q);
			seq_d.append(x, '-');
			while(x--) {
				seq_a.append(1, g_nuc[myrand.uniform(4)]);
			}
		} else if(p <  1.0-model.p_end) {
			// V
			unsigned int x = myrand.geometric(q);
			seq_a.append(x, '-');
			while(x--) {
				seq_d.append(1, g_nuc[myrand.uniform(4)]);
			}
		} else {
			// E
			break;
		}
	}
}


// Boost.Geometry (aka GGL, Generic Geometry Library)
// QuickBook Example

// Copyright (c) 2011-2012 Barend Gehrels, Amsterdam, the Netherlands.

// Use, modification and distribution is subject to the Boost Software License,
// Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)

//[register_multi_point_templated
//` Show the use of the macro BOOST_GEOMETRY_REGISTER_MULTI_POINT_TEMPLATED

#include <iostream>
#include <boost/geometry.hpp>
#include <boost/geometry/geometries/adapted/boost_tuple.hpp>
#include <boost/geometry/geometries/register/multi_point.hpp>

BOOST_GEOMETRY_REGISTER_MULTI_POINT_TEMPLATED(std::deque)

BOOST_GEOMETRY_REGISTER_BOOST_TUPLE_CS(cs::cartesian)

int main()
{
    // Normal usage of std::
    std::deque<boost::tuple<float, float> > multi_point;
    multi_point.push_back(boost::tuple<float, float>(1, 1));
    multi_point.push_back(boost::tuple<float, float>(3, 2));

    // Usage of Boost.Geometry
    std::cout << "WKT: "  << boost::geometry::wkt(multi_point) << std::endl;

    return 0;
}

//]

//[register_multi_point_templated_output
/*`
Output:
[pre
WKT: MULTIPOINT((1 1),(3 2))
]
*/
//]


/*
 * cg.cpp
 *
 * Author: H. Hofbauer (hhofbaue@cosy.sbg.ac.at), E. Pschernig (epschern@cosy.sbg.ac.at), P. Wild (pwild@cosy.sbg.ac.at)
 *
 * Generates an iris code from iris texture using complex gabor filters with different filter size and wavelength.
 *
 */
#include "version.h"
#include <cstdio>
#include <map>
#include <vector>
#include <string>
#include <cstring>
#include <ctime>
#include <fstream>
#include <iostream>
#include <opencv2/core/core.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <boost/regex.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/filesystem.hpp>
#include <boost/date_time/posix_time/posix_time_types.hpp>
#ifndef M_PI
#define M_PI 3.14159265358979323846
#endif

using namespace std;
using namespace cv;

/** no globbing in win32 mode **/
int _CRT_glob = 0;

/** Program modes **/
static const int MODE_MAIN = 1, MODE_HELP = 2;

int FILTER_HEIGHT=31;

/*
 * Print command line usage for this program
 */
void printUsage() {
    printVersion();
	printf("+-----------------------------------------------------------------------------+\n");
	printf("| cg - Iris-code generation (feature extraction) using complex gabor filters  |\n");
	printf("|      with varying wavelengths.                                              |\n");
	printf("|                                                                             |\n");
	printf("| John Daugman. The importance of being random: statistical principles of     |\n");
	printf("| iris recognition. Pattern Recognition 36(2003) 279--291.                    |\n");
	printf("|                                                                             |\n");
	printf("| MODES                                                                       |\n");
	printf("|                                                                             |\n");
    printf("| (# 1) cg iris code extraction from iris textures                            |\n");
    printf("| (# 2) usage                                                                 |\n");
    printf("|                                                                             |\n");
    printf("| ARGUMENTS                                                                   |\n");
    printf("|                                                                             |\n");
    printf("+------+------------+---+---+-------------------------------------------------+\n");
    printf("| Name | Parameters | # | ? | Description                                     |\n");
    printf("+------+------------+---+---+-------------------------------------------------+\n");
    printf("| -i   | infile     | 1 | N | input iris texture (use * as wildcard, all other|\n");
    printf("|      |            |   |   | file parameters may refer to n-th * with ?n)    |\n");
    printf("| -o   | outfile    | 1 | N | output iris code image                          |\n");
//    printf("| -m   | inmaskfile | 1 | Y | noise pixelmask (0: noise, 255: noise-free)     |\n");
//    printf("|      | outmaskfile|   |   | code bitmask (0: noise, 1: noise-free, off)     |\n");
    printf("| -q   |            | 1 | Y | quiet mode on (off)                             |\n");
    printf("| -t   |            | 1 | Y | time progress on (off)                          |\n");
    printf("| -h   |            | 2 | N | prints usage                                    |\n");
    printf("|      |            |   |   |                                                 |\n");
    printf("| -pwl | wavelength | 1 | N | Base wavelength of the gabor filter in pixel.   |\n");
    printf("|      |            |   |   | Size of base filter will then be 2*wl+1.        |\n");
    printf("|      |            |   |   | (default 6)                                     |\n");
    printf("| -pbp | borderpower| 1 | N | Remaining power of the gauss-part of the filter |\n");
    printf("|      |            |   |   | relative to the center (default 0.01).          |\n");
    printf("| -psx | x-samples  | 1 | N | Number of horizontal samples (default 256)      |\n");
    printf("| -x   | width      | 1 | N | Width of input texture (default 512).           |\n");
    printf("| -y   | height     | 1 | N | Height of input texture (default 64).           |\n");
    printf("|      |            |   |   |                                                 |\n");
    printf("| -wf  |            | 1 | N | Write filterbank images                         |\n");
    printf("| -ws  |            | 1 | N | Write image sample with filtered versions       |\n");
    printf("| -wp  |            | 1 | N | Write iris image with extraction points         |\n");
    printf("+------+------------+---+---+-------------------------------------------------+\n");
    printf("|                                                                             |\n");
    printf("| EXAMPLE USAGE                                                               |\n");
    printf("|                                                                             |\n");
    printf("| -i s1.tiff -o s1.png                                                        |\n");
    printf("| -i *.tiff -o ?1.png -q -t                                                   |\n");
    printf("| -i *.tiff -im ?1_mask.png -o ?1_code.png -om ?1_codemask.png -q -t          |\n");
    printf("|                                                                             |\n");
    printf("| AUTHORS                                                                     |\n");
    printf("|                                                                             |\n");
    printf("| Heinz Hofbauer (hhofbaue@cosy.sbg.ac.at)                                    |\n");
    printf("| Elias Pschernig (epschern@cosy.sbg.ac.at)                                   |\n");
    printf("| Peter Wild (pwild@cosy.sbg.ac.at)                                           |\n");
    printf("|                                                                             |\n");
    printf("| COPYRIGHT                                                                   |\n");
    printf("|                                                                             |\n");
    printf("| (C) 2012 All rights reserved. Do not distribute without written permission. |\n");
    printf("+-----------------------------------------------------------------------------+\n");
}

/** --------------- writerstuff for debugging/control --------------- **/

void writeNormMatToFile(const Mat &what, const string &where, const string &txtmessage = "", bool write_sign=false){
    if( write_sign){
        Mat img = ( what >= 0);
        imwrite( where, img);
    } else {
        Mat img(what.rows, what.cols, CV_8U);
        normalize( what, img, 0, 255, NORM_MINMAX);
        imwrite( where, img);
    }
    if( !txtmessage.empty()) cout << txtmessage << " file("<<where<<")"<<endl;
}

void writeExtractionsequence( const Mat &img, const vector<Point> &extractionsequence, bool &writeagain){
    Mat pointoverlay = img.clone();
    for( vector<Point>::const_iterator eit = extractionsequence.begin(); eit != extractionsequence.end(); advance(eit, 1) )
        circle( pointoverlay, *eit, 1, Scalar(255,128,128));
    writeagain = false;
    imwrite( "extractionpointlocation.png", pointoverlay);
    cout << "Extraction point locations written to: extractionpointlocation.png"<<endl;
}
void writeFilterbank( const vector<Mat> &filterbank){
    for( vector<Mat>::const_iterator it = filterbank.begin(); it != filterbank.end(); advance(it,1)){
        string filename = "filter_";
        int idx = distance( filterbank.begin(), it);
        if ( idx % 2 == 0 ) filename+="re";
        else filename+="im";
        filename += boost::lexical_cast<string>(idx/2+1) + "_wl" + boost::lexical_cast<string>((it->cols-1)/2);
        filename += ".png";
        writeNormMatToFile( *it, filename);
        cout << "filterbank written: "<<filename<<endl;
    }
}
/** ------------------------------- image processing functions ------------------------------- **/

Mat extendByWrap (const Mat &src,  int ext_width,  int  ext_height=0){
    CV_Assert(ext_width >=0 && ext_height >= 0);
    int w = src.cols + ext_width;
    int woff = ext_width/2;
    int wrest = ext_width-woff;
    int h = src.rows + ext_height;
    int hoff = ext_height/2;
    int hrest = ext_height-hoff;

    Mat dst(h,w,src.type());
    copyMakeBorder( src, dst, hoff, hrest, woff, wrest, BORDER_WRAP);
    return dst;
}

void featureExtract(Mat &code, Mat &codeMask, const Mat &img, const Mat &mask, const vector<Mat> &filterbank, const vector<Point> &extractionsequence, bool &write_samples){
    CV_Assert(code.size() == Size( filterbank.size()  * extractionsequence.size()/8,1));
    CV_Assert(mask.empty() || codeMask.empty() || codeMask.size() == code.size());
    bool useMask = !mask.empty() && !codeMask.empty();

    int filter_w=0, filter_h=0;
    for( vector<Mat>::const_iterator it=filterbank.begin(); it != filterbank.end(); advance(it,1)){
        if( it->cols > filter_w) filter_w = it->cols;
        if( it->rows > filter_h) filter_h = it->rows;
    }
    Mat extimg = extendByWrap( img, filter_w/* , filter_h */);
    Point extoff(filter_w/2, 0);

    vector<Mat> filteredExtImg;
    for( unsigned int i=0; i < filterbank.size(); ++i){
        filteredExtImg.push_back(extimg.clone());
        filter2D(extimg, filteredExtImg[i], CV_32F, filterbank[i]);
        if( write_samples){
            writeNormMatToFile(filteredExtImg[i], "filtered_ext_fb"+boost::lexical_cast<string>(i)+".png", "sample written:");
            writeNormMatToFile(filteredExtImg[i], "filtered_ext_fb"+boost::lexical_cast<string>(i)+"_bitmap.png", "sample written:", true);
        }
    }
    if( write_samples) writeNormMatToFile(extimg, "filtered_ext_orig.png", "sample written");

    uchar *codeMaskData = NULL;
	if (useMask){
        codeMask.setTo(255);
        codeMaskData = codeMask.data;
    }
	code.setTo(0);
    uchar * codeData = code.data;
    int bitpos = 0;
    float fval;
    for( vector<Mat>::const_iterator fit = filteredExtImg.begin(); fit != filteredExtImg.end(); advance( fit, 1) ){
        for( vector<Point>::const_iterator eit = extractionsequence.begin(); eit != extractionsequence.end(); advance( eit,1) ){
            // true = 1 for sign, 0 is default
            fval = fit->at<float>(*eit + extoff);
            if( fval >= 0) codeData[0] |= 1<<(7-bitpos);
            if( useMask ){
                if( ( fval*fval < 0.000000001 ) ||    // values are very small, discard as possible error
                    ( codeMask.at<uchar>(*eit) > 0 ) )     // texture is masked
                            codeMaskData[0] &= 0xff ^ 1 << (7-bitpos);
            }
            bitpos++;
            if( bitpos == 8){
                codeData++;
                bitpos = 0;
                if (useMask) codeMaskData++;
            }
        }
    }
    write_samples = false; // only write one set
}

/** ------------------------------- commandline functions ------------------------------- **/

/**
 * Parses a command line
 * This routine should be called for parsing command lines for executables.
 * Note, that all options require '-' as prefix and may contain an arbitrary
 * number of optional arguments.
 *
 * cmd: commandline representation
 * argc: number of parameters
 * argv: string array of argument values
 */
void cmdRead(map<string ,vector<string> >& cmd, int argc, char *argv[]){
	for (int i=1; i< argc; i++){
		char * argument = argv[i];
		if (strlen(argument) > 1 && argument[0] == '-' && (argument[1] < '0' || argument[1] > '9')){
			cmd[argument]; // insert
			char * argument2;
			while (i + 1 < argc && (strlen(argument2 = argv[i+1]) <= 1 || argument2[0] != '-'  || (argument2[1] >= '0' && argument2[1] <= '9'))){
				cmd[argument].push_back(argument2);
				i++;
			}
		}
		else {
			CV_Error(CV_StsBadArg,"Invalid command line format");
		}
	}
}

/**
 * Checks, if each command line option is valid, i.e. exists in the options array
 *
 * cmd: commandline representation
 * validOptions: list of valid options separated by pipe (i.e. |) character
 */
void cmdCheckOpts(map<string ,vector<string> >& cmd, const string &validOptions){
	vector<string> tokens;
	const string delimiters = "|";
	string::size_type lastPos = validOptions.find_first_not_of(delimiters,0); // skip delimiters at beginning
	string::size_type pos = validOptions.find_first_of(delimiters, lastPos); // find first non-delimiter
	while (string::npos != pos || string::npos != lastPos){
		tokens.push_back(validOptions.substr(lastPos,pos - lastPos)); // add found token to vector
		lastPos = validOptions.find_first_not_of(delimiters,pos); // skip delimiters
		pos = validOptions.find_first_of(delimiters,lastPos); // find next non-delimiter
	}
	sort(tokens.begin(), tokens.end());
	for (map<string, vector<string> >::iterator it = cmd.begin(); it != cmd.end(); std::advance(it,1)){
		if (!binary_search(tokens.begin(),tokens.end(),it->first)){
			CV_Error(CV_StsBadArg,"Command line parameter '" + it->first + "' not allowed.");
			tokens.clear();
			return;
		}
	}
	tokens.clear();
}

/*
 * Checks, if a specific required option exists in the command line
 *
 * cmd: commandline representation
 * option: option name
 */
void cmdCheckOptExists(map<string ,vector<string> >& cmd, const string &option){
	map<string, vector<string> >::iterator it = cmd.find(option);
	if (it == cmd.end()) CV_Error(CV_StsBadArg,"Command line parameter '" + option + "' is required, but does not exist.");
}

/*
 * Checks, if a specific option has the appropriate number of parameters
 *
 * cmd: commandline representation
 * option: option name
 * size: appropriate number of parameters for the option
 */
void cmdCheckOptSize(map<string ,vector<string> >& cmd, const string &option, const unsigned int size = 1){
	map<string, vector<string> >::iterator it = cmd.find(option);
	if (it->second.size() != size) CV_Error(CV_StsBadArg,"Command line parameter '" + option + "' has unexpected size.");
}

/*
 * Checks, if a specific option has the appropriate number of parameters
 *
 * cmd: commandline representation
 * option: option name
 * min: minimum appropriate number of parameters for the option
 * max: maximum appropriate number of parameters for the option
 */
void cmdCheckOptRange(map<string ,vector<string> >& cmd, string option, unsigned int min = 0, unsigned int max = 1){
	map<string, vector<string> >::iterator it = cmd.find(option);
	unsigned int size = it->second.size();
	if (size < min || size > max) CV_Error(CV_StsBadArg,"Command line parameter '" + option + "' is out of range.");
}

/*
 * Returns the list of parameters for a given option
 *
 * cmd: commandline representation
 * option: name of the option
 */
vector<string> * cmdGetOpt(map<string ,vector<string> >& cmd, const string &option){
	map<string, vector<string> >::iterator it = cmd.find(option);
	return (it != cmd.end()) ? &(it->second) : 0;
}

/*
 * Returns number of parameters in an option
 *
 * cmd: commandline representation
 * option: name of the option
 */
unsigned int cmdSizePars(map<string ,vector<string> >& cmd, const string &option){
	map<string, vector<string> >::iterator it = cmd.find(option);
	return (it != cmd.end()) ? it->second.size() : 0;
}

/*
 * Returns a specific parameter type (int) given an option and parameter index
 *
 * cmd: commandline representation
 * option: name of option
 * param: name of parameter
 */
int cmdGetParInt(map<string ,vector<string> >& cmd, string option, unsigned int param = 0){
	map<string, vector<string> >::iterator it = cmd.find(option);
	if (it != cmd.end()) {
		if (param < it->second.size()) {
			return atoi(it->second[param].c_str());
		}
	}
	return 0;
}

/*
 * Returns a specific parameter type (float) given an option and parameter index
 *
 * cmd: commandline representation
 * option: name of option
 * param: name of parameter
 */
float cmdGetParFloat(map<string ,vector<string> >& cmd, const string &option, const unsigned int param = 0){
	map<string, vector<string> >::iterator it = cmd.find(option);
	if (it != cmd.end()) {
		if (param < it->second.size()) {
			return atof(it->second[param].c_str());
		}
	}
	return 0;
}

/*
 * Returns a specific parameter type (string) given an option and parameter index
 *
 * cmd: commandline representation
 * option: name of option
 * param: name of parameter
 */
string cmdGetPar(map<string ,vector<string> >& cmd, const string &option, const unsigned int param = 0){
	map<string, vector<string> >::iterator it = cmd.find(option);
	if (it != cmd.end()) {
		if (param < it->second.size()) {
			return it->second[param];
		}
	}
	return 0;
}

/** ------------------------------- timing functions ------------------------------- **/

/**
 * Class for handling timing progress information
 */
class Timing{
public:
	/** integer indicating progress with respect tot total **/
	int progress;
	/** total count for progress **/
	int total;

	/*
	 * Default constructor for timing initializing time.
	 * Automatically calls init()
	 *
	 * seconds: update interval in seconds
	 * eraseMode: if true, outputs sends erase characters at each print command
	 */
	Timing(long seconds, bool eraseMode){
		updateInterval = seconds;
		progress = 1;
		total = 100;
		eraseCount=0;
		erase = eraseMode;
		init();
	}

	/*
	 * Destructor
	 */
	~Timing(){}

	/*
	 * Initializes timing variables
	 */
	void init(void){
		start = boost::posix_time::microsec_clock::universal_time();
		lastPrint = start - boost::posix_time::seconds(updateInterval);
	}

	/*
	 * Clears printing (for erase option only)
	 */
	void clear(void){
		string erase(eraseCount,'\r');
		erase.append(eraseCount,' ');
		erase.append(eraseCount,'\r');
		printf("%s",erase.c_str());
		eraseCount = 0;
	}

	/*
	 * Updates current time and returns true, if output should be printed
	 */
	bool update(void){
		current = boost::posix_time::microsec_clock::universal_time();
		return ((current - lastPrint > boost::posix_time::seconds(updateInterval)) || (progress == total));
	}

	/*
	 * Prints timing object to STDOUT
	 */
	void print(void){
		lastPrint = current;
		float percent = 100.f * progress / total;
		boost::posix_time::time_duration passed = (current - start);
		boost::posix_time::time_duration togo = passed * (total - progress) / max(1,progress);
		if (erase) {
			string erase(eraseCount,'\r');
			printf("%s",erase.c_str());
			int newEraseCount = (progress != total) ? printf("Progress ... %3.2f%% (%i/%i Total %i:%02i:%02i.%03i Remaining ca. %i:%02i:%02i.%03i)",percent,progress,total,passed.hours(),passed.minutes(),passed.seconds(),(int)(passed.total_milliseconds()%1000),togo.hours(),togo.minutes(),togo.seconds(),(int)(togo.total_milliseconds() % 1000)) : printf("Progress ... %3.2f%% (%i/%i Total %i:%02i:%02i.%03d)",percent,progress,total,passed.hours(),passed.minutes(),passed.seconds(),(int)(passed.total_milliseconds()%1000));
			if (newEraseCount < eraseCount) {
				string erase(newEraseCount-eraseCount,' ');
				erase.append(newEraseCount-eraseCount,'\r');
				printf("%s",erase.c_str());
			}
			eraseCount = newEraseCount;
		}
		else {
			eraseCount = (progress != total) ? printf("Progress ... %3.2f%% (%i/%i Total %i:%02i:%02i.%03i Remaining ca. %i:%02i:%02i.%03i)\n",percent,progress,total,passed.hours(),passed.minutes(),passed.seconds(),(int)(passed.total_milliseconds()%1000),togo.hours(),togo.minutes(),togo.seconds(),(int)(togo.total_milliseconds() % 1000)) : printf("Progress ... %3.2f%% (%i/%i Total %i:%02i:%02i.%03d)\n",percent,progress,total,passed.hours(),passed.minutes(),passed.seconds(),(int)(passed.total_milliseconds()%1000));
		}
	}
private:
	long updateInterval;
	boost::posix_time::ptime start;
	boost::posix_time::ptime current;
	boost::posix_time::ptime lastPrint;
	int eraseCount;
	bool erase;
};

/** ------------------------------- file pattern matching functions ------------------------------- **/

/*
 * Formats a given string, such that it can be used as a regular expression
 * I.e. escapes special characters and uses * and ? as wildcards
 *
 * pattern: regular expression path pattern
 * pos: substring starting index
 * n: substring size
 *
 * returning: escaped substring
 */
string patternSubstrRegex(string& pattern, size_t pos, size_t n){
	string result;
	for (size_t i=pos, e=pos+n; i < e; i++ ) {
		char c = pattern[i];
		if ( c == '\\' || c == '.' || c == '+' || c == '[' || c == '{' || c == '|' || c == '(' || c == ')' || c == '^' || c == '$' || c == '}' || c == ']') {
			result.append(1,'\\');
			result.append(1,c);
		}
		else if (c == '*'){
			result.append("([^/\\\\]*)");
		}
		else if (c == '?'){
			result.append("([^/\\\\])");
		}
		else {
			result.append(1,c);
		}
	}
	return result;
}

/*
 * Converts a regular expression path pattern into a list of files matching with this pattern by replacing wildcards
 * starting in position pos assuming that all prior wildcards have been resolved yielding intermediate directory path.
 * I.e. this function appends the files in the specified path according to yet unresolved pattern by recursive calling.
 *
 * pattern: regular expression path pattern
 * files: the list to which new files can be applied
 * pos: an index such that positions 0...pos-1 of pattern are already considered/matched yielding path
 * path: the current directory (or empty)
 */
void patternToFiles(string& pattern, vector<string>& files, const size_t& pos, const string& path){
	size_t first_unknown = pattern.find_first_of("*?",pos); // find unknown * in pattern
	if (first_unknown != string::npos){
		size_t last_dirpath = pattern.find_last_of("/\\",first_unknown);
		size_t next_dirpath = pattern.find_first_of("/\\",first_unknown);
		if (next_dirpath != string::npos){
			boost::regex expr((last_dirpath != string::npos && last_dirpath > pos) ? patternSubstrRegex(pattern,last_dirpath+1,next_dirpath-last_dirpath-1) : patternSubstrRegex(pattern,pos,next_dirpath-pos));
			boost::filesystem::directory_iterator end_itr; // default construction yields past-the-end
			try {
				for ( boost::filesystem::directory_iterator itr( ((path.length() > 0) ? path + pattern[pos-1] : (last_dirpath != string::npos && last_dirpath > pos) ? "" : "./") + ((last_dirpath != string::npos && last_dirpath > pos) ? pattern.substr(pos,last_dirpath-pos) : "")); itr != end_itr; ++itr )
				{
					if (boost::filesystem::is_directory(itr->path())){
						boost::filesystem::path p = itr->path().filename();
						string s =  p.string();
						if (boost::regex_match(s.c_str(), expr)){
							patternToFiles(pattern,files,(int)(next_dirpath+1),((path.length() > 0) ? path + pattern[pos-1] : "") + ((last_dirpath != string::npos && last_dirpath > pos) ? pattern.substr(pos,last_dirpath-pos) + pattern[last_dirpath] : "") + s);
						}
					}
				}
			}
			catch (boost::filesystem::filesystem_error &e){}
		}
		else {
			boost::regex expr((last_dirpath != string::npos && last_dirpath > pos) ? patternSubstrRegex(pattern,last_dirpath+1,pattern.length()-last_dirpath-1) : patternSubstrRegex(pattern,pos,pattern.length()-pos));
			boost::filesystem::directory_iterator end_itr; // default construction yields past-the-end
			try {
				for ( boost::filesystem::directory_iterator itr(((path.length() > 0) ? path +  pattern[pos-1] : (last_dirpath != string::npos && last_dirpath > pos) ? "" : "./") + ((last_dirpath != string::npos && last_dirpath > pos) ? pattern.substr(pos,last_dirpath-pos) : "")); itr != end_itr; ++itr )
				{
					boost::filesystem::path p = itr->path().filename();
					string s =  p.string();
					if (boost::regex_match(s.c_str(), expr)){
						files.push_back(((path.length() > 0) ? path + pattern[pos-1] : "") + ((last_dirpath != string::npos && last_dirpath > pos) ? pattern.substr(pos,last_dirpath-pos) + pattern[last_dirpath] : "") + s);
					}
				}
			}
			catch (boost::filesystem::filesystem_error &e){}
		}
	}
	else { // no unknown symbols
		boost::filesystem::path file(((path.length() > 0) ? path + "/" : "") + pattern.substr(pos,pattern.length()-pos));
		if (boost::filesystem::exists(file)){
			files.push_back(file.string());
		}
	}
}
/**
 * Converts a regular expression path pattern into a list of files matching with this pattern
 *
 * pattern: regular expression path pattern
 * files: the list to which new files can be applied
 */
void patternToFiles(string& pattern, vector<string>& files){
	patternToFiles(pattern,files,0,"");
}

/*
 * Renames a given filename corresponding to the actual file pattern using a renaming pattern.
 * Wildcards can be referred to as ?1, ?2, ... in the order they appeared in the file pattern.
 *
 * pattern: regular expression path pattern
 * renamePattern: renaming pattern using ?1, ?2, ... as placeholders for wildcards
 * infile: path of the file (matching with pattern) to be renamed
 * outfile: path of the renamed file
 * par: used parameter (default: '?')
 */
void patternFileRename(string& pattern, const string& renamePattern, const string& infile, string& outfile, const char par = '?'){
	size_t first_unknown = renamePattern.find_first_of(par,0); // find unknown ? in renamePattern
	if (first_unknown != string::npos){
		string formatOut = "";
		for (size_t i=0, e=renamePattern.length(); i < e; i++ ) {
			char c = renamePattern[i];
			if ( c == par && i+1 < e) {
				c = renamePattern[i+1];
				if (c > '0' && c <= '9'){
					formatOut.append(1,'$');
					formatOut.append(1,c);
				}
				else {
					formatOut.append(1,par);
					formatOut.append(1,c);
				}
				i++;
			}
			else {
				formatOut.append(1,c);
			}
		}
		boost::regex patternOut(patternSubstrRegex(pattern,0,pattern.length()));
		outfile = boost::regex_replace(infile,patternOut,formatOut,boost::match_default | boost::format_perl);
	} else {
		outfile = renamePattern;
	}
}

/**
 * First filter ist square with width wl*2+1, for the other filter horizontal
 * wavelength increase by one octave for each filter.
 * A total of 8 filters are generated, 3 octaves from base with real and
 * imaginary parts of the complex gabor wavelet as separate filter.
 *
 * base_wavelength:       Is the base wavelength of the underlying cosine/sine.
 * border_gauss_residual: Adjust gauss sigma such that this percentage of
 *                        central energy is left at the border.
 */
void generateFilterbank( vector<Mat> &filterbank, const int &base_wavelength, const float &border_gauss_residual, int max_octave=3){
    float amp = -1*M_PI;
    float borderfactor = sqrtf(amp/logf(border_gauss_residual));
    int hr = FILTER_HEIGHT; // height radius and filter height
    int h = 2*hr+1;
    float alpha = hr*borderfactor;

    for( int octave =0, wavelength = base_wavelength; octave <= max_octave; wavelength*=2, octave++){
        int wr = wavelength;
        int w = wr*2+1;
        float beta = wr*borderfactor;
        cout << "generating filter for wavelength: "<<wavelength<<" and size "<<w<<"x"<<h ;
        cout <<" alpha: "<<alpha<<" beta: "<<beta<<endl;
        Mat filter_re( h,w,CV_32F), filter_im( h,w,CV_32F);
        float G,F_RE,F_IM;
        int xp,yp;
        for( int y = 0; y < h; ++y)
        for( int x = 0; x < w; ++x){
            xp = x-wr;
            yp = y-hr;
            G = exp( amp*( powf( float(xp)/beta,2) + pow(float(yp)/alpha,2)) );
            F_RE = cos( xp * 2*M_PI/wavelength );
            F_IM = sin( xp * 2*M_PI/wavelength );
            filter_re.at<float>(y,x) = G*F_RE;
            filter_im.at<float>(y,x) = G*F_IM;
        }
        filter_re -= sum(filter_re)[0]/(w*h); // correct for non/zero response  of real filter
        filterbank.push_back(filter_re);
        filterbank.push_back(filter_im);
    }
}

/**
 * Specifies which points to use for the iris code, points are taken as given each point is taken from each filter, then the next point is taken.
 *
 * This function specifies and equally spaced grid for extraction
 *
 * grid_x: number of points in x-direction to extract
 *
 * returns the number of grouped points for a single angle
 */
void generateExtractsequenceGrid( vector<Point> &extractsequence, const int &base_wavelength, const int &grid_x,  const int size_x, const int size_y){
    int sample_x = (grid_x >= 1) ? grid_x : size_x/base_wavelength ;
    int sample_y = size_y/(FILTER_HEIGHT*2+1) ;
    int dx = size_x/sample_x; // horizontal equally spaced, image is rotationally extended for filtering
    float offy = FILTER_HEIGHT+0.5;
    float resty= size_y - 2.*offy;
    float dy = resty/(sample_y-1); // keep filter in image for vertical
    if (dy!=dy || sample_y == 1) dy=0; //NaN comparissons are always false or smaple_y ==1 ,i.e., dy = inf
    for( int y=0; y<sample_y; ++y)
    for( int x=0; x<sample_x; ++x)
        extractsequence.push_back( Point( int(dx/2. + x*dx),int(offy + y*dy)));

    cout << "Extractsequence samples: "<<sample_x<<"x"<<sample_y<<endl;
}

/** ------------------------------- Program ------------------------------- **/

/*
 * Main program
 */
int main(int argc, char *argv[])
{
	int mode = MODE_HELP;
	map<string,vector<string> > cmd;
	try {
		cmdRead(cmd,argc,argv);
    	if (cmd.empty() || cmdGetOpt(cmd,"-h") != 0) mode = MODE_HELP;
    	else mode = MODE_MAIN;
    	if (mode == MODE_MAIN){
			// validate command line
			cmdCheckOpts(cmd,"-i|-o|-q|-t|-wf|-ws|-wp|-pwl|-pbp|-psx|-x|-y");
			cmdCheckOptExists(cmd,"-i");
			cmdCheckOptSize(cmd,"-i",1);
			string inFiles = cmdGetPar(cmd,"-i");
			cmdCheckOptExists(cmd,"-o");
			cmdCheckOptSize(cmd,"-o",1);
			string outFiles = cmdGetPar(cmd,"-o");
			string imaskFiles, omaskFiles;
			if (cmdGetOpt(cmd,"-m") != 0){
				cmdCheckOptSize(cmd,"-m",2);
				imaskFiles = cmdGetPar(cmd,"-m", 0);
				omaskFiles = cmdGetPar(cmd,"-m", 1);
			}
            /** parmeters **/
            int base_wavelength = 6;
			if (cmdGetOpt(cmd,"-pwl") != 0){
				cmdCheckOptSize(cmd,"-pwl",1);
				base_wavelength = cmdGetParInt(cmd,"-pwl", 0);
			}
            float border_gauss_residual = 0.01;
			if (cmdGetOpt(cmd,"-pbp") != 0){
				cmdCheckOptSize(cmd,"-pbp",1);
				border_gauss_residual = cmdGetParFloat(cmd,"-pbp", 0);
                if( border_gauss_residual <= 0 || border_gauss_residual >= 1){
                    cout << "WARNING: Border power does not make much sense, reverting to 0.01"<<endl;
                    border_gauss_residual = 0.01;
                }
			}
            int grid_x=256;
			if (cmdGetOpt(cmd,"-psx") != 0){
				cmdCheckOptSize(cmd,"-psx",1);
				grid_x = cmdGetParInt(cmd,"-psx", 0);
                if( grid_x <= 0){
                    cout << "WARNING: grid_x does not make much sense, setting to sampling with 50% base filter overlap" << endl;
                    grid_x = -1;
                }
			}
            cout <<" grid_x from parameters" << grid_x << endl;
            int size_x=512;
			if (cmdGetOpt(cmd,"-x") != 0){
				cmdCheckOptSize(cmd,"-x",1);
				size_x = cmdGetParInt(cmd,"-x", 0);
			}
            int size_y=64;
			if (cmdGetOpt(cmd,"-y") != 0){
				cmdCheckOptSize(cmd,"-y",1);
				size_y = cmdGetParInt(cmd,"-y", 0);
			}
            FILTER_HEIGHT=(size_y-1)/2 ;
            /** options **/
			bool quiet = false;
			if (cmdGetOpt(cmd,"-q") != 0){
				cmdCheckOptSize(cmd,"-q",0);
				quiet = true;
			}
			bool time = false;
			if (cmdGetOpt(cmd,"-t") != 0){
				cmdCheckOptSize(cmd,"-t",0);
				time = true;
			}
			bool write_filterbank = false;
			if (cmdGetOpt(cmd,"-wf") != 0){
				cmdCheckOptSize(cmd,"-wf",0);
				write_filterbank = true;
			}
			bool write_samples = false;
			if (cmdGetOpt(cmd,"-ws") != 0){
				cmdCheckOptSize(cmd,"-ws",0);
				write_samples = true;
			}
			bool write_points = false;
			if (cmdGetOpt(cmd,"-wp") != 0){
				cmdCheckOptSize(cmd,"-wp",0);
				write_points = true;
			}
			// starting routine
			Timing timing(1,quiet);
			vector<string> files;
			patternToFiles(inFiles,files);
			CV_Assert(files.size() > 0);
			timing.total = files.size();
            vector<Mat> filterbank;
            generateFilterbank( filterbank, base_wavelength, border_gauss_residual);
            if( write_filterbank) writeFilterbank(filterbank);
            vector<Point> extractsequence;
            generateExtractsequenceGrid( extractsequence, base_wavelength, grid_x,  size_x, size_y);
			Mat code (1,extractsequence.size()*filterbank.size()/8,CV_8UC1);
            cout << "Code size (bytes): "<< code.size()<<endl;
			Mat codeMask(1,extractsequence.size()*filterbank.size()/8,CV_8UC1);
			for (vector<string>::iterator inFile = files.begin(); inFile != files.end(); std::advance( inFile, 1), timing.progress++){
				if (!quiet) printf("Loading texture '%s' ...\n", (*inFile).c_str());;
				Mat img = imread(*inFile, CV_LOAD_IMAGE_GRAYSCALE);
                if( write_points) writeExtractionsequence(img, extractsequence, write_points); // write_points is reset after one use
				CV_Assert(img.data != 0);
				CV_Assert(img.size() == Size(size_x,size_y));
				Mat mask = Mat();
				if (!imaskFiles.empty()) {
					string imaskfile;
					patternFileRename(inFiles,imaskFiles,*inFile,imaskfile);
					if (!quiet) printf("Loading mask image '%s' ...\n", imaskfile.c_str());;
					mask = imread(imaskfile, CV_LOAD_IMAGE_GRAYSCALE);
					CV_Assert(mask.data != 0);
					CV_Assert(mask.size() == Size(size_x,size_y));
				}
				if (!quiet) printf("Creating iris-code ...\n");

				//featureExtract(code, codeMask, img, mask, m, n);
				featureExtract(code, codeMask, img, mask, filterbank, extractsequence, write_samples);
				string outfile;
				patternFileRename(inFiles,outFiles,*inFile,outfile);
				if (!quiet) printf("Storing code '%s' ...\n", outfile.c_str());
				if (!imwrite(outfile,code)) CV_Error(CV_StsError,"Could not save image '" + outfile + "'");
				if (!imaskFiles.empty()) {
					string omaskfile;
					patternFileRename(inFiles,omaskFiles,*inFile,omaskfile);
					if (!quiet) printf("Storing code-mask '%s' ...\n", omaskfile.c_str());
					if (!imwrite(omaskfile,codeMask)) CV_Error(CV_StsError,"Could not save image '" + omaskfile + "'");
				}
				if (time && timing.update()) timing.print();
			}
			if (time && quiet) timing.clear();
    	}
    	else if (mode == MODE_HELP){
			// validate command line
			cmdCheckOpts(cmd,"-h");
			if (cmdGetOpt(cmd,"-h") != 0) cmdCheckOptSize(cmd,"-h",0);
			// starting routine
			printUsage();
    	}
    }
	catch (...){
	   	printf("Exit with errors.\n");
	   	exit(EXIT_FAILURE);
	}
    return EXIT_SUCCESS;
}


/*
 * Copyright 2009-2020 The VOTCA Development Team (http://www.votca.org)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

#define BOOST_TEST_MAIN

#define BOOST_TEST_MODULE correlate_test

// Standard includes
#include <cmath>
#include <exception>
#include <iostream>

// Third party includes
#include <boost/test/unit_test.hpp>

// Local VOTCA includes
#include "votca/tools/correlate.h"

using namespace votca;
using namespace votca::tools;

BOOST_AUTO_TEST_SUITE(correlate_test)

BOOST_AUTO_TEST_CASE(correlation_test) {

  DataCollection<double> d;
  DataCollection<double>::array* x = d.CreateArray("first");
  x->resize(50);
  for (Index i = 0; i < Index(x->size()); i++) {
    (*x)[i] = double(i) * 0.5;
  }

  DataCollection<double>::array* y = d.CreateArray("second");
  *y = *x;

  DataCollection<double>::array* z = d.CreateArray("third");
  *z = *x;
  std::reverse(z->begin(), z->end());
  DataCollection<double>::selection s;
  s.push_back(x);
  s.push_back(y);
  s.push_back(z);
  Correlate cor;
  cor.CalcCorrelations(s);
  BOOST_CHECK_CLOSE(cor.getData()[0], 1, 1e-9);
  BOOST_CHECK_CLOSE(cor.getData()[1], -1, 1e-9);
}

BOOST_AUTO_TEST_SUITE_END()


/* Copyright (C) 2010-2019, The Regents of The University of Michigan.
 All rights reserved.

 This software was developed as part of the The Vulcan project in the Intelligent Robotics Lab
 under the direction of Benjamin Kuipers, kuipers@umich.edu. Use of this code is governed by an
 MIT-style License that can be found at "https://github.com/h2ssh/Vulcan".
*/

/**
* \file     summarize_label_results.cpp
* \author   Collin Johnson
*
* A simple program to load the results of running evaluate_labels on a variety of maps. It accumulates the data from
* each map to produce overall means and stds for the edit distance and cell-by-cell comparisons.
*/

#include <boost/accumulators/framework/accumulator_set.hpp>
#include <boost/accumulators/statistics/stats.hpp>
#include <boost/accumulators/statistics/weighted_mean.hpp>
#include <boost/accumulators/statistics/weighted_variance.hpp>
#include <boost/accumulators/statistics/max.hpp>
#include <boost/accumulators/statistics/min.hpp>
#include <boost/accumulators/statistics/variance.hpp>
#include <boost/accumulators/statistics/weighted_median.hpp>
#include <boost/tuple/tuple.hpp>
#include <gnuplot-iostream.h>
#include <fstream>
#include <iostream>
#include <string>
#include <sstream>

struct map_results_t
{
    std::string name;
    int numPaths = 0;
    double meanTopoEdit = 0.0;
    double varTopoEdit = 0.0;
    double meanRouteEdit = 0.0;
    double varRouteEdit = 0.0;
    double zeroErrorRoutePercent = 0.0;
    double cellSuccessRate = 0.0;
    int64_t totalCells = 0;
    int64_t correctCells = 0;
    int numDecisionAsPath = 0;
    int numDecisionAsDest = 0;
    int numDestAsPath = 0;
    int numDestAsDecision = 0;
    int numPathAsDecision = 0;
    int numPathAsDest = 0;
};

map_results_t operator+(const map_results_t& lhs, const map_results_t& rhs);
std::istream& operator>>(std::istream& in, map_results_t& results);
std::ostream& operator<<(std::ostream& out, map_results_t& results);

void plot_cell_by_cell(std::vector<map_results_t>& results);
void plot_topological_edit_distance(std::vector<map_results_t>& results);
void plot_route_edit_distance(std::vector<map_results_t>& results);

/**
* summarize_label_results takes the name of the results file as its only argument.
*/
int main(int argc, char** argv)
{
    if(argc < 2)
    {
        std::cout << "INFO: Expected command-line: summarize_label_results 'results filename' 'plots/no_plots (default)'\n";
        return -1;
    }

    bool showPlots = false;
    if(argc > 2)
    {
        showPlots = std::string(argv[2]) == std::string("plots");
    }

    using namespace boost::accumulators;
    using StatsAcc = accumulator_set<double, stats<tag::min,
                                                   tag::max,
                                                   tag::weighted_mean,
                                                   tag::weighted_variance,
                                                   tag::weighted_median>, int>;
    StatsAcc cellAcc;
    StatsAcc topoAcc;
    StatsAcc routeAcc;
    StatsAcc errorAcc;

    std::ifstream in(argv[1]);
    std::string resultsLine;

    map_results_t total;
    map_results_t individual;
    std::vector<map_results_t> allResults;

    while(true)
    {
        std::getline(in, resultsLine);

        // Break as soon as finished reading file
        if(resultsLine.length() < 10)
        {
            break;
        }

        std::istringstream inStr(resultsLine);
        inStr >> individual;
        total = total + individual;
        std::cout << individual << '\n';
        allResults.push_back(individual);

        cellAcc(individual.cellSuccessRate, weight = individual.totalCells);
        topoAcc(individual.meanTopoEdit, weight = 1);
        routeAcc(individual.meanRouteEdit, weight = 1);
        errorAcc(individual.zeroErrorRoutePercent, weight = 1);
    }

    std::cout << "\n================== Overall ====================\n"
        << "Topo edit (min, max, mean, std, median): " << min(topoAcc) << ", " << max(topoAcc) << ", " << weighted_mean(topoAcc) << ", " << std::sqrt(weighted_variance(topoAcc)) << ", " << weighted_median(topoAcc) << '\n'
        << "Route edit (min, max, mean, std, median): " << min(routeAcc) << ", " << max(routeAcc) << ", " << weighted_mean(routeAcc) << ", " << std::sqrt(weighted_variance(routeAcc)) << ", " << weighted_median(routeAcc) << '\n'
        << "Zero error (min, max, mean, std, median): " << min(errorAcc) << ", " << max(errorAcc) << ", " << weighted_mean(errorAcc) << ", " << std::sqrt(weighted_variance(errorAcc)) << ", " << weighted_median(errorAcc) << '\n'
        << "Cell % (min, max, mean, std, median): " << min(cellAcc) << ", " << max(cellAcc) << ", " << weighted_mean(cellAcc) << ", " << std::sqrt(weighted_variance(cellAcc)) << ", " << weighted_median(cellAcc) << '\n';

    std::cout << "\n================== Latex ====================\n";
    printf("Norm edit: %.3f & %.3f & %.3f & %.3f & %.3f & \\\\ \n", min(topoAcc), max(topoAcc), weighted_mean(topoAcc), std::sqrt(weighted_variance(topoAcc)), weighted_median(topoAcc));
    printf("Raw edit:  %.3f & %.3f & %.3f & %.3f & %.3f & \\\\ \n", min(routeAcc), max(routeAcc), weighted_mean(routeAcc), std::sqrt(weighted_variance(routeAcc)), weighted_median(routeAcc));
    printf("Cell:      %.3f & %.3f & %.3f & %.3f & %.3f & \\\\ \n", min(cellAcc), max(cellAcc), weighted_mean(cellAcc), std::sqrt(weighted_variance(cellAcc)), weighted_median(cellAcc));

    if(showPlots)
    {
        plot_cell_by_cell(allResults);
        plot_topological_edit_distance(allResults);
        plot_route_edit_distance(allResults);
    }

    return 0;
}

map_results_t operator+(const map_results_t& lhs, const map_results_t& rhs)
{
    map_results_t sumResults;
    sumResults.name = "Total:";

    // Accumulate the edit distance values
    sumResults.numPaths = lhs.numPaths + rhs.numPaths;
    sumResults.meanTopoEdit = ((lhs.numPaths * lhs.meanTopoEdit) + (rhs.numPaths * rhs.meanTopoEdit)) / sumResults.numPaths;
    sumResults.varTopoEdit = lhs.varTopoEdit + rhs.varTopoEdit;
    sumResults.meanRouteEdit = ((lhs.numPaths * lhs.meanRouteEdit) + (rhs.numPaths * rhs.meanRouteEdit)) / sumResults.numPaths;
    sumResults.varRouteEdit = lhs.varRouteEdit + rhs.varRouteEdit;
    sumResults.zeroErrorRoutePercent = ((lhs.zeroErrorRoutePercent * lhs.numPaths)
        + (rhs.zeroErrorRoutePercent * rhs.numPaths)) / (lhs.numPaths + rhs.numPaths);

    // Accumulate the cell-by-cell values
    sumResults.totalCells = lhs.totalCells + rhs.totalCells;
    sumResults.correctCells = lhs.correctCells + rhs.correctCells;
    sumResults.cellSuccessRate = static_cast<double>(sumResults.correctCells) / sumResults.totalCells;
    sumResults.numDecisionAsPath = lhs.numDecisionAsPath + rhs.numDecisionAsPath;
    sumResults.numDecisionAsDest = lhs.numDecisionAsDest + rhs.numDecisionAsDest;
    sumResults.numDestAsPath = lhs.numDestAsPath + rhs.numDestAsPath;
    sumResults.numDestAsDecision = lhs.numDestAsDecision + rhs.numDestAsDecision;
    sumResults.numPathAsDecision = lhs.numPathAsDecision + rhs.numPathAsDecision;
    sumResults.numPathAsDest = lhs.numPathAsDest + rhs.numPathAsDest;

    return sumResults;
}

std::istream& operator>>(std::istream& in, map_results_t& results)
{
    in >> results.name >> results.numPaths >> results.meanTopoEdit >> results.varTopoEdit >> results.meanRouteEdit
        >> results.varRouteEdit >> results.zeroErrorRoutePercent >> results.cellSuccessRate >> results.totalCells
        >> results.correctCells >> results.numDecisionAsPath >> results.numDecisionAsDest >> results.numDestAsPath
        >> results.numDestAsDecision >> results.numPathAsDecision >> results.numPathAsDest;
    return in;
}

std::ostream& operator<<(std::ostream& out, map_results_t& results)
{
    out << "================ " << results.name << "================\n"
        << "Norm edit (mean, std):" << results.meanTopoEdit << ", " << std::sqrt(results.varTopoEdit) << '\n'
        << "Raw edit (mean, std):" << results.meanRouteEdit << ", " << std::sqrt(results.varRouteEdit) << '\n'
        << "Zero error percent: " << (results.zeroErrorRoutePercent * 100) << '\n'
        << "Cell-by-cell (overall %):" << results.cellSuccessRate << '\n';
    return out;
}

void plot_cell_by_cell(std::vector<map_results_t>& results)
{
    std::sort(results.begin(), results.end(), [](auto& lhs, auto& rhs) {
        return lhs.cellSuccessRate > rhs.cellSuccessRate;
    });

    Gnuplot plot;
    plot << "set yrange [0:100]\n";
    plot << "set style fill solid border lc black\n";
    plot << "set title 'Cell-by-Cell Results for MCMC Place Classification'\n";
    plot << "set ylabel 'Accuracy (% correct cells)'\n";
    plot << "set xtics rotate by -60\n";
    plot << "set xtics format \"%s\"\n";
    plot << "set xtics noenhanced\n";   // display underscores
    plot << "set style data histograms\n";
    plot << "set style histogram clustered gap 1\n";
    plot << "set style fill solid 1.0\n";

    plot << "plot '-' using 1:xticlabels(2) fillcolor \"skyblue\" notitle\n";
    plot << std::endl;

    // Data is col 1 = value, col 2 = tic value
    std::vector<double> accuracy;
    std::vector<char*> xtics;

    for(auto& r : results)
    {
        accuracy.push_back(r.cellSuccessRate * 100);
        std::string name = r.name;  // print a short version of the name
        auto end = name.find("_result");
        if(end == std::string::npos)
        {
            end = name.find("_best");

            if(end == std::string::npos)
            {
                end = name.find_first_of("_.");
            }
        }
        name.erase(end, std::string::npos);
        char *strname = strdup(name.c_str());
        xtics.push_back(strname);
    }

    plot.send1d(boost::make_tuple(accuracy, xtics));
    sleep(1);

    for(auto& str : xtics)
    {
        free(str);
    }
}

void plot_topological_edit_distance(std::vector<map_results_t>& results)
{
    Gnuplot plot;
    plot << "set yrange [0:60]\n";
    plot << "set style fill solid border lc black\n";
    plot << "set title 'Mean Normalized Edit Distance for MCMC Place Classification'\n";
    plot << "set ylabel 'Normalized Edit Distance'\n";
    plot << "set xtics rotate by -60\n";
    plot << "set xtics format \"%s\"\n";
    plot << "set xtics noenhanced\n";   // display underscores
    plot << "set style data histograms\n";
    plot << "set style histogram clustered gap 1\n";
    plot << "set style fill solid 1.0\n";

    plot << "plot '-' using 1:xticlabels(2) fillcolor \"skyblue\" notitle\n";
    plot << std::endl;

    // Data is col 1 = value, col 2 = tic value
    std::vector<double> mean;
    std::vector<char*> xtics;

    for(auto& r : results)
    {
        mean.push_back(r.meanTopoEdit);
        std::string name = r.name;  // print a short version of the name
        auto end = name.find("_result");
        if(end == std::string::npos)
        {
            end = name.find("_best");

            if(end == std::string::npos)
            {
                end = name.find_first_of("_.");
            }
        }
        name.erase(end, std::string::npos);
        char *strname = strdup(name.c_str());
        xtics.push_back(strname);
    }

    plot.send1d(boost::make_tuple(mean, xtics));
    sleep(1);

    for(auto& str : xtics)
    {
        free(str);
    }
}

void plot_route_edit_distance(std::vector<map_results_t>& results)
{
    std::sort(results.begin(), results.end(), [](auto& lhs, auto& rhs) {
        return lhs.meanRouteEdit < rhs.meanRouteEdit;
    });

    Gnuplot plot;
    plot << "set yrange [0:5]\n";
    plot << "set style fill solid border lc black\n";
    plot << "set title 'Mean Route Edit Distance for MCMC Place Classification'\n";
    plot << "set ylabel 'Route Edit Distance'\n";
    plot << "set xtics rotate by -60\n";
    plot << "set xtics format \"%s\"\n";
    plot << "set xtics noenhanced\n";   // display underscores
    plot << "set style data histograms\n";
    plot << "set style histogram clustered gap 1\n";
    plot << "set style fill solid 1.0\n";

    plot << "plot '-' using 1:xticlabels(2) fillcolor \"skyblue\" notitle\n";
    plot << std::endl;

    // Data is col 1 = value, col 2 = tic value
    std::vector<double> mean;
    std::vector<char*> xtics;

    for(auto& r : results)
    {
        mean.push_back(r.meanRouteEdit);
        std::string name = r.name;  // print a short version of the name
        auto end = name.find("_result");
        if(end == std::string::npos)
        {
            end = name.find("_best");

            if(end == std::string::npos)
            {
                end = name.find_first_of("_.");
            }
        }
        name.erase(end, std::string::npos);
        char *strname = strdup(name.c_str());
        xtics.push_back(strname);
    }

    plot.send1d(boost::make_tuple(mean, xtics));
    sleep(1);

    for(auto& str : xtics)
    {
        free(str);
    }
}


#define BOOST_TEST_MODULE communicator_alltoallv

#include <boost/test/included/unit_test.hpp>
#include <mpl/mpl.hpp>
#include "test_helper.hpp"

template<typename T>
bool alltoallv_with_displacements_test(const T &val) {
  const mpl::communicator &comm_world{mpl::environment::comm_world()};
  const int N_processes{comm_world.size()};
  const int N_send{comm_world.rank() + 1};  // number of elements to send to each process
  const int N_recv{(N_processes * N_processes + N_processes) /
                   2};  // total number of elements to receive
  std::vector<T> send_data;
  std::vector<T> recv_data(N_recv);
  std::vector<T> expected;
  mpl::layouts<T> sendls;
  mpl::layouts<T> recvls;
  mpl::displacements senddispls;
  mpl::displacements recvdispls;
  T send_val{val};
  T expected_val{val};
  for (int i{0}; i < comm_world.rank(); ++i)
    ++expected_val;
  for (int j{0}; j < N_processes; ++j) {
    for (int i{0}; i < N_send; ++i)
      send_data.push_back(send_val);
    ++send_val;
    sendls.push_back(mpl::contiguous_layout<T>(N_send));
    senddispls.push_back(sizeof(T) * j * N_send);
    recvls.push_back(mpl::contiguous_layout<T>(j + 1));
    recvdispls.push_back(sizeof(T) * (j * j + j) / 2);
    for (int i{0}; i < j + 1; ++i)
      expected.push_back(expected_val);
  }
  comm_world.alltoallv(send_data.data(), sendls, senddispls, recv_data.data(), recvls,
                       recvdispls);
  return recv_data == expected;
}

template<typename T>
bool alltoallv_without_displacements_test(const T &val) {
  const mpl::communicator &comm_world{mpl::environment::comm_world()};
  const int N_processes{comm_world.size()};
  const int N_send{comm_world.rank() + 1};  // number of elements to send to each process
  const int N_recv{(N_processes * N_processes + N_processes) /
                   2};  // total number of elements to receive
  std::vector<T> send_data;
  std::vector<T> recv_data(N_recv);
  std::vector<T> expected;
  mpl::layouts<T> sendls;
  mpl::layouts<T> recvls;
  T send_val{val};
  T expected_val{val};
  for (int i{0}; i < comm_world.rank(); ++i)
    ++expected_val;
  for (int j{0}; j < N_processes; ++j) {
    for (int i{0}; i < N_send; ++i)
      send_data.push_back(send_val);
    ++send_val;
    sendls.push_back(mpl::indexed_layout<T>({{N_send, j * N_send}}));
    recvls.push_back(mpl::indexed_layout<T>({{j + 1, (j * j + j) / 2}}));
    for (int i{0}; i < j + 1; ++i)
      expected.push_back(expected_val);
  }
  comm_world.alltoallv(send_data.data(), sendls, recv_data.data(), recvls);
  return recv_data == expected;
}

template<typename T>
bool ialltoallv_with_displacements_test(const T &val) {
  const mpl::communicator &comm_world{mpl::environment::comm_world()};
  const int N_processes{comm_world.size()};
  const int N_send{comm_world.rank() + 1};  // number of elements to send to each process
  const int N_recv{(N_processes * N_processes + N_processes) /
                   2};  // total number of elements to receive
  std::vector<T> send_data;
  std::vector<T> recv_data(N_recv);
  std::vector<T> expected;
  mpl::layouts<T> sendls;
  mpl::layouts<T> recvls;
  mpl::displacements senddispls;
  mpl::displacements recvdispls;
  T send_val{val};
  T expected_val{val};
  for (int i{0}; i < comm_world.rank(); ++i)
    ++expected_val;
  for (int j{0}; j < N_processes; ++j) {
    for (int i{0}; i < N_send; ++i)
      send_data.push_back(send_val);
    ++send_val;
    sendls.push_back(mpl::contiguous_layout<T>(N_send));
    senddispls.push_back(sizeof(T) * j * N_send);
    recvls.push_back(mpl::contiguous_layout<T>(j + 1));
    recvdispls.push_back(sizeof(T) * (j * j + j) / 2);
    for (int i{0}; i < j + 1; ++i)
      expected.push_back(expected_val);
  }
  auto r{comm_world.ialltoallv(send_data.data(), sendls, senddispls, recv_data.data(), recvls,
                               recvdispls)};
  r.wait();
  return recv_data == expected;
}

template<typename T>
bool ialltoallv_without_displacements_test(const T &val) {
  const mpl::communicator &comm_world{mpl::environment::comm_world()};
  const int N_processes{comm_world.size()};
  const int N_send{comm_world.rank() + 1};  // number of elements to send to each process
  const int N_recv{(N_processes * N_processes + N_processes) /
                   2};  // total number of elements to receive
  std::vector<T> send_data;
  std::vector<T> recv_data(N_recv);
  std::vector<T> expected;
  mpl::layouts<T> sendls;
  mpl::layouts<T> recvls;
  T send_val{val};
  T expected_val{val};
  for (int i{0}; i < comm_world.rank(); ++i)
    ++expected_val;
  for (int j{0}; j < N_processes; ++j) {
    for (int i{0}; i < N_send; ++i)
      send_data.push_back(send_val);
    ++send_val;
    sendls.push_back(mpl::indexed_layout<T>({{N_send, j * N_send}}));
    recvls.push_back(mpl::indexed_layout<T>({{j + 1, (j * j + j) / 2}}));
    for (int i{0}; i < j + 1; ++i)
      expected.push_back(expected_val);
  }
  auto r{comm_world.ialltoallv(send_data.data(), sendls, recv_data.data(), recvls)};
  r.wait();
  return recv_data == expected;
}

template<typename T>
bool alltoallv_in_place_with_displacements_test(const T &val) {
  const mpl::communicator &comm_world{mpl::environment::comm_world()};
  const int N_processes{comm_world.size()};
  std::vector<T> sendrecv_data;
  std::vector<T> expected;
  mpl::layouts<T> sendrecvls;
  mpl::displacements sendrecvdispls;
  T send_val{val};
  T expected_val{val};
  for (int i{0}; i < comm_world.rank(); ++i)
    ++send_val;
  int displ{0};
  for (int j{0}; j < N_processes; ++j) {
    const int N_sendrecv{j + comm_world.rank() + 1};  // must be symmetric in j and rank
    for (int i{0}; i < N_sendrecv; ++i) {
      sendrecv_data.push_back(send_val);
      expected.push_back(expected_val);
    }
    sendrecvls.push_back(mpl::contiguous_layout<T>(N_sendrecv));
    sendrecvdispls.push_back(sizeof(T) * displ);
    displ += N_sendrecv;
    ++expected_val;
  }
  comm_world.alltoallv(sendrecv_data.data(), sendrecvls, sendrecvdispls);
  return sendrecv_data == expected;
}

template<typename T>
bool alltoallv_in_place_without_displacements_test(const T &val) {
  const mpl::communicator &comm_world{mpl::environment::comm_world()};
  const int N_processes{comm_world.size()};
  std::vector<T> sendrecv_data;
  std::vector<T> expected;
  mpl::layouts<T> sendrecvls;
  T send_val{val};
  T expected_val{val};
  for (int i{0}; i < comm_world.rank(); ++i)
    ++send_val;
  int displ{0};
  for (int j{0}; j < N_processes; ++j) {
    const int N_sendrecv{j + comm_world.rank() + 1};  // must be symmetric in j and rank
    for (int i{0}; i < N_sendrecv; ++i) {
      sendrecv_data.push_back(send_val);
      expected.push_back(expected_val);
    }
    sendrecvls.push_back(mpl::indexed_layout<T>({{N_sendrecv, displ}}));
    displ += N_sendrecv;
    ++expected_val;
  }
  comm_world.alltoallv(sendrecv_data.data(), sendrecvls);
  return sendrecv_data == expected;
}

template<typename T>
bool ialltoallv_in_place_with_displacements_test(const T &val) {
  const mpl::communicator &comm_world{mpl::environment::comm_world()};
  const int N_processes{comm_world.size()};
  std::vector<T> sendrecv_data;
  std::vector<T> expected;
  mpl::layouts<T> sendrecvls;
  mpl::displacements sendrecvdispls;
  T send_val{val};
  T expected_val{val};
  for (int i{0}; i < comm_world.rank(); ++i)
    ++send_val;
  int displ{0};
  for (int j{0}; j < N_processes; ++j) {
    const int N_sendrecv{j + comm_world.rank() + 1};  // must be symmetric in j and rank
    for (int i{0}; i < N_sendrecv; ++i) {
      sendrecv_data.push_back(send_val);
      expected.push_back(expected_val);
    }
    sendrecvls.push_back(mpl::contiguous_layout<T>(N_sendrecv));
    sendrecvdispls.push_back(sizeof(T) * displ);
    displ += N_sendrecv;
    ++expected_val;
  }
  auto r{comm_world.ialltoallv(sendrecv_data.data(), sendrecvls, sendrecvdispls)};
  r.wait();
  return sendrecv_data == expected;
}

template<typename T>
bool ialltoallv_in_place_without_displacements_test(const T &val) {
  const mpl::communicator &comm_world{mpl::environment::comm_world()};
  const int N_processes{comm_world.size()};
  std::vector<T> sendrecv_data;
  std::vector<T> expected;
  mpl::layouts<T> sendrecvls;
  T send_val{val};
  T expected_val{val};
  for (int i{0}; i < comm_world.rank(); ++i)
    ++send_val;
  int displ{0};
  for (int j{0}; j < N_processes; ++j) {
    const int N_sendrecv{j + comm_world.rank() + 1};  // must be symmetric in j and rank
    for (int i{0}; i < N_sendrecv; ++i) {
      sendrecv_data.push_back(send_val);
      expected.push_back(expected_val);
    }
    sendrecvls.push_back(mpl::indexed_layout<T>({{N_sendrecv, displ}}));
    displ += N_sendrecv;
    ++expected_val;
  }
  auto r{comm_world.ialltoallv(sendrecv_data.data(), sendrecvls)};
  r.wait();
  return sendrecv_data == expected;
}

BOOST_AUTO_TEST_CASE(alltoallv) {
  BOOST_TEST(alltoallv_with_displacements_test(1.0));
  BOOST_TEST(alltoallv_with_displacements_test(tuple{1, 2.0}));

  BOOST_TEST(alltoallv_without_displacements_test(1.0));
  BOOST_TEST(alltoallv_without_displacements_test(tuple{1, 2.0}));

  BOOST_TEST(ialltoallv_with_displacements_test(1.0));
  BOOST_TEST(ialltoallv_with_displacements_test(tuple{1, 2.0}));

  BOOST_TEST(ialltoallv_without_displacements_test(1.0));
  BOOST_TEST(ialltoallv_without_displacements_test(tuple{1, 2.0}));

  BOOST_TEST(alltoallv_in_place_with_displacements_test(1.0));
  BOOST_TEST(alltoallv_in_place_with_displacements_test(tuple{1, 2.0}));

  BOOST_TEST(alltoallv_in_place_without_displacements_test(1.0));
  BOOST_TEST(alltoallv_in_place_without_displacements_test(tuple{1, 2.0}));

  BOOST_TEST(ialltoallv_in_place_with_displacements_test(1.0));
  BOOST_TEST(ialltoallv_in_place_with_displacements_test(tuple{1, 2.0}));

  BOOST_TEST(ialltoallv_in_place_without_displacements_test(1.0));
  BOOST_TEST(ialltoallv_in_place_without_displacements_test(tuple{1, 2.0}));
}


//==============================================================================
//         Copyright 2003 - 2012   LASMEA UMR 6602 CNRS/Univ. Clermont II
//         Copyright 2009 - 2012   LRI    UMR 8623 CNRS/Univ Paris Sud XI
//
//          Distributed under the Boost Software License, Version 1.0.
//                 See accompanying file LICENSE.txt or copy at
//                     http://www.boost.org/LICENSE_1_0.txt
//==============================================================================
#include <nt2/include/functions/valuation.hpp>
#include <nt2/include/functions/isequal.hpp>
#include <nt2/sdk/functor/meta/call.hpp>
#include <nt2/sdk/unit/tests/relation.hpp>
#include <nt2/sdk/unit/module.hpp>
#include <nt2/table.hpp>
#include <nt2/include/functions/complexify.hpp>

#include <boost/array.hpp>

NT2_TEST_CASE_TPL ( valuation,  NT2_REAL_TYPES)
{

  typedef std::complex<T> cT;
  using nt2::valuation;
  using nt2::tag::valuation_;
  nt2::table<cT> a =  nt2::complexify(nt2::_(T(1), T(4)));
  nt2::table<cT> b =  nt2::complexify(nt2::_(T(0), T(0), T(3)));
  nt2::table<cT> c(nt2::of_size(1, 0));
  NT2_TEST_EQUAL(0, valuation(a));
  NT2_TEST_EQUAL(-1, valuation(b));
  b(2) = b(1) = cT(1);
  NT2_TEST_EQUAL(1, valuation(b));

} // end of test for floating_

NT2_TEST_CASE_TPL ( valuation_2,  NT2_REAL_TYPES)
{

  typedef std::complex<T> cT;
  using nt2::valuation;
  using nt2::tag::valuation_;
  cT b = cT(1.0);
  cT a = cT(0.0);
  NT2_TEST_EQUAL(valuation(a), -1);
  NT2_TEST_EQUAL(valuation(b), 0);
  nt2::container::table<cT> bb =  nt2::complexify(nt2::_(T(1), T(1)));
  nt2::container::table<cT> aa =  nt2::complexify(nt2::_(T(0), T(0)));
  NT2_TEST_EQUAL(valuation(aa), -1);
  NT2_TEST_EQUAL(valuation(bb), 0);
  nt2::container::table<cT> cc;
  NT2_TEST_EQUAL(valuation(cc), -1);

}


/* -*- mode: c++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */

/*

 Copyright (C) 2016 Klaus Spanderen

 This file is part of QuantLib, a free-software/open-source library
 for financial quantitative analysts and developers - http://quantlib.org/

 QuantLib is free software: you can redistribute it and/or modify it
 under the terms of the QuantLib license.  You should have received a
 copy of the license along with this program; if not, please email
 <quantlib-dev@lists.sf.net>. The license is also available online at
 <http://quantlib.org/license.shtml>.

 This program is distributed in the hope that it will be useful, but WITHOUT
 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 FOR A PARTICULAR PURPOSE.  See the license for more details.
*/

/*! \file stochasticcollocationinvcdf.hpp
    Stochastic collocation inverse cumulative distribution function
*/

#ifndef quantlib_stochastic_collation_inv_cdf_hpp
#define quantlib_stochastic_collation_inv_cdf_hpp

#include <ql/math/distributions/normaldistribution.hpp>
#include <ql/math/interpolations/lagrangeinterpolation.hpp>

#include <boost/function.hpp>
#include <functional>

namespace QuantLib {
    //! Stochastic collocation inverse cumulative distribution function

    /*! References:
        L.A. Grzelak, J.A.S. Witteveen, M.Suárez-Taboada, C.W. Oosterlee,
        The Stochastic Collocation Monte Carlo Sampler: Highly efficient
        sampling from “expensive” distributions
        http://papers.ssrn.com/sol3/papers.cfm?abstract_id=2529691
     */

    class StochasticCollocationInvCDF : public std::unary_function<Real,Real> {
      public:
        StochasticCollocationInvCDF(
            const boost::function<Real(Real)>& invCDF,
            Size lagrangeOrder,
            Real pMax = Null<Real>(),
            Real pMin = Null<Real>());

        Real value(Real x) const;
        Real operator()(Real u) const;

      private:
        const Array x_;
        const Volatility sigma_;
        const Array y_;
        const LagrangeInterpolation interpl_;
    };
}

/* -*- mode: c++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */

/*

 Copyright (C) 2016 Klaus Spanderen

 This file is part of QuantLib, a free-software/open-source library
 for financial quantitative analysts and developers - http://quantlib.org/

 QuantLib is free software: you can redistribute it and/or modify it
 under the terms of the QuantLib license.  You should have received a
 copy of the license along with this program; if not, please email
 <quantlib-dev@lists.sf.net>. The license is also available online at
 <http://quantlib.org/license.shtml>.

 This program is distributed in the hope that it will be useful, but WITHOUT
 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 FOR A PARTICULAR PURPOSE.  See the license for more details.
*/

/*! \file stochasticcollationcdf.cpp
*/

#include <ql/mathconstants.hpp>
#include <ql/math/integrals/gaussianquadratures.hpp>

namespace QuantLib {

    namespace {
        Disposable<Array> g(Real sigma, const Array& x,
                            const boost::function<Real(Real)>& invCDF) {

            Array y(x.size());
            const CumulativeNormalDistribution normalCDF;

            for (Size i=0, n=x.size(); i < n; ++i) {
                y[i] = invCDF(normalCDF(x[i]/sigma));
            }

            return y;
        }
    }

    inline StochasticCollocationInvCDF::StochasticCollocationInvCDF(
        const boost::function<Real(Real)>& invCDF,
        Size lagrangeOrder, Real pMax, Real pMin)
    : x_(M_SQRT2*GaussHermiteIntegration(lagrangeOrder).x()),
      sigma_( (pMax != Null<Real>())
              ? x_.back() / InverseCumulativeNormal()(pMax)
              : (pMin != Null<Real>())
                  ? x_.front() / InverseCumulativeNormal()(pMin)
                  : 1.0),
      y_(g(sigma_, x_, invCDF)),
      interpl_(x_.begin(), x_.end(), y_.begin()) {
    }

    inline Real StochasticCollocationInvCDF::value(Real x) const {
        return interpl_(x*sigma_, true);
    }
    inline Real StochasticCollocationInvCDF::operator()(Real u) const {
        return value(InverseCumulativeNormal()(u));
    }
}

#endif


#include <boost/serialization/export.hpp>
#include "GatedRecurrentUnit.hpp"

using namespace std;
using namespace snn;
using namespace internal;

BOOST_CLASS_EXPORT(GatedRecurrentUnit)

GatedRecurrentUnit::GatedRecurrentUnit(NeuronModel model, shared_ptr<NeuralNetworkOptimizer> optimizer)
    : numberOfInputs(model.numberOfInputs),
      resetGate({model.numberOfInputs, model.numberOfWeights, activation::sigmoid}, optimizer),
      updateGate({model.numberOfInputs, model.numberOfWeights, activation::sigmoid}, optimizer),
      outputGate({model.numberOfInputs, model.numberOfWeights, activation::tanh}, optimizer)
{
}

float GatedRecurrentUnit::output(const vector<float>& inputs, bool temporalReset)
{
    if (temporalReset)
        this->reset();
    float resetGateOutput = this->resetGate.output(inputs, temporalReset);
    this->updateGateOutput = this->updateGate.output(inputs, temporalReset);
    this->outputGate.lastOutput *= resetGateOutput;
    this->outputGateOutput = this->outputGate.output(inputs, temporalReset);

    float output = (1 - this->updateGateOutput) * this->previousOutput + this->updateGateOutput * outputGateOutput;

    this->resetGate.lastOutput = output;
    this->updateGate.lastOutput = output;
    this->outputGate.lastOutput = output;

    return output;
}

std::vector<float>& GatedRecurrentUnit::backOutput(float error)
{
    float d3 = error;
    float d8 = d3 * this->updateGateOutput;
    float d7 = d3 * this->outputGateOutput;
    float d9 = d7 + d8;

    this->errors = this->outputGate.backOutput(d8);
    auto e2 = this->updateGate.backOutput(d9);
    float d13 = this->errors.back();
    float d16 = d13 * this->previousOutput;
    auto e3 = this->resetGate.backOutput(d16);

    transform(this->errors.begin(), this->errors.end(), e2.begin(), this->errors.begin(), plus<float>());
    transform(this->errors.begin(), this->errors.end(), e3.begin(), this->errors.begin(), plus<float>());
    return this->errors;
}

void GatedRecurrentUnit::train(float error)
{
    float d3 = error;
    float d8 = d3 * this->updateGateOutput;
    float d7 = d3 * this->outputGateOutput;
    float d9 = d7 + d8;

    this->errors = this->outputGate.backOutput(d8);
    auto e2 = this->updateGate.backOutput(d9);
    float d13 = this->errors.back();
    float d16 = d13 * this->previousOutput;
    auto e3 = this->resetGate.backOutput(d16);
}

vector<float> GatedRecurrentUnit::getWeights() const
{
    vector<float> allWeights;
    allWeights.insert(allWeights.end(), this->resetGate.weights.begin(), this->resetGate.weights.end());
    allWeights.insert(allWeights.end(), this->updateGate.weights.begin(), this->updateGate.weights.end());
    allWeights.insert(allWeights.end(), this->outputGate.weights.begin(), this->outputGate.weights.end());
    return allWeights;
}

int GatedRecurrentUnit::getNumberOfParameters() const
{
    return this->resetGate.getNumberOfParameters()
        + this->updateGate.getNumberOfParameters()
        + this->outputGate.getNumberOfParameters();
}

int GatedRecurrentUnit::getNumberOfInputs() const
{
    return this->numberOfInputs;
}

inline
void GatedRecurrentUnit::reset()
{
    this->previousOutput = 0;
    this->recurrentError = 0;
    this->updateGateOutput = 0;
}

int GatedRecurrentUnit::isValid() const
{
    auto err = resetGate.isValid();
    if (err != 0)
        return err;
    err = updateGate.isValid();
    if (err != 0)
        return err;
    err = outputGate.isValid();
    if (err != 0)
        return err;
    return 0;
}

bool GatedRecurrentUnit::operator==(const GatedRecurrentUnit& neuron) const
{
    return this->numberOfInputs == neuron .numberOfInputs
        && this->previousOutput == neuron.previousOutput
        && this->recurrentError == neuron.recurrentError
        && this->updateGateOutput == neuron.updateGateOutput
        && this->outputGateOutput == neuron.outputGateOutput
        && this->resetGate == neuron.resetGate
        && this->updateGate == neuron.updateGate
        && this->outputGate == neuron.outputGate;
}

bool GatedRecurrentUnit::operator!=(const GatedRecurrentUnit& neuron) const
{
    return !(*this == neuron);
}


#include <Eigen/Core>
#include <Eigen/SparseCore>
#include <Eigen/QR>
#include <Eigen/Householder>
#include <Eigen/SVD>
#include "toperations.h"

using namespace std;
using namespace Eigen;

/* QR */
void tQR::fwd(){
	int i=0;
	Segment xs = c->xt->segment(0, ccols);
	assert((*c->edgesOut.begin())->n2 == c);
    Edge* eself = *(c->edgesOut.begin());
	MatrixXd R = (eself->A21)->topRows(xs.size());

	trsv(&R, &xs, CblasUpper, CblasTrans, CblasNonUnit); // Do inv(R')*xs
	for (auto e: c->edgesIn){
		assert(e->A21 != nullptr);
		int s = A12_indices[i];
		assert(e->A21->cols() == s);
		(e->n1->xt->segment(0, s)) -= (e->A21->topRows(ccols)).transpose()*xs;
		++i;
	}
}

void tQR::bwd(){
	MatrixXd Q = MatrixXd::Zero(nrows, ncols);
	VectorXd xc = VectorXd::Zero(nrows);

	int curr_row = 0;
	int i=0;
	for (auto e: c->edgesOut){
		if (e->A21 != nullptr){
			int s = A21_indices[i];
			xc.segment(curr_row, s) = e->n2->xt->segment(0, s);
			Q.middleRows(curr_row, s) = *(e->A21);
			curr_row += s;
			++i;
		}
	}

	larfb(&Q, c->get_T(), &xc, 'L', 'N', 'F', 'C');
	curr_row = 0;
	i=0;
	for (auto e: c->edgesOut){
		if (e->A21 != nullptr){
			int s = A21_indices[i];
			e->n2->xt->segment(0, s) = xc.segment(curr_row, s);
			curr_row += s;
			++i;
		}
	}
}

/* Reassign rows */
void tReassign::bwd(){
	// cout << c->get_id() << " " << n->get_id() << endl;
	assert(indices.size() == nrows);
	for (int i= nstart; i < nstart+nrows; ++i){
		(*c->xt)[indices[i-nstart]] = (*n->xt)[i];
	}

}

/* Scaling */
void tScale::fwd(){
    trsv(R, &xs, CblasUpper, CblasTrans, CblasNonUnit);
}

void tScaleD::fwd(){
	MatrixXd R = Q->topRows(xs.size());
	trsv(&R, &xs, CblasUpper, CblasTrans, CblasNonUnit);
}

/* Sparsification using Orthogonal transformations */
void tOrthogonal::fwd(){ormqr_trans(V, tau, &xs);}

void tOrthogonalD::fwd(){
	ormqr_trans(V, tau, &xs);
}
void tOrthogonalD::bwd(){
	ormqr_notrans(V, tau, &xs);
}

/* Merging in the cluster heirarchy */
void tMerge::fwd(){
	int k=0;
	for (auto c: parent->children){
		for (int i=0; i < c->cols(); ++i){
			(*parent->xt)[k] = (*c->xt)[i];
			++k;
		}
	}
}

void tMerge::bwd(){
	int k=0;
	for (auto c: parent->children){
		for (int i=0; i < c->rows(); ++i){
			(*c->xt)[i] = (*parent->xt)[k];
			++k;
		}
	}
	// assert(k== parent->get_x()->size());
}

/* Split between coarse and fine nodes */
void tSplit::fwd(){xsf = xsc_head.bottomRows(xsf.size());}
void tSplit::bwd(){xsc.bottomRows(xsf.size()) = xsf;}

void tSplitD::fwd(){
	xsf = xsc.middleRows(rank, ccols-rank);
	VectorXd xsc_rem = xsc.bottomRows(crows-ccols);
	xsc.middleRows(rank, crows-ccols) = xsc_rem;
	xsc.bottomRows(ccols-rank) = xsf;
}
void tSplitD::bwd(){
	// VectorXd xsf = xsc.bottomRows(ccols-rank);
	VectorXd xsc_rem = xsc.middleRows(rank, crows-ccols);
	xsc.middleRows(rank, ccols-rank) = xsf;
	xsc.bottomRows(crows-ccols) = xsc_rem;
}


#include <iostream>
#include <Eigen/Dense>
#include <time.h>
#include <random>
#include <NTL/BasicThreadPool.h>

using namespace Eigen;
using namespace std;

typedef Matrix<double, Dynamic, Dynamic> MatrixXI;

// This is to perform SRLC2 test
int test()
{

  int numthread = 1; // please use single thread here and make it multi-threaded externally
  NTL::SetNumThreads(numthread);
  int k = 50;
  int c = 5;
  int m = 2*k;
  int rounds = (1<<25)/numthread;

  std::random_device rd; // obtain a random number from hardware
  std::mt19937 gen(rd()); // seed the generator
  std::uniform_int_distribution<> distr(0, m-1); // define the range
  // rng_type::result_type const seedval = get_seed();

  NTL_EXEC_INDEX(numthread, index)
    int supercounter = rounds;
    int errorctr = 0;
  while(supercounter > 0){

    if(supercounter % (1<<12) == 0) cout << index << ": " << supercounter << endl;
  	MatrixXI mtx(m,k);
    for(int i = 0; i < k; i++){
	    for(int j = 0; j < 3;){
	    	int bkt = distr(gen);
	    	if(mtx(bkt,i) == 0){
	    		mtx(bkt, i) = rand()%65537+1;
	    		j++;
	    		}
	    	}

	    }
  supercounter--;
  if(mtx.colPivHouseholderQr().rank() < k) {errorctr += 1; cout << "\n!!!\n" << endl;
  if (errorctr>= 1) break;}

  for(int i = 0; i < m;i++){
    for(int j = 0; j < k; j++){
      mtx(i,j) = 0;
    }
  }
  }
  std::cout << rounds-supercounter << " " << errorctr << std::endl;
  NTL_EXEC_INDEX_END

	return 0;
}

int main(){
  test();
}