/**
Haha, not all of this code is written by Aman Agrawal
 (Indian Institute of Technology, Delhi)
*/

#include <bits/stdc++.h>
#include <Eigen/Dense>
#include <Eigen/Core>
#include <Eigen/LU>
#include <Eigen/SVD>
#include <Eigen/QR>
#include <math.h>
#include <boost/math/distributions/students_t.hpp>

#include "time.h"
#include <thread>
#include <chrono>

#include "global.h"
#include "genotype.h"
#include "mailman.h"
#include "helper.h"
#include "storage.h"
#include "Goptions.hpp"
#include "mailbox.h"
#include "EigenGWAS.hpp"

using namespace Eigen;
using namespace std;

// Storing in RowMajor Form
typedef Matrix<double, Dynamic, Dynamic, RowMajor> MatrixXdr;

//options command_line_opts;
extern Goptions goptions;
extern genotype g;
//MatrixXdr geno_matrix; //(p, n)

//Intermediate Variables
//
//How to batch columns:
//int blocksize;
extern double **partialsums;
extern double *sum_op;

// Intermediate computations in E-step.
// Size = 3^(log_3(n)) * k
extern double **yint_e;
// n X k
extern double ***y_e;

// Intermediate computations in M-step.
// Size = nthreads X 3^(log_3(n)) * k
extern double **yint_m;
//  nthreads X log_3(n) X k
extern double ***y_m;

struct timespec t0;

MatrixXdr c; //(p,k)
MatrixXdr means; //(p,1)
MatrixXdr stds; //(p,1)
MatrixXdr eveP; //(n,k) //projection matrix for EigenGWAS
std::vector<std::vector<double> > pheVal;
string phe_File = "";

pair<double, double> get_error_norm(MatrixXdr &c) {

	int k = goptions.GetGenericMailmanBlockSize();
	int Nsnp = g.Nsnp;
	int Nindv = g.Nindv;
	int k_orig = goptions.GetGenericEigenvecNumber();

	HouseholderQR<MatrixXdr> qr(c);
	MatrixXdr Q;
	Q = qr.householderQ() * MatrixXdr::Identity(Nsnp, k);
	MatrixXdr q_t(k, Nsnp);
	q_t = Q.transpose();
	MatrixXdr b(k, Nindv);
	// Need this for subtracting the correct mean in case of missing data
	if (goptions.IsGenericMissing()) {
		multiply_y_post(q_t, k, b, false);
		// Just calculating b from seen data
		MatrixXdr M_temp(k, 1);
		M_temp = q_t * means;
		for (int j = 0; j < Nindv; j++) {
			MatrixXdr M_to_remove(k, 1);
			M_to_remove = MatrixXdr::Zero(k, 1);
			for (int i = 0; i < g.not_O_j[j].size(); i++) {
				int idx = g.not_O_j[j][i];
				M_to_remove = M_to_remove + (Q.row(idx).transpose() * g.get_col_mean(idx));
			}
			b.col(j) -= (M_temp - M_to_remove);
		}
	} else {
		multiply_y_post(q_t, k, b, true);
	}

	JacobiSVD<MatrixXdr> b_svd(b, ComputeThinU | ComputeThinV);
	MatrixXdr u_l, d_l, v_l;
	if (goptions.IsGenericFastMode())
        u_l = b_svd.matrixU();
    else
        u_l = Q * b_svd.matrixU();
	v_l = b_svd.matrixV();
	d_l = MatrixXdr::Zero(k, k);
	for (int kk = 0; kk < k; kk++)
		d_l(kk, kk) = (b_svd.singularValues())(kk);

	MatrixXdr u_k, v_k, d_k;
	u_k = u_l.leftCols(k_orig);
	v_k = v_l.leftCols(k_orig);
	d_k = MatrixXdr::Zero(k_orig, k_orig);
	for (int kk = 0; kk < k_orig; kk++)
		d_k(kk, kk) = (b_svd.singularValues())(kk);

	MatrixXdr b_l, b_k;
    b_l = u_l * d_l * (v_l.transpose());
    b_k = u_k * d_k * (v_k.transpose());

    if (goptions.IsGenericFastMode()) {
        double temp_k = b_k.cwiseProduct(b).sum();
        double temp_l = b_l.cwiseProduct(b).sum();
        double b_knorm = b_k.norm();
        double b_lnorm = b_l.norm();
        double norm_k = (b_knorm * b_knorm) - (2 * temp_k);
        double norm_l = (b_lnorm * b_lnorm) - (2 * temp_l);
        return make_pair(norm_k, norm_l);
    } else {
        MatrixXdr e_l(Nsnp, Nindv);
        MatrixXdr e_k(Nsnp, Nindv);
        for (int p_iter = 0; p_iter < Nsnp; p_iter++) {
            for (int n_iter = 0; n_iter < Nindv; n_iter++) {
                e_l(p_iter, n_iter) = g.get_geno(p_iter, n_iter, goptions.IsGenericVarNorm()) - b_l(p_iter, n_iter);
                e_k(p_iter, n_iter) = g.get_geno(p_iter, n_iter, goptions.IsGenericVarNorm()) - b_k(p_iter, n_iter);
            }
        }

        double ek_norm = e_k.norm();
        double el_norm = e_l.norm();
        return make_pair(ek_norm, el_norm);
    }
}

/* Run one iteration of EM when genotypes are not missing
 * c_orig : p X k matrix
 * Output: c_new : p X k matrix
 */
MatrixXdr run_EM_not_missing(MatrixXdr &c_orig) {

	int k = goptions.GetGenericMailmanBlockSize();
	int Nsnp = g.Nsnp;
	int Nindv = g.Nindv;

	#if DEBUG==1
		if (debug) {
			print_time();
			cout << "Enter: run_EM_not_missing" << endl;
		}
	#endif

 	// c_temp : k X p matrix: (C^T C)^{-1} C^{T}
	MatrixXdr c_temp(k, Nsnp);
	MatrixXdr c_new(Nsnp, k);
	c_temp = ((c_orig.transpose() * c_orig).inverse()) * (c_orig.transpose());

	#if DEBUG == 1
		if (debug) {
			print_timenl();
		}
	#endif

 	/*E-step: Compute X = Z G
 	* G: p X n genotype matrix
 	* Z: k X p matrix: (C^T C)^{-1} C^{T}
 	* X: k X n matrix
 	* x_fn: X
 	* c_temp: Z
 	*/
	MatrixXdr x_fn(k, Nindv);
	multiply_y_post(c_temp, k, x_fn, true);

	#if DEBUG == 1
		if (debug) {
			print_timenl();
		}
	#endif

	//x_temp: n X k matrix X^{T} (XX^{T})^{-1}
	MatrixXdr x_temp(Nindv, k);
	x_temp = (x_fn.transpose()) * ((x_fn*(x_fn.transpose())).inverse());

	/* M-step: X = G Z
 	* G: p X n genotype matrix
 	* Z: n X k matrix: X^{T}(XX^{T})^{-1}
 	* X = p X k matrix
 	* c_new: X
 	* x_temp: Z
 	*/
	multiply_y_pre(x_temp, k, c_new, true);

	#if DEBUG==1
		if(debug) {
			print_time();
			cout << "Exiting: run_EM_not_missing" << endl;
		}
	#endif
	return c_new;
}

MatrixXdr run_EM_missing(MatrixXdr &c_orig) {
	int k = goptions.GetGenericMailmanBlockSize();
	int Nsnp = g.Nsnp;
	int Nindv = g.Nindv;
	MatrixXdr c_new(Nsnp, k);
	MatrixXdr mu(k, Nindv);

	// E step
	MatrixXdr c_temp(k, k);
	c_temp = c_orig.transpose() * c_orig;

	MatrixXdr T(k, Nindv);
	MatrixXdr c_fn;
	c_fn = c_orig.transpose();
	multiply_y_post(c_fn, k, T, false);

	MatrixXdr M_temp(k, 1);
	M_temp = c_orig.transpose() * means;

	for (int j = 0; j < Nindv; j++) {
		MatrixXdr D(k, k);
		MatrixXdr M_to_remove(k, 1);
		D = MatrixXdr::Zero(k, k);
		M_to_remove = MatrixXdr::Zero(k, 1);
		for (int i = 0; i < g.not_O_j[j].size(); i++) {
			int idx = g.not_O_j[j][i];
			D = D + (c_orig.row(idx).transpose() * c_orig.row(idx));
			M_to_remove = M_to_remove + (c_orig.row(idx).transpose() * g.get_col_mean(idx));
		}
		mu.col(j) = (c_temp - D).inverse() * (T.col(j) - M_temp + M_to_remove);
	}

	#if DEBUG == 1
		if (debug) {
			ofstream x_file;
//			x_file.open((string(command_line_opts.OUTPUT_PATH)+string("x_in_fn_vals.txt")).c_str());
			x_file.open((goptions.GetGenericOutFile() + string("x_in_fn_vals.txt")).c_str());
			x_file<<std::setprecision(15)<<mu<<endl;
			x_file.close();
		}
	#endif

	// M step
	MatrixXdr mu_temp(k, k);
	mu_temp = mu * mu.transpose();
	MatrixXdr T1(Nsnp, k);
	MatrixXdr mu_fn;
	mu_fn = mu.transpose();
	multiply_y_pre(mu_fn, k, T1, false);
	MatrixXdr mu_sum(k, 1);
	mu_sum = MatrixXdr::Zero(k, 1);
	mu_sum = mu.rowwise().sum();

	for (int i = 0; i < Nsnp; i++) {
		MatrixXdr D(k, k);
		MatrixXdr mu_to_remove(k, 1);
		D = MatrixXdr::Zero(k, k);
		mu_to_remove = MatrixXdr::Zero(k, 1);
		for (int j = 0; j < g.not_O_i[i].size(); j++) {
			int idx = g.not_O_i[i][j];
			D = D + (mu.col(idx) * mu.col(idx).transpose());
			mu_to_remove = mu_to_remove + (mu.col(idx));
		}
		c_new.row(i) = (((mu_temp-D).inverse()) * (T1.row(i).transpose() - (g.get_col_mean(i) * (mu_sum-mu_to_remove)))).transpose();
		double mean;
		mean = g.get_col_sum(i);
		mean = mean -  (c_orig.row(i) * (mu_sum-mu_to_remove))(0, 0);
		mean = mean * 1.0 / (Nindv - g.not_O_i[i].size());
		g.update_col_mean(i, mean);
	}

	// IMPORTANT: Update the value of means variable present locally, so that for next iteration, updated value of means is used.
	for (int i = 0; i < Nsnp; i++) {
		means(i, 0) = g.get_col_mean(i);
		// Also updating std, just for consistency, though, it is not used presently.
		stds(i, 0) = g.get_col_std(i);
	}

	return c_new;
}

MatrixXdr run_EM(MatrixXdr &c_orig) {
	if (goptions.IsGenericMissing()) {
		return run_EM_missing(c_orig);
	} else {
		return run_EM_not_missing(c_orig);
	}
}

void print_vals() {
	int k = goptions.GetGenericMailmanBlockSize();
	int Nsnp = g.Nsnp;
	int Nindv = g.Nindv;
	int k_orig = goptions.GetGenericEigenvecNumber();

	HouseholderQR<MatrixXdr> qr(c);
	MatrixXdr Q;
	Q = qr.householderQ() * MatrixXdr::Identity(Nsnp, k);
	MatrixXdr q_t(k, Nsnp);
	q_t = Q.transpose();
	MatrixXdr b(k, Nindv);

	// Need this for subtracting the correct mean in case of missing data
	if (goptions.IsGenericMissing()) {
		multiply_y_post(q_t, k, b, false);
		// Just calculating b from seen data
		MatrixXdr M_temp(k, 1);
		M_temp = q_t * means;
		for (int j = 0; j < Nindv; j++) {
			MatrixXdr M_to_remove(k, 1);
			M_to_remove = MatrixXdr::Zero(k, 1);
			for (int i = 0; i < g.not_O_j[j].size(); i++) {
				int idx = g.not_O_j[j][i];
				M_to_remove = M_to_remove + (Q.row(idx).transpose() * g.get_col_mean(idx));
			}
			b.col(j) -= (M_temp - M_to_remove);
		}
	} else {
		multiply_y_post(q_t, k, b, true);
	}

	JacobiSVD<MatrixXdr> b_svd(b, ComputeThinU | ComputeThinV);
	MatrixXdr u_l;
	u_l = b_svd.matrixU();
	MatrixXdr v_l;
	v_l = b_svd.matrixV();
	MatrixXdr u_k;
	MatrixXdr v_k, d_k;
	u_k = u_l.leftCols(k_orig);
	v_k = v_l.leftCols(k_orig);

	ofstream evec_file;
	evec_file.open((goptions.GetGenericOutFile() + string("evecs.txt")).c_str());
	evec_file << std::setprecision(15) << Q * u_k << endl;
	evec_file.close();
	ofstream eval_file;
	eval_file.open((goptions.GetGenericOutFile() + string("evals.txt")).c_str());
	for(int kk = 0; kk < k_orig; kk++)
		eval_file << std::setprecision(15) << (b_svd.singularValues())(kk) * (b_svd.singularValues())(kk)/g.Nsnp<<endl;
	eval_file.close();

	eveP = v_l.leftCols(k_orig);
	ofstream proj_file;
	proj_file.open((goptions.GetGenericOutFile() + string("projections.txt")).c_str());
	proj_file << std::setprecision(15)<< v_k << endl;
	proj_file.close();

	if (goptions.IsGenericDebug()) {
		ofstream c_file;
		c_file.open((goptions.GetGenericOutFile()+string("cvals.txt")).c_str());
		c_file << std::setprecision(15) << c << endl;
		c_file.close();

		ofstream means_file;
		means_file.open((goptions.GetGenericOutFile()+string("means.txt")).c_str());
		means_file << std::setprecision(15) << means << endl;
		means_file.close();

		d_k = MatrixXdr::Zero(k_orig, k_orig);
		for (int kk =0; kk < k_orig; kk++)
			d_k(kk,kk)  =(b_svd.singularValues())(kk);
		MatrixXdr x_k;
		x_k = d_k * (v_k.transpose());
		ofstream x_file;
		x_file.open((goptions.GetGenericOutFile() + string("xvals.txt")).c_str());
		x_file << std::setprecision(15) << x_k.transpose() << endl;
		x_file.close();
	}
}

void RHE_read_pheno() {
   	ifstream ifs(goptions.GetGenericPhenoFile().c_str(), ios::in);
	std::string temp;

	while (std::getline(ifs, temp)) {
    std::istringstream buffer(temp);
    std::vector<double> line((std::istream_iterator<double>(buffer)),
                             std::istream_iterator<double>());
    	pheVal.push_back(line);
	}

	// for (auto it = numbers.begin(); it != numbers.end(); it++) {
	// 	vector<double> n1 = *it;
	// 	for (auto it1 = n1.begin(); it1 != n1.end(); it1++) {
	// 		cout << (*it1) << " ";
	// 	}
	// 	cout << endl;
	// }
}

void RHE_reg(int seed, int iter, int phe_idx) {

	cout << "Randomized HE (mailman), iteration for " << iter << " times" << endl;
	clock_t heReg_begin = clock();

	MatrixXdr yval(g.Nindv, 1);
	int cnt = 0;
	for (int i = 0; i < yval.rows(); i++) {
		yval(i, 0) = pheVal[i][phe_idx];
	}
	cout<<yval(0, 0)<<" "<<yval(1, 0)<<endl;

	double LB = 0;
	MatrixXdr Bz(g.Nindv, iter);
	srand(goptions.GetGenericSeed());
	std::default_random_engine generator(goptions.GetGenericSeed());
	std::normal_distribution<double> norm_dist(0, 1.0);
	for (int i = 0; i < Bz.rows(); i++) {
		for (int j = 0; j < Bz.cols(); j++) {
			Bz(i, j) = norm_dist(generator);
		}
	}

	//geno_matrix * Bz; //(p x n) * (n x iter) = p x iter
	MatrixXdr T1(g.Nsnp, iter);
	cout << "Here T1=G^T * z" << endl;
	multiply_y_pre(Bz, iter, T1, true);
	MatrixXdr T1_transpose(iter, g.Nsnp);
	T1_transpose = T1.transpose(); //iter X p
	cout << "Here T1.transpose" << endl;

	MatrixXdr T2(iter, g.Nindv);
	cout << "Here T2=T1^T * G^T" << endl;
	cout << T1_transpose.rows() << " " << T1_transpose.cols() << " " << endl;
	multiply_y_post(T1_transpose, iter, T2, true);

	for (int i = 0; i < T2.rows(); i++) {
		for (int j = 0; j < T2.cols(); j++) {
			LB += T2(i, j) * T2(i, j);
		}
	}

	cout << "LB " << LB << endl;
	LB = LB / (1.0 * iter * g.Nsnp * g.Nsnp);
	cout << "LB2 " << LB << endl;
	double me = (LB - g.Nindv) / (1.0 * g.Nindv * g.Nindv);
	cout << "Me " << 1/me << endl;
	clock_t heReg_end = clock();

	double heReg_time = double(heReg_end - heReg_begin) / CLOCKS_PER_SEC;
	cout << "RHE time " << heReg_time << endl;
}

void ENC(int seed, int kval) {
	cout << "ENC generates " << kval << " tags for " << g.Nindv <<" samples" <<endl;
	clock_t ENC_begin = clock();

	srand(goptions.GetGenericSeed());
	std::default_random_engine generator(goptions.GetGenericSeed());
	std::normal_distribution<double> norm_dist(0, 1.0);

	MatrixXdr Bz(kval, g.Nsnp);
	for (int i = 0; i < Bz.rows(); i++)
		for (int j = 0; j < Bz.cols(); j++)
			Bz(i, j) = norm_dist(generator);

	MatrixXdr encG(kval, g.Nindv);
	multiply_y_post(Bz, kval, encG, true);

	ofstream e_file;
	e_file.open((goptions.GetGenericOutFile() + string(".enc.txt")).c_str());
	for (int i = 0; i < encG.cols(); i++) {
		for (int j = 0; j < encG.rows(); j++) {
			e_file<<encG(j, i);
			if (j != (encG.rows() - 1)) e_file << " ";
		}
		e_file << endl;
	}

	e_file.close();
	clock_t ENC_end = clock();
	double ENC_time = double(ENC_end - ENC_begin) / CLOCKS_PER_SEC;
	cout<< "Save encG to " <<(goptions.GetGenericOutFile() + string(".enc.txt")).c_str() <<endl;
	cout << "ENC time " << ENC_time << endl;
}

void CLD() {

}

void ProPC() {

	int k = goptions.GetGenericMailmanBlockSize();
	int Nsnp = g.Nsnp;
	clock_t pc_begin = clock();

	pair<double,double> prev_error = make_pair(0.0, 0.0);
	bool toStop = false;

	//	if (convergence_limit != -1)
	if (goptions.GetPropcConvergenceLimit() != -1)
		toStop = true;

	double prevnll = 0.0;

	c.resize(Nsnp, k);
	means.resize(Nsnp, 1);
	stds.resize(Nsnp, 1);
	for (int i = 0; i < Nsnp; i++) {
		means(i, 0) = g.get_col_mean(i);
		stds(i, 0) = g.get_col_std(i);
	}

	std::default_random_engine generator(goptions.GetGenericSeed());
	std::normal_distribution<double> norm_dist(0, 1.0);
	for (int i = 0; i < c.rows(); i++)
		for (int j = 0; j < c.cols(); j++)
			c(i, j) = norm_dist(generator);
	// Initial intermediate data structures
	// Operate in blocks to improve caching
	//

	ofstream c_file;
	if (goptions.IsGenericDebug()) {
		// c_file.open((string(command_line_opts.OUTPUT_PATH) + string("cvals_orig.txt")).c_str());
		c_file.open((goptions.GetGenericOutFile() + string("cvals_orig.txt")).c_str());
		c_file << std::setprecision(15) << c << endl;
		c_file.close();
		printf("Read Matrix\n");
	}

	cout << "Running on Dataset of " << g.Nsnp << " SNPs and " << g.Nindv << " Individuals" << endl;

	#if SSE_SUPPORT == 1
		if (fast_mode)
			cout<<"Using Optimized SSE FastMultiply"<<endl;
	#endif

	clock_t it_begin = clock();
	for (int i = 0; i < goptions.GetPropcMaxIteration(); i++) {
		MatrixXdr c1, c2, cint, r, v;
		double a, nll;
		if (goptions.IsGenericDebug()) {
			print_time ();
			cout << "*********** Begin epoch " << i << "***********" << endl;
		}

//		if (accelerated_em != 0) {
		if (goptions.GetPropcAcceleratedEM() != 0) {
			#if DEBUG == 1
			if (debug) {
				print_time();
				cout << "Before EM" << endl;
			}
			#endif
			c1 = run_EM(c);
			c2 = run_EM(c1);
			#if DEBUG == 1
			if (debug) {
				print_time();
				cout << "After EM but before acceleration" << endl;
			}
			#endif
			r = c1 - c;
			v = (c2 - c1) - r;
			a = -1.0 * r.norm() / (v.norm()) ;
			if (goptions.GetPropcAcceleratedEM() == 1) {
				if (a > -1) {
					a = -1;
					cint = c2;
				} else {
					cint = c - 2 * a * r + a * a * v;
					nll = get_error_norm(cint).second;
					if ( i > 0 ) {
						while ( nll>prevnll && a < -1) {
							a = 0.5 * (a-1);
							cint = c - 2 * a * r + (a * a * v);
							nll = get_error_norm(cint).second;
						}
					}
				}
				c = cint;
			} else if (goptions.GetPropcAcceleratedEM() == 2) {
				cint = c - 2 * a * r + a * a * v;
				c = cint;
				// c = run_EM(cint);
			}
		} else {
			c = run_EM(c);
		}

		if (goptions.GetPropcAcceleratedEM() == 1 || goptions.IsPropcAccuracy() || toStop) {
			pair<double, double> e = get_error_norm(c);
				prevnll = e.second;
				if (goptions.IsPropcAccuracy())
					cout << "Iteration " << i+1 << "  " << std::setprecision(15) << e.first << "  " << e.second << endl;
				if (abs(e.first - prev_error.first) <= goptions.GetPropcConvergenceLimit()) {
					cout << "Breaking after " << i+1 << " iterations" << endl;
					break;
				}
				prev_error = e;
		}

		if (goptions.IsGenericDebug()) {
				print_time();
				cout << "*********** End epoch " << i << "***********" << endl;
		}
	}

	clock_t it_end = clock();

	print_vals();

	clock_t pc_end = clock();
	double avg_it_time = double(it_end - it_begin) / (goptions.GetPropcMaxIteration() * 1.0 * CLOCKS_PER_SEC);
	double total_time = double(pc_end - pc_begin) / CLOCKS_PER_SEC;
	cout << "\nAVG Iteration Time: " << avg_it_time << "\nTotal runtime: " << total_time << endl;
}

int main(int argc, char const *argv[]) {

	try {
    	goptions.ParseOptions(argc, argv);
//      PrintOptions(goptions);
	}
	catch (OptionsExitsProgram){}

	auto start = std::chrono::system_clock::now();
	clock_t io_begin = clock();
    clock_gettime(CLOCK_REALTIME, &t0);

	srand(goptions.GetGenericSeed());

	g.read_geno(goptions.GetGenericGenoFile(),
	goptions.IsGenericTextMode(),
	goptions.IsGenericFastMode(),
	goptions.IsGenericMissing(),
	goptions.IsGenericMemoryEfficient(),
	goptions.IsGenericVarNorm());

	//TODO: Implement these codes.
/*
	if (missing && !fast_mode) {
		cout << "Missing version works only with mailman i.e. fast mode\n EXITING..." << endl;
		exit(-1);
	}
	if (fast_mode && memory_efficient) {
		cout << "Memory effecient version for mailman EM not yet implemented" << endl;
		cout << "Ignoring Memory effecient Flag" << endl;
	}
	if (missing && var_normalize) {
		cout << "Missing version works only without variance normalization\n EXITING..." << endl;
		exit(-1);
	}
*/

	if (!goptions.IsGenericFastMode() && !goptions.IsGenericMemoryEfficient()) {
		cout << "Genotype standardization..." << endl;
	}

	clock_t io_end = clock();
	double io_time = double(io_end - io_begin) / CLOCKS_PER_SEC;
	cout << "IO Time: " << io_time << endl;

	if (goptions.CheckPropcMasterOption()) {
		setMem();
		ProPC();
		cleanMem();
	} else if (goptions.CheckEigenGWASMasterOption()) {
		setMem();
		ProPC();
		EigenGWAS eg;
		eg.Scan(eveP);
//		EigenGWAS(eveP);
		cleanMem();
	} else if (goptions.CheckRandHEMasterOption()) {
//		RHE_read_pheno(command_line_opts.PHENO_FILE);
//		setMem(command_line_opts.rhe_it);
//		RHE_reg(seed, command_line_opts.rhe_it, command_line_opts.pheno_num);
		RHE_read_pheno();
		setMem();
		RHE_reg(goptions.GetGenericSeed(), goptions.GetGenericIteration(), goptions.GetGenericPhenoNum()[0]);
		cleanMem();
	} else if (goptions.CheckEncMasterOption()) {
		setMem();

//		gmanENC(seed, goptions.GetEncK(), gman);
		ENC(goptions.GetGenericSeed(), goptions.GetEncK());
		cleanMem();
	} else if (goptions.CheckCLDMasterOption()) {
		CLD();
	}

	std::chrono::duration<double> wctduration = std::chrono::system_clock::now() - start;
	cout << "Wall clock time = " << wctduration.count() << endl;
	return 0;
}


#include <pybind11/pybind11.h>
#include <pybind11/stl.h>
#include <pybind11/complex.h>
#include <pybind11/numpy.h>

#include <vector>
#include <string>
#include <cstdarg>
#include <cstring>
#include <cstddef>
#include <iterator>

#include <boost/lexical_cast.hpp>

#include <CGAL/Exact_predicates_inexact_constructions_kernel.h>
#include <CGAL/Delaunay_triangulation_3.h>
#include <CGAL/Triangulation_vertex_base_with_info_3.h>

namespace py = pybind11;

using K = CGAL::Exact_predicates_inexact_constructions_kernel;
using Vb = CGAL::Triangulation_vertex_base_with_info_3<unsigned int, K>;
using Tds = CGAL::Triangulation_data_structure_3<Vb>;
using DT = CGAL::Delaunay_triangulation_3<K, Tds>;

using Point = K::Point_3;
using Vertex_handle = DT::Vertex_handle;
using Vi = DT::Finite_vertices_iterator;

template <typename T>
class TypedInputIterator
{
public:
    using iterator_category = std::input_iterator_tag;
    using difference_type = std::ptrdiff_t;
    using value_type = T;
    using pointer = T*;
    using reference = T&;

    explicit TypedInputIterator(py::iterator& py_iter) :
        py_iter_(py_iter)
    {
    }

    explicit TypedInputIterator(py::iterator&& py_iter) :
        py_iter_(py_iter)
    {
    }

    value_type operator*()
    {
        return (*py_iter_).template cast<value_type>();
    }

    TypedInputIterator operator++(int)
    {
        auto copy = *this;
        ++py_iter_;
        return copy;
    }

    TypedInputIterator& operator++()
    {
        ++py_iter_;
        return *this;
    }

    bool operator!=(TypedInputIterator &rhs)
    {
        return py_iter_ != rhs.py_iter_;
    }

    bool operator==(TypedInputIterator &rhs)
    {
        return py_iter_ == rhs.py_iter_;
    }

private:
    py::iterator py_iter_;
};

PYBIND11_MODULE(delaunay_class3, m)
{
    py::class_<Point>(m, "Point")
            .def(py::init<int, int, int>(),  py::arg("x"), py::arg("y"), py::arg("z"))
            .def(py::init<double, double, double>(), py::arg("x"), py::arg("y"), py::arg("z"))
            .def_property_readonly("x", &Point::x)
            .def_property_readonly("y", &Point::y)
            .def_property_readonly("z", &Point::z)
            .def("__repr__",
            [](const Point &p) {
                std::string r("Point(");
                r += boost::lexical_cast<std::string>(p.x());
                r += ", ";
                r += boost::lexical_cast<std::string>(p.y());
                r += ", ";
                r += boost::lexical_cast<std::string>(p.z());
                r += ")";
                return r;
            })
            ;

    py::class_<Vertex_handle>(m, "VertexHandle")
        .def_property_readonly(
        "point",
        [](const Vertex_handle& vertex_handle)
        {
            return vertex_handle->point();
        })
        ;

        py::class_<DT>(m, "DelaunayTriangulation3")

            .def(py::init())

            .def("insert", [](DT & dt, const std::vector<double> & p) {
                  std::vector< std::pair<Point,unsigned> > points;
                  int num_points = p.size()/3;
                  // start adding at the end of the current table
                  int start = dt.number_of_vertices();
                  for(std::size_t i = 0; i < num_points; ++i)
                  {
                    // add index information to form face table later
                     points.push_back( std::make_pair( Point(p[i*3+0],p[i*3+1], p[i*3+2]), start) );
                     start += 1;
                  }
                  return dt.insert(points.begin(),points.end());
                })

            .def("move", [](DT & dt, const std::vector<unsigned int> & to_move, const std::vector<double> & new_positions){
                    std::vector<Vertex_handle> handles;
                    std::vector<Point> new_pos;
                    int num_to_move= to_move.size();
                    // store all vertex handles
                    // Should this be finite_vertices_begin and finite_vertices_end?
                    for (Vi vi = dt.finite_vertices_begin(); vi != dt.finite_vertices_end(); vi++){
                        handles.push_back(vi);
                    }
                    // store new positions as a vector of Point
                    for(std::size_t i = 0; i < num_to_move; ++i)
                    {
                       new_pos.push_back(Point(new_positions[3*i], new_positions[3*i+1], new_positions[3*i+2]));
                    }
                    //
                    for(std::size_t i = 0; i < num_to_move; ++i)
                    {
                       dt.move( handles[to_move[i]], new_pos[i] );
                    }
                    return dt;
                    })

            .def("remove", [](DT & dt, const std::vector<unsigned int> & to_remove) {
                    int num_to_remove= to_remove.size();
                    std::vector<Vertex_handle> handles;
                    for (Vi vi = dt.finite_vertices_begin(); vi != dt.finite_vertices_end(); vi++){
                        handles.push_back(vi);
                    }
                    for(std::size_t i=0; i < num_to_remove; ++i){
                        dt.remove(handles[to_remove[i]]);
                    }
                    return dt;
                })

            .def("number_of_vertices", [](DT & dt){
                    return dt.number_of_vertices();
                })

            .def("number_of_cells", [](DT & dt){
                int count=0;
                for(DT::Finite_cells_iterator fit = dt.finite_cells_begin();
                fit != dt.finite_cells_end(); ++fit) {
                    count += 1;
                }
                    return count;
                })

            .def("finite_vertices", [](DT & dt) -> py::iterator
             {
                 return py::make_iterator(dt.finite_vertices_begin(), dt.finite_vertices_end());
             })

            .def("get_finite_cells", [](DT & dt)
            {
              // ouput the cell table
              // YOU MUST CALL get_finite_vertices before if any incremental operations
              // were performed
              std::vector<int> cells;
              cells.resize(dt.number_of_finite_cells()*4);

              int i=0;
              for(DT::Finite_cells_iterator fit = dt.finite_cells_begin();
                fit != dt.finite_cells_end(); ++fit) {

                DT::Cell_handle cell = fit;
                cells[i*4]=cell->vertex(0)->info();
                cells[i*4+1]=cell->vertex(1)->info();
                cells[i*4+2]=cell->vertex(2)->info();
                cells[i*4+3]=cell->vertex(3)->info();
                i+=1;
              }
              ssize_t              soint      = sizeof(int);
              ssize_t              num_cells = cells.size()/4;
              ssize_t              ndim      = 2;
              std::vector<ssize_t> shape     = {num_cells, 4};
              std::vector<ssize_t> strides   = {soint*4, soint};

              // return 2-D NumPy array
              return py::array(py::buffer_info(
                cells.data(),                           /* data as contiguous array  */
                sizeof(int),                          /* size of one scalar        */
                py::format_descriptor<int>::format(), /* data type                 */
                2,                                    /* number of dimensions      */
                shape,                                   /* shape of the matrix       */
                strides                                  /* strides for each axis     */
              ));
            })

            .def("get_finite_vertices", [](DT & dt)
             {
               // ouput the vertices
               std::vector<double> vertices;
               vertices.resize(dt.number_of_vertices()*3);

               int i=0;
               for(DT::Finite_vertices_iterator fit = dt.finite_vertices_begin();
                 fit != dt.finite_vertices_end(); ++fit) {

                 Vertex_handle vertex = fit;
                 // critical! update the point index table so faces comes out correctly
                 vertex->info() = i;
                 vertices[i*3]=vertex->point().x();
                 vertices[i*3+1]=vertex->point().y();
                 vertices[i*3+2]=vertex->point().z();
                 i+=1;
               }
               ssize_t              sdble   = sizeof(double);
               ssize_t              num_vertices = vertices.size()/3;
               ssize_t              ndim      = 2;
               std::vector<ssize_t> shape     = {num_vertices, 3};
               std::vector<ssize_t> strides   = {sdble*3, sdble};

               // return 2-D NumPy array
               return py::array(py::buffer_info(
                 vertices.data(),                           /* data as contiguous array  */
                 sizeof(double),                          /* size of one scalar        */
                 py::format_descriptor<double>::format(), /* data type                 */
                 2,                                    /* number of dimensions      */
                 shape,                                   /* shape of the matrix       */
                 strides                                  /* strides for each axis     */
               ));
             })
            ;

    py::class_<DT::Finite_vertices_iterator::value_type>(m, "Vertex")
            .def_property_readonly(
                "point", [](DT::Finite_vertices_iterator::value_type& vertex)
                {
                    return vertex.point();
                }
            )
            ;

    py::class_<DT::Finite_cells_iterator::value_type>(m, "Cell")
            .def("vertex_handle",
                [](DT::Finite_cells_iterator::value_type& cell, int index)
                {
                    return cell.vertex(index);
                },
                py::arg("index")
            )
            ;

}


// __BEGIN_LICENSE__
//  Copyright (c) 2006-2013, United States Government as represented by the
//  Administrator of the National Aeronautics and Space Administration. All
//  rights reserved.
//
//  The NASA Vision Workbench is licensed under the Apache License,
//  Version 2.0 (the "License"); you may not use this file except in
//  compliance with the License. You may obtain a copy of the License at
//  http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
// __END_LICENSE__

/// \file CameraCurve.cc
///
/// Functions for deducing the camera response curve by comparing
/// relative brightness values from several images of the same scene.

#include <vw/Camera/Exif.h>
#include <vw/HDR/CameraCurve.h>
#include <vw/Math/LinearAlgebra.h>
#include <boost/algorithm/string.hpp>

namespace vw {
namespace hdr {

  /// Generate a set of brightness values based on a known ratio of
  /// exposures between images.  Note that this brightness value will
  /// only be correct in a relative sense.  Given this, we choose
  /// (arbitrarily) a set of brightness values that correspond to a
  /// "typical" exposure of 1/60th of a second at ISO 100 and an
  /// aperture of f/5.6 (B = 235.2 in this case)
  ///
  ///
  std::vector<double> brightness_values_from_exposure_ratio(double exposure_ratio, int size) {
    const double base_brightness = 235.2; // average luminance for
                                          // 1/60th s exposure at ISO
                                          // 100 and f/5.6
    std::vector<double> brightness_values(size);
    for ( unsigned i=0; i < brightness_values.size(); ++i ) {
      brightness_values[i] = base_brightness*pow(exposure_ratio, (double)i);
    }
    return brightness_values;
  }

  /// Generate a list of brightness values from EXIF information
  /// stored in a list of files.
  std::vector<double> brightness_values_from_exif(std::vector<std::string> const& filenames) {
    int num_images = filenames.size();
    std::vector<double> brightness_values(num_images);

    for (int i = 0; i < num_images; i++) {
      vw::camera::ExifView exif(filenames[i].c_str());
      brightness_values[i] = exif.get_average_luminance();
    }
    return brightness_values;
  }

  // A simple gaussian weighting function for use in camera curve
  // estimation (below)
  inline double gaussian_weighting_func(double x) {
    return exp(-pow((x-0.5),2)/(0.07));
  }

  Vector<double> estimate_camera_curve(vw::Matrix<double> const& pixels,
                                       std::vector<double> const& brightness_values) {

    const int n = 256;                    // Create a solution with 256 points
    const int smoothing_factor = 10;      // Smoothing by a factor of 10

    // Initialize storage
    Matrix<double> A(pixels.rows()*pixels.cols()+n+1, n+pixels.rows());
    Vector<double> b(A.rows());

    // Include data fitting equations
    int k = 0;
    for (unsigned i = 0; i < pixels.rows(); ++i) {
      for (unsigned j = 0; j < pixels.cols(); ++j) {
        int idx = int(pixels(i,j)*(n-1));
        double wij = gaussian_weighting_func(pixels(i,j));
        A(k,idx) = wij;
        A(k,n+i) = -wij;
        b(k) = wij * log(1/brightness_values[j]);
        ++k;
      }
    }

    // Fix the curve by setting its middle value to 0
    A(k,n/2) = 1;
    ++k;

    // Include the smoothness equations
    for (int i = 0; i < n-2; ++i) {
      double weight = gaussian_weighting_func(double(i+1)/(n-1));
      A(k,i) = smoothing_factor*weight;
      A(k,i+1) = -2*smoothing_factor*weight;
      A(k,i+2) = smoothing_factor*weight;
      ++k;
    }

    // Solve the system using linear least squares
    Vector<double> x = least_squares(A, b);
    return subvector(x,0,n);
  }

  void write_curves(std::string const& curves_file,
                    CameraCurveFn const &curves) {

    FILE* output_file = fopen(curves_file.c_str(), "w");
    if ( !output_file ) vw_throw( IOErr() << "write_curves: failed to open file for writing." );
    for (unsigned i = 0; i < curves.num_channels(); ++i) {
      for ( unsigned j = 0; j < curves.lookup_table(0).size(); ++j ) {
        fprintf(output_file, "%f ", curves.lookup_table(i)[j]);
      }
      fprintf(output_file, "\n");
    }
    fclose(output_file);
  }

  CameraCurveFn read_curves(std::string const& curves_file) {
    FILE* input_file = fopen(curves_file.c_str(), "r");
    if ( !input_file ) vw_throw( IOErr() << "read_curves: failed to open file for reading." );

    char c_line[10000];

    std::vector<vw::Vector<double> > lookup_tables;
    while ( !feof(input_file) ) {
      if ( !fgets(c_line, 10000, input_file) )
        break;
      std::string line = c_line;
      boost::trim_left(line);
      boost::trim_right(line);

      std::vector< std::string > split_vec; // #2: Search for individual values
      boost::split( split_vec, line, boost::is_any_of(" ") );
      Vector<double> curve(split_vec.size());
      for ( unsigned i = 0; i < split_vec.size(); ++i ) {
        curve[i] = atof(split_vec[i].c_str());
      }
      lookup_tables.push_back(curve);
    }
    fclose(input_file);

    return CameraCurveFn(lookup_tables);
  }

}} // namespace vw::hdr


// Copyright 2015 XLGAMES Inc.
//
// Distributed under the MIT License (See
// accompanying file "LICENSE" or the website
// http://www.opensource.org/licenses/mit-license.php)

#include "DualContour.h"
#include "../Math/Matrix.h"
#include "../Math/Transformations.h"
#include "../Math/Geometry.h"
#include "../Utility/PtrUtils.h"

#pragma warning(disable:4714)
#pragma push_macro("new")
#undef new
#include <Eigen/Dense>
#pragma pop_macro("new")

#pragma warning(disable:4127)       // conditional expression is constant

namespace SceneEngine
{
    class EdgeIntersection
    {
    public:
        Float3 _pt;
        Float3 _normal;

        EdgeIntersection(const Float3& intersectionPt, const Float3& intersectionNormal)
            : _pt(intersectionPt), _normal(intersectionNormal) {}
    };

// #define STORE_INTERSECTIONS

    class GridElement
    {
    public:
            //  Store the grid element values that we will
            //  need to combine together adjacent elements during
            //  the simplification
        Float3x3    _Ahat;      // (upper triangular -- maybe we don't have to store all of it?
        Float3      _Bhat;
        float       _r;
        Float3      _massPointAccum;
        unsigned    _massPointCount;

        #if defined(STORE_INTERSECTIONS)
            std::vector<EdgeIntersection> _intersectionPts;
        #endif

        GridElement()
            : _Ahat(Zero<Float3x3>()), _Bhat(Zero<Float3>()), _r(0.f)
            , _massPointAccum(Zero<Float3>()), _massPointCount(0) {}
    };

    static EdgeIntersection  TestEdge(  const Float3& e0, const Float3& e1, float d0, float d1,
                                        const IVolumeDensityFunction& fn)
    {
            //  Test the edge between these points, and attempt to find the point where
            //  the surface passes through.
            //
            //  The caller should have filtered out edges
            //  that don't pass through the surface.
        assert((d0 < 0.f) != (d1 < 0.f));

            //      It might be a good idea to further improve the
            //      result by taking a few steps to try to get to the
            //      smallest density value. note that a very strange
            //      density function might behave very non-linearly,
            //      and so produce strange results when trying to
            //      find the intersection (particularly if there are
            //      really multiple intersections).

        float x0 = 0.f, x1 = 1.f;
        float x = 1.f, d = FLT_MAX;
        unsigned maxImprovementSteps = 6;
        for (unsigned c=0; ; ++c) {
            float prevD = d; float prevX = x;
            x = LinearInterpolate(x0, x1, -d0 / (d1 - d0));
            d = fn.GetDensity(LinearInterpolate(e0, e1, x));

                // along noisy edges we could end up getting a worse result after a step
                //  In these cases, just give up at the last reasonable result
            if (XlAbs(d) > XlAbs(prevD)) {
                x = prevX;
                break;
            }

            if (XlAbs(d) < 1e-6f) break;   // if we get close enough, just stop
            if ((c+1)>=maxImprovementSteps) break;

                //  We're going to attempt another improvement.
                //  Divide the search area again, depending on where
                //  the origin falls
            if ((d < 0.f) != (d1 < 0.f)) {
                x0 = x;
                d0 = d;
            } else {
                x1 = x;
                d1 = d;
            }
        }

        assert(x>=0.f && x <= 1.f);
        Float3 bestIntersection = LinearInterpolate(e0, e1, x);
        Float3 normal = fn.GetNormal(bestIntersection);     // note -- we might need to tell the function the sampling density
        return EdgeIntersection(bestIntersection, normal);
    }

    static void RotateToUpperTriangle(Eigen::Matrix<float,5,4>& A)
    {
        // Eigen::ColPivHouseholderQR<Eigen::Matrix<float,5,4>> qr(A);
        // A = qr.matrixR().triangularView<Eigen::Upper>();
        Eigen::HouseholderQR<Eigen::Matrix<float,5,4>> qr2(A);
        A = qr2.matrixQR().triangularView<Eigen::Upper>();
    }

    static void MergeInEdgeIntersection(GridElement& gridElement, const EdgeIntersection& intersection, const Float3& gridElementCenter)
    {
            //  The edge intersections defines the a plane through the grid element.
            //  The matrices in the grid element define a set of linear equations for
            //  calculating the best point for the given planes. We want to merge
            //  the plane from this new intersection into the equations already in
            //  the grid element.
            //  See the original dual contour paper for a description of this.
            //  Basically, we're using QR decomposition to define an orthogonal
            //  matrix and a upper triangular matrix. The "Q" orthogonal matrix
            //  gets factored out when solving the linear equations. It leaves only
            //  the "R" upper triangular matrix.
            //  The original paper describes using "Given's rotations" to perform
            //  the QR decomposition. But I'm using the Householder method, here
            //  -- which is very similar, but requires fewer operations.
            //  Also note that everything is translated to end up relative
            //  to the grid center. This is just to guarantee small & simple numbers.

        typedef Eigen::Matrix<float,5,4> Float5x4;
        Float5x4 mat = Float5x4::Zero();

            // fill in the existing values from the grid element
        mat(0, 0) = gridElement._Ahat(0, 0);
        mat(0, 1) = gridElement._Ahat(0, 1);
        mat(0, 2) = gridElement._Ahat(0, 2);
        mat(1, 0) = gridElement._Ahat(1, 0);
        mat(1, 1) = gridElement._Ahat(1, 1);
        mat(1, 2) = gridElement._Ahat(1, 2);
        mat(2, 0) = gridElement._Ahat(2, 0);
        mat(2, 1) = gridElement._Ahat(2, 1);
        mat(2, 2) = gridElement._Ahat(2, 2);
        mat(0, 3) = gridElement._Bhat[0];
        mat(1, 3) = gridElement._Bhat[1];
        mat(2, 3) = gridElement._Bhat[2];
        mat(3, 3) = gridElement._r;

        mat(4, 0) = intersection._normal[0];        // normal can be positive or negative direction; we'll still get the same plane equation
        mat(4, 1) = intersection._normal[1];
        mat(4, 2) = intersection._normal[2];
        mat(4, 3) = Dot(intersection._pt - gridElementCenter, intersection._normal);
        RotateToUpperTriangle(mat);

            // extract new grid element values
        gridElement._Ahat(0, 0) = mat(0, 0);
        gridElement._Ahat(0, 1) = mat(0, 1);
        gridElement._Ahat(0, 2) = mat(0, 2);
        gridElement._Ahat(1, 0) = mat(1, 0);
        gridElement._Ahat(1, 1) = mat(1, 1);
        gridElement._Ahat(1, 2) = mat(1, 2);
        gridElement._Ahat(2, 0) = mat(2, 0);
        gridElement._Ahat(2, 1) = mat(2, 1);
        gridElement._Ahat(2, 2) = mat(2, 2);
        gridElement._Bhat[0] = mat(0, 3);
        gridElement._Bhat[1] = mat(1, 3);
        gridElement._Bhat[2] = mat(2, 3);
        gridElement._r = mat(3, 3);

        assert( Equivalent(mat(3, 0), 0.0f, 1e-6f)
            &&  Equivalent(mat(3, 1), 0.0f, 1e-6f)
            &&  Equivalent(mat(3, 2), 0.0f, 1e-6f));

        gridElement._massPointAccum += intersection._pt - gridElementCenter;
        ++gridElement._massPointCount;

        #if defined(STORE_INTERSECTIONS)
            gridElement._intersectionPts.push_back(
                EdgeIntersection(intersection._pt - gridElementCenter, intersection._normal));
        #endif
    }

    static Float3 CalculateCellPoint(const GridElement& gridElement, const Float3& gridElementSize)
    {
        Float3 massPoint = gridElement._massPointAccum / float(gridElement._massPointCount);
        // assert(XlAbs(massPoint[0]) <= 0.5f*gridElementSize[0]
        //     && XlAbs(massPoint[1]) <= 0.5f*gridElementSize[1]
        //     && XlAbs(massPoint[2]) <= 0.5f*gridElementSize[2]);

        #if defined(STORE_INTERSECTIONS)
            typedef Eigen::Matrix<float,5,4> Float5x4;
            Float5x4 mat = Float5x4::Zero();

            for (unsigned c=0; c<unsigned(gridElement._intersectionPts.size()); ++c) {
                mat(4, 0) = gridElement._intersectionPts[c]._normal[0];
                mat(4, 1) = gridElement._intersectionPts[c]._normal[1];
                mat(4, 2) = gridElement._intersectionPts[c]._normal[2];
                mat(4, 3) = Dot(gridElement._intersectionPts[c]._pt, gridElement._intersectionPts[c]._normal);

                assert(XlAbs(gridElement._intersectionPts[c]._pt[0]) <= 0.5f*gridElementSize[0]);
                assert(XlAbs(gridElement._intersectionPts[c]._pt[1]) <= 0.5f*gridElementSize[1]);
                assert(XlAbs(gridElement._intersectionPts[c]._pt[2]) <= 0.5f*gridElementSize[2]);
                RotateToUpperTriangle(mat);
            }

            Eigen::Matrix<float,3,3> Ahat;
            Eigen::Matrix<float,3,1> Bhat;
            for (unsigned c=0; c<3; ++c) {
                Ahat(c,0) = mat(c,0);
                Ahat(c,1) = mat(c,1);
                Ahat(c,2) = mat(c,2);
                Bhat(c,0) = mat(c,3);
            }

            Eigen::Matrix<float,3,1> massPointVec;
            massPointVec(0,0) = massPoint[0];
            massPointVec(1,0) = massPoint[1];
            massPointVec(2,0) = massPoint[2];

            Eigen::MatrixXf x(3, 1);

                //  The original dual contour multiplies through with the transpose of A. This might
                //  reduce the work when calculating the SVD, but the Eigen library doesn't seem to
                //  support this optimisation, however. So I'm not sure if we need to multiply
                //  through with AHatTranspose here.
            static bool useTranspose = true;
            if (useTranspose) {
                auto AhatTranspose = Ahat.transpose();
                Eigen::JacobiSVD<Eigen::MatrixXf> svd(AhatTranspose * Ahat, Eigen::ComputeFullU | Eigen::ComputeFullV);
                svd.solve(AhatTranspose * Bhat - AhatTranspose * Ahat * massPointVec).evalTo(x);
            } else {
                Eigen::JacobiSVD<Eigen::MatrixXf> svd(Ahat, Eigen::ComputeFullU | Eigen::ComputeFullV);
                svd.solve(Bhat - Ahat * massPointVec).evalTo(x);
            }

            // assert(XlAbs(x(0,0)) < .5f * gridElementSize[0]
            //     && XlAbs(x(1,0)) < .5f * gridElementSize[1]
            //     && XlAbs(x(2,0)) < .5f * gridElementSize[2]);

            auto result = Float3(x(0, 0) + massPoint[0], x(1, 0) + massPoint[1], x(2, 0) + massPoint[2]);
            return result;

        #else

            Eigen::Matrix<float,3,1> massPointVec;
            massPointVec(0,0) = massPoint[0];
            massPointVec(1,0) = massPoint[1];
            massPointVec(2,0) = massPoint[2];

            Eigen::Matrix<float,3,1> x;

            Eigen::Matrix<float,3,3> Ahat;
            Eigen::Matrix<float,3,1> Bhat;
            Ahat(0,0) = gridElement._Ahat(0,0);
            Ahat(0,1) = gridElement._Ahat(0,1);
            Ahat(0,2) = gridElement._Ahat(0,2);
            Ahat(1,0) = gridElement._Ahat(1,0);
            Ahat(1,1) = gridElement._Ahat(1,1);
            Ahat(1,2) = gridElement._Ahat(1,2);
            Ahat(2,0) = gridElement._Ahat(2,0);
            Ahat(2,1) = gridElement._Ahat(2,1);
            Ahat(2,2) = gridElement._Ahat(2,2);

            Bhat(0, 0) = gridElement._Bhat[0];
            Bhat(1, 0) = gridElement._Bhat[1];
            Bhat(2, 0) = gridElement._Bhat[2];

                //  The original dual contour multiplies through with the transpose of A. This might
                //  reduce the work when calculating the SVD, but the Eigen library doesn't seem to
                //  support this optimisation, however. So I'm not sure if we need to multiply
                //  through with AHatTranspose here.
            const bool useTranspose = true;
            if (useTranspose) {
                auto AhatTranspose = Ahat.transpose();
                Eigen::JacobiSVD<Eigen::Matrix<float, 3, 3>> svd(AhatTranspose * Ahat, Eigen::ComputeFullU | Eigen::ComputeFullV);
                svd.solve(AhatTranspose * Bhat - AhatTranspose * Ahat * massPointVec).evalTo(x);
            } else {
                    //  note that if we know the mass point when we're calculating Ahat, we can probably
                    //  just take into account the mass point then. However, if we're using a marching
                    //  cubes-like algorithm to move through the data field, and so visiting each cube
                    //  element multiple times, we might not be able to calculate the mass point until
                    //  after AHat has been fully built. But we can compensate during the solution,
                    //  as so ---
                Eigen::JacobiSVD<Eigen::Matrix<float, 3, 3>> svd(Ahat, Eigen::ComputeFullU | Eigen::ComputeFullV);
                svd.solve(Bhat - Ahat * massPointVec).evalTo(x);
            }

            auto result = Float3(x(0, 0) + massPoint[0], x(1, 0) + massPoint[1], x(2, 0) + massPoint[2]);

            static bool preventBadResults = true;
            if (preventBadResults) {
                Float3 halfSize = 0.5f * gridElementSize;
                if (result[0] < -halfSize[0] || result[0] > halfSize[0] ||
                    result[1] < -halfSize[1] || result[1] > halfSize[1] ||
                    result[2] < -halfSize[2] || result[2] > halfSize[2]) {
                    return massPoint;       // got a poor result from the QEF test -- too much curvature in a small area.
                }
            }

            return result;

        #endif
    }

#if 0
    static void CheckWindingOrder(DualContourMesh::Quad& q, const std::vector<DualContourMesh::Vertex>& vertices)
    {
            //  Check the winding order of the given quad, and correct if necessary
            //  There are various ways we could attempt to find the correct winding order.
            //  But they all have problems.
            //
            //  It's possible that that input quad could have a twist in it. We also have to
            //  be careful about input quads that are near-degenerate.
            //
            //  We want to avoid more queries of the density field to generate more normals.
            //  So let's use the normals we already have access to. We can take some kind of
            //  weighted mean of the vertex normals and use this as an approximation of the
            //  face normal, Then we can test the vertex winding order to see if it agrees
            //  with the face normal we've picked.
            //
            //  How do we weight the normals? We could use the interior angle of the corner.

            //  The vertices in our quads are arranged in a "Z" pattern...
            //  We can't just +1 or -1 to get "prev" and "next". Use the following table:
        unsigned prevVertices[] = { 2, 0, 3, 1 };
        unsigned nextVertices[] = { 1, 3, 0, 2 };

        const bool useWeightedMean = false;
        if (constant_expression<useWeightedMean>::result()) {

            Float3 faceNormal(0.f, 0.f, 0.f);
            for (unsigned c=0; c<4; ++c) {
                unsigned prevVertex = q._verts[prevVertices[c]];
                unsigned thisVertex = q._verts[c];
                unsigned nextVertex = q._verts[nextVertices[c]];
                float dot = Dot(
                    vertices[prevVertex]._pt - vertices[thisVertex]._pt,
                    vertices[nextVertex]._pt - vertices[thisVertex]._pt);
                float angle = XlACos(dot);  // because it's a quad, angle must be less than 180. An angle of 180 just gives a straight line
                assert(angle < gPI);
                faceNormal += angle * vertices[q._verts[c]]._normal;
            }
            faceNormal = Normalize(faceNormal);

                //  We can use the cross product to check if the winding is going
                //  in the correct direction. The cross product gives us directional
                //  information about the 2 input vectors. We can compare the result
                //  of the cross product to the direction of the face normal to see
                //  if the winding order is correct for this normal.
            unsigned correctCount = 0;
            for (unsigned c=0; c<4; ++c) {
                unsigned prevVertex = q._verts[prevVertices[c]];
                unsigned thisVertex = q._verts[c];
                unsigned nextVertex = q._verts[nextVertices[c]];

                auto crs = Cross(
                    vertices[prevVertex]._pt - vertices[thisVertex]._pt,
                    vertices[nextVertex]._pt - vertices[thisVertex]._pt);
                bool correct = Dot(faceNormal, crs) <= 0.f;
                correctCount += unsigned(correct);
            }

            if (correctCount < 2) {
                    //  Some tolerance if only one or two vertices are a problem.
                    //  But, otherwise, we need to reverse the order. Quads verts are
                    //  in a "Z" pattern... so just swap 1 & 2
                std::swap(q._verts[1], q._verts[2]);
            }

        } else {

            unsigned correctCount = 0;
            for (unsigned c=0; c<4; ++c) {
                unsigned prevVertex = q._verts[prevVertices[c]];
                unsigned thisVertex = q._verts[c];
                unsigned nextVertex = q._verts[nextVertices[c]];

                auto testDirection = Cross(
                    vertices[prevVertex]._pt - vertices[thisVertex]._pt,
                    vertices[nextVertex]._pt - vertices[thisVertex]._pt);
                for (unsigned v=0; v<4; ++v) {
                    bool correct = Dot(vertices[q._verts[v]]._normal, testDirection) <= 0.f;
                    correctCount += unsigned(correct);
                }
            }

                // if we're correct less than half of the time, then flip
            if (correctCount < 4*4/2) {
                std::swap(q._verts[1], q._verts[2]);
            }

        }

    }

    static bool NeedsFlip(DualContourMesh& mesh, unsigned a, unsigned b, unsigned c, const IVolumeDensityFunction& fn)
    {
        auto plane = PlaneFit(mesh._vertices[a]._pt, mesh._vertices[b]._pt, mesh._vertices[c]._pt);
        auto averagePoint = 0.25f * (mesh._vertices[a]._pt + mesh._vertices[b]._pt + mesh._vertices[c]._pt);

        static float testOffset = 0.1f;

        float density0 = fn.GetDensity(averagePoint);
        float density1 = fn.GetDensity(averagePoint + Truncate(plane) * testOffset);
        return density0 > density1;
    }

    static void AddQuad(DualContourMesh& mesh, const DualContourMesh::Quad& quad, const IVolumeDensityFunction& fn)
    {
        // Our quad should be reasonbly close to coplanear. When we convert it into
        // triangles, we want the winding order of both triangles to be the same...
        // so let's do a winding order test on the quad, and apply the result to
        // both triangles.
        // Float3 pts[4] =
        // {
        //     mesh._vertices[quad._verts[0]]._pt,
        //     mesh._vertices[quad._verts[1]]._pt,
        //     mesh._vertices[quad._verts[2]]._pt,
        //     mesh._vertices[quad._verts[3]]._pt
        // };
        // Float3 averagePoint = 0.25f * (pts[0] + pts[1] + pts[2] + pts[3]);
        // Float4 plane = PlaneFit(pts, dimof(pts));
        //
        // // the average point should be almost (if not quite) on both triangles.
        // static float testOffset = 0.1f;
        //
        // auto q = quad;
        // float density0 = fn.GetDensity(averagePoint);
        // float density1 = fn.GetDensity(averagePoint + Truncate(plane) * testOffset);
        // if (density0 > density1)
        //     std::swap(q._verts[1], q._verts[2]);
        //
        // mesh._quads.push_back(q);

        bool flip0 = NeedsFlip(mesh, quad._verts[0], quad._verts[1], quad._verts[2], fn);
        bool flip1 = NeedsFlip(mesh, quad._verts[2], quad._verts[1], quad._verts[3], fn);
        // assert(flip0 == flip1);

        auto q = quad;
        if (flip0 || flip1)
            std::swap(q._verts[1], q._verts[2]);
        mesh._quads.push_back(q);
    }

#endif

    static void AddQuad(DualContourMesh& mesh, const DualContourMesh::Quad& quad, bool flipDirection)
    {
        auto q = quad;
        if (flipDirection)
            std::swap(q._verts[1], q._verts[2]);
        mesh._quads.push_back(q);
    }

    DualContourMesh     DualContourMesh_Build(  unsigned samplingGridDimensions,
                                                const IVolumeDensityFunction& fn)
    {
            //  Build a mesh of triangles from the given input function
            //      (using dual contouring method)
            //
            //  First we'll build a grid containing information for each
            //  voxel. Then we'll go through a calculate the QEF's at
            //  each grid point -- that will give us enough information
            //  to generate the triangles needed. Note that the algorithm
            //  should naturally build quads most of the time. They'll need
            //  to be split up into triangles.
            //
            //  Ideally, we would also do simplification before we calculate
            //  the QEF's and generate the triangles. But currently, no
            //  simplification.
        auto boundary = fn.GetBoundary();
        auto gridElements = std::make_unique<GridElement[]>(
            samplingGridDimensions*samplingGridDimensions*samplingGridDimensions);

            //  For each grid element, let's fill it in with the values from the
            //  density function. Note that we could reduce the work here slightly
            //  by finding a point within the grid that lies on the surface, and
            //  then marching along the surface, into each new grid that takes us.
            //
            //  The current method will create many redundant tests of the volume
            //  function -- because most grids are probably not on the surface of the
            //  volume.
            //
            //  Let's find and test each edge. When we find a edge that crosses the
            //  boundary, we can merge that into the QEF's for that adjacent grid
            //  elements.
            //
            //  It's a good idea to calculate the density results at each corner first
            //  This will help reduce the number of times we need to call the
            //  GetDensity() function.

        Float3x4 gridToSampleSpace = Zero<Float3x4>();
        gridToSampleSpace(0,0) = (boundary.second[0] - boundary.first[0]) / float(samplingGridDimensions);
        gridToSampleSpace(1,1) = (boundary.second[1] - boundary.first[1]) / float(samplingGridDimensions);
        gridToSampleSpace(2,2) = (boundary.second[2] - boundary.first[2]) / float(samplingGridDimensions);
        gridToSampleSpace(0,3) = boundary.first[0];
        gridToSampleSpace(1,3) = boundary.first[1];
        gridToSampleSpace(2,3) = boundary.first[2];

        auto densityResults = std::make_unique<float[]>(
            (samplingGridDimensions+1)*(samplingGridDimensions+1)*(samplingGridDimensions+1));
        for (int z=0; z<int(samplingGridDimensions+1); ++z)
            for (int y=0; y<int(samplingGridDimensions+1); ++y)
                for (int x=0; x<int(samplingGridDimensions+1); ++x) {
                    Float3 p0 = TransformPoint(gridToSampleSpace, Float3(float(x), float(y), float(z)));
                    densityResults[(z * (samplingGridDimensions+1) + y) * (samplingGridDimensions+1) + x] = fn.GetDensity(p0);
                }

            // note --  The order of the cell offsets here is important, because it
            //          determines the order of the vertices in the quad.
        Int3 cellOffsetsX[] = { Int3(0, 0, 0), Int3(0, -1, 0), Int3(0, 0, -1), Int3(0, -1, -1) };
        Int3 cellOffsetsY[] = { Int3(0, 0, 0), Int3(0, 0, -1), Int3(-1, 0, 0), Int3(-1, 0, -1) };
        Int3 cellOffsetsZ[] = { Int3(0, 0, 0), Int3(-1, 0, 0), Int3(0, -1, 0), Int3(-1, -1, 0) };

        for (int z=0; z<int(samplingGridDimensions); ++z) {
            for (int y=0; y<int(samplingGridDimensions); ++y) {
                for (int x=0; x<int(samplingGridDimensions); ++x) {

                        //  For each grid element, we're going to test 3 edges.
                        //  we'll add the effect of those edges to adjacent grids
                        //  as well. This means each edges gets tested once.
                        //  However, some edges on the extreme positive boundary
                        //  of the sampling area will never be tested. We'll assume
                        //  that the function doesn't go through these boundary edges.

                    Float3 p0 = TransformPoint(gridToSampleSpace, Float3(float(x), float(y), float(z)));
                    Float3 p1 = TransformPoint(gridToSampleSpace, Float3(float(x+1), float(y), float(z)));
                    Float3 p2 = TransformPoint(gridToSampleSpace, Float3(float(x), float(y+1), float(z)));
                    Float3 p3 = TransformPoint(gridToSampleSpace, Float3(float(x), float(y), float(z+1)));

                    float d0 = densityResults[(z * (samplingGridDimensions+1) + y) * (samplingGridDimensions+1) + x];
                    float d1 = densityResults[(z * (samplingGridDimensions+1) + y) * (samplingGridDimensions+1) + x + 1];
                    float d2 = densityResults[(z * (samplingGridDimensions+1) + y + 1) * (samplingGridDimensions+1) + x];
                    float d3 = densityResults[((z + 1) * (samplingGridDimensions+1) + y) * (samplingGridDimensions+1) + x];

                        //  Look for edges that contain intersections with the surface, and then merge
                        //  those edge into all of the grid cells that contain them.
                        //  Note that TestEdge() will do extra calls to GetDensity to improve the
                        //  intersection point.

                    if ((d0 < 0.f) != (d1 < 0.f)) {
                        auto intersection = TestEdge(p0, p1, d0, d1, fn);
                        for (unsigned c=0; c<dimof(cellOffsetsX); ++c) {
                            Int3 g(x + cellOffsetsX[c][0], y + cellOffsetsX[c][1], z + cellOffsetsX[c][2]);
                            if (g[0] >= 0 && g[1] >= 0 && g[2] >= 0) {
                                const auto cellCenter = Float3(
                                    LinearInterpolate(boundary.first[0], boundary.second[0], (float(g[0]) + .5f) / float(samplingGridDimensions)),
                                    LinearInterpolate(boundary.first[1], boundary.second[1], (float(g[1]) + .5f) / float(samplingGridDimensions)),
                                    LinearInterpolate(boundary.first[2], boundary.second[2], (float(g[2]) + .5f) / float(samplingGridDimensions)));

                                MergeInEdgeIntersection(
                                    gridElements[(g[2] * samplingGridDimensions + g[1]) * samplingGridDimensions + g[0]],
                                    intersection, cellCenter);
                            }
                        }
                    }

                    if ((d0 < 0.f) != (d2 < 0.f)) {
                        auto intersection = TestEdge(p0, p2, d0, d2, fn);
                        for (unsigned c=0; c<dimof(cellOffsetsY); ++c) {
                            Int3 g(x + cellOffsetsY[c][0], y + cellOffsetsY[c][1], z + cellOffsetsY[c][2]);
                            if (g[0] >= 0 && g[1] >= 0 && g[2] >= 0) {
                                const auto cellCenter = Float3(
                                    LinearInterpolate(boundary.first[0], boundary.second[0], (float(g[0]) + .5f) / float(samplingGridDimensions)),
                                    LinearInterpolate(boundary.first[1], boundary.second[1], (float(g[1]) + .5f) / float(samplingGridDimensions)),
                                    LinearInterpolate(boundary.first[2], boundary.second[2], (float(g[2]) + .5f) / float(samplingGridDimensions)));

                                MergeInEdgeIntersection(
                                    gridElements[(g[2] * samplingGridDimensions + g[1]) * samplingGridDimensions + g[0]],
                                    intersection, cellCenter);
                            }
                        }
                    }

                    if ((d0 < 0.f) != (d3 < 0.f)) {
                        auto intersection = TestEdge(p0, p3, d0, d3, fn);
                        for (unsigned c=0; c<dimof(cellOffsetsZ); ++c) {
                            Int3 g(x + cellOffsetsZ[c][0], y + cellOffsetsZ[c][1], z + cellOffsetsZ[c][2]);
                            if (g[0] >= 0 && g[1] >= 0 && g[2] >= 0) {
                                const auto cellCenter = Float3(
                                    LinearInterpolate(boundary.first[0], boundary.second[0], (float(g[0]) + .5f) / float(samplingGridDimensions)),
                                    LinearInterpolate(boundary.first[1], boundary.second[1], (float(g[1]) + .5f) / float(samplingGridDimensions)),
                                    LinearInterpolate(boundary.first[2], boundary.second[2], (float(g[2]) + .5f) / float(samplingGridDimensions)));

                                MergeInEdgeIntersection(
                                    gridElements[(g[2] * samplingGridDimensions + g[1]) * samplingGridDimensions + g[0]],
                                    intersection, cellCenter);
                            }
                        }
                    }

                }
            }
        }

            //  Now, we've calculated the error functions for all of the grid elements.
            //  For each grid element, we can calculate the appropriate point for that
            //  element. Let's make sure we do this only one per grid element (because
            //  typically each vertex will be used in multiple quads.

        const auto cellSize = Float3(
            (boundary.second[0] - boundary.first[0]) / float(samplingGridDimensions),
            (boundary.second[1] - boundary.first[1]) / float(samplingGridDimensions),
            (boundary.second[2] - boundary.first[2]) / float(samplingGridDimensions));

        std::vector<DualContourMesh::Vertex> vertices;
        auto vertexIndices = std::make_unique<unsigned[]>(
            samplingGridDimensions*samplingGridDimensions*samplingGridDimensions);
        for (int z=0; z<int(samplingGridDimensions); ++z) {
            for (int y=0; y<int(samplingGridDimensions); ++y) {
                for (int x=0; x<int(samplingGridDimensions); ++x) {
                    auto index = (z * samplingGridDimensions + y) * samplingGridDimensions + x;
                    const auto& g = gridElements[index];
                    if (!g._massPointCount) {
                        vertexIndices[index] = 0xffffffff;
                        continue;
                    }

                    const auto cellCenter = Float3(
                        LinearInterpolate(boundary.first[0], boundary.second[0], (float(x) + .5f) / float(samplingGridDimensions)),
                        LinearInterpolate(boundary.first[1], boundary.second[1], (float(y) + .5f) / float(samplingGridDimensions)),
                        LinearInterpolate(boundary.first[2], boundary.second[2], (float(z) + .5f) / float(samplingGridDimensions)));
                    auto pt = CalculateCellPoint(g, cellSize) + cellCenter;

                        //  We need the normal at this location, also.
                        //  We've lost the locations of the edge intersections -- so we can't
                        //  just add together the normals from them. However. We can
                        //  query the density field again to get the normal at this location.
                    auto normal = fn.GetNormal(pt);
                    vertices.push_back(DualContourMesh::Vertex(pt, normal));
                    vertexIndices[index] = unsigned(vertices.size()-1);
                }
            }
        }

            //  We just need to calculate the triangles.
            //  For each edge with an intersection, we want to create a quad.
            //  we start at one here, because the edge cells have nothing to join
            //  on to.

        DualContourMesh mesh;
        mesh._vertices = std::move(vertices);

        for (int z=1; z<int(samplingGridDimensions); ++z) {
            for (int y=1; y<int(samplingGridDimensions); ++y) {
                for (int x=1; x<int(samplingGridDimensions); ++x) {

                    float d0 = densityResults[(z * (samplingGridDimensions+1) + y) * (samplingGridDimensions+1) + x];
                    float d1 = densityResults[(z * (samplingGridDimensions+1) + y) * (samplingGridDimensions+1) + x + 1];
                    float d2 = densityResults[(z * (samplingGridDimensions+1) + y + 1) * (samplingGridDimensions+1) + x];
                    float d3 = densityResults[((z + 1) * (samplingGridDimensions+1) + y) * (samplingGridDimensions+1) + x];

                        //  If the edge has a intersection point. We want to create a
                        //  quad by joining together all of the cells that use this edge.
                    if ((d0 < 0.f) != (d1 < 0.f)) {
                        DualContourMesh::Quad q;
                        for (unsigned c=0; c<4; ++c) {
                            Int3 g(x + cellOffsetsX[c][0], y + cellOffsetsX[c][1], z + cellOffsetsX[c][2]);
                            auto index = (g[2] * samplingGridDimensions + g[1]) * samplingGridDimensions + g[0];
                            q._verts[c] = vertexIndices[index];
                            assert(q._verts[c] < mesh._vertices.size());
                        }
                        AddQuad(mesh, q, d0 < 0.f);
                    }

                    if ((d0 < 0.f) != (d2 < 0.f)) {
                        DualContourMesh::Quad q;
                        for (unsigned c=0; c<4; ++c) {
                            Int3 g(x + cellOffsetsY[c][0], y + cellOffsetsY[c][1], z + cellOffsetsY[c][2]);
                            auto index = (g[2] * samplingGridDimensions + g[1]) * samplingGridDimensions + g[0];
                            q._verts[c] = vertexIndices[index];
                            assert(q._verts[c] < mesh._vertices.size());
                        }
                        AddQuad(mesh, q, d0 < 0.f);
                    }

                    if ((d0 < 0.f) != (d3 < 0.f)) {
                        DualContourMesh::Quad q;
                        for (unsigned c=0; c<4; ++c) {
                            Int3 g(x + cellOffsetsZ[c][0], y + cellOffsetsZ[c][1], z + cellOffsetsZ[c][2]);
                            auto index = (g[2] * samplingGridDimensions + g[1]) * samplingGridDimensions + g[0];
                            q._verts[c] = vertexIndices[index];
                            assert(q._verts[c] < mesh._vertices.size());
                        }
                        AddQuad(mesh, q, d0 < 0.f);
                    }

                }
            }
        }

        return mesh;
    }

    DualContourMesh::DualContourMesh() {}
    DualContourMesh::DualContourMesh(DualContourMesh&& moveFrom)
    : _vertices(std::move(moveFrom._vertices))
    , _quads(std::move(moveFrom._quads))
    {}
    DualContourMesh& DualContourMesh::operator=(DualContourMesh&& moveFrom)
    {
        _vertices = std::move(moveFrom._vertices);
        _quads = std::move(moveFrom._quads);
        return *this;
    }
    DualContourMesh::~DualContourMesh() {}

}


#pragma once

#include <bitset>
#include <functional>
#include <numeric>
#include <stdexcept>
#include <vector>

#include <dtl/dtl.hpp>
#include <dtl/bits.hpp>
#include <dtl/math.hpp>
#include <dtl/simd.hpp>
#include <dtl/filter/blocked_bloomfilter/vector_helper.hpp>

#include "immintrin.h"

#include <boost/integer/static_min_max.hpp>

namespace dtl {

namespace {

//===----------------------------------------------------------------------===//
// Recursive template to work with multi-word sectors.
//
// Specialization for the case where sector count < word_cnt ('sltw'),
// which results in a random access pattern within the block (and sector).
// I.e., for every bit to set, the corresponding word needs to be determined
// and loaded.
//
// Same as in the 'sgew' case, we process the block sector by sector,
// however, a sector consists of more than one word.
//===----------------------------------------------------------------------===//
template<
    typename key_t,               // the key type
    typename word_t,              // the word type
    u32 word_cnt,                 // the number of words per sector
    u32 k,                        // the number of bits to set/test
    template<typename Ty, u32 i> class hasher,      // the hash function family to use
    typename hash_value_t,        // the hash value type to use

    u32 hash_fn_idx,              // current hash function index (used for recursion)
    u32 remaining_hash_bit_cnt,   // the number of remaining hash bits (used for recursion)
    u32 remaining_k_cnt           // the remaining number of bits to set in the sector (used for recursion)
>
struct multiword_sector {

  //===----------------------------------------------------------------------===//
  // Static part
  //===----------------------------------------------------------------------===//

  static constexpr u32 word_cnt_log2 = dtl::ct::log_2<word_cnt>::value;
  static constexpr u32 word_cnt_log2_mask = (1u << word_cnt_log2) - 1;
  static_assert(dtl::is_power_of_two(word_cnt), "Parameter 'word_cnt' must be a power of two.");
  static constexpr u32 word_bitlength = sizeof(word_t) * 8;
  static constexpr u32 word_bitlength_log2 = dtl::ct::log_2<word_bitlength>::value;
  static constexpr u32 word_bitlength_log2_mask = (1u << word_bitlength_log2) - 1;

  static constexpr u32 hash_value_bitlength = sizeof(hash_value_t) * 8;

  static constexpr u32 k_cnt_per_sector = k;

  static constexpr u32 current_k_idx() { return k - remaining_k_cnt; }

  static constexpr u32 hash_bit_cnt_per_k = word_cnt_log2 + word_bitlength_log2;

  static constexpr u1 rehash = remaining_hash_bit_cnt < hash_bit_cnt_per_k;
  static constexpr u32 remaining_hash_bit_cnt_after_rehash = rehash ? hash_value_bitlength : remaining_hash_bit_cnt;
  //===----------------------------------------------------------------------===//

  //===----------------------------------------------------------------------===//
  // Insert
  //===----------------------------------------------------------------------===//
  __forceinline__
  static void
  insert(word_t* __restrict sector_ptr, const key_t key) noexcept {

    hash_value_t hash_val = 0;

    // Call the recursive function
    static constexpr u32 remaining_hash_bits = 0;
    multiword_sector<key_t, word_t, word_cnt, k,
                     hasher, hash_value_t, hash_fn_idx, remaining_hash_bits,
                     remaining_k_cnt>
                     ::insert_atomic(sector_ptr, key, hash_val);
//                     ::insert(sector_ptr, key, hash_val);
  }
  //===----------------------------------------------------------------------===//

  //===----------------------------------------------------------------------===//
  // Insert (Recursive)
  //===----------------------------------------------------------------------===//
  __forceinline__
  static void
  insert(word_t* __restrict sector_ptr, const key_t key, hash_value_t& hash_val) noexcept {

    hash_val = rehash ? hasher<key_t, hash_fn_idx>::hash(key) : hash_val;

    // Determine the word of interest
    constexpr u32 word_idx_shift = remaining_hash_bit_cnt_after_rehash - word_cnt_log2;
    $u32 word_idx = ((hash_val >> word_idx_shift) & word_cnt_log2_mask);

    // Load the word of interest
    word_t word = sector_ptr[word_idx];

    // Set a bit in the given word
    constexpr u32 bit_idx_shift = remaining_hash_bit_cnt_after_rehash - word_cnt_log2 - word_bitlength_log2;
    $u32 bit_idx = ((hash_val >> bit_idx_shift) & word_bitlength_log2_mask);
    word |= word_t(1) << bit_idx;

    // Update the bit vector
    sector_ptr[word_idx] = word;

    // Process remaining k's recursively, if any
    multiword_sector<key_t, word_t, word_cnt, k,
                     hasher, hash_value_t,
                     rehash ? hash_fn_idx + 1 : hash_fn_idx,
                     remaining_hash_bit_cnt_after_rehash >= hash_bit_cnt_per_k ? remaining_hash_bit_cnt_after_rehash - hash_bit_cnt_per_k : 0,
                     remaining_k_cnt - 1>
      ::insert(sector_ptr, key, hash_val);
  }
  //===----------------------------------------------------------------------===//

  //===----------------------------------------------------------------------===//
  // Insert Atomic (Recursive)
  //===----------------------------------------------------------------------===//
  __forceinline__
  static void
  insert_atomic(word_t* __restrict sector_ptr, const key_t key, hash_value_t& hash_val) noexcept {

    hash_val = rehash ? hasher<key_t, hash_fn_idx>::hash(key) : hash_val;

    // Determine the word of interest
    constexpr u32 word_idx_shift = remaining_hash_bit_cnt_after_rehash - word_cnt_log2;
    u32 word_idx = ((hash_val >> word_idx_shift) & word_cnt_log2_mask);

    // Set a bit in the given word
    constexpr u32 bit_idx_shift = remaining_hash_bit_cnt_after_rehash - word_cnt_log2 - word_bitlength_log2;
    u32 bit_idx = ((hash_val >> bit_idx_shift) & word_bitlength_log2_mask);
    const word_t which_bit = word_t(1) << bit_idx;

    word_t* word_ptr = &sector_ptr[word_idx];
    std::atomic<word_t>* atomic_word_ptr = reinterpret_cast<std::atomic<word_t>*>(word_ptr);
    $u1 success = false;
    do {
      // Load the word of interest
      word_t word = atomic_word_ptr->load();
      // Update the bit vector
      word_t updated_word = word | which_bit;
      success = atomic_word_ptr->compare_exchange_weak(word, updated_word);
    } while (!success);

    // Process remaining k's recursively, if any
    multiword_sector<key_t, word_t, word_cnt, k,
                     hasher, hash_value_t,
                     rehash ? hash_fn_idx + 1 : hash_fn_idx,
                     remaining_hash_bit_cnt_after_rehash >= hash_bit_cnt_per_k ? remaining_hash_bit_cnt_after_rehash - hash_bit_cnt_per_k : 0,
                     remaining_k_cnt - 1>
      ::insert_atomic(sector_ptr, key, hash_val);
  }
  //===----------------------------------------------------------------------===//

  //===----------------------------------------------------------------------===//
  // Contains
  //===----------------------------------------------------------------------===//
  __forceinline__ __unroll_loops__ __host__ __device__
  static u1
  contains(const word_t* __restrict sector_ptr, const key_t key) noexcept {

    hash_value_t hash_val = 0;

    // Call the recursive function
    static constexpr u32 remaining_hash_bits = 0;
    return multiword_sector<key_t, word_t, word_cnt, k,
                            hasher, hash_value_t,
                            hash_fn_idx,
                            remaining_hash_bits,
                            remaining_k_cnt>
      ::contains(sector_ptr, key, hash_val, true);
  }
  //===----------------------------------------------------------------------===//

  //===----------------------------------------------------------------------===//
  // Contains (Recursive)
  //===----------------------------------------------------------------------===//
  __forceinline__ __unroll_loops__ __host__ __device__
  static u1
  contains(const word_t* __restrict sector_ptr, const key_t key, hash_value_t& hash_val, u1 is_contained_in_sector) noexcept {

    hash_val = rehash ? hasher<key_t, hash_fn_idx>::hash(key) : hash_val;

    // Determine the word of interest
    constexpr u32 word_idx_shift = remaining_hash_bit_cnt_after_rehash - word_cnt_log2;
    $u32 word_idx = ((hash_val >> word_idx_shift) & word_cnt_log2_mask);

    // Load the word of interest
    word_t word = sector_ptr[word_idx];

    // Test a bit in the given word
    constexpr u32 bit_idx_shift = remaining_hash_bit_cnt_after_rehash - word_cnt_log2 - word_bitlength_log2;
    $u32 bit_idx = ((hash_val >> bit_idx_shift) & word_bitlength_log2_mask);
    u1 found_in_word = word & (word_t(1) << bit_idx);

    // Process remaining k's recursively, if any
    return multiword_sector<key_t, word_t, word_cnt, k,
                            hasher, hash_value_t,
                            rehash ? hash_fn_idx + 1 : hash_fn_idx,
                            remaining_hash_bit_cnt_after_rehash >= hash_bit_cnt_per_k ? remaining_hash_bit_cnt_after_rehash - hash_bit_cnt_per_k : 0,
                            remaining_k_cnt - 1>
                            ::contains(sector_ptr, key, hash_val, found_in_word & is_contained_in_sector);
  }
  //===----------------------------------------------------------------------===//

  //===----------------------------------------------------------------------===//
  // Contains (SIMD)
  //===----------------------------------------------------------------------===//
  template<u64 n>
  __forceinline__ __unroll_loops__
  static auto
  contains(const vec<key_t,n>& keys,
           const word_t* __restrict bitvector_base_address,
           const vec<key_t,n>& sector_start_word_idxs) noexcept {

    vec<hash_value_t, n> hash_vals(0);
    auto is_contained_in_sector_mask = vec<word_t,n>::mask::make_all_mask();

    // Call recursive function
    static constexpr u32 remaining_hash_bits = 0;
    return multiword_sector<key_t, word_t, word_cnt, k,
                            hasher, hash_value_t,
                            hash_fn_idx,
                            remaining_hash_bits,
                            remaining_k_cnt>
      ::contains(keys, hash_vals, bitvector_base_address, sector_start_word_idxs, is_contained_in_sector_mask);
  }

  //===----------------------------------------------------------------------===//
  // Contains (SIMD, Recursive)
  //===----------------------------------------------------------------------===//
  template<u64 n>
  __forceinline__ __unroll_loops__
  static auto
  contains(const vec<key_t,n>& keys,
           vec<hash_value_t,n>& hash_vals,
           const word_t* __restrict bitvector_base_address,
           const vec<hash_value_t,n>& sector_start_word_idxs,
           const typename vec<word_t,n>::mask is_contained_in_sector_mask) noexcept {

    // Typedef the vector types
    using key_vt = vec<key_t, n>;
    using word_vt = vec<word_t, n>;

    hash_vals = rehash ? hasher<key_vt, hash_fn_idx>::hash(keys) : hash_vals;

    // Determine the word of interest
    constexpr u32 word_idx_shift = remaining_hash_bit_cnt_after_rehash - word_cnt_log2;
    const auto in_sector_word_idxs = (hash_vals >> word_idx_shift) & static_cast<hash_value_t>(word_cnt_log2_mask);
    const auto word_idxs = sector_start_word_idxs + in_sector_word_idxs;

    // Gather the words of interest
    const word_vt words = internal::vector_gather<word_t, hash_value_t, n>::gather(bitvector_base_address, word_idxs);

    // Test a bit in the given word
    constexpr u32 bit_idx_shift = remaining_hash_bit_cnt_after_rehash - word_cnt_log2 - word_bitlength_log2;
    const auto bit_idx = (hash_vals >> bit_idx_shift) & static_cast<hash_value_t>(word_bitlength_log2_mask);
    const word_vt bits_to_test = word_vt(1) << internal::vector_convert<hash_value_t, word_t, n>::convert(bit_idx);
    const auto found_in_word_mask = (words & bits_to_test) == bits_to_test;

    // Process remaining k's recursively, if any
    return multiword_sector<key_t, word_t, word_cnt, k,
                            hasher, hash_value_t,
                            rehash ? hash_fn_idx + 1 : hash_fn_idx,
                            remaining_hash_bit_cnt_after_rehash >= hash_bit_cnt_per_k ? remaining_hash_bit_cnt_after_rehash - hash_bit_cnt_per_k : 0,
                            remaining_k_cnt - 1>
      ::contains(keys, hash_vals, bitvector_base_address, sector_start_word_idxs, found_in_word_mask & is_contained_in_sector_mask);

  }
  //===----------------------------------------------------------------------===//

  //===----------------------------------------------------------------------===//
  // The number of required hash functions.
  //===----------------------------------------------------------------------===//
  static constexpr u32 hash_fn_idx_end =
      multiword_sector<key_t, word_t, word_cnt, k,
                       hasher, hash_value_t,
                       (rehash ? hash_fn_idx + 1 : hash_fn_idx), // increment the hash function index
                       remaining_hash_bit_cnt_after_rehash >= hash_bit_cnt_per_k
                         ? remaining_hash_bit_cnt_after_rehash - hash_bit_cnt_per_k
                         : 0, // the number of remaining hash bits
                       remaining_k_cnt - 1> // decrement the remaining k counter
      ::hash_fn_idx_end;
  //===----------------------------------------------------------------------===//

  //===----------------------------------------------------------------------===//
  // The number of required hash bits.
  //===----------------------------------------------------------------------===//
  static constexpr u32 remaining_hash_bits =
      multiword_sector<key_t, word_t, word_cnt, k,
                 hasher, hash_value_t,
                 (rehash ? hash_fn_idx + 1 : hash_fn_idx), // increment the hash function index
                 remaining_hash_bit_cnt_after_rehash >= hash_bit_cnt_per_k
                   ? remaining_hash_bit_cnt_after_rehash - hash_bit_cnt_per_k
                   : 0, // the number of remaining hash bits
                 remaining_k_cnt - 1> // decrement the remaining k counter
      ::remaining_hash_bits;
  //===----------------------------------------------------------------------===//

};

template<
    typename key_t,               // the key type
    typename word_t,              // the word type
    u32 word_cnt,                 // the number of words per sector
    u32 k,                        // the number of bits to set/test
    template<typename Ty, u32 i> class hasher,      // the hash function family to use
    typename hash_value_t,        // the hash value type to use

    u32 hash_fn_idx,              // current hash function index (used for recursion)
    u32 remaining_hash_bit_cnt    // the number of remaining hash bits (used for recursion)
>
struct multiword_sector<key_t, word_t, word_cnt, k, hasher, hash_value_t, hash_fn_idx, remaining_hash_bit_cnt, 0 /* no more k's */> {

  //===----------------------------------------------------------------------===//
  // Insert
  //===----------------------------------------------------------------------===//
  __forceinline__
  static void
  insert(word_t* __restrict sector_ptr, const key_t key, hash_value_t& hash_val) noexcept {
    // End of recursion.
  }
  __forceinline__
  static void
  insert_atomic(word_t* __restrict sector_ptr, const key_t key, hash_value_t& hash_val) noexcept {
    // End of recursion.
  }
  //===----------------------------------------------------------------------===//

  //===----------------------------------------------------------------------===//
  // Contains
  //===----------------------------------------------------------------------===//
  __forceinline__ __host__ __device__
  static u1
  contains(const word_t* __restrict sector_ptr, const key_t key, hash_value_t& hash_val, u1 is_contained_in_sector) noexcept {
    // End of recursion.
    return is_contained_in_sector;
  }
  //===----------------------------------------------------------------------===//

  //===----------------------------------------------------------------------===//
  // Contains (SIMD)
  //===----------------------------------------------------------------------===//
  template<u64 n>
  __forceinline__ __unroll_loops__
  static auto
  contains(const vec<key_t,n>& keys,
           vec<hash_value_t,n>& hash_vals,
           const word_t* __restrict bitvector_base_address,
           const vec<hash_value_t,n>& sector_start_word_idxs,
           const typename vec<word_t,n>::mask is_contained_in_sector_mask) noexcept {
    // End of recursion.
    return is_contained_in_sector_mask;
  }
  //===----------------------------------------------------------------------===//

  static constexpr u32 hash_fn_idx_end = hash_fn_idx;

  static constexpr u32 remaining_hash_bits = remaining_hash_bit_cnt;

};

} // anonymous namespace

//===----------------------------------------------------------------------===//
// Recursive template to work with multiple multi-word sectors.
//
// Specialization for the case where sector count < word_cnt ('sltw'),
// which results in a random access pattern within the block (and sector).
// I.e., for every bit to set/test, the corresponding word needs to be determined
// and loaded.
//
// Same as in the 'sgew' case, we process the block sector by sector,
// however, a sector consists of more than one word.
//===----------------------------------------------------------------------===//
template<
    typename _key_t,              // the key type
    typename _word_t,             // the word type
    u32 _word_cnt,                // the number of words per block
    u32 _s,                       // the numbers of sectors (must be a power of two)
    u32 _k,                       // the number of bits to set/test
    template<typename Ty, u32 i> class hasher,      // the hash function family to use
    typename hash_value_t,        // the hash value type to use

    u32 hash_fn_idx,              // current hash function index (used for recursion)
    u32 remaining_hash_bit_cnt,   // the number of remaining hash bits (used for recursion)
    u32 remaining_sector_cnt,     // the remaining number of sector (used for recursion)

    u1 early_out = false          // allows for branching out during lookups (before the next sector is tested)
>
struct multisector_block {

  //===----------------------------------------------------------------------===//
  // Static part
  //===----------------------------------------------------------------------===//
  using key_t = _key_t;
  using word_t = _word_t;

  static constexpr u32 k = _k;
  static constexpr u32 word_cnt = _word_cnt;
  static constexpr u32 sector_cnt = _s;
  static_assert(dtl::is_power_of_two(sector_cnt), "Parameter 'sector_cnt' must be a power of two.");
  static constexpr u32 sector_cnt_log2 = dtl::ct::log_2<sector_cnt>::value;

  static constexpr u32 current_sector_idx = sector_cnt - remaining_sector_cnt;

  static_assert(dtl::is_power_of_two(word_cnt), "Parameter 'word_cnt' must be a power of two.");
  static constexpr u32 word_cnt_per_sector = word_cnt / sector_cnt;
  static constexpr u32 k_cnt_per_sector = k / sector_cnt;
  static_assert(k % sector_cnt == 0, "Parameter 'k' must be dividable by 's'.");

  //===----------------------------------------------------------------------===//
  // Insert
  //===----------------------------------------------------------------------===//
  __forceinline__
  static void
  insert(word_t* __restrict block_ptr, const key_t key) noexcept {

    hash_value_t hash_val = 0;

    // Call the recursive function
    static constexpr u32 remaining_hash_bits = 0;
    multisector_block<key_t, word_t, word_cnt, sector_cnt, k,
                      hasher, hash_value_t, hash_fn_idx, remaining_hash_bits,
                      remaining_sector_cnt, early_out>
      ::insert(block_ptr, key, hash_val);
  }
  //===----------------------------------------------------------------------===//

  //===----------------------------------------------------------------------===//
  // Insert (Recursive)
  //===----------------------------------------------------------------------===//
  __forceinline__
  static void
  insert(word_t* __restrict block_ptr, const key_t key, hash_value_t& hash_val) noexcept {

    // Sector pointer
    word_t* sector_ptr = block_ptr + (word_cnt_per_sector * current_sector_idx);

    // Process remaining k's recursively, if any
    using sector_t =
      multiword_sector<key_t, word_t, word_cnt_per_sector, k_cnt_per_sector,
                       hasher, hash_value_t, hash_fn_idx, remaining_hash_bit_cnt,
                       k_cnt_per_sector>;
    sector_t::insert_atomic(sector_ptr, key, hash_val);
//    sector_t::insert(sector_ptr, key, hash_val);

    multisector_block<key_t, word_t, word_cnt, sector_cnt, k,
                      hasher, hash_value_t, sector_t::hash_fn_idx_end, sector_t::remaining_hash_bits,
                      remaining_sector_cnt - 1, early_out>
      ::insert(block_ptr, key, hash_val);
  }
  //===----------------------------------------------------------------------===//

  //===----------------------------------------------------------------------===//
  // Contains
  //===----------------------------------------------------------------------===//
  __forceinline__ __unroll_loops__ __host__ __device__
  static u1
  contains(const word_t* __restrict block_ptr, const key_t key) noexcept {

    hash_value_t hash_val = 0;

    // Call the recursive function
    static constexpr u32 remaining_hash_bits = 0;
    return multisector_block<key_t, word_t, word_cnt, sector_cnt, k,
                             hasher, hash_value_t, hash_fn_idx, remaining_hash_bits,
                             remaining_sector_cnt, early_out>
      ::contains(block_ptr, key, hash_val, true);
  }
  //===----------------------------------------------------------------------===//

  //===----------------------------------------------------------------------===//
  // Contains (Recursive)
  //===----------------------------------------------------------------------===//
  __forceinline__ __unroll_loops__ __host__ __device__
  static u1
  contains(const word_t* __restrict block_ptr, const key_t key, hash_value_t& hash_val, u1 is_contained_in_block) noexcept {

    // Sector pointer
    const word_t* sector_ptr = block_ptr + (word_cnt_per_sector * current_sector_idx);

    // Process the current sector
    using sector_t =
      multiword_sector<key_t, word_t, word_cnt_per_sector, k_cnt_per_sector,
                       hasher, hash_value_t, hash_fn_idx, remaining_hash_bit_cnt,
                       k_cnt_per_sector>;
    auto found_in_sector = sector_t::contains(sector_ptr, key, hash_val, true);

    // Early out
    if (early_out) {
      if (likely(!found_in_sector)) return false;
    }

    // Process remaining sectors recursively, if any
    return multisector_block<key_t, word_t, word_cnt, sector_cnt, k,
                      hasher, hash_value_t, sector_t::hash_fn_idx_end, sector_t::remaining_hash_bits,
                      remaining_sector_cnt - 1, early_out>
        ::contains(block_ptr, key, hash_val, found_in_sector & is_contained_in_block);
  }
  //===----------------------------------------------------------------------===//

  //===----------------------------------------------------------------------===//
  // Contains (SIMD)
  //===----------------------------------------------------------------------===//
  template<u64 n>
  __forceinline__ __unroll_loops__
  static auto
  contains(const vec<key_t,n>& keys,
           const word_t* __restrict bitvector_base_address,
           const vec<hash_value_t,n>& block_start_word_idxs) noexcept {

    vec<hash_value_t, n> hash_vals(0);
    const auto is_contained_in_block_mask = vec<word_t,n>::mask::make_all_mask(); // true

    // Call recursive function
    static constexpr u32 remaining_hash_bits = 0;
    return multisector_block<key_t, word_t, word_cnt, sector_cnt, k,
                             hasher, hash_value_t, hash_fn_idx, remaining_hash_bits,
                             remaining_sector_cnt, early_out>
      ::contains(keys, hash_vals, bitvector_base_address, block_start_word_idxs, is_contained_in_block_mask);

  }
  //===----------------------------------------------------------------------===//

  //===----------------------------------------------------------------------===//
  // Contains (SIMD, Recursive)
  //===----------------------------------------------------------------------===//
  template<u64 n>
  __forceinline__ __unroll_loops__
  static auto
  contains(const vec<key_t,n>& keys,
           vec<hash_value_t,n>& hash_vals,
           const word_t* __restrict bitvector_base_address,
           const vec<hash_value_t,n>& block_start_word_idxs,
           const typename vec<word_t,n>::mask is_contained_in_block_mask) noexcept {

    // Sector pointers
    auto sector_start_word_idxs = block_start_word_idxs + (word_cnt_per_sector * current_sector_idx);

    // Process the current sector
    using sector_t =
            multiword_sector<key_t, word_t, word_cnt_per_sector, k_cnt_per_sector,
                             hasher, hash_value_t, hash_fn_idx, remaining_hash_bit_cnt,
                             k_cnt_per_sector>;
    auto found_in_sector_mask = sector_t::contains(keys, hash_vals, bitvector_base_address, sector_start_word_idxs, vec<word_t,n>::mask::make_all_mask());

    // Early out
    if (early_out) {
      if (likely(found_in_sector_mask.none())) return found_in_sector_mask;
    }

    // Process remaining sectors recursively, if any
    return multisector_block<key_t, word_t, word_cnt, sector_cnt, k,
                             hasher, hash_value_t, sector_t::hash_fn_idx_end, sector_t::remaining_hash_bits,
                             remaining_sector_cnt - 1, early_out>
      ::contains(keys, hash_vals, bitvector_base_address, block_start_word_idxs, found_in_sector_mask & is_contained_in_block_mask);
  }
  //===----------------------------------------------------------------------===//

};

//===----------------------------------------------------------------------===//
template<
    typename key_t,               // the key type
    typename word_t,              // the word type
    u32 word_cnt,                 // the number of words per block
    u32 s,                        // the numbers of sectors (must be a power of two)
    u32 k,                        // the number of bits to set/test
    template<typename Ty, u32 i> class hasher,      // the hash function family to use
    typename hash_value_t,        // the hash value type to use

    u32 hash_fn_idx,              // current hash function index (used for recursion)
    u32 remaining_hash_bit_cnt,   // the number of remaining hash bits (used for recursion)

    u1 early_out                  // allows for branching out during lookups (before the next sector is tested)
>
struct multisector_block<key_t, word_t, word_cnt, s, k, hasher, hash_value_t, hash_fn_idx, remaining_hash_bit_cnt,
                         0 /* no more remaining sectors */, early_out>  {

  //===----------------------------------------------------------------------===//
  // Insert
  //===----------------------------------------------------------------------===//
  __forceinline__
  static void
  insert(word_t* __restrict block_ptr, const key_t key, hash_value_t& hash_val) noexcept {
    // End of recursion.
  }
  //===----------------------------------------------------------------------===//

  //===----------------------------------------------------------------------===//
  // Contains
  //===----------------------------------------------------------------------===//
  __forceinline__ __host__ __device__
  static u1
  contains(const word_t* __restrict block_ptr, const key_t key, hash_value_t& hash_val, u1 is_contained_in_block) noexcept {
    // End of recursion.
    return is_contained_in_block;
  }
  //===----------------------------------------------------------------------===//

  //===----------------------------------------------------------------------===//
  // Contains (SIMD)
  //===----------------------------------------------------------------------===//
  template<u64 n>
  __forceinline__ __unroll_loops__
  static auto
  contains(const vec<key_t,n>& keys,
           vec<hash_value_t,n>& hash_vals,
           const word_t* __restrict bitvector_base_address,
           const vec<key_t,n>& block_start_word_idxs,
           const typename vec<word_t,n>::mask is_contained_in_block_mask) noexcept {
    // End of recursion.
    return is_contained_in_block_mask;
  }
  //===----------------------------------------------------------------------===//

};

} // namespace dtl