#include <fstream>
#include <iostream>
#include <string>
#include <stdlib.h>
#include <vector>
//#include <random>

#include <bits/stdc++.h>
#include <Eigen/Dense>
#include <Eigen/Core>
#include <Eigen/LU>
#include <Eigen/SVD>
#include <Eigen/QR>
#include "time.h"

#include "genotype.h"
#include "mailman.h"
#include "arguments.h"
//#include "helper.h"
#include "storage.h"

#include "/usr/include/boost/numeric/ublas/matrix.hpp"
#include "/usr/include/boost/numeric/ublas/io.hpp"
#include "/usr/include/boost/random.hpp"
#include "/usr/include/boost/numeric/ublas/lu.hpp"
#include "/usr/include/boost/numeric/ublas/vector.hpp"
#include "/usr/include/boost/numeric/ublas/vector_proxy.hpp"
#include "/usr/include/boost/numeric/ublas/triangular.hpp"
#include <boost/lexical_cast.hpp>

#if SSE_SUPPORT==1
	#define fastmultiply fastmultiply_sse
	#define fastmultiply_pre fastmultiply_pre_sse
#else
	#define fastmultiply fastmultiply_normal
	#define fastmultiply_pre fastmultiply_pre_normal
#endif

using namespace Eigen;
using namespace std;

// Storing in RowMajor Form
typedef Matrix<double, Dynamic, Dynamic, RowMajor> MatrixXdr;
//typedef Matrix<int, Dynamic, Dynamic, RowMajor> MatrixXdrInt;

class data {

 public:
     MatrixXdr gen;
     int index;
     MatrixXdr maf;
};

//Intermediate Variables
int blocksize;
int hsegsize;
double *partialsums;
double *sum_op;
double *yint_e;
double *yint_m;
double **y_e;
double **y_m;

struct timespec t0;

clock_t total_begin = clock();
MatrixXdr pheno;
MatrixXdr mask;
MatrixXdr covariate;
MatrixXdr Q;
MatrixXdr v1; //W^ty
MatrixXdr v2;            //QW^ty
MatrixXdr v3;    //WQW^ty
MatrixXdr new_pheno;

genotype g;
genotype g1;
genotype g2;
MatrixXdr geno_matrix; //(p,n)
genotype* Geno;
int MAX_ITER;
int k,p,n;
int k_orig;

MatrixXdr c; //(p,k)
MatrixXdr x; //(k,n)
MatrixXdr v; //(p,k)
MatrixXdr means; //(p,1)
MatrixXdr stds; //(p,1)
MatrixXdr sum2;
MatrixXdr sum;

Eigen::RowVectorXd means_na;
Eigen::RowVectorXd stds_na;
////////
//related to phenotype
double y_sum;
double y_mean;

options command_line_opts;

bool debug = false;
bool check_accuracy = false;
bool var_normalize=false;
int accelerated_em=0;
double convergence_limit;
bool memory_efficient = false;
bool missing=false;
bool fast_mode = true;
bool text_version = false;
bool use_cov=false;

/////
bool mean_impute=true;
std::vector<int> miss_index;
/////
bool dominance=false;
bool only_dominance=false;
int nongen_Nbin;
int total_Nbin;

///
//// jackknife index wich are computed based on annotation file
MatrixXdr dic_index;
MatrixXdr jack_bin_size;
vector<int> len;
vector<int> Annot;
int Njack;
int Nbin;
int Nz=10;
///////

//define random vector z's
MatrixXdr  all_zb;
MatrixXdr  all_Uzb;
MatrixXdr res;
MatrixXdr XXz;
MatrixXdr Xy;
MatrixXdr yXXy;

///// for simulation
MatrixXdr maf_ld;
MatrixXdr A_effect;
MatrixXdr A_effect_multi;
vector<string> fid;
MatrixXdr simul_pheno;
MatrixXdr simul_par;
int num_simul;
/////vars for dominace

MatrixXdr global_MAF;

///
//Matrix<int, Dynamic, Dynamic, RowMajor> gen;
MatrixXdr gen;
bool read_header;
//read variables
unsigned char mask2;
int wordsize;
unsigned int unitsperword;
int unitsize;
int nrow, ncol;
unsigned char *gtype;
int Nsnp;
int Nindv;
bool **bin_annot;
int step_size;
int step_size_rem;
std::vector<std::vector<bool> > annot_bool;
std::vector<std::vector<int> > jack_bin;
vector <data> allgen;
vector <genotype> allgen_mail;
int global_snp_index;
bool use_mailman=true;

///variable for non geno vc

vector <data> nongen_all;
MatrixXdr nongen_vc;

///var for analytic se

MatrixXdr wt;
MatrixXdr vt;

///reading single col annot
vector <int> SNP_annot;
bool use_1col_annot=false;

///Variables for reg out cov on both side of LM
bool both_side_cov=false;
MatrixXdr UXXz;
MatrixXdr XXUz;
MatrixXdr Xz;
MatrixXdr trVK;

std::istream& newline(std::istream& in)
{
    if ((in >> std::ws).peek() != std::char_traits<char>::to_int_type('\n')) {
        in.setstate(std::ios_base::failbit);
    }
    return in.ignore();
}

int read_cov(bool std,int Nind, std::string filename, std::string covname){
	ifstream ifs(filename.c_str(), ios::in);
	std::string line;
	std::istringstream in;
	int covIndex = 0;
	std::getline(ifs,line);
	in.str(line);
	string b;
	vector<vector<int> > missing;
	int covNum=0;
	while(in>>b)
	{
		if(b!="FID" && b !="IID"){
		missing.push_back(vector<int>()); //push an empty row
		if(b==covname && covname!="")
			covIndex=covNum;
		covNum++;
		}
	}
	vector<double> cov_sum(covNum, 0);
	if(covname=="")
	{
		covariate.resize(Nind, covNum);
		cout<< "Read in "<<covNum << " Covariates.. "<<endl;
	}
	else
	{
		covariate.resize(Nind, 1);
		cout<< "Read in covariate "<<covname<<endl;
	}

	int j=0;
	while(std::getline(ifs, line)){
		in.clear();
		in.str(line);
		string temp;
		in>>temp; in>>temp; //FID IID
		for(int k=0; k<covNum; k++){

			in>>temp;
			if(temp=="NA")
			{
				missing[k].push_back(j);
				continue;
			}
			double cur = atof(temp.c_str());
			if(cur==-9)
			{
				missing[k].push_back(j);
				continue;
			}
			if(covname=="")
			{
				cov_sum[k]= cov_sum[k]+ cur;
				covariate(j,k) = cur;
			}
			else
				if(k==covIndex)
				{
					covariate(j, 0) = cur;
					cov_sum[k] = cov_sum[k]+cur;
				}
		}
		//if(j<10)
		//	cout<<covariate.block(j,0,1, covNum)<<endl;
		j++;
	}
	//compute cov mean and impute
	for (int a=0; a<covNum ; a++)
	{
		int missing_num = missing[a].size();
		cov_sum[a] = cov_sum[a] / (Nind - missing_num);

		for(int b=0; b<missing_num; b++)
		{
                        int index = missing[a][b];
                        if(covname=="")
                                covariate(index, a) = cov_sum[a];
                        else if (a==covIndex)
                                covariate(index, 0) = cov_sum[a];
                }
	}
	if(std)
	{
		MatrixXdr cov_std;
		cov_std.resize(1,covNum);
		MatrixXdr sum = covariate.colwise().sum();
		MatrixXdr sum2 = (covariate.cwiseProduct(covariate)).colwise().sum();
		MatrixXdr temp;
//		temp.resize(Nind, 1);
//		for(int i=0; i<Nind; i++)
//			temp(i,0)=1;
		for(int b=0; b<covNum; b++)
		{
			cov_std(0,b) = sum2(0,b) + Nind*cov_sum[b]*cov_sum[b]- 2*cov_sum[b]*sum(0,b);
			cov_std(0,b) =sqrt((Nind- 1)/cov_std(0,b)) ;
			double scalar=cov_std(0,b);
			for(int j=0; j<Nind; j++)
			{
				covariate(j,b) = covariate(j,b)-cov_sum[b];
				covariate(j,b) =covariate(j,b)*scalar;
			}
			//covariate.col(b) = covariate.col(b) -temp*cov_sum[b];

		}
	}
	return covNum;
}

void multiply_y_pre_fast(MatrixXdr &op, int Ncol_op ,MatrixXdr &res,bool subtract_means){

        for(int k_iter=0;k_iter<Ncol_op;k_iter++){
                sum_op[k_iter]=op.col(k_iter).sum();
        }

                        //cout << "Nops = " << Ncol_op << "\t" <<g.Nsegments_hori << endl;
        #if DEBUG==1
                if(debug){
                        print_time ();
                        cout <<"Starting mailman on premultiply"<<endl;
                        cout << "Nops = " << Ncol_op << "\t" <<g.Nsegments_hori << endl;
                        cout << "Segment size = " << g.segment_size_hori << endl;
                        cout << "Matrix size = " <<g.segment_size_hori<<"\t" <<g.Nindv << endl;
                        cout << "op = " <<  op.rows () << "\t" << op.cols () << endl;
                }
        #endif

        //TODO: Memory Effecient SSE FastMultipy

        for(int seg_iter=0;seg_iter<g.Nsegments_hori-1;seg_iter++){
                mailman::fastmultiply(g.segment_size_hori,g.Nindv,Ncol_op,g.p[seg_iter],op,yint_m,partialsums,y_m);
                int p_base = seg_iter*g.segment_size_hori;
                for(int p_iter=p_base; (p_iter<p_base+g.segment_size_hori) && (p_iter<g.Nsnp) ; p_iter++ ){
                        for(int k_iter=0;k_iter<Ncol_op;k_iter++)
                                res(p_iter,k_iter) = y_m[p_iter-p_base][k_iter];
                }
        }

        int last_seg_size = (g.Nsnp%g.segment_size_hori !=0 ) ? g.Nsnp%g.segment_size_hori : g.segment_size_hori;
        mailman::fastmultiply(last_seg_size,g.Nindv,Ncol_op,g.p[g.Nsegments_hori-1],op,yint_m,partialsums,y_m);
        int p_base = (g.Nsegments_hori-1)*g.segment_size_hori;
        for(int p_iter=p_base; (p_iter<p_base+g.segment_size_hori) && (p_iter<g.Nsnp) ; p_iter++){
                for(int k_iter=0;k_iter<Ncol_op;k_iter++)
                        res(p_iter,k_iter) = y_m[p_iter-p_base][k_iter];
        }

        #if DEBUG==1
                if(debug){
                        print_time ();
                        cout <<"Ending mailman on premultiply"<<endl;
                }
        #endif

        if(!subtract_means)
                return;

        for(int p_iter=0;p_iter<p;p_iter++){
                for(int k_iter=0;k_iter<Ncol_op;k_iter++){
                        res(p_iter,k_iter) = res(p_iter,k_iter) - (g.get_col_mean(p_iter)*sum_op[k_iter]);
                        if(var_normalize)
                                res(p_iter,k_iter) = res(p_iter,k_iter)/(g.get_col_std(p_iter));
                }
        }

}

void multiply_y_post_fast(MatrixXdr &op_orig, int Nrows_op, MatrixXdr &res,bool subtract_means){

        MatrixXdr op;
        op = op_orig.transpose();

        if(var_normalize && subtract_means){
                for(int p_iter=0;p_iter<p;p_iter++){
                        for(int k_iter=0;k_iter<Nrows_op;k_iter++)
                                op(p_iter,k_iter) = op(p_iter,k_iter) / (g.get_col_std(p_iter));
                }
        }

        #if DEBUG==1
                if(debug){
                        print_time ();
                        cout <<"Starting mailman on postmultiply"<<endl;
                }
        #endif

        int Ncol_op = Nrows_op;

        //cout << "ncol_op = " << Ncol_op << endl;

        int seg_iter;
        for(seg_iter=0;seg_iter<g.Nsegments_hori-1;seg_iter++){
mailman::fastmultiply_pre(g.segment_size_hori,g.Nindv,Ncol_op, seg_iter * g.segment_size_hori, g.p[seg_iter],op,yint_e,partialsums,y_e);
        }
        int last_seg_size = (g.Nsnp%g.segment_size_hori !=0 ) ? g.Nsnp%g.segment_size_hori : g.segment_size_hori;
        mailman::fastmultiply_pre(last_seg_size,g.Nindv,Ncol_op, seg_iter * g.segment_size_hori, g.p[seg_iter],op,yint_e,partialsums,y_e);

        for(int n_iter=0; n_iter<n; n_iter++)  {
                for(int k_iter=0;k_iter<Ncol_op;k_iter++) {
                        res(k_iter,n_iter) = y_e[n_iter][k_iter];
                        y_e[n_iter][k_iter] = 0;
                }
        }

        #if DEBUG==1
                if(debug){
                        print_time ();
                        cout <<"Ending mailman on postmultiply"<<endl;
                }
        #endif

        if(!subtract_means)
                return;

        double *sums_elements = new double[Ncol_op];
        memset (sums_elements, 0, Nrows_op * sizeof(int));

        for(int k_iter=0;k_iter<Ncol_op;k_iter++){
                double sum_to_calc=0.0;
                for(int p_iter=0;p_iter<p;p_iter++)
                        sum_to_calc += g.get_col_mean(p_iter)*op(p_iter,k_iter);
                sums_elements[k_iter] = sum_to_calc;
        }
        for(int k_iter=0;k_iter<Ncol_op;k_iter++){
                for(int n_iter=0;n_iter<n;n_iter++)
                        res(k_iter,n_iter) = res(k_iter,n_iter) - sums_elements[k_iter];
        }

}

void initial_var()
{
    /*if(key==1)
        g=g1;
    if(key==2)
        g=g2;*/
   // g=Geno[key];

	p = g.Nsnp;
        n = g.Nindv;

        c.resize(p,k);
        x.resize(k,n);
        v.resize(p,k);
        //means.resize(p,1);
        //stds.resize(p,1);
        sum2.resize(p,1);
        sum.resize(p,1);

        if(!fast_mode && !memory_efficient){
                geno_matrix.resize(p,n);
                g.generate_eigen_geno(geno_matrix,var_normalize);
        }

        //TODO: Initialization of c with gaussian distribution
        c = MatrixXdr::Random(p,k);

        // Initial intermediate data structures
        blocksize = k;
         hsegsize = g.segment_size_hori;        // = log_3(n)
        int hsize = pow(3,hsegsize);
        int vsegsize = g.segment_size_ver;              // = log_3(p)
        int vsize = pow(3,vsegsize);

        partialsums = new double [blocksize];
        sum_op = new double[blocksize];
        yint_e = new double [hsize*blocksize];
        yint_m = new double [hsize*blocksize];
        memset (yint_m, 0, hsize*blocksize * sizeof(double));
        memset (yint_e, 0, hsize*blocksize * sizeof(double));

        y_e  = new double*[g.Nindv];
        for (int i = 0 ; i < g.Nindv ; i++) {
                y_e[i] = new double[blocksize];
                memset (y_e[i], 0, blocksize * sizeof(double));
        }

        y_m = new double*[hsegsize];
        for (int i = 0 ; i < hsegsize ; i++)
                y_m[i] = new double[blocksize];
      /*  for(int i=0;i<p;i++){
                means(i,0) = g.get_col_mean(i);
                stds(i,0) =1/g.get_col_std(i);
                //sum2(i,0) =g.get_col_sum2(i);
                sum(i,0)= g.get_col_sum(i);
        }

*/

}

/*void read_cov(int Nind, std::string filename, std::string covname){
	ifstream ifs(filename.c_str(), ios::in);
	std::string line;
	std::istringstream in;
	int covIndex = 0;
	std::getline(ifs,line);
	in.str(line);
	string b;
	vector<vector<int> > missing;
	int covNum=0;
	while(in>>b)
	{
		missing.push_back(vector<int>()); //push an empty row
		if(b==covname && covname!="")
			covIndex=covNum;
		covNum++;
	}
	vector<double> cov_sum(covNum, 0);
	if(covname=="")
	{
		covariate.resize(Nind, covNum);
		cout<< "Read in "<<covNum << " Covariates.. "<<endl;
	}
	else
	{
		covariate.resize(Nind, 1);
		cout<< "Read in covariate "<<covname<<endl;
	}

	int j=0;
	while(std::getline(ifs, line)){
		in.clear();
		in.str(line);
		string temp;
		for(int k=0; k<covNum; k++){
			in>>temp;
			if(temp=="NA")
			{
				missing[k].push_back(j);
				continue;
			}
			int cur = atof(temp.c_str());
			if(cur==-9)
			{
				missing[k].push_back(j);
				continue;
			}
			if(covname=="")
			{
				cov_sum[k]= cov_sum[k]+ cur;
				covariate(j,k) = cur;
			}
			else
				if(k==covIndex)
				{
					covariate(j, 0) = cur;
					cov_sum[k] = cov_sum[k]+cur;
				}
		}
		j++;
	}
	//compute cov mean and impute
	for (int a=0; a<covNum ; a++)
	{
		int missing_num = missing[a].size();
		cov_sum[a] = cov_sum[a] / (covNum - missing_num);

		for(int b=0; b<missing_num; b++)
		{
                        int index = missing[a][b];
                        if(covname=="")
                                covariate(index, a) = cov_sum[a];
                        else if (a==covIndex)
                                covariate(index, 0) = cov_sum[a];
                }
	}
}*/
void read_pheno2(int Nind, std::string filename){
//	pheno.resize(Nind,1);
	ifstream ifs(filename.c_str(), ios::in);

	std::string line;
	std::istringstream in;
	int phenocount=0;
//read header
	std::getline(ifs,line);
	in.str(line);
	string b;
	while(in>>b)
	{
		if(b!="FID" && b !="IID")
			phenocount++;
	}
	pheno.resize(Nind, phenocount);
	mask.resize(Nind, phenocount);
	int i=0;
	while(std::getline(ifs, line)){
		in.clear();
		in.str(line);
		string temp;
		//fid,iid
		//todo: fid iid mapping;
		//todo: handle missing phenotype
		in>>temp; in>>temp;
		for(int j=0; j<phenocount;j++) {
			in>>temp;
			double cur = atof(temp.c_str());
			if(temp=="NA" || cur==-9){
			pheno(i,j)=0;
			mask(i,j)=0;
			}
			else{
			pheno(i,j)=atof(temp.c_str());
			mask(i,j)=1;

			}

		}
		i++;
	}
	//cout<<pheno;
}

double compute_yXXy(int num_snp){

        MatrixXdr res(num_snp, 1);

	if(use_mailman==true)
		multiply_y_pre_fast(pheno,1,res,false);
	else
		 res=gen*pheno;

	res = res.cwiseProduct(stds);
        MatrixXdr resid(num_snp, 1);
        resid = means.cwiseProduct(stds);
        resid = resid *y_sum;
        MatrixXdr Xy(num_snp,1);
        Xy = res-resid;

        double yXXy = (Xy.array()* Xy.array()).sum();

	//cout<<yXXy<<endl;

        return yXXy;

}

double compute_yVXXVy(int num_snp){
        MatrixXdr new_pheno_sum = new_pheno.colwise().sum();
        MatrixXdr res(num_snp, 1);

        if(use_mailman==true)
                multiply_y_pre_fast(new_pheno,1,res,false);
        else
                 res=gen*new_pheno;

        res = res.cwiseProduct(stds);
        MatrixXdr resid(num_snp, 1);
        resid = means.cwiseProduct(stds);
        resid = resid *new_pheno_sum;
        MatrixXdr Xy(num_snp,1);
        Xy = res-resid;
        double ytVXXVy = (Xy.array()* Xy.array()).sum();
        return ytVXXVy;

}

MatrixXdr  compute_XXz (int num_snp){
	//mask
	/*for (int i=0;i<Nz;i++)
	   for(int j=0;j<Nindv;j++)
		 all_zb(j,i)=all_zb(j,i)*mask(j,0);
*/
         res.resize(num_snp, Nz);

	if(use_mailman==true)
		multiply_y_pre_fast(all_zb,Nz,res, false);
	else
        	res=gen*all_zb;

        MatrixXdr zb_sum = all_zb.colwise().sum();

	for(int j=0; j<num_snp; j++)
            for(int k=0; k<Nz;k++)
                res(j,k) = res(j,k)*stds(j,0);

        MatrixXdr resid(num_snp, Nz);
        MatrixXdr inter = means.cwiseProduct(stds);
        resid = inter * zb_sum;
        MatrixXdr inter_zb = res - resid;

	for(int k=0; k<Nz; k++)
            for(int j=0; j<num_snp;j++)
                inter_zb(j,k) =inter_zb(j,k) *stds(j,0);
       MatrixXdr new_zb = inter_zb.transpose();
       MatrixXdr new_res(Nz, Nindv);

	 if(use_mailman==true)
	    multiply_y_post_fast(new_zb, Nz, new_res, false);
	 else
	    new_res=new_zb*gen;

       MatrixXdr new_resid(Nz, num_snp);
       MatrixXdr zb_scale_sum = new_zb * means;
       new_resid = zb_scale_sum * MatrixXdr::Constant(1,Nindv, 1);

                      /// new zb
       MatrixXdr temp=new_res - new_resid;

	for (int i=0;i<Nz;i++)
           for(int j=0;j<Nindv;j++)
                 temp(i,j)=temp(i,j)*mask(j,0);

	return temp.transpose();

}

MatrixXdr  compute_XXUz (int num_snp){
        //mask
/*        for (int i=0;i<Nz;i++)
           for(int j=0;j<Nindv;j++)
                 all_Uzb(j,i)=all_Uzb(j,i)*mask(j,0);
*/
         res.resize(num_snp, Nz);

        if(use_mailman==true)
                multiply_y_pre_fast(all_Uzb,Nz,res, false);
        else
                res=gen*all_Uzb;

        MatrixXdr zb_sum = all_Uzb.colwise().sum();

        for(int j=0; j<num_snp; j++)
            for(int k=0; k<Nz;k++)
                res(j,k) = res(j,k)*stds(j,0);

        MatrixXdr resid(num_snp, Nz);
        MatrixXdr inter = means.cwiseProduct(stds);
        resid = inter * zb_sum;
        MatrixXdr inter_zb = res - resid;

        for(int k=0; k<Nz; k++)
            for(int j=0; j<num_snp;j++)
                inter_zb(j,k) =inter_zb(j,k) *stds(j,0);
       MatrixXdr new_zb = inter_zb.transpose();
       MatrixXdr new_res(Nz, Nindv);

         if(use_mailman==true)
            multiply_y_post_fast(new_zb, Nz, new_res, false);
         else
            new_res=new_zb*gen;

       MatrixXdr new_resid(Nz, num_snp);
       MatrixXdr zb_scale_sum = new_zb * means;
       new_resid = zb_scale_sum * MatrixXdr::Constant(1,Nindv, 1);

                      /// new zb
       MatrixXdr temp=new_res - new_resid;

        for (int i=0;i<Nz;i++)
           for(int j=0;j<Nindv;j++)
                 temp(i,j)=temp(i,j)*mask(j,0);

        return temp.transpose();

}

MatrixXdr  compute_Xz (int num_snp){

         MatrixXdr new_zb= MatrixXdr::Random(Nz,num_snp);
         new_zb = new_zb * sqrt(3);

	 MatrixXdr new_res(Nz, Nindv);

        if(use_mailman==true)
                multiply_y_post_fast(new_zb,Nz,new_res, false);
        else
                new_res=new_zb*gen;

        MatrixXdr new_resid(Nz, num_snp);
       MatrixXdr zb_scale_sum = new_zb * means;
       new_resid = zb_scale_sum * MatrixXdr::Constant(1,Nindv, 1);

                      /// new zb
       MatrixXdr temp=new_res - new_resid;

        for (int i=0;i<Nz;i++)
           for(int j=0;j<Nindv;j++)
                 temp(i,j)=temp(i,j)*mask(j,0);

	return temp.transpose();
}

void read_annot (string filename){

//	int step_size=Nsnp/Njack;
  //      int step_size_rem=Nsnp%Njack;
        vector<bool> snp_annot;
	//jack_bin.resize(Njack, vector<int>(Nbin,0));

//	cout<<step_size<<endl;

	ifstream inp(filename.c_str());
        if (!inp.is_open()){
                cerr << "Error reading file "<< filename <<endl;
                exit(1);
        }
        string line;
        int j = 0 ;
        int linenum = 0 ;
        int num_parti;
        stringstream check1(line);
        string intermediate;
        vector <string> tokens;
        while(std::getline (inp, line)){
                //linenum ++;
                char c = line[0];
                if (c=='#')
                        continue;
                istringstream ss (line);
                if (line.empty())
                        continue;
                j++;
                //cout<<line<<endl;

                stringstream check1(line);
                string intermediate;
                vector <string> tokens;
                // Tokenizing w.r.t. space ' '
                while(getline(check1, intermediate, ' '))
                 {
                      tokens.push_back(intermediate);
                 }
                 if(linenum==0){
                 num_parti=tokens.size();
		 Nbin=num_parti;
                  snp_annot.resize(Nbin,0);
          	  jack_bin.resize(Njack, vector<int>(Nbin,0));
	         len.resize(num_parti,0);
                }
                int index_annot=0;
                for(int i = 0; i < tokens.size(); i++){
		      snp_annot[i]=0;
		      if (tokens[i]=="1"){
                            len[i]++;
			    snp_annot[i]=1;
 		      }
                }
		annot_bool.push_back(snp_annot);
        linenum++;
       }

	  if(Nsnp!=linenum){
          cout<<"Number of the rows in bim file and annotation file does not match"<<endl;
        }

	Nsnp=linenum;
	//cout<<"Total number of SNPs : "<<Nsnp<<endl;
	int selected_snps=0;
	for (int i=0;i<num_parti;i++){
                cout<<len[i]<<" SNPs in "<<i<<"-th bin"<<endl;
		selected_snps+=len[i];
        }

	cout<<" Number of selected SNPs w.r.t  annot file : " <<selected_snps<<endl;

	 step_size=Nsnp/Njack;
         step_size_rem=Nsnp%Njack;
	cout<<"Number of SNPs per block : "<<step_size<<endl;
      //  cout<<"stepsize : "<<step_size_rem<<endl;
	jack_bin.resize(Njack, vector<int>(Nbin,0));
	int temp;
	for (int i=0;i<Nsnp;i++)
	   for(int j=0;j<Nbin;j++)
		 if (annot_bool[i][j]==1){
			temp=i/step_size;
			if (temp==Njack)
				temp--;
			//cout<<i<<"xxx"<<j<<"xxx"<<temp<<endl;
			jack_bin[temp][j]++;
		 }
/*
cout<<"jackbin"<<endl;
	for (int i=0;i<Njack;i++){
	   for(int j=0;j<Nbin;j++)
                cout<<jack_bin[i][j]<<" ";
	  cout<<endl;
        }*/
/*
for (int i=0;i<linenum;i++){
  for(int j=0;j<Nbin;j++)
	cout<<annot_bool[i][j]<<" ";
  cout<<endl;
}
*/

}

//vector <int> SNP_annot;

void read_annot_1col (string filename){

        ifstream ifs(filename.c_str(), ios::in);

        std::string line;
        std::istringstream in;

	len.resize(Nbin,0);
	 step_size=Nsnp/Njack;
         step_size_rem=Nsnp%Njack;
        cout<<"Number of SNPs per block : "<<step_size<<endl;
	jack_bin.resize(Njack, vector<int>(Nbin,0));
	int i=0;
        while(std::getline(ifs, line)){
                in.clear();
                in.str(line);
                string temp;

		in>>temp;
	        int  cur = atoi(temp.c_str());
		SNP_annot.push_back(cur);
		len[cur-1]++;

		int jack_val=i/step_size;
		 if (jack_val==Njack)
                    jack_val--;
		jack_bin[jack_val][SNP_annot[i]-1]++;

		 i++;
        }

	if(Nsnp!=i){
	  cout<<"Number of the rows in bim file and annotation file does not match"<<endl;
	}

        cout<<"Total number of SNPs : "<<Nsnp<<endl;
        for (int i=0;i<Nbin;i++){
                cout<<len[i]<<" SNPs in "<<i<<"-th bin"<<endl;
        }

        /*for (int i=0;i<Njack;i++){
           for(int j=0;j<Nbin;j++)
                cout<<jack_bin[i][j]<<" ";
          cout<<endl;
        }*/

}

void read_bim (string filename){
        ifstream inp(filename.c_str());
        if (!inp.is_open()){
                cerr << "Error reading file "<< filename <<endl;
                exit(1);
        }
        string line;
        int j = 0 ;
        int linenum = 0 ;
        while(std::getline (inp, line)){
                linenum ++;
                char c = line[0];
                if (c=='#')
                        continue;
                istringstream ss (line);
                if (line.empty())
                        continue;
                j++;
        }
        Nsnp = j;
        inp.close();
	cout<<"#SNP in bim file "<<Nsnp<<endl;
}

void count_pheno(std::string filename){
        ifstream ifs(filename.c_str(), ios::in);

        std::string line;
        int i=0;
        while(std::getline(ifs, line)){
                i++;
        }
        Nindv=i-1;
}

int  count_fam(std::string filename){
        ifstream ifs(filename.c_str(), ios::in);

        std::string line;
        int i=0;
        while(std::getline(ifs, line)){
                i++;
        }
        return i;
}

void  create_simul_pheno(std::string filename){

	//vector<string> fid;
	 ifstream ifs(filename.c_str(), ios::in);

        std::string line;
        std::istringstream in;
        string temp;

        double val;
        int i=0;
        while(std::getline(ifs, line)){
                  in.clear();
                  in.str(line);

                  in >> temp;
                  fid.push_back(temp.c_str());
			//val = atof(temp.c_str());
		//	cout<<val<<endl;
        //          cout<<fid[i]<<endl;
		  i++;
        }
	Nindv=i;

}

//// functions related to reading without mailman

template<typename T>
static std::istream & binary_read(std::istream& stream, T& value){
        return stream.read(reinterpret_cast<char*>(&value), sizeof(T));
}

void set_metadata() {
        wordsize = sizeof(char) * 8;
        unitsize = 2;
        unitsperword = wordsize/unitsize;
        mask2 = 0;
        for (int i = 0 ; i < unitsize; i++)
                mask2 = mask2 |(0x1<<i);
    nrow = Nsnp;
    ncol = ceil(1.0*Nindv/unitsperword);
}

int simulate2_geno_from_random(float p_j){
        float rval = static_cast <float> (rand()) / static_cast <float> (RAND_MAX);
        float dist_pj[3] = { (1-p_j)*(1-p_j), 2*p_j*(1-p_j), p_j*p_j };
        if(rval < dist_pj[0] )
                return 0;
        else if( rval >= dist_pj[0] && rval < (dist_pj[0]+dist_pj[1]))
                return 1;
        else
                return 2;
}

float get_observed_pj(const unsigned char* line){
        int y[4];
        int observed_sum=0;
        int observed_ct=0;

        for (int k = 0 ;k < ncol ; k++) {
                unsigned char c = line [k];
                y[0] = (c)&mask2;
                y[1] = (c>>2)&mask2;
                y[2] = (c>>4)&mask2;
                y[3] = (c>>6)&mask2;
                int j0 = k * unitsperword;
                int lmax = 4;
                if (k == ncol - 1)  {
                        lmax = Nindv%4;
                        lmax = (lmax==0)?4:lmax;
                }
                for ( int l = 0 ; l < lmax; l++){
                        int j = j0 + l ;
                        // Extract  PLINK coded genotype and convert into 0/1/2
                        // // PLINK coding:
                        // // 00->0
                        // // 01->missing
                        // // 10->1
                        // // 11->2
                        int val = y[l];
                        val-- ;
                        if(val != 0){
                                val =  (val < 0 ) ? 0 :val ;
                                observed_sum += val;
                                observed_ct ++;
                        }
                }
        }
        return observed_sum*0.5/observed_ct;

}

void read_bed (std::istream& ifs,bool allow_missing,int num_snp)  {
         //ifstream ifs (filename.c_str(), ios::in|ios::binary);
        char magic[3];
        set_metadata ();

    gtype =  new unsigned char[ncol];

     if(read_header)
      binary_read(ifs,magic);

        int sum=0;

        // Note that the coding of 0 and 2 can get flipped relative to plink because plink uses allele frequency (minor)
        // allele to code a SNP as 0 or 1.
        // This flipping does not matter for results.
        int y[4];

for(int i=0;i<num_snp;i++){
		global_snp_index++;
                ifs.read (reinterpret_cast<char*>(gtype), ncol*sizeof(unsigned char));
                float p_j = get_observed_pj(gtype);
        for (int k = 0 ;k < ncol ; k++) {
                unsigned char c = gtype [k];
                        // Extract PLINK genotypes
                y[0] = (c)&mask2;
                y[1] = (c>>2)&mask2;
                y[2] = (c>>4)&mask2;
                y[3] = (c>>6)&mask2;
                        int j0 = k * unitsperword;
                        // Handle number of individuals not being a multiple of 4
                        int lmax = 4;
                        if (k == ncol - 1)  {
                                lmax = Nindv%4;
                                lmax = (lmax==0)?4:lmax;
                        }
                        for ( int l = 0 ; l < lmax; l++){
                                int j = j0 + l ;
                                // Extract  PLINK coded genotype and convert into 0/1/2
                                // PLINK coding:
                                // 00->0
                                // 01->missing
                                // 10->1
                                // 11->2
                                int val = y[l];
                                if(val==1 && !allow_missing){
                                        val = simulate2_geno_from_random(p_j);
                                        val++;
                                        val = (val==1) ? 0 : val;
  //                                 val=0;
				 }
                                val-- ;
                                val =  (val < 0 ) ? 0 :val ;
				sum += val;

			    for(int bin_index=0;bin_index<Nbin;bin_index++){
				if(annot_bool[global_snp_index][bin_index]==1){

				      int snp_index;
				     //int snp_index=allgen[bin_index].index;
				     //allgen[bin_index].gen(snp_index,j)=val;

				     if(use_mailman==true){
				 	 snp_index=allgen_mail[bin_index].index;
					 int horiz_seg_no = snp_index/allgen_mail[bin_index].segment_size_hori;
					 allgen_mail[bin_index].p[horiz_seg_no][j] = 3 *allgen_mail[bin_index].p[horiz_seg_no][j]  + val;
				     // computing sum for every snp to compute mean
				         allgen_mail[bin_index].columnsum[snp_index]+=val;

				      }
				     else{
					 snp_index=allgen[bin_index].index;
                                         allgen[bin_index].gen(snp_index,j)=val;
				     }

				}

			    }

                    }
        }

    for(int bin_index=0;bin_index<Nbin;bin_index++)
       if(annot_bool[global_snp_index][bin_index]==1){
       		//cout<<"global"<<global_snp_index<<endl;
		        //cout<<allgen[bin_index].index<<endl;
		//       allgen[bin_index].index++;

	      if(use_mailman==true)
		  allgen_mail[bin_index].index++;
	       else
	           allgen[bin_index].index++;
	}

   }

	sum = 0 ;
        delete[] gtype;
}

void read_bed2 (std::istream& ifs,bool allow_missing,int num_snp)  {
         //ifstream ifs (filename.c_str(), ios::in|ios::binary);
        char magic[3];
        set_metadata ();

    gtype =  new unsigned char[ncol];

     if(read_header)
      binary_read(ifs,magic);

        int sum=0;
	double sum_dom=0;
        // Note that the coding of 0 and 2 can get flipped relative to plink because plink uses allele frequency (minor)
        // allele to code a SNP as 0 or 1.
        // This flipping does not matter for results.
        int y[4];

int bin_pointer;

vector<int> pointer_bins;

for(int i=0;i<num_snp;i++){
                global_snp_index++;
                ifs.read (reinterpret_cast<char*>(gtype), ncol*sizeof(unsigned char));
                float p_j = get_observed_pj(gtype);

	   pointer_bins.clear();
       	  for(int bin_index=0;bin_index<Nbin;bin_index++)
       		if(annot_bool[global_snp_index][bin_index]==1)
			  pointer_bins.push_back(bin_index);
			//bin_pointer=bin_index;

	  for (int k = 0 ;k < ncol ; k++) {
                unsigned char c = gtype [k];
                        // Extract PLINK genotypes
                y[0] = (c)&mask2;
                y[1] = (c>>2)&mask2;
                y[2] = (c>>4)&mask2;
                y[3] = (c>>6)&mask2;
                        int j0 = k * unitsperword;
                        // Handle number of individuals not being a multiple of 4
                        int lmax = 4;
                        if (k == ncol - 1)  {
                                lmax = Nindv%4;
                                lmax = (lmax==0)?4:lmax;
                        }
                        for ( int l = 0 ; l < lmax; l++){
                                int j = j0 + l ;
                                // Extract  PLINK coded genotype and convert into 0/1/2
                                // PLINK coding:
                                // 00->0
                                // 01->missing
                                // 10->1
                                // 11->2
                                double val = y[l];
				double val_dom;
                                if(val==1 && !allow_missing){

					if(use_mailman==false & mean_impute==true){
						miss_index.push_back(j);
						val=0;
					}
					else{
					val = simulate2_geno_from_random(p_j);
                                        val++;
                                        val = (val==1) ? 0 : val;
                                        //val=0;
                                        }

                                 }
                                val-- ;
                                val =  (val < 0 ) ? 0 :val ;

				//	/cout<<val<<endl;
				if(only_dominance==true){
					if(val==1)
					   val=0;
				}

			         sum += val;

				if(dominance==true){
				    if (val==1)
					val_dom=2*global_MAF(global_snp_index,0);
				    else if (val==2)
					val_dom=(4*global_MAF(global_snp_index,0))-2;
				    else if (val==0)
					  val_dom=val;

				  sum_dom+=val_dom;
 				}

                            for(int bin_index=0;bin_index<pointer_bins.size();bin_index++){

				       bin_pointer=pointer_bins[bin_index];
                                      int snp_index;

                                     if(use_mailman==true){
                                         snp_index=allgen_mail[bin_pointer].index;
                                         int horiz_seg_no = snp_index/allgen_mail[bin_pointer].segment_size_hori;
                                         allgen_mail[bin_pointer].p[horiz_seg_no][j] = 3 *allgen_mail[bin_pointer].p[horiz_seg_no][j]  + val;
                                     // computing sum for every snp to compute mean
                                         allgen_mail[bin_pointer].columnsum[snp_index]+=val;

                                      }
                                     else{
                                         snp_index=allgen[bin_pointer].index;
					 if(dominance==false)
                                         	allgen[bin_pointer].gen(snp_index,j)=val;
					 else{
						allgen[bin_pointer].gen(snp_index,j)=val_dom;
						allgen[bin_pointer].maf(snp_index,0)+=(double)val/Nindv/2;

					}

                                     }

                            }

                    }
        }

      if(use_mailman==false & mean_impute==true){
                 for(int bin_index=0;bin_index<pointer_bins.size();bin_index++){
			int bin_pointer=pointer_bins[bin_index];
			int snp_index=allgen[bin_pointer].index;

			for(int indv_index=0;indv_index<miss_index.size();indv_index++){
				int temp=miss_index[indv_index];
				if(dominance==false)
					allgen[bin_pointer].gen(snp_index,temp)=(double)sum/Nindv;
        			else
					allgen[bin_pointer].gen(snp_index,temp)=(double)sum_dom/Nindv;

			}

	         }
      }

	std::vector<int>().swap(miss_index);

     for(int bin_index=0;bin_index<pointer_bins.size();bin_index++){
     		bin_pointer=pointer_bins[bin_index];
	         if(use_mailman==true)
                  allgen_mail[bin_pointer].index++;
               else
                   allgen[bin_pointer].index++;
     }

   if(dominance==false)
  	global_MAF(global_snp_index,0)=(double)sum/Nindv/2;

  sum=0;
  sum_dom=0;
}

      //  sum = 0 ;
        delete[] gtype;
}

void read_bed_1colannot (std::istream& ifs,bool allow_missing,int num_snp)  {
         //ifstream ifs (filename.c_str(), ios::in|ios::binary);
        char magic[3];
        set_metadata ();

    gtype =  new unsigned char[ncol];

     if(read_header)
      binary_read(ifs,magic);

        int sum=0;

        // Note that the coding of 0 and 2 can get flipped relative to plink because plink uses allele frequency (minor)
        // allele to code a SNP as 0 or 1.
        // This flipping does not matter for results.
        int y[4];

int bin_pointer;

//vector<int> pointer_bins;

for(int i=0;i<num_snp;i++){
                global_snp_index++;
                ifs.read (reinterpret_cast<char*>(gtype), ncol*sizeof(unsigned char));
                float p_j = get_observed_pj(gtype);

          for (int k = 0 ;k < ncol ; k++) {
                unsigned char c = gtype [k];
                        // Extract PLINK genotypes
                y[0] = (c)&mask2;
                y[1] = (c>>2)&mask2;
                y[2] = (c>>4)&mask2;
                y[3] = (c>>6)&mask2;
                        int j0 = k * unitsperword;
                        // Handle number of individuals not being a multiple of 4
                        int lmax = 4;
                        if (k == ncol - 1)  {
                                lmax = Nindv%4;
                                lmax = (lmax==0)?4:lmax;
                        }
                        for ( int l = 0 ; l < lmax; l++){
                                int j = j0 + l ;
                                // Extract  PLINK coded genotype and convert into 0/1/2
                                // PLINK coding:
                                // 00->0
                                // 01->missing
                                // 10->1
                                // 11->2
                                int val = y[l];
                                if(val==1 && !allow_missing){
                                        val = simulate2_geno_from_random(p_j);
                                        val++;
                                        val = (val==1) ? 0 : val;
                                   //val=0;
                                 }
                                val-- ;
                                val =  (val < 0 ) ? 0 :val ;
                                sum += val;
//cout<<"sss"<<endl;
//cout<<global_snp_index<<endl;

                                       bin_pointer=SNP_annot[global_snp_index]-1;

//	cout<<bin_pointer<<endl;
			           int snp_index;
                                     if(use_mailman==true){
                                         snp_index=allgen_mail[bin_pointer].index;
                                         int horiz_seg_no = snp_index/allgen_mail[bin_pointer].segment_size_hori;
                                         allgen_mail[bin_pointer].p[horiz_seg_no][j] = 3 *allgen_mail[bin_pointer].p[horiz_seg_no][j]  + val;
                                     // computing sum for every snp to compute mean
                                         allgen_mail[bin_pointer].columnsum[snp_index]+=val;

                                      }
                                     else{
                                         snp_index=allgen[bin_pointer].index;
                                         allgen[bin_pointer].gen(snp_index,j)=val;
                                     }

                    }
        }

        	bin_pointer=SNP_annot[global_snp_index]-1;
	         if(use_mailman==true)
                  allgen_mail[bin_pointer].index++;
               else
                   allgen[bin_pointer].index++;
}

  sum = 0 ;
  delete[] gtype;

}

MatrixXdr jack_se(MatrixXdr jack){

int nrows=jack.rows();
int ncols=jack.cols();
MatrixXdr sum_row=jack.rowwise().mean();
MatrixXdr SEjack;
SEjack=MatrixXdr::Zero(nrows,1);
double temp_val=0;
for (int i=0;i<nrows;i++){
    for (int j=0;j<ncols;j++){
        temp_val=jack(i,j)-sum_row(i);
        temp_val= temp_val* temp_val;
        SEjack(i,0)+=temp_val;
    }
    SEjack(i,0)=SEjack(i,0)*(Njack-1)/Njack;
    SEjack(i,0)=sqrt(SEjack(i,0));
}

return SEjack;
}

//MatrixXdr maf_ld;
//MatrixXdr A_effect;
void read_mafld(std::string filename){

	maf_ld.resize(Nsnp,3);
	ifstream ifs(filename.c_str(), ios::in);

        std::string line;
        std::istringstream in;
	string temp;

	std::getline(ifs,line);
	in.str(line);
	cout<<"reading MAF and LD scores"<<endl;
	while(in>>temp){
		cout<<temp<<" ";
	}

	cout<<endl;
	double val;
	int i=0;
	while(std::getline(ifs, line)){
		  in.clear();
                  in.str(line);

		  in >> temp;
		  val = atof(temp.c_str());
		  maf_ld(i,0)=val*(1-val);  ///MAF(1-MAF)
		  maf_ld(i,2)=val;

		  in >> temp;
		  val = atof(temp.c_str());
                  maf_ld(i,1)=val;

		  i++;
	}

}

void read_simul_par(std::string filename){

//double p_casual,double ld_ex,double maf_ex,double min_maf,double max_maf,double h2,int num

	int num_par=7;
	int num_vc=2;
	simul_par.resize(num_par,num_vc);
        ifstream ifs(filename.c_str(), ios::in);

        std::string line;
        std::istringstream in;
        string temp;

        std::getline(ifs,line);
        in.str(line);
	cout<<"reading simulation parameters"<<endl;
        while(in>>temp){
                cout<<temp<<" ";
        }
	cout<<endl;
        double val;
        int i=0;
        while(std::getline(ifs, line)){
                  in.clear();
                  in.str(line);

		  for (int j=0;j<num_par;j++){
                  in >> temp;
                  val = atof(temp.c_str());
                  simul_par(j,i)=val;
		  cout<<val<<" ";
		  }
		  cout<<endl;
		  i++;
        }
	cout<<endl;

}

void make_maf_ld_effect(double p_casual,double ld_ex,double maf_ex,double min_maf,double max_maf,double h2,int num){

        std::ofstream outfile;
	if(dominance==true){
	outfile.open("causal_snp_dom.txt", std::ios_base::out);
	}
	else{
        outfile.open("causal_snp_add.txt", std::ios_base::out);
	}
	//MatrixXdr A_effect;
	A_effect.resize(Nsnp,1);
        A_effect_multi.resize(Nsnp,num);

	boost::mt19937 seedr;
	seedr.seed(std::time(0));
        boost::bernoulli_distribution<> b_dist(p_casual);
        boost::variate_generator<boost::mt19937&, boost::bernoulli_distribution<>  > coin(seedr, b_dist);

	MatrixXdr cas;
	cas.resize(Nsnp,1);
	for (int i=0;i<Nsnp;i++){
	 cas(i,0)=coin();
	 if(maf_ld(i,2)<min_maf || maf_ld(i,2)>max_maf)
			cas(i,0)=0;

	}
	cout<<cas.sum()<<endl;

	///write causal snps in files
	outfile<<cas<<endl;
	outfile.close();
	///

	maf_ld.col(0)=maf_ld.col(0).array().pow(maf_ex);
	maf_ld.col(1)=maf_ld.col(1).array().pow(ld_ex);

	/*cout<<"f(1-f)"<<maf_ld(0,0)<<endl;
	cout<<"ld"<<maf_ld(0,1)<<endl;
	cout<<"c"<<cas(0,0)<<endl;
	*/

	A_effect=maf_ld.col(0).array()*maf_ld.col(1).array()*cas.array();

	//cout<<"c_ld_f"<<effect(0,0)<<endl;

	double cons_fact=(double)h2/A_effect.sum();

	//cout<<"const"<<cons_fact<<endl;

	A_effect=A_effect*cons_fact;

	//cout<<"normalized eff"<<effect(0,0)<<endl;
	cout<<"sum  "<<A_effect.sum()<<endl;

	 for (int i=0;i<Nsnp;i++){
		boost::normal_distribution<> dist(0,sqrt(A_effect(i,0)));
        	boost::variate_generator<boost::mt19937&, boost::normal_distribution<> > effect_size(seedr, dist);
	//	cout<<i<<" "<<sqrt(A_effect(i,0))<<endl;
		for (int j=0;j<num;j++){
		A_effect_multi(i,j)=effect_size();

		}
	}

}

void make_dom_effect(double p_casual,double h2d,int num){

	boost::mt19937 seedr;
        seedr.seed(std::time(0));
        boost::bernoulli_distribution<> b_dist(p_casual);
        boost::variate_generator<boost::mt19937&, boost::bernoulli_distribution<>  > coin(seedr, b_dist);

}

void  read_nongen(std::string filename,int num_indv){
        ifstream ifs(filename.c_str(), ios::in);
        std::string line;
        std::istringstream in;

        int j=0;
        while(std::getline(ifs, line)){
                in.clear();
                in.str(line);
                string temp;

		for (int i=0; i<num_indv;i++){
                in>>temp;
                        /*if(temp=="NA")
                        {
                                missing[k].push_back(j);
                                continue;
                        }*/
		double cur = atof(temp.c_str());
		//nongen_vc(j,i)=cur;
		gen(j,i)=cur;
               }
        j++;
        }

}

MatrixXdr  compute_XXy (int num_snp,MatrixXdr pheno){

	res.resize(num_snp, 1);
        if(use_mailman==true)
                multiply_y_pre_fast(pheno,1,res, false);
        else
                res=gen*pheno;

        MatrixXdr zb_sum = pheno.colwise().sum();

        for(int j=0; j<num_snp; j++)
            for(int k=0; k<1;k++)
                res(j,k) = res(j,k)*stds(j,0);

        MatrixXdr resid(num_snp, 1);
        MatrixXdr inter = means.cwiseProduct(stds);
        resid = inter * zb_sum;
        MatrixXdr inter_zb = res - resid;

        for(int k=0; k<1; k++)
            for(int j=0; j<num_snp;j++)
                inter_zb(j,k) =inter_zb(j,k) *stds(j,0);
       MatrixXdr new_zb = inter_zb.transpose();
       MatrixXdr new_res(1, Nindv);

         if(use_mailman==true)
            multiply_y_post_fast(new_zb, 1, new_res, false);
         else
            new_res=new_zb*gen;

       MatrixXdr new_resid(1, num_snp);
       MatrixXdr zb_scale_sum = new_zb * means;
       new_resid = zb_scale_sum * MatrixXdr::Constant(1,Nindv, 1);

                      /// new zb
       MatrixXdr temp=new_res - new_resid;

        for (int i=0;i<1;i++)
           for(int j=0;j<Nindv;j++)
                 temp(i,j)=temp(i,j)*mask(j,0);

        return temp.transpose();

}

MatrixXdr compute_Xtw(int num_snp,MatrixXdr w){

        MatrixXdr res(num_snp, 1);

        if(use_mailman==true)
                multiply_y_pre_fast(w,1,res,false);
        else
                 res=gen*w;

        double w_sum=w.sum();

        res = res.cwiseProduct(stds);
        MatrixXdr resid(num_snp, 1);
        resid = means.cwiseProduct(stds);
        resid = resid *w_sum;
        MatrixXdr Xy(num_snp,1);
        Xy = res-resid;

        return Xy;

}

MatrixXdr  compute_Xv (int num_snp,MatrixXdr new_zb){

	Nz=num_simul;
	for (int j=0;j<Nz;j++)
        for (int i=0;i<num_snp;i++)
                new_zb(j,i)= new_zb(j,i)*stds(i,0);

         MatrixXdr new_res(Nz, Nindv);

        if(use_mailman==true)
                multiply_y_post_fast(new_zb,Nz,new_res, false);
        else
                new_res=new_zb*gen;

        MatrixXdr new_resid(Nz, num_snp);
       MatrixXdr zb_scale_sum = new_zb * means;
       new_resid = zb_scale_sum * MatrixXdr::Constant(1,Nindv, 1);

                      /// new zb
       MatrixXdr temp=new_res - new_resid;

        return temp.transpose();
}

void process(string name, int round){

ifstream ifs (name.c_str(), ios::in|ios::binary);
read_header=true;
global_snp_index=-1;

//////////////////////////

int point_index=0;
for (int jack_index=0;jack_index<Njack;jack_index++){

	cout<<"jack index"<<jack_index<<endl;
        int read_Nsnp=(jack_index<(Njack-1)) ? (step_size) : (step_size+step_size_rem);

       if(use_mailman==true){
        for (int i=0;i<Nbin;i++){
        allgen_mail[i].segment_size_hori = floor(log(Nindv)/log(3)) - 2 ;
        allgen_mail[i].Nsegments_hori = ceil(jack_bin[jack_index][i]*1.0/(allgen_mail[i].segment_size_hori*1.0));
        allgen_mail[i].p.resize(allgen_mail[i].Nsegments_hori,std::vector<int>(Nindv));
        allgen_mail[i].not_O_i.resize(jack_bin[jack_index][i]);
        allgen_mail[i].not_O_j.resize(Nindv);
        allgen_mail[i].index=0;
        allgen_mail[i].Nsnp=jack_bin[jack_index][i];
        allgen_mail[i].Nindv=Nindv;

         allgen_mail[i].columnsum.resize(jack_bin[jack_index][i],1);
          for (int index_temp=0;index_temp<jack_bin[jack_index][i];index_temp++)
                    allgen_mail[i].columnsum[index_temp]=0;

         }
       }
       else{
           for (int bin_index=0;bin_index<Nbin;bin_index++){
                allgen[bin_index].gen.resize(jack_bin[jack_index][bin_index],Nindv);
                allgen[bin_index].index=0;
		//allgen[bin_index].maf.resize(jack_bin[jack_index][bin_index],1);
		allgen[bin_index].maf=MatrixXdr::Zero(jack_bin[jack_index][bin_index],1);
           }
       }

        if(use_1col_annot==true)
                read_bed_1colannot(ifs,missing,read_Nsnp);
        else
                read_bed2(ifs,missing,read_Nsnp);
       read_header=false;
     for (int bin_index=0;bin_index<Nbin;bin_index++){
          int num_snp;
          if (use_mailman==true)
                num_snp=allgen_mail[bin_index].index;
          else
                num_snp=allgen[bin_index].index;

          if(num_snp!=0){
          stds.resize(num_snp,1);
          means.resize(num_snp,1);
	  Eigen::VectorXd mean;
          if(use_mailman==true){
                for (int i=0;i<num_snp;i++)
                   means(i,0)=(double)allgen_mail[bin_index].columnsum[i]/Nindv;
         }
          else{
		 means=allgen[bin_index].gen.rowwise().mean();
                mean=allgen[bin_index].gen.rowwise().mean();
	  }

         if(mean_impute==true & use_mailman==false){

		 stds=1/((allgen[bin_index].gen.colwise()-mean).array().square().rowwise().sum()/(allgen[bin_index].gen.cols()-1)).sqrt();
         }
	 else{

		if(dominance==false){
	 		 for (int i=0;i<num_snp;i++)
               			stds(i,0)=1/sqrt((means(i,0)*(1-(0.5*means(i,0)))));
		}
		else{
			for (int i=0;i<num_snp;i++){
				double mf=allgen[bin_index].maf(i,0);
				stds(i,0)=1/((2*mf)*(1-mf));
			//cout<<stds(i,0)<<endl;
			}
		}
        }

	if (use_mailman==true){
                g=allgen_mail[bin_index];
                 g.segment_size_hori = floor(log(Nindv)/log(3)) - 2 ;
                 g.Nsegments_hori = ceil(jack_bin[jack_index][bin_index]*1.0/(g.segment_size_hori*1.0));
                 g.p.resize(g.Nsegments_hori,std::vector<int>(Nindv));
                 g.not_O_i.resize(jack_bin[jack_index][bin_index]);
                 g.not_O_j.resize(Nindv);
                initial_var();
           }
          else{
                 gen=allgen[bin_index].gen;

          }

////////////////////

/// save dominance grm
/*      if(dominance==true){
	MatrixXdr grm= gen*gen.transpose();
	 std::ofstream outfile;
        string add_output;
        add_output="dominance_smallscale.grm.gz";
        outfile.open(add_output.c_str(), std::ios_base::out);
	      for (int i=0;i<Nindv;i++)
                for (int j=0;j<=i;j++){
                        double grmval=(double)grm(i,j)/Nsnp;
                        outfile<<i+1<<" "<<j+1<<" "<<Nsnp<<" "<<grmval<<endl;
                }
     }*/
///////

	MatrixXdr vec_eff=A_effect_multi.block(point_index,0,num_snp,num_simul).transpose();
	point_index=point_index+num_snp;
	simul_pheno+=compute_Xv(num_snp,vec_eff);

/////////////////

        //cout<<num_snp<< "SNPs in bin "<<bin_index<<"of jack "<<jack_index<<endl;
        //cout<<" Reading and computing bin "<<bin_index <<"  of "<< jack_index<<"-th is finished"<<endl;

            if(use_mailman==true){
                delete[] sum_op;
                delete[] partialsums;
                 delete[] yint_e;
                delete[] yint_m;
                for (int i  = 0 ; i < hsegsize; i++)
                        delete[] y_m [i];
                delete[] y_m;

                for (int i  = 0 ; i < g.Nindv; i++)
                        delete[] y_e[i];
                delete[] y_e;

                std::vector< std::vector<int> >().swap(g.p);
                std::vector< std::vector<int> >().swap(g.not_O_j);
                std::vector< std::vector<int> >().swap(g.not_O_i);
                std::vector< std::vector<int> >().swap(allgen_mail[bin_index].p);
                std::vector< std::vector<int> >().swap(allgen_mail[bin_index].not_O_j);
                std::vector< std::vector<int> >().swap(allgen_mail[bin_index].not_O_i);
        //g.p.clear();
        //g.not_O_j.clear();
        //g.not_O_i.clear();
                g.columnsum.clear();
                g.columnsum2.clear();
                g.columnmeans.clear();
                 g.columnmeans2.clear();
                 allgen_mail[bin_index].columnsum.clear();
                allgen_mail[bin_index].columnsum2.clear();
                allgen_mail[bin_index].columnmeans.clear();
                 allgen_mail[bin_index].columnmeans2.clear();
            }
        }

     }
//cout<<" Reading and computing  of "<< jack_index<<"-th is finished"<<endl;
}

}

int main(int argc, char const *argv[]){

	parse_args(argc,argv);
////////////////////////////////////////////
///////////////////////////////////////////

        int B = command_line_opts.batchNum;
        k_orig = command_line_opts.num_of_evec ;
        debug = command_line_opts.debugmode ;
        check_accuracy = command_line_opts.getaccuracy;
        var_normalize = false;
        accelerated_em = command_line_opts.accelerated_em;
        k = k_orig + command_line_opts.l;
        k = (int)ceil(k/10.0)*10;
        command_line_opts.l = k - k_orig;
        srand((unsigned int) time(0));
	Nz=command_line_opts.num_of_evec;
        k=Nz;

	Njack=command_line_opts.jack_number;

////
string filename;
//////////////////////////// Read multi genotypes
string line;
int cov_num;
int num_files=0;
string geno_name=command_line_opts.GENOTYPE_FILE_PATH;

/////////Read bim file to count # SNPs
std::stringstream f1;
f1 << geno_name << ".bim";
read_bim (f1.str());

//////Read annotation files
filename=command_line_opts.Annot_PATH;

if(use_1col_annot==true){
read_annot_1col(filename);
}
else{
read_annot(filename);
}
//filename=command_line_opts.Annot_PATH;

///reading phnotype and save the number of indvs
filename=command_line_opts.PHENOTYPE_FILE_PATH;
count_pheno(filename);

std::stringstream f0;
f0 << geno_name << ".fam";
string name_fam=f0.str();
int fam_lines=count_fam(name_fam);

create_simul_pheno(name_fam);

cout<<"Number of Indvs :"<<Nindv<<endl;

///// code for handeling overlapping annotations
std::stringstream f3;
f3 << geno_name << ".bed";
string name=f3.str();
cout<<name<<endl;
ifstream ifs (name.c_str(), ios::in|ios::binary);
read_header=true;
global_snp_index=-1;

////read maf ld file

filename=command_line_opts.maf_ld_PATH;
read_mafld(filename);

global_MAF.resize(Nsnp,1);

allgen_mail.resize(Nbin);
//allgen.resize(Nbin);

double h2A=0;
double h2D=0;
filename=command_line_opts.simul_par_PATH;
read_simul_par(filename);
make_maf_ld_effect(simul_par(0,0),simul_par(1,0),simul_par(2,0),simul_par(3,0),simul_par(4,0),simul_par(5,0),simul_par(6,0));

num_simul=simul_par(6,0);

Nz=num_simul;
k=Nz;
cout<<"number of simulations"<<num_simul<<endl;

simul_pheno=MatrixXdr::Zero(Nindv,num_simul);

use_mailman=true;
dominance=false;
process(name,1);
h2A=simul_par(5,0);

bool simul_dom=false;

if(simul_dom==true){
dominance=true;

make_maf_ld_effect(simul_par(0,1),simul_par(1,1),simul_par(2,1),simul_par(3,1),simul_par(4,1),simul_par(5,1),simul_par(6,1));

//dominance=true;
process(name,1);
 h2D=simul_par(5,1);
}

 boost::mt19937 seedr;
boost::normal_distribution<> dist(0,sqrt(1-h2A-h2D));
boost::variate_generator<boost::mt19937&, boost::normal_distribution<> > effect_env(seedr, dist);

std::ofstream outfile;
string add_output=command_line_opts.OUTPUT_FILE_PATH;

cout<<"writing phenos in "<<add_output<<endl;

for ( int i=0;i<num_simul;i++){
	cout<<"writing "<<i<<"-th simulation"<<endl;

	string index=boost::lexical_cast<string>(i);
	string name=add_output+index+".pheno";
	std::ofstream outfile;
	outfile.open(name, std::ios_base::out);

	cout<<name<<endl;
	outfile<<"FID IID pheno"<<endl;
	for (int j=0;j<Nindv;j++){
		outfile<<fid[j]<<" "<<fid[j]<<" "<<simul_pheno(j,i)+effect_env()<<endl;
	}
	outfile.close();
}

/*
cout<<"xxxxxxxxx"<<endl;
use_mailman=false;
dominance=true;
process(name,1);
*/

return 0;

}


#include <base/cgal_typedefs.h>
#include <IO/fileIO.h>
#include <util/helper.h>
#include <util/geometricOperations.h>

#include <processing/tetIntersection.h>
#include <processing/rayTracingTet.h>
#include <processing/meshProcessing.h>
#include <processing/graphCut.h>
#include <processing/edgeManifoldness.h>
#include <processing/pointSetProcessing.h>
#include <processing/meshProcessing.h>

#include <learning/learning.h>
#include <learning/learningMath.h>
#include <learning/learningRayTracing.h>
#include <learning/learningIO.h>

#include <CGAL/point_generators_3.h>
#include <CGAL/Random.h>
#include <CGAL/boost/graph/convert_nef_polyhedron_to_polygon_mesh.h>
#include <CGAL/boost/graph/graph_traits_Polyhedron_3.h>
#include <boost/foreach.hpp>
#include <CGAL/Polygon_mesh_processing/distance.h>
#include <CGAL/polygon_mesh_processing.h>
#include <CGAL/optimal_bounding_box.h>
#include <CGAL/Side_of_triangle_mesh.h>

#include <util/helper.h>

#include <random>

using namespace std;

int labelObjectWithOpenGroundTruth(dirHolder dir, dataHolder& data, int sampling_points){

    auto start = std::chrono::high_resolution_clock::now();

    if(data.gt_poly.size_of_vertices() == 0){
        cout << "\nERROR: you need to load a ground truth polygon (with -g) for scanning it" << endl;
        return 1;
    }

    assert(sampling_points > 0);

    cout << "\nLabelling cells with open ground truth and ray tracing..." << endl;
    cout << "\t-sample " << sampling_points << endl;

    // Initialize the point-in-polyhedron tester
    // Construct AABB tree with a KdTree

    SurfaceMesh smesh;
    CGAL::copy_face_graph(data.gt_poly, smesh);
    CGAL::Polygon_mesh_processing::remove_connected_components_of_negligible_size(smesh);
//    dir.suffix = "_repaired_mesh";
//    exportOFF(dir,smesh);
//    cout << "gt centroid: " << data.gt_centroid << endl;

//    Tree tree(faces(smesh).first, faces(smesh).second, smesh);

    vector<face_descriptor> fds;
    for(face_descriptor fd : faces(smesh)){
        if(!CGAL::Polygon_mesh_processing::is_degenerate_triangle_face(fd,smesh))
            fds.push_back(fd);

    }
    Tree tree(fds.begin(), fds.end(), smesh);
    tree.accelerate_distance_queries();

//    Polyhedron poly;
//    CGAL::copy_face_graph(data.sm_obb, poly);
//    Tree bb_tree(faces(data.sm_obb).first, faces(data.sm_obb).second, data.sm_obb);
//    const Point_inside inside_bb(data.sm_obb);

    int icount = 0;
    int ncount = 0;
    Delaunay::All_cells_iterator aci;
    // random sampler
    CGAL::Random random(42);
    vector<Point> sampled_points;
    double iperc;
    double operc;
    int newCellIndex = 0;
    vector<Point> all_sampled_points;
    vector<vertex_info> all_sampled_infos;
    int progress = 0;
    int total = data.Dt.number_of_cells();
    for(aci = data.Dt.all_cells_begin(); aci != data.Dt.all_cells_end(); aci++){
//        cout << progress++ << endl;
//        if((progress*100/total) % 10 == 0 && (++progress*100/total) % 10 != 0)
//            cout << progress << "/" << total << endl;

        aci->info().global_idx=newCellIndex++;

        if(data.Dt.is_infinite(aci)){
            aci->info().gc_label = 1; // more likely that it is outside
            aci->info().inside_score = 0.0;
            aci->info().outside_score = 1.0;
            continue;
        }

        // sample points inside the tet and check if they are inside or outside
        double cinside=0;
        double coutside=0;
        Tetrahedron current_tet = data.Dt.tetrahedron(aci);
        CGAL::Random_points_in_tetrahedron_3<Point> tet_point_sampler(current_tet, random);
        sampled_points.clear();
        CGAL::cpp11::copy_n(tet_point_sampler, sampling_points, std::back_inserter(sampled_points));
        for(auto const& sampled_point : sampled_points){

//            all_sampled_points.push_back(sampled_point);
//            vertex_info vi;
//            if(inside_bb(sampled_point) == CGAL::ON_UNBOUNDED_SIDE){
//                vi.color = CGAL::violet();
//                all_sampled_infos.push_back(vi);
//                cinside+=1.0;
//            }
//            else{
            Segment seg(sampled_point,data.gt_centroid);
            int n = tree.number_of_intersected_primitives(seg);
//            cout << "number of intersected primitives: " << n << endl;
            if(n % 2){
                cinside+=1.0;
//                vi.color = CGAL::red();
//                all_sampled_infos.push_back(vi);
            }
            else{
                coutside+=1.0;
//                vi.color = CGAL::blue();
//                all_sampled_infos.push_back(vi);
            }
//            if(n == 0)
//                vi.color = CGAL::yellow();
//            else if(n == 1)
//                vi.color = CGAL::green();
//            else if(n == 2)
//                vi.color = CGAL::blue();
//            else if(n == 3)
//                vi.color = CGAL::red();
//            else
//                vi.color = CGAL::black();
//            all_sampled_infos.push_back(vi);

//            }
        }
        iperc = cinside/(cinside+coutside);
        operc = coutside/(cinside+coutside);
        aci->info().outside_score = operc;
        aci->info().inside_score = iperc;
//        aci->info().gt_outside = operc;
//        aci->info().gt_inside = iperc;
        if(operc>iperc){
            ncount++;
//            aci->info().gc_label = 1;
        }
        else{
            icount++;
//            aci->info().gc_label = 0;
        }

    }
//    dir.suffix = "_sampled";
//    exportOptions eo;
//    eo.color = true;
//    exportPLY(dir,all_sampled_points,all_sampled_infos,eo);

    auto stop = chrono::high_resolution_clock::now();
    auto duration = chrono::duration_cast<std::chrono::seconds>(stop - start);
    cout << "\t-Labelled object with " << icount << "/" << ncount << "(inside/outside) = " << icount+ncount << " cells" << endl;
    cout << "\t-in "<<duration.count() << "s" << endl;
    if(icount == 0 || ncount == 0){
        cout << "\nERROR: cells could not be labelled" << endl;
        return 1;
    }
    return 0;
}

int labelObjectWithClosedGroundTruth(dirHolder dir, dataHolder& data, int sampling_points){

    auto start = std::chrono::high_resolution_clock::now();

    if(data.gt_poly.size_of_vertices() == 0){
        cout << "\nERROR: you need to load a ground truth polygon (with -g) for scanning it" << endl;
        return 1;
    }
    if(!data.gt_poly.is_closed()){
        cout << "\nERROR: ground truth is not closed" << endl;
        return 1;
    }

    assert(sampling_points > 0);

    cout << "\nLabelling cells by sampling points inside cells..." << endl;
    cout << "\t-sample " << sampling_points << endl;

    // Initialize the point-in-polyhedron tester
    // Construct AABB tree with a KdTree
    SurfaceMesh smesh;
    CGAL::copy_face_graph(data.gt_poly, smesh);
    Tree tree(faces(smesh).first, faces(smesh).second, smesh);
//    Tree tree(faces(data.gt_poly).first, faces(data.gt_poly).second, data.gt_poly);
    tree.accelerate_distance_queries();
    // Initialize the point-in-polyhedron tester
    const Point_inside inside_tester(tree);
//    Polyhedron poly;
//    CGAL::copy_face_graph(data.sm_obb, poly);
////    Tree bb_tree(faces(data.sm_obb).first, faces(data.sm_obb).second, data.sm_obb);
//    const Point_inside inside_bb(poly);

    int icount = 0;
    int ncount = 0;
    Delaunay::All_cells_iterator aci;
    // random sampler
    CGAL::Random random(42);
    vector<Point> sampled_points;
    double iperc;
    double operc;
    int newCellIndex = 0;
    vector<Point> all_sampled_points;
    vector<vertex_info> all_sampled_infos;
    for(aci = data.Dt.all_cells_begin(); aci != data.Dt.all_cells_end(); aci++){

        aci->info().global_idx=newCellIndex++;

        if(data.Dt.is_infinite(aci)){
            aci->info().gc_label = 1; // more likely that it is outside
            aci->info().inside_score = 0.0;
            aci->info().outside_score = 1.0;
            continue;
        }

        // sample points inside the tet and check if they are inside or outside
        double cinside=0;
        double coutside=0;
        Tetrahedron current_tet = data.Dt.tetrahedron(aci);
        CGAL::Random_points_in_tetrahedron_3<Point> tet_point_sampler(current_tet, random);
        sampled_points.clear();
        CGAL::cpp11::copy_n(tet_point_sampler, sampling_points, std::back_inserter(sampled_points));
        for(auto const& sampled_point : sampled_points){
//            all_sampled_points.push_back(sampled_point);
//            vertex_info vi;

////            cout<<"\t check point "<< pcount++ <<"/"<<sampling_points<<endl;
//            if(inside_bb(sampled_point) != CGAL::ON_BOUNDED_SIDE){
//                vi.color = CGAL::green();
//                all_sampled_infos.push_back(vi);
//                coutside+=1.0;
//                continue;
//            }

            if(inside_tester(sampled_point) == CGAL::ON_BOUNDED_SIDE){
                cinside+=1.0;
//                vi.color = CGAL::red();
//                all_sampled_infos.push_back(vi);
            }
            else{
                coutside+=1.0;
//                vi.color = CGAL::blue();
//                all_sampled_infos.push_back(vi);
            }
        }
        iperc = cinside/(cinside+coutside);
        operc = coutside/(cinside+coutside);
        aci->info().outside_score = operc;
        aci->info().inside_score = iperc;

        if(operc>iperc){
            ncount++;
//            aci->info().gc_label = 1;
        }
        else{
            icount++;
//            aci->info().gc_label = 0;
        }

    }
//    dir.suffix = "_sampled";
//    exportOptions eo;
//    eo.color = true;
//    exportPLY(dir,all_sampled_points,all_sampled_infos,eo);

    auto stop = chrono::high_resolution_clock::now();
    auto duration = chrono::duration_cast<std::chrono::seconds>(stop - start);
    cout << "\t-Labelled object with " << icount << "/" << ncount << "(inside/outside) = " << icount+ncount << " cells" << endl;
    cout << "\t-in "<<duration.count() << "s" << endl;
    if(icount == 0 || ncount == 0){
        cout << "\nERROR: cells could not be labelled" << endl;
        return 1;
    }
    return 0;
}

#ifdef RECONBENCH
int labelObjectWithImplicit(dataHolder& data, int sampling_points){

    auto start = std::chrono::high_resolution_clock::now();

    assert(sampling_points > 0);

    cout << "\nLabelling cells by sampling points inside cells: " << endl;
    cout << "\t-sample " << sampling_points << endl;

    int icount = 0;
    int ncount = 0;
    Delaunay::All_cells_iterator aci;
    // random sampler
    CGAL::Random random(42);
    vector<Point> sampled_points;
    double iperc;
    double operc;
    int newCellIndex = 0;
    for(aci = data.Dt.all_cells_begin(); aci != data.Dt.all_cells_end(); aci++){

        if(data.Dt.is_infinite(aci)){
            aci->info().gc_label = 1; // more likely that it is outside
            aci->info().inside_score = 0.2;
            aci->info().outside_score = 0.8;
            aci->info().global_idx=newCellIndex++;
            continue;
        }
        aci->info().global_idx=newCellIndex++;

        // sample points inside the tet and check if they are inside or outside
        double cinside=0;
        double coutside=0;
        Tetrahedron current_tet = data.Dt.tetrahedron(aci);
        CGAL::Random_points_in_tetrahedron_3<Point> tet_point_sampler(current_tet, random);
        sampled_points.clear();
        CGAL::cpp11::copy_n(tet_point_sampler, sampling_points, std::back_inserter(sampled_points));
        for(auto const& sampled_point : sampled_points){

            auto sampled_point_rb = point2Vector3RB(sampled_point);

            if(data.implicit->function(sampled_point_rb) <= 0.0){
                cinside+=1.0;
//                Color col = CGAL::make_array((unsigned char)255, (unsigned char)0, (unsigned char)0);
//                vertex_info vi;
//                vi.color = col;
//                all_sampled_infos.push_back(vi);
            }
            else{
                coutside+=1.0;
//                Color col = CGAL::make_array((unsigned char)0, (unsigned char)0, (unsigned char)255);
//                vertex_info vi;
//                vi.color = col;
//                all_sampled_infos.push_back(vi);
            }
        }
        iperc = cinside/(cinside+coutside);
        operc = coutside/(cinside+coutside);
        aci->info().outside_score = operc;
        aci->info().inside_score = iperc;
        if(operc>iperc){
            ncount++;
            aci->info().gc_label = 1;
        }
        else{
            icount++;
            aci->info().gc_label = 0;
        }

    }

    auto stop = chrono::high_resolution_clock::now();
    auto duration = chrono::duration_cast<std::chrono::seconds>(stop - start);
    cout << "\t-Labelled object with " << icount << "/" << ncount << "(inside/outside) = " << icount+ncount << " cells" << endl;
    cout << "\t-in "<<duration.count() << "s" << endl;
    if(icount == 0 || ncount == 0)
        return 0;
    return 1;
}
#endif


/* Copyright (C) 2012-2017 IBM Corp.
 * This program is Licensed under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at
 *   http://www.apache.org/licenses/LICENSE-2.0
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License. See accompanying LICENSE file.
 */

/* Test_General.cpp - A general test program that uses a mix of operations over four ciphertexts.
 */
#include <NTL/ZZ.h>
#include "FHE.h"
#include "timing.h"
#include "EncryptedArray.h"
#include <NTL/lzz_pXFactoring.h>

#include <cassert>
#include <cstdio>

#ifdef DEBUG_PRINTOUT
#define debugCompare(ea,sk,p,c) {\
  NewPlaintextArray pp(ea);\
  ea.decrypt(c, sk, pp);\
  if (!equals(ea, pp, p)) { \
    cerr << "oops:\n"; cerr << p << "\n"; \
    cerr << pp << "\n"; \
    exit(0); \
  }}
#else
#define debugCompare(ea,sk,p,c)
#endif

void  TestIt(long R, long p, long r, long d, long c, long k, long w,
               long L, long m, const Vec<long>& gens, const Vec<long>& ords)
{
  char buffer[32];
  cerr << "\n\n******** TestIt" << (isDryRun()? "(dry run):" : ":");
  cerr << " R=" << R
       << ", p=" << p
       << ", r=" << r
       << ", d=" << d
       << ", c=" << c
       << ", k=" << k
       << ", w=" << w
       << ", L=" << L
       << ", m=" << m
       << ", gens=" << gens
       << ", ords=" << ords
       << endl;

  vector<long> gens1, ords1;
  convert(gens1, gens);
  convert(ords1, ords);

  FHEcontext context(m, p, r, gens1, ords1);
  buildModChain(context, L, c);

  context.zMStar.printout();
  cerr << endl;

  cerr << "# ctxt primes = " << context.ctxtPrimes.card() << "\n";
  cerr << "# bits in ctxt primes = "
       << long(context.logOfProduct(context.ctxtPrimes)/log(2.0) + 0.5) << "\n";
  cerr << "# special primes = " << context.specialPrimes.card() << "\n";
  cerr << "# bits in special primes = "
       << long(context.logOfProduct(context.specialPrimes)/log(2.0) + 0.5) << "\n";

  FHESecKey secretKey(context);
  const FHEPubKey& publicKey = secretKey;
  secretKey.GenSecKey(w); // A Hamming-weight-w secret key

  ZZX G;

  if (d == 0)
    G = context.alMod.getFactorsOverZZ()[0];
  else
    G = makeIrredPoly(p, d);

  cerr << "G = " << G << "\n";
  cerr << "generating key-switching matrices... ";
  addSome1DMatrices(secretKey); // compute key-switching matrices that we need
  cerr << "done\n";

  cerr << "computing masks and tables for rotation...";
  EncryptedArray ea(context, G);
  cerr << "done\n";

  long nslots = ea.size();

  NewPlaintextArray p0(ea);
  NewPlaintextArray p1(ea);
  NewPlaintextArray p2(ea);
  NewPlaintextArray p3(ea);

  random(ea, p0);
  random(ea, p1);
  random(ea, p2);
  random(ea, p3);

  Ctxt c0(publicKey), c1(publicKey), c2(publicKey), c3(publicKey);
  ea.encrypt(c0, publicKey, p0);
  ea.encrypt(c1, publicKey, p1); // real encryption
  ea.encrypt(c2, publicKey, p2); // real encryption
  ea.encrypt(c3, publicKey, p3); // real encryption

  c0.cleanUp();
  c1.cleanUp();

  resetAllTimers();

  FHE_NTIMER_START(Circuit);

  for (long i = 0; i < R; i++) {

    cerr << "*** round " << i << "..."<<endl;

     mul(ea, p1, p0);     // c1.multiplyBy(c0)
     c1.multiplyBy(c0);
     CheckCtxt(c1, "c1*=c0");
     debugCompare(ea,secretKey,p1,c1);

  }

  c1.cleanUp();

  FHE_NTIMER_STOP(Circuit);

  cerr << endl;
  printAllTimers();
  cerr << endl;

  resetAllTimers();
  FHE_NTIMER_START(Check);

  NewPlaintextArray pp0(ea);
  NewPlaintextArray pp1(ea);
  NewPlaintextArray pp2(ea);
  NewPlaintextArray pp3(ea);

  ea.decrypt(c0, secretKey, pp0);
  ea.decrypt(c1, secretKey, pp1);
  ea.decrypt(c2, secretKey, pp2);
  ea.decrypt(c3, secretKey, pp3);

  if (!equals(ea, pp0, p0))  cerr << "oops 0\n";
  if (!equals(ea, pp1, p1))  cerr << "oops 1\n";
  if (!equals(ea, pp2, p2))  cerr << "oops 2\n";
  if (!equals(ea, pp3, p3))  cerr << "oops 3\n";

  FHE_NTIMER_STOP(Check);

  cerr << endl;
  printAllTimers();
  cerr << endl;

#if 0

  vector<Ctxt> vc(L,c0);            // A vector of L ciphertexts
  vector<NewPlaintextArray> vp(L, p0); // A vector of L plaintexts
  for (long i=0; i<L; i++) {
    vp[i].random();                     // choose a random plaintext
    ea.encrypt(vc[i], publicKey, vp[i]); // encrypt it
    if (i>0) vp[i].mul(vp[i-1]); // keep a running product of plaintexts
  }
  incrementalProduct(vc); // Compute the same running product homomorphically

  // Check that the products match
  bool fail = false;
  for (long i=0; i<L; i++) {
    ea.decrypt(vc[i], secretKey, p0); // decrypt it
    if (!p0.equals(vp[i])) {
      fail = true;
      cerr << "incrementalProduct oops "<<i<< endl;
    }
  }
  if (!fail) cerr << "incrementalProduct works\n";
#endif

}

/* A general test program that uses a mix of operations over four ciphertexts.
 * Usage: Test_General_x [ name=value ]...
 *   R       number of rounds  [ default=1 ]
 *   p       plaintext base  [ default=2 ]
 *   r       lifting  [ default=1 ]
 *   d       degree of the field extension  [ default=1 ]
 *              d == 0 => factors[0] defines extension
 *   c       number of columns in the key-switching matrices  [ default=2 ]
 *   k       security parameter  [ default=80 ]
 *   L       # of levels in the modulus chain  [ default=heuristic ]
 *   s       minimum number of slots  [ default=0 ]
 *   repeat  number of times to repeat the test  [ default=1 ]
 *   m       use specified value as modulus
 *   mvec    use product of the integers as  modulus
 *              e.g., mvec='[5 3 187]' (this overwrite the m argument)
 *   gens    use specified vector of generators
 *              e.g., gens='[562 1871 751]'
 *   ords    use specified vector of orders
 *              e.g., ords='[4 2 -4]', negative means 'bad'
 */
int main(int argc, char **argv)
{
  setTimersOn();

  ArgMapping amap;

  bool dry=false;
  amap.arg("dry", dry, "dry=1 for a dry-run");

  long R=1;
  amap.arg("R", R, "number of rounds");

  long p=2;
  amap.arg("p", p, "plaintext base");

  long r=1;
  amap.arg("r", r,  "lifting");

  long d=1;
  amap.arg("d", d, "degree of the field extension");
  amap.note("d == 0 => factors[0] defines extension");

  long c=2;
  amap.arg("c", c, "number of columns in the key-switching matrices");

  long k=80;
  amap.arg("k", k, "security parameter");

  long L=0;
  amap.arg("L", L, "# of levels in the modulus chain",  "heuristic");

  long s=0;
  amap.arg("s", s, "minimum number of slots");

  long repeat=1;
  amap.arg("repeat", repeat,  "number of times to repeat the test");

  long chosen_m=0;
  amap.arg("m", chosen_m, "use specified value as modulus", NULL);

  Vec<long> mvec;
  amap.arg("mvec", mvec, "use product of the integers as  modulus", NULL);
  amap.note("e.g., mvec='[5 3 187]' (this overwrite the m argument)");

  Vec<long> gens;
  amap.arg("gens", gens, "use specified vector of generators", NULL);
  amap.note("e.g., gens='[562 1871 751]'");

  Vec<long> ords;
  amap.arg("ords", ords, "use specified vector of orders", NULL);
  amap.note("e.g., ords='[4 2 -4]', negative means 'bad'");

  long seed=0;
  amap.arg("seed", seed, "PRG seed");

  amap.parse(argc, argv);

  SetSeed(ZZ(seed));

  if (L==0) { // determine L based on R,r
    L = 3*R+3;
    if (p>2 || r>1) { // add some more primes for each round
      long addPerRound = 2*ceil(log((double)p)*r*3)/(log(2.0)*FHE_p2Size) +1;
      L += R * addPerRound;
    }
  }

  long w = 64; // Hamming weight of secret key
  //  long L = z*R; // number of levels

  if (mvec.length()>0)
    chosen_m = computeProd(mvec);
  long m = FindM(k, L, c, p, d, s, chosen_m, true);

  setDryRun(dry);
  for (long repeat_cnt = 0; repeat_cnt < repeat; repeat_cnt++) {
    TestIt(R, p, r, d, c, k, w, L, m, gens, ords);
  }
}

// call to get our running test case:
// Test_General_x p=23 m=20485 L=10 R=5
//
// another call to get an example where phi(m) is very
// close to m:
// Test_General_x m=18631 L=10 R=5


#include <boost/test/unit_test.hpp>
#include <boost/test/data/test_case.hpp>

#include <vector>
#include <random>
#include <limits>
#include <cassert>
#include <iostream>
#include <algorithm>

namespace bdata = boost::unit_test::data;

using vi = std::vector<int>;
using uint = std::uint32_t;

/**
 * Create a length-n vector<int> of random integers, use seed s;
 * Optional:
 *  Exclude all integers, if any, specified in xs
 *  Include all integers, if any, specified in ys
 *  Use random integers from the range [lo, hi)
 */
vi create_vector(uint n, uint s, vi xs, vi ys, int lo, int hi) {
    assert(n >= ys.size());

    std::mt19937 engine(s);
    std::uniform_int_distribution<> uniform(lo, hi);

    vi zs(n - ys.size());

    for (uint i = 0; i < n - ys.size(); ++i) {
        auto z = uniform(engine);

        while (std::find(xs.begin(), xs.end(), z) != xs.end()) {
            z = uniform(engine);
        }

        zs[i] = z;
    }

    zs.insert(zs.end(), ys.begin(), ys.end());

    return zs;
}

vi create_vector(uint n, uint s, vi xs, vi ys) {
    auto l = std::numeric_limits<int>::min();
    auto h = std::numeric_limits<int>::max();
    return create_vector(n, s, xs, ys, l, h);
}

vi create_vector(uint n, uint s) {
    return create_vector(n, s, {}, {});
}

bool equal_contents(vi xs, vi ys) {
    std::sort(xs.begin(), xs.end());
    std::sort(ys.begin(), ys.end());

    return std::equal(xs.begin(), xs.end(), ys.begin(), ys.end());
}


// Boost.Geometry (aka GGL, Generic Geometry Library)
// QuickBook Example

// Copyright (c) 2014 Barend Gehrels, Amsterdam, the Netherlands.

// Use, modification and distribution is subject to the Boost Software License,
// Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)

//[read_wkt
//` Shows the usage of read_wkt

#include <boost/geometry.hpp>
#include <boost/geometry/geometries/point_xy.hpp>
#include <boost/geometry/geometries/linestring.hpp>
#include <boost/geometry/geometries/polygon.hpp>

int main()
{
    typedef boost::geometry::model::d2::point_xy<double> point_type;

    point_type a;
    boost::geometry::model::linestring<point_type> b;
    boost::geometry::model::polygon<point_type> c;
    boost::geometry::model::box<point_type> d;
    boost::geometry::model::segment<point_type> e;

    boost::geometry::read_wkt("POINT(1 2)", a);
    boost::geometry::read_wkt("LINESTRING(0 0,2 2,3 1)", b);
    boost::geometry::read_wkt("POLYGON((0 0,0 7,4 2,2 0,0 0))", c);
    boost::geometry::read_wkt("BOX(0 0,3 3)", d);
    boost::geometry::read_wkt("SEGMENT(1 0,3 4)", e);

    return 0;
}

//]


/*ckwg +29
 * Copyright 2012-2013 by Kitware, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *  * Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *
 *  * Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 *  * Neither name of Kitware, Inc. nor the names of any contributors may be used
 *    to endorse or promote products derived from this software without specific
 *    prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS''
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "statistics.h"

#include <boost/accumulators/accumulators.hpp>
#include <boost/accumulators/statistics/count.hpp>
#include <boost/accumulators/statistics/max.hpp>
#include <boost/accumulators/statistics/mean.hpp>
#include <boost/accumulators/statistics/median.hpp>
#include <boost/accumulators/statistics/min.hpp>
#include <boost/accumulators/statistics/stats.hpp>
#include <boost/accumulators/statistics/variance.hpp>

/**
 * \file statistics.cxx
 *
 * \brief Implementation of a statistics class.
 */

namespace ba = boost::accumulators;

namespace sprokit
{

class statistics::priv
{
  public:
    priv();
    ~priv();

    typedef ba::accumulator_set<data_point_t, ba::stats
      < ba::tag::count
      , ba::tag::sum
      , ba::tag::min
      , ba::tag::max
      , ba::tag::mean
      , ba::tag::median(ba::with_p_square_quantile)
      , ba::tag::variance(ba::lazy)
      > > stats_t;

    stats_t acc;
    data_points_t data;
};

statistics
::statistics(data_points_t const& pts)
  : d(new priv)
{
  add_points(pts);
}

statistics
::~statistics()
{
}

void
statistics
::add_point(data_point_t pt)
{
  d->acc(pt);
  d->data.push_back(pt);
}

void
statistics
::add_points(data_points_t const& pts)
{
  d->acc = std::for_each(pts.begin(), pts.end(), d->acc);
  d->data.insert(d->data.end(), pts.begin(), pts.begin());
}

statistics::data_points_t
statistics
::data() const
{
  return d->data;
}

size_t
statistics
::count() const
{
  return ba::count(d->acc);
}

statistics::data_point_t
statistics
::sum() const
{
  return ba::sum(d->acc);
}

statistics::data_point_t
statistics
::minimum() const
{
  return ba::min(d->acc);
}

statistics::data_point_t
statistics
::maximum() const
{
  return ba::max(d->acc);
}

statistics::data_point_t
statistics
::range() const
{
  return (maximum() - minimum());
}

double
statistics
::mean() const
{
  return ba::mean(d->acc);
}

double
statistics
::median() const
{
  return ba::median(d->acc);
}

double
statistics
::variance() const
{
  return ba::variance(d->acc);
}

double
statistics
::standard_deviation() const
{
  return sqrt(variance());
}

statistics::priv
::priv()
  : acc()
  , data()
{
}

statistics::priv
::~priv()
{
}

}