//          Copyright Gavin Band 2008 - 2012.
// Distributed under the Boost Software License, Version 1.0.
//    (See accompanying file LICENSE_1_0.txt or copy at
//          http://www.boost.org/LICENSE_1_0.txt)

#include <vector>
#include <unordered_map>
#include <ctime>
#include <boost/function.hpp>
#include <boost/format.hpp>
#include <boost/filesystem.hpp>
#include <boost/timer/timer.hpp>
#include <boost/functional/hash.hpp>
#include <boost/random/mersenne_twister.hpp>
#include <boost/random/discrete_distribution.hpp>
#include <boost/bind.hpp>

#include "config/package_revision_autogenerated.hpp"

#include "metro/constants.hpp"
#include "metro/ShotgunStochasticSearch.hpp"
#include "metro/regression/Design.hpp"
#include "metro/regression/Logistic.hpp"
#include "metro/IndependentParameterDistribution.hpp"
#include "metro/distributions/LogF.hpp"
#include "metro/distributions/Flat.hpp"
#include "metro/regression/LogUnnormalisedPosterior.hpp"
#include "metro/intersect_ranges.hpp"
#include "metro/fit_model.hpp"
#include "metro/CholeskyStepper.hpp"
#include "metro/SampleRange.hpp"
#include "metro/log_sum_exp.hpp"

#include "appcontext/CmdLineOptionProcessor.hpp"
#include "appcontext/ApplicationContext.hpp"

#include "genfile/MissingValue.hpp"
#include "genfile/bgen/View.hpp"
#include "genfile/bgen/Query.hpp"
#include "genfile/SNPDataSource.hpp"
#include "genfile/db/Error.hpp"
#include "genfile/ToGP.hpp"
#include "genfile/CrossCohortCovariateValueMapping.hpp"
#include "genfile/CommonSNPFilter.hpp"
#include "genfile/VariantIdentifyingDataFilteringSNPDataSource.hpp"

#include "qcdb/FlatFileMultiVariantOutputter.hpp"

namespace bfs = boost::filesystem ;

// #define DEBUG 1
// #define DEBUG_DOSAGESTORE 1

namespace globals {
	std::string const program_name = "multifinemap" ;
	std::string const program_version = package_version ;
	std::string const program_revision =  std::string( package_revision ).substr( 0, 7 ) ;
}

namespace {
	std::string print_state( std::size_t N, std::vector< std::size_t > const& states ) {
		std::string result = "[";
		std::size_t begin = 0 ;
		for( std::size_t i = 0; i <= states.size(); ++i ) {
			std::size_t end = (i==states.size()) ? N : states[i] ;
			for( std::size_t j = begin; j < end; ++j ) {
				result.push_back( '0' ) ;
			}
			if( end != N ) {
				result.push_back( '1' ) ;
			}
			begin = end+1 ;
		}
		result += ']' ;
		return result ;
	}
}

struct MFMOptions: public appcontext::CmdLineOptionProcessor
{
public:
	std::string get_program_name() const { return globals::program_name ; }

	void declare_options( appcontext::OptionProcessor& options ) {
		// Meta-options
		options.set_help_option( "-help" ) ;
		options.set_spec_option( "-spec" ) ;

		// File options
		options.declare_group( "Input file options" ) ;
	    options[ "-g" ]
	        .set_description( 	"Path to BGEN file giving genotypes." )
			.set_takes_values( 1 )
			.set_minimum_multiplicity( 1 )
			.set_maximum_multiplicity( 1 ) ;
	    options[ "-filetype" ]
			.set_description(
				"Specify the filetype of the genotype files specified by -g. "
				"By default, qctool will guess the file type.  Use this option to override that guess. "
				"Possible types are: \"" + genfile::string_utils::join( genfile::SNPDataSource::get_file_types(), "\",\"" ) + "\"." )
			.set_takes_single_value()
			.set_default_value( "guess" ) ;

	    options[ "-assume-chromosome" ]
	        .set_description( "Chromosome to assume if not in genetic data." )
			.set_takes_single_value() ;
	    options[ "-s" ]
	        .set_description( "Path of sample file to input.  If specified, this option must occur as often as the -g option"
							" to specify one sample file per cohort." )
			.set_takes_values( 1 )
			.set_minimum_multiplicity( 1 )
			.set_maximum_multiplicity( 1000 ) ;
		options[ "-outcome" ]
			.set_description( "Name of column containing outcome.  Currently must be a binary outcome." )
			.set_takes_single_value()
			.set_is_required() ;

		options.declare_group( "Input filtering options" ) ;
		options[ "-incl-range" ]
			.set_description( "Specify a range of SNPs (or comma-separated list of ranges of SNPs) to operate on. "
				"Each range should be in the format CC:xxxx-yyyy where CC is the chromosome and xxxx and yyyy are the "
				"start and end coordinates, or just xxxx-yyyy which matches that range from all chromosomes. "
				"You can also omit either of xxxx or yyyy to get all SNPs from the start or to the end of a chromosome." )
			.set_takes_values_until_next_option() ;
		options[ "-excl-range" ]
			.set_description( "Specify a range of SNPs (or comma-separated list of ranges of SNPs) to exclude from operation. "
				"Each range should be in the format CC:xxxx-yyyy where CC is the chromosome and xxxx and yyyy are the "
				"start and end coordinates, or just xxxx-yyyy which matches that range from all chromosomes. "
				"You can also omit either of xxxx or yyyy to get all SNPs from the start or to the end of a chromosome." )
			.set_takes_values_until_next_option() ;
		options[ "-incl-ranges" ]
			.set_description( "Like -incl-range, but read the ranges from the specified file."
				"The file must contain a whitespace-separated list of genomic ranges to include in the analysis." )
			.set_takes_values_until_next_option() ;
		options[ "-excl-ranges" ]
			.set_description( "Like -excl-range, but read the ranges from the specified file."
				"The file must contain a whitespace-separated list of genomic ranges to exclude from the analysis." )
			.set_takes_values_until_next_option() ;

		options.declare_group( "Output file options" ) ;
		options[ "-o" ]
			.set_description( "Path to output file.  Use \"-\" for stdout." )
			.set_takes_single_value()
			.set_default_value( "-" ) ;

		options.declare_group( "Model options" ) ;
		options[ "-incl-samples"]
			.set_description( "Filter out samples whose sample ID does not lie in the given file(s).")
			.set_takes_values( 1 )
			.set_minimum_multiplicity( 0 )
			.set_maximum_multiplicity( 100 ) ;
		options[ "-excl-samples"]
			.set_description( "Filter out samples whose sample ID lies in the given file.")
			.set_takes_values( 1 )
			.set_minimum_multiplicity( 0 )
			.set_maximum_multiplicity( 100 ) ;
		options[ "-incl-samples-where"]
			.set_description( "Include samples by specifying conditions on the values of columns in the sample file.")
			.set_takes_values( 1 )
			.set_minimum_multiplicity( 0 )
			.set_maximum_multiplicity( 100 ) ;
		options[ "-excl-samples-where"]
			.set_description( "Exclude samples by specifying conditions on the values of columns in the sample file.")
			.set_takes_values( 1 )
			.set_minimum_multiplicity( 0 )
			.set_maximum_multiplicity( 100 ) ;
		options[ "-minimum-allele-count" ]
			.set_description( "Skip predictors that have less than this minor allele count" )
			.set_takes_values(1)
			.set_default_value( 10 ) ;
		options[ "-covariates" ]
			.set_description( "Specify the name of one or more covariates to include in the model."
				" These must be columns named in the sample file." )
			.set_takes_values_until_next_option() ;

		options.declare_group( "Model fitting options" ) ;
		options[ "-tolerance" ]
			.set_description( "Tolerance to use when model fitting" )
			.set_takes_values(1)
			.set_default_value( 0.001 ) ;
		options[ "-max-iterations" ]
			.set_description( "Maximum iterations to allow when model fitting" )
			.set_takes_values(1)
			.set_default_value( 100 ) ;
		options[ "-max-causal-variants" ]
			.set_description( "Set the maximum number of variants allowed in a causal configuration")
			.set_takes_values(1)
			.set_default_value(5)
		;
		options[ "-additive" ]
			.set_description( "Specify that only additive effects are allowed for" ) ;
		;
		options.declare_group( "Miscellaneous options" ) ;
		options[ "-analysis-name" ]
			.set_description( "Specify a name to label results from this analysis with." )
			.set_takes_single_value()
			.set_default_value( "qctool analysis" ) ;
		options[ "-analysis-chunk" ]
			.set_description( "Specify a name denoting the current genomic region or chunk on which this is run.  This is intended for use in parallel environments." )
			.set_takes_single_value()
			.set_default_value( genfile::MissingValue() ) ;
		options.declare_group( "Miscellaneous options" ) ;
		options[ "-debug" ]
			.set_description( "Output debugging information." ) ;
		options [ "-log" ]
			.set_description( "Specify that " + globals::program_name + " should write a log file to the given file." )
			.set_takes_single_value() ;
	}
} ;

struct DosageStore {
public:
	typedef boost::function< void( std::size_t const, boost::optional< std::size_t > const ) > ProgressCallback ;
	typedef std::unique_ptr< DosageStore > UniquePtr ;

	static UniquePtr create() {
		return UniquePtr( new DosageStore() ) ;
	}
public:
	DosageStore() {}

	void load( genfile::SNPDataSource::UniquePtr source, ProgressCallback callback ) {
		load_unsafe( *source, callback ) ;
	}

	std::size_t number_of_samples() const { return m_ploidy[0].size() ; }
	std::size_t number_of_variants() const { return m_variants.size() ; }
	genfile::VariantIdentifyingData const& variant( std::size_t i ) const { return m_variants[i] ; }

	std::size_t estimate_memory_usage() const {
		std::size_t result = 0 ;
		for( std::size_t i = 0; i < m_variants.size(); ++i ) {
			result += m_variants[i].estimate_bytes_used() ;
			result += m_ploidy[i].size() ;
			result += m_AB[i].size() ;
			result += m_BB[i].size() ;
		}
		return result ;
	}

	std::string get_summary() const {
		using boost::format ;
		return (
			boost::format( "DosageStore: %d variants, %d samples, approx %.2fMb memory usage.\n" )
				% m_variants.size()
				% (m_variants.size() > 0 ? m_AB[0].size() : 0 )
				% (estimate_memory_usage() / 1000000.0)
		).str() ;
	}

	genfile::VariantIdentifyingData const get_variant( std::size_t i ) const {
		assert( i < m_variants.size() ) ;
		return m_variants[i] ;
	}

	template< typename Callback >
	void get_dosages( std::size_t i, Callback callback ) const {
		assert( i < m_variants.size() ) ;
		double const constant = double(0xFFFFFFFF >> 24) ;
		assert( i < m_variants.size() ) ;
		for( std::size_t j = 0; j < m_AB[i].size(); ++j ) {
			// unpack 8 bit bgen encoding
#if DEBUG_DOSAGESTORE
			if( j < 10 ) {
				std::cerr << "get_dosages(): " << j << ": " << int( m_AB[i][j] ) << ", " << int( m_BB[i][j] ) << ".\n" ;
			}
#endif
			callback( j, double( m_AB[i][j] ) / constant, double( m_BB[i][j] ) / constant ) ;
		}
	}

	std::vector< metro::SampleRange > const nonmissing_samples( std::size_t i ) const {
		assert( i < m_variants.size() ) ;
		return m_nonmissing_samples[i] ;
	}

private:

	/* Store genotype probabilities in 8 bit, BGEN-like encoding */
	struct DosageSetter {
		DosageSetter(
			std::vector< uint8_t >* ploidy,
			std::vector< uint8_t >* AB,
			std::vector< uint8_t >* BB
		):
			m_ploidy( ploidy ),
			m_AB( AB ),
			m_BB( BB ),
			m_bits(8),
			m_constant( double(0xFFFFFFFF >> (32 - m_bits))),
			m_sample_i(0),
			m_order_type( genfile::eUnknownOrderType )
		{}

		void initialise( std::size_t nSamples, std::size_t nAlleles ) {
			assert( nAlleles == 2 ) ;
			m_ploidy->resize( nSamples ) ;
			std::fill( m_ploidy->begin(), m_ploidy->end(), 0x80 ) ;
			m_AB->resize( nSamples ) ;
			std::fill( m_AB->begin(), m_AB->end(), 0 ) ;
			m_BB->resize( nSamples ) ;
			std::fill( m_BB->begin(), m_BB->end(), 0 ) ;
			m_last_nonmissing_sample_i = 0 ;
		}

		bool set_sample( std::size_t n ) {
			m_sample_i = n ;
			m_order_type = genfile::eUnknownOrderType ;
			return true ;
		}

		void set_number_of_entries(
			uint32_t ploidy, std::size_t n,
			genfile::OrderType const order_type,
			genfile::ValueType const value_type
		) {
			assert( ploidy <= 2 ) ;
			assert( (order_type == genfile::ePerOrderedHaplotype || genfile::ePerUnorderedGenotype) && value_type == genfile::eProbability ) ;
			(*m_ploidy)[m_sample_i] = ploidy ;
			m_order_type = order_type ;
		}

		void set_value( std::size_t entry_i, genfile::MissingValue const value ) {
			uint8_t& ploidy = (*m_ploidy)[m_sample_i] ;
			uint8_t& AB = (*m_AB)[m_sample_i] ;
			uint8_t& BB = (*m_BB)[m_sample_i] ;
			ploidy |= 0x80 ;
			AB = BB = 0 ;
			record_missing_sample( m_sample_i ) ;
		}

		void set_value( std::size_t entry_i, double const value ) {
			uint8_t& ploidy = (*m_ploidy)[m_sample_i] ;
			uint8_t& AB = (*m_AB)[m_sample_i] ;
			uint8_t& BB = (*m_BB)[m_sample_i] ;

#if DEBUG_DOSAGESTORE
			if( m_sample_i < 10 ) {
				std::cerr << "DosageSetter::set_value(): " << m_sample_i << ", " << entry_i << ": " << value << ".\n" ;
			}
#endif
			if( !(ploidy & 0x80) ) {
				if( entry_i == 1 ) {
					AB = value * m_constant ;
				} else if (entry_i == 2 ) {
					BB = value * m_constant ;
				}
			}
		}

		void set_value( std::size_t entry_i, genfile::Integer const value ) {
			assert(0) ; // expecting GT field
		}

		void finalise() {
			// add the final sample range if needed.
			record_missing_sample( m_ploidy->size() ) ;
		}

		std::vector< metro::SampleRange > const nonmissing_samples() { return m_nonmissing_samples ; }

		private:
			std::vector< uint8_t >* m_ploidy ;
			std::vector< uint8_t >* m_AB ;
			std::vector< uint8_t >* m_BB ;
			uint32_t m_bits ;
			double m_constant ;
			std::size_t m_sample_i ;
			std::size_t m_last_nonmissing_sample_i ;
			genfile::OrderType m_order_type ;
			std::vector< metro::SampleRange > m_nonmissing_samples ;

		void record_missing_sample( std::size_t sample_i ) {
			if( sample_i >= m_last_nonmissing_sample_i ) {
				m_nonmissing_samples.push_back(
					metro::SampleRange( m_last_nonmissing_sample_i, m_sample_i )
				) ;
			}
			m_last_nonmissing_sample_i = sample_i + 1 ;
		}
	} ;

	void load_unsafe( genfile::SNPDataSource& source, ProgressCallback callback ) {
		if( source.total_number_of_snps() ) {
			std::size_t N = *source.total_number_of_snps() ;
			m_variants.reserve( N ) ;
			m_ploidy.reserve( N ) ;
			m_AB.reserve( N ) ;
			m_BB.reserve( N ) ;
		}

		genfile::VariantIdentifyingData variant ;
		std::vector< uint16_t > data ;
		std::size_t count = 0 ;
		std::vector< uint8_t > ploidy, AB, BB ;

		while( source.get_snp_identifying_data( &variant )) {
			// ok, read dosages.
			std::cerr << "LOADING " << variant << ".\n" ;
			DosageSetter setter( &ploidy, &AB, &BB ) ;
			source.read_variant_data()->get( ":genotypes:", genfile::to_GP_unphased( setter ) ) ;
			m_variants.push_back( variant ) ;
			m_nonmissing_samples.push_back( setter.nonmissing_samples() ) ;
			m_ploidy.push_back( ploidy ) ;
			m_AB.push_back( AB ) ;
			m_BB.push_back( BB ) ;
			//m_data.push_back( data ) ;
			if( callback ) {
				callback( ++count, source.total_number_of_snps() ) ;
			}
		}
	}

	std::vector< genfile::VariantIdentifyingData > m_variants ;
	// To avoid using too much space, we encode dosage data as 16-bit integer
	// and expand to required format when requested.
	std::vector< std::vector< metro::SampleRange > > m_nonmissing_samples ;
	std::vector< std::vector< uint8_t > > m_ploidy ;
	std::vector< std::vector< uint8_t > > m_AB ;
	std::vector< std::vector< uint8_t > > m_BB ;
} ;

struct MFMApplication: public appcontext::ApplicationContext {
	MFMApplication( int argc, char** argv ):
		appcontext::ApplicationContext(
			globals::program_name,
			"version: " + globals::program_version + ", revision: " + globals::program_revision,
			std::auto_ptr< appcontext::OptionProcessor >( new MFMOptions ),
			argc,
			argv,
			"-log"
		)
	{
		process() ;
	}

private:

	void process() {
		try {
			unsafe_process() ;
		} catch( genfile::InputError const& e ) {
			ui().logger() << "!! Error (" << e.what() << "): " << e.format_message() << ".\n" ;
			throw appcontext::HaltProgramWithReturnCode( -1 ) ;
		}
		catch( genfile::FileNotFoundError const& e ) {
			ui().logger() << "\nError: No file matching \"" << e.filespec() << "\" could be found.\n" ;
			throw appcontext::HaltProgramWithReturnCode( -1 ) ;
		}
		catch( genfile::db::Error const& e ) {
			ui().logger() << "!! Error (" << e.what() << ") with the following statement: \""
				<< e.sql()
				<< "\".\n" ;
			throw appcontext::HaltProgramWithReturnCode( -1 ) ;
		}
	}

	void unsafe_process() {
		DosageStore::UniquePtr store = load_genotypes() ;

		genfile::CohortIndividualSource::UniquePtr
			samples = genfile::CohortIndividualSource::create( options().get< std::string >( "-s" ) ) ;

		metro::regression::LogLikelihood::UniquePtr
			ll = create_loglikelihood( samples->size() ) ;

		if( options().check( "-covariates" )) {
			add_covariates( ll->design(), *samples, options().get_values< std::string >( "-covariates" ) ) ;
		}

		set_outcome( *ll, *samples, options().get_value< std::string >( "-outcome" )) ;

		std::cerr << ll->design().get_summary() << "\n" ;

		run_search( *store, *ll ) ;
	}

	DosageStore::UniquePtr load_genotypes() {
		appcontext::UIContext::ProgressContext progress_context = ui().get_progress_context( "Loading data" ) ;
		DosageStore::UniquePtr store = DosageStore::create() ;
		store->load(
			open_snp_data_source( options().get< std::string >( "-g" ) ),
			progress_context
		) ;
		ui().logger() << store->get_summary() ;
		return store ;
	}

	genfile::SNPDataSource::UniquePtr
	open_snp_data_source( std::string const& filename ) const {
		genfile::Chromosome chromosome_hint ;
		std::string chromosome_indicator ;
		if( options().check_if_option_was_supplied( "-assume-chromosome" )) {
			chromosome_indicator = options().get_value< std::string >( "-assume-chromosome" ) ;
		}
		if( chromosome_indicator != "" ) {
			chromosome_hint = chromosome_indicator ;
		}

		genfile::SNPDataSource::UniquePtr source ;

		std::pair< std::string, std::string > uf = genfile::uniformise( filename ) ;

		{
			boost::optional< genfile::vcf::MetadataParser::Metadata > metadata ;
//			if( m_options.check( "-metadata" )) {
//				metadata = genfile::vcf::StrictMetadataParser(
//					m_options.get_value< std::string >( "-metadata" )
//				).get_metadata() ;
//			}
			source = genfile::SNPDataSource::create(
				filename,
				chromosome_hint,
				metadata,
				options().get< std::string >( "-filetype" )
			) ;
		}

		genfile::CommonSNPFilter::UniquePtr snp_filter = construct_snp_filter() ;
		// Filter SNPs if necessary
		if( snp_filter.get() ) {
			genfile::VariantIdentifyingDataFilteringSNPDataSource::UniquePtr snp_filtering_source
				= genfile::VariantIdentifyingDataFilteringSNPDataSource::create(
					source,
					genfile::VariantIdentifyingDataTest::UniquePtr( snp_filter.release() )
				) ;

			source.reset(
				snp_filtering_source.release()
			) ;
		}

		return source ;
	}

	genfile::CommonSNPFilter::UniquePtr construct_snp_filter() const {
		genfile::CommonSNPFilter::UniquePtr snp_filter ;

		if( options().check_if_option_was_supplied_in_group( "Input filtering options" )) {
			snp_filter.reset( new genfile::CommonSNPFilter ) ;
			if( options().check_if_option_was_supplied( "-incl-range" )) {
				std::vector< std::string > specs = options().get_values< std::string >( "-incl-range" ) ;
				for ( std::size_t i = 0; i < specs.size(); ++i ) {
					snp_filter->include_snps_in_range(
						genfile::GenomePositionRange::parse( specs[i] )
					) ;
				}
			}

			if( options().check_if_option_was_supplied( "-excl-range" )) {
				std::vector< std::string > specs = options().get_values< std::string >( "-excl-range" ) ;
				for ( std::size_t i = 0; i < specs.size(); ++i ) {
					snp_filter->exclude_snps_in_range(
						genfile::GenomePositionRange::parse( specs[i] )
					) ;
				}
			}

			if( options().check_if_option_was_supplied( "-incl-ranges" )) {
				std::vector< std::string > files = options().get_values< std::string >( "-incl-ranges" ) ;
				for( std::string const& filename: files ) {
					std::auto_ptr< std::istream > in = genfile::open_text_file_for_input( filename ) ;
					std::string range ;
					while( (*in) >> range ) {
						snp_filter->include_snps_in_range(
							genfile::GenomePositionRange::parse( range )
						) ;
					}
				}
			}

			if( options().check_if_option_was_supplied( "-excl-ranges" )) {
				std::vector< std::string > files = options().get_values< std::string >( "-excl-ranges" ) ;
				for( std::string const& filename: files ) {
					std::auto_ptr< std::istream > in = genfile::open_text_file_for_input( filename ) ;
					std::string range ;
					while( (*in) >> range ) {
						snp_filter->exclude_snps_in_range(
							genfile::GenomePositionRange::parse( range )
						) ;
					}
				}
			}
		}
		return snp_filter ;
	}

	metro::regression::LogLikelihood::UniquePtr create_loglikelihood( std::size_t number_of_samples ) {
		typedef std::vector< std::string > Names ;
		typedef std::vector< metro::SampleRange > SampleRanges ;
		using namespace metro::regression ;

		Eigen::MatrixXd outcome = Eigen::MatrixXd::Zero( number_of_samples, 2 ) ;

		Design::UniquePtr design = Design::create(
			outcome, SampleRanges(), Names({ "outcome=0", "outcome=1"} ),
			Names({
				"add1", "het1",
				"add2", "het2",
				"add3", "het3",
				"add4", "het4",
				"add5", "het5"
			})
		) ;
		LogLikelihood::UniquePtr ll( metro::regression::Logistic::create( design ).release() ) ;
		return ll ;
	}

	void add_covariates(
		metro::regression::Design& design,
		genfile::CohortIndividualSource const& samples,
		std::vector< std::string > const& covariates
	) {
		ui().logger() << "Adding covariates...\n" ;
		{
			std::set< std::string > uniqueCovariates( covariates.begin(), covariates.end() ) ;
			if( uniqueCovariates.size() != covariates.size() ) {
				throw genfile::BadArgumentError(
					"add_covariates()",
					"covariates=\"" + genfile::string_utils::join( covariates, " " ) + "\"",
					"Covariates should not be duplicated."
				) ;
			}
		}
		genfile::CohortIndividualSource::ColumnSpec const& spec = samples.get_column_spec() ;
		for( std::size_t i = 0; i < covariates.size(); ++i ) {
			std::string const& covariateName = covariates[i] ;
			std::size_t const column_index = spec.find_column( covariateName ) ;
			genfile::CohortIndividualSource::SingleColumnSpec columnSpec = spec[ column_index ] ;
			genfile::CrossCohortCovariateValueMapping::UniquePtr mapping
				= genfile::CrossCohortCovariateValueMapping::create( columnSpec, true ) ;
			mapping->add_source( samples ) ;
			if( columnSpec.is_discrete() ) {
				design.add_discrete_covariate(
					covariateName,
					[&samples, &column_index, &mapping] ( std::size_t sample_index ) {
						int value = -1 ;
						genfile::VariantEntry entry = samples.get_entry( sample_index, column_index ) ;
						if( !entry.is_missing() ) {
							value = mapping->get_mapped_value( entry ).as< int >() ;
						}
						return value ;
					},
					[&mapping]( int level ) {
						genfile::VariantEntry entry = mapping->get_unmapped_value( level ) ;
						return entry.as< std::string >() ;
					},
					mapping->get_number_of_distinct_mapped_values()
				) ;
			} else {
				design.add_single_covariate(
					covariateName,
					[&samples, &column_index, &mapping]( std::size_t sample_index ) {
						double value = metro::NA ;
						genfile::VariantEntry entry = samples.get_entry( sample_index, column_index ) ;
						if( !entry.is_missing() ) {
							value = mapping->get_mapped_value( entry ).as< double >() ;
						}
						return value ;
					}
				) ;
			}

			ui().logger()
				<< "++ Added covariate: \"" + covariateName + "\":\n"
				<<  mapping->get_summary( "     " )
				<< "\n\n" ;
		}
	}

	void set_outcome(
		metro::regression::LogLikelihood& ll,
		genfile::CohortIndividualSource const& samples,
		std::string const& column
	) const {
		typedef std::vector< std::string > Names ;
		typedef std::vector< metro::SampleRange > SampleRanges ;
		Eigen::MatrixXd outcome = Eigen::MatrixXd::Zero( samples.size(), 2 ) ;
		SampleRanges nonmissingness ;
		get_binary_outcome( samples, column, &outcome, &nonmissingness ) ;
		ll.design().set_outcome(
			outcome, nonmissingness,
			Names({ column + "=0", column + "=1" })
		) ;
	}

	void get_binary_outcome(
		genfile::CohortIndividualSource const& samples,
		std::string const& column,
		Eigen::MatrixXd* outcome,
		std::vector< metro::SampleRange >* nonmissingness
	) const {
		std::size_t column_i = samples.get_column_spec().find_column( column ) ;
		assert( samples.get_column_spec()[ column_i ].is_discrete() ) ;
		std::size_t last_nonmissing_sample_i = 0 ;
		outcome->setZero( samples.size(), 2 ) ;
		for( std::size_t i = 0; i < samples.size(); ++i ) {
			genfile::VariantEntry const& entry = samples.get_entry( i, column_i ) ;
			if( entry.is_missing() ) {
				nonmissingness->push_back( metro::SampleRange( last_nonmissing_sample_i, i )) ;
				last_nonmissing_sample_i = i+1 ;
			} else {
				int value = entry.as< int >() ;
				assert( value == 0 || value == 1 ) ;
				(*outcome)(i,value) = 1 ;
			}
		}
		if( last_nonmissing_sample_i < samples.size() ) {
			nonmissingness->push_back( metro::SampleRange( last_nonmissing_sample_i, samples.size() )) ;
		}
#if DEBUG
		std::cerr << "get_outcome(): outcome is:\n"
			<< outcome->block(0,0,10,2 ) << "\n"
			<< "nonmissingness: " << *nonmissingness << "\n" ;
#endif
	}

	typedef std::map< std::vector< metro::SampleRange >, std::pair< Eigen::VectorXd, double > > NullLLStore ;

	double test_variant(
		metro::ShotgunStochasticSearch::SelectedStates const& pick,
		DosageStore const& store,
		metro::regression::LogLikelihood& ll,
		NullLLStore& null_ll_store
	) {
#if DEBUG
		std::cerr << "   TESTING: " << print_state( store.number_of_variants(), pick ) << ".\n" ;
#endif
		using namespace metro::regression ;
		using metro::SampleRange ;

		double const minus_infinity = -std::numeric_limits< double >::infinity() ;
		double const log_weight = ( (pick.size() == 0) || (pick.size() > 5) )
			? minus_infinity
			: (pick.size() * std::log( 1.0 / store.number_of_variants() )) ;

		if( log_weight == minus_infinity ) {
			// Disallowed state, don't bother fitting.
			return minus_infinity ;
		}

		boost::format parameter_format( "%s%d/%s" ) ;
		bool const debug = options().check( "-debug" ) ;

		assert( pick.size() < 6 ) ;
		Eigen::MatrixXd predictors = Eigen::MatrixXd::Zero( store.number_of_samples(), 10 ) ;
		std::vector< metro::SampleRange > nonmissing_samples( 1, metro::SampleRange( 0, store.number_of_samples() )) ;

		metro::IndependentParameterDistribution prior( ll.get_parameter_names() ) ;
		metro::IndependentParameterDistribution null_prior( ll.get_parameter_names() ) ;

		double const add_prior_obs = 32.0 ;
		double const het_prior_obs = options().check( "-additive" ) ? 1000.0 : 32.0 ;

		for( std::size_t i = 0; i < 5; ++i ) {
			null_prior.set_prior(
				(parameter_format % "add" % (i+1) % ll.design().get_outcome_name(1) ).str(),
				metro::distributions::LogF::create( 10000.0, 10000.0 )
			) ;
			null_prior.set_prior(
				(parameter_format % "het" % (i+1) % ll.design().get_outcome_name(1) ).str(),
				metro::distributions::LogF::create( 10000.0, 10000.0 )
			) ;
		}

		for( std::size_t i = 0; i < pick.size(); ++i ) {
			store.get_dosages(
				pick[i],
				[&predictors,i] ( int sample, double ab, double bb ){
					predictors( sample, i*2+0) = ab + 2*bb ;
					predictors( sample, i*2+1) = ab ;
				}
			) ;
			nonmissing_samples = metro::impl::intersect_ranges(
				nonmissing_samples,
				store.nonmissing_samples(pick[i])
			) ;
			prior.set_prior(
				(parameter_format % "add" % (i+1) % ll.design().get_outcome_name(1) ).str(),
				metro::distributions::LogF::create( add_prior_obs, add_prior_obs )
			) ;
			prior.set_prior(
				(parameter_format % "het" % (i+1) % ll.design().get_outcome_name(1) ).str(),
				metro::distributions::LogF::create( het_prior_obs, het_prior_obs )
			) ;
		}
		for( std::size_t i = pick.size(); i < 5; ++i ) {
			prior.set_prior(
				(parameter_format % "add" % (i+1) % ll.design().get_outcome_name(1) ).str(),
				metro::distributions::LogF::create( add_prior_obs, add_prior_obs )
			) ;
			prior.set_prior(
				(parameter_format % "het" % (i+1) % ll.design().get_outcome_name(1) ).str(),
				metro::distributions::LogF::create( het_prior_obs, het_prior_obs )
			) ;
		}

		// Set up a trace for model fitting iterations.
		metro::CholeskyStepper::Tracer tracer ;
		if( debug ) {
			tracer = [this] (
				int iteration,
				double ll,
				double target_ll,
				metro::CholeskyStepper::Vector const& point,
				metro::CholeskyStepper::Vector const& first_derivative,
				metro::CholeskyStepper::Vector const& step,
				bool converged
			) {
				this->ui().logger()
					<< " ITERATION: " << iteration << "\n"
					<< "        LL: " << ll << "\n"
					<< "    TARGET: " << target_ll << "\n"
					<< "     POINT: " << point.transpose() << "\n"
					<< "DERIVATIVE: " << first_derivative.transpose() << "\n"
					<< "      STEP: " << step.transpose() << "\n"
					<< " CONVERGED: " << ( converged ? "yes" : "no" ) << "\n" ;
			} ;
		}

		// To compute the BF we need the null model.
		// The null model does not depend on the SNPs, but might depend on the missing samples.
		// We cache the value for speed if the nonmissing samples don't change.
		std::vector< std::string > comments ;
		// null_ll will be log-marginal likelihood of null model
		// i.e. logarithm of prior x likelihood, integrated over parameters.
		double null_ll = 0.0 ;
		Eigen::VectorXd null_parameters ;
		{
			NullLLStore::const_iterator where = null_ll_store.find( nonmissing_samples ) ;
			if( where == null_ll_store.end() ) {
				if( debug ) {
					this->ui().logger() << "---> fitting NULL...\n" ;
				}
				ll.design().set_predictors( Eigen::MatrixXd::Zero( store.number_of_samples(), 10 ), nonmissing_samples ) ;
				LogUnnormalisedPosterior posterior( ll, null_prior ) ;
				metro::CholeskyStepper stopping_condition( 0.01, 100, tracer ) ;
				std::pair< bool, int > fit = metro::fit_model(
					posterior,
					"null",
					Eigen::VectorXd::Zero( ll.identify_parameters().rows() ),
					stopping_condition,
					&comments
				) ;

				null_ll = laplace_approximate( posterior ) ;
				null_parameters = posterior.parameters() ;
				//null_ll = posterior.get_value_of_function() ;
				where = null_ll_store.insert(
					NullLLStore::value_type(
						nonmissing_samples,
						NullLLStore::mapped_type(
							ll.parameters(),
							null_ll
						)
					)
				).first ;
			} else {
				null_parameters = where->second.first ;
				null_ll = where->second.second ;
			}
		}

		ll.design().set_predictors( predictors, nonmissing_samples ) ;
		LogUnnormalisedPosterior posterior( ll, prior ) ;

		double posterior_weight = minus_infinity ;
		double result = minus_infinity ;

		if( log_weight > minus_infinity ) {
			if( debug ) {
				this->ui().logger() << "---> fitting full...\n" ;
			}
			metro::CholeskyStepper stopping_condition( 0.01, 100, tracer ) ;
			std::pair< bool, int > fit = metro::fit_model(
				posterior,
				"full",
				null_parameters,
				stopping_condition,
				&comments
			) ;
			if( fit.first ) {
				result = laplace_approximate( posterior ) - null_ll + log_weight ;
				//result = posterior.get_value_of_function() - null_ll + log_weight ;
			}

#if DEBUG
			std::cerr << print_state( store.number_of_variants(), pick ) << ".\n" ;
			//std::cerr << nonmissing_samples << ".\n" ;
			// std::cerr << "PREDICTORS:\n" << predictors.block( 0, 0, 10, predictors.cols() ) << ".\n" ;
			std::cerr << posterior.get_summary() << "\n" ;
			std::cerr << "  INCLUDED: " << posterior.ll().design().nonmissing_samples() << "\n" ;
			std::cerr << " CONVERGED: " << (fit.first ? "1" : "0") << "\n" ;
			std::cerr << "        LL: " << posterior.get_value_of_function() << "\n" ;
			std::cerr << "      NULL: " << null_ll << "\n" ;
			std::cerr << " pick.size: " << pick.size() << "\n" ;
			std::cerr << "LOG WEIGHT: " << log_weight << "\n" ;
			std::cerr << "    RESULT: " << result << "\n" ;

			std::cerr << "\n+++++++++++++++++++++++++\n" ;
#endif
		}

		return result ;
	}

	// Compute laplace approximation.
	// It is assumed that the function has already been maximised.
	double laplace_approximate( metro::SmoothFunction const& function ) const {
		double const fx = function.get_value_of_function() ;
		metro::SmoothFunction::Matrix const& H = function.get_value_of_second_derivative() ;
		double const log_2pi = std::log( 2.0 * 3.1415926535897932384626 ) ;
		Eigen::ColPivHouseholderQR< metro::SmoothFunction::Matrix > solver( -H ) ;
		return
			fx + 0.5 * ( function.number_of_parameters() * log_2pi - solver.logAbsDeterminant() ) ;
	}

	void run_search( DosageStore& store, metro::regression::LogLikelihood& ll ) {
		// Do a test search for now
		NullLLStore null_ll_store ;

		ui().logger() << "Running shotgun stochastic search...\n" ;

		metro::ShotgunStochasticSearch ss(
			store.number_of_variants(),
			[this, &store, &ll, &null_ll_store]( metro::ShotgunStochasticSearch::SelectedStates const& s ) {
				return this->test_variant( s, store, ll, null_ll_store ) ;
			},
			static_cast<std::uint32_t>(std::time(0))
		) ;

		for( std::size_t i = 0; i < 100; ++i ) {
			boost::timer::cpu_timer timer ;
			metro::ShotgunStochasticSearch::SelectedStates const& pick = ss.update() ;

			ui().logger() << "Iteration " << i << ": "
				<< "took " << boost::timer::format( timer.elapsed(), 2, "%ts" )
				<< ", state is: " << print_state( store.number_of_variants(), pick )
				<< "\n" ;
		}

		ui().logger() << "Outputting search results...\n" ;
		output_search_results( ss, store ) ;

		ui().logger() << "Fine-mapping complete.\n" ;
	}

	void output_search_results(
		metro::ShotgunStochasticSearch const& ss,
		DosageStore const& store
	) {
		qcdb::MultiVariantStorage::UniquePtr outputter
			= qcdb::MultiVariantStorage::create(
				options().get< std::string >( "-o" ),
				5,
				options().get< std::string >( "-analysis-name" ),
				options().get< std::string >( "-analysis-chunk" ),
				get_application_metadata()
		) ;
		outputter->add_variable( "loglikelihood" ) ;
		outputter->add_variable( "logposterior" ) ;
		outputter->add_variable( "posterior" ) ;

		std::vector< genfile::VariantIdentifyingData > variants ;

		auto comparator = []( double a, double b ) { return a > b ; } ;
		typedef std::multimap< double, metro::ShotgunStochasticSearch::SelectedStates, decltype(comparator) > OrderedStates ;
		OrderedStates ordered_states( comparator ) ;
		std::vector< double > posterior_weights ;
		ss.visited_states(
			[&ordered_states, &posterior_weights](
				metro::ShotgunStochasticSearch::SelectedStates const& states,
				double value
			) {
				std::cerr << "GOT: " << states.size() << ", " << value << "\n" ;
				ordered_states.insert( std::make_pair( value, states )) ;
				posterior_weights.push_back( value ) ;
			}
		) ;

		double const total_log_weight = metro::log_sum_exp( posterior_weights ) ;
		ui().logger() << "TOTAL LOG WEIGHT:" << total_log_weight << "\n" ;
		int count = 0 ;
		for( OrderedStates::const_iterator i = ordered_states.begin(); i != ordered_states.end(); ++i, ++count ) {
			metro::ShotgunStochasticSearch::SelectedStates const& states = i->second ;
			variants.resize( states.size() ) ;
			for( std::size_t i = 0; i < states.size(); ++i ) {
				variants[i] = store.variant( states[i] ) ;
			}
			outputter->store_data_for_key(
				variants,
				"rank",
				count
			) ;
			outputter->store_data_for_key(
				variants,
				"loglikelihood",
				i->first
			) ;
			outputter->store_data_for_key(
				variants,
				"logposterior",
				i->first - total_log_weight
			) ;
			outputter->store_data_for_key(
				variants,
				"posterior",
				std::exp( i->first - total_log_weight )
			) ;
		}
		outputter->finalise() ;
	}

#if 0
	void create_query( genfile::bgen::Query* query ) {
		if( options().check( "-incl-range" )) {
			auto const elts = options().get_values< std::string >( "-incl-range" ) ;
			for( std::string const& elt: elts ) {
				query->include_range( parse_range( elt )) ;
			}
		}
		if( options().check( "-excl-range" )) {
			auto const elts = options().get_values< std::string >( "-excl-range" ) ;
			for( std::string const& elt: elts ) {
				query->exclude_range( parse_range( elt )) ;
			}
		}
		if( options().check( "-incl-rsids" )) {
			auto const ids = options().get_values< std::string >( "-incl-rsids" ) ;
			query->include_rsids( ids ) ;
		}

		if( options().check( "-excl-rsids" )) {
			auto const ids = options().get_values< std::string >( "-excl-rsids" ) ;
			query->exclude_rsids( ids ) ;
		}
	}

	genfile::bgen::IndexQuery::GenomicRange parse_range( std::string const& spec ) const {
		std::size_t colon_pos = spec.find( ':' ) ;
		if ( colon_pos == std::string::npos ) {
			throw std::invalid_argument( "spec=\"" + spec + "\"" ) ;
		}

		std::vector< std::string > pieces ;
		pieces.push_back( spec.substr( 0, colon_pos )) ;
		pieces.push_back( spec.substr( colon_pos+1, spec.size() )) ;

		if( pieces.size() != 2 ) {
			throw std::invalid_argument( "spec=\"" + spec + "\"" ) ;
		}

		std::size_t separator_pos = pieces[1].find( '-' ) ;
		if ( separator_pos == std::string::npos ) {
			throw std::invalid_argument( "spec=\"" + spec + "\"" ) ;
		}

		std::string chromosome( pieces[0] ) ;
		int pos1 = (separator_pos == 0) ? 0 : std::stoi( pieces[1].substr( 0, separator_pos ) ) ;
		int pos2 = (separator_pos == (pieces[1].size()-1)) ? std::numeric_limits< int >::max() : std::stoi( pieces[1].substr( separator_pos + 1, pieces[1].size() ) ) ;
		assert( pos1 >= 0 ) ;
		assert( pos2 >= pos1 ) ;

		return genfile::bgen::IndexQuery::GenomicRange( chromosome, pos1, pos2 ) ;
	}
#endif
} ;

int main( int argc, char** argv ) {
	try {
		MFMApplication app( argc, argv ) ;
	}
	catch( appcontext::HaltProgramWithReturnCode const& e ) {
		return e.return_code() ;
	}
	return 0 ;
}


/*!
@file
Forward declares `boost::hana::Monad`.

@copyright Louis Dionne 2013-2016
Distributed under the Boost Software License, Version 1.0.
(See accompanying file LICENSE.md or copy at http://boost.org/LICENSE_1_0.txt)
 */

#ifndef BOOST_HANA_FWD_CONCEPT_MONAD_HPP
#define BOOST_HANA_FWD_CONCEPT_MONAD_HPP

#include <boost/hana/config.hpp>

BOOST_HANA_NAMESPACE_BEGIN
    //! @ingroup group-concepts
    //! @defgroup group-Monad Monad
    //! The `Monad` concept represents `Applicative`s with the ability to
    //! flatten nested levels of structure.
    //!
    //! Historically, Monads are a construction coming from category theory,
    //! an abstract branch of mathematics. The functional programming
    //! community eventually discovered how Monads could be used to
    //! formalize several useful things like side effects, which led
    //! to the wide adoption of Monads in that community. However, even
    //! in a multi-paradigm language like C++, there are several constructs
    //! which turn out to be Monads, like `std::optional`, `std::vector` and
    //! others.
    //!
    //! Everybody tries to introduce `Monad`s with a different analogy, and
    //! most people fail. This is called the [Monad tutorial fallacy][1]. We
    //! will try to avoid this trap by not presenting a specific intuition,
    //! and we will instead present what monads are mathematically.
    //! For specific intuitions, we will let readers who are new to this
    //! concept read one of the many excellent tutorials available online.
    //! Understanding Monads might take time at first, but once you get it,
    //! a lot of patterns will become obvious Monads; this enlightening will
    //! be your reward for the hard work.
    //!
    //! There are different ways of defining a Monad; Haskell uses a function
    //! called `bind` (`>>=`) and another one called `return` (it has nothing
    //! to do with C++'s `return` statement). They then introduce relationships
    //! that must be satisfied for a type to be a Monad with those functions.
    //! Mathematicians sometimes use a function called `join` and another one
    //! called `unit`, or they also sometimes use other category theoretic
    //! constructions like functor adjunctions and the Kleisli category.
    //!
    //! This library uses a composite approach. First, we use the `flatten`
    //! function (equivalent to `join`) along with the `lift` function from
    //! `Applicative` (equivalent to `unit`) to introduce the notion of
    //! monadic function composition. We then write the properties that must
    //! be satisfied by a Monad using this monadic composition operator,
    //! because we feel it shows the link between Monads and Monoids more
    //! clearly than other approaches.
    //!
    //! Roughly speaking, we will say that a `Monad` is an `Applicative` which
    //! also defines a way to compose functions returning a monadic result,
    //! as opposed to only being able to compose functions returning a normal
    //! result. We will then ask for this composition to be associative and to
    //! have a neutral element, just like normal function composition. For
    //! usual composition, the neutral element is the identity function `id`.
    //! For monadic composition, the neutral element is the `lift` function
    //! defined by `Applicative`. This construction is made clearer in the
    //! laws below.
    //!
    //! @note
    //! Monads are known to be a big chunk to swallow. However, it is out of
    //! the scope of this documentation to provide a full-blown explanation
    //! of the concept. The [Typeclassopedia][2] is a nice Haskell-oriented
    //! resource where more information about Monads can be found.
    //!
    //!
    //! Minimal complete definitions
    //! ----------------------------
    //! First, a `Monad` must be both a `Functor` and an `Applicative`.
    //! Also, an implementation of `flatten` or `chain` satisfying the
    //! laws below for monadic composition must be provided.
    //!
    //! @note
    //! The `ap` method for `Applicatives` may be derived from the minimal
    //! complete definition of `Monad` and `Functor`; see below for more
    //! information.
    //!
    //!
    //! Laws
    //! ----
    //! To simplify writing the laws, we use the comparison between functions.
    //! For two functions `f` and `g`, we define
    //! @code
    //!     f == g  if and only if  f(x) == g(x) for all x
    //! @endcode
    //!
    //! With the usual composition of functions, we are given two functions
    //! @f$ f : A \to B @f$ and @f$ g : B \to C @f$, and we must produce a
    //! new function @f$ compose(g, f) : A \to C @f$. This composition of
    //! functions is associative, which means that
    //! @code
    //!     compose(h, compose(g, f)) == compose(compose(h, g), f)
    //! @endcode
    //!
    //! Also, this composition has an identity element, which is the identity
    //! function. This simply means that
    //! @code
    //!     compose(f, id) == compose(id, f) == f
    //! @endcode
    //!
    //! This is probably nothing new if you are reading the `Monad` laws.
    //! Now, we can observe that the above is equivalent to saying that
    //! functions with the composition operator form a `Monoid`, where the
    //! neutral element is the identity function.
    //!
    //! Given an `Applicative` `F`, what if we wanted to compose two functions
    //! @f$ f : A \to F(B) @f$ and @f$ g : B \to F(C) @f$? When the
    //! `Applicative` `F` is also a `Monad`, such functions taking normal
    //! values but returning monadic values are called _monadic functions_.
    //! To compose them, we obviously can't use normal function composition,
    //! since the domains and codomains of `f` and `g` do not match properly.
    //! Instead, we'll need a new operator -- let's call it `monadic_compose`:
    //! @f[
    //!     \mathtt{monadic\_compose} :
    //!         (B \to F(C)) \times (A \to F(B)) \to (A \to F(C))
    //! @f]
    //!
    //! How could we go about implementing this function? Well, since we know
    //! `F` is an `Applicative`, the only functions we have are `transform`
    //! (from `Functor`), and `lift` and `ap` (from `Applicative`). Hence,
    //! the only thing we can do at this point while respecting the signatures
    //! of `f` and `g` is to set (for `x` of type `A`)
    //! @code
    //!     monadic_compose(g, f)(x) = transform(f(x), g)
    //! @endcode
    //!
    //! Indeed, `f(x)` is of type `F(B)`, so we can map `g` (which takes `B`'s)
    //! on it. Doing so will leave us with a result of type `F(F(C))`, but what
    //! we wanted was a result of type `F(C)` to respect the signature of
    //! `monadic_compose`. If we had a joker of type @f$ F(F(C)) \to F(C) @f$,
    //! we could simply set
    //! @code
    //!     monadic_compose(g, f)(x) = joker(transform(f(x), g))
    //! @endcode
    //!
    //! and we would be happy. It turns out that `flatten` is precisely this
    //! joker. Now, we'll want our joker to satisfy some properties to make
    //! sure this composition is associative, just like our normal composition
    //! was. These properties are slightly cumbersome to specify, so we won't
    //! do it here. Also, we'll need some kind of neutral element for the
    //! composition. This neutral element can't be the usual identity function,
    //! because it does not have the right type: our neutral element needs to
    //! be a function of type @f$ X \to F(X) @f$ but the identity function has
    //! type @f$ X \to X @f$. It is now the right time to observe that `lift`
    //! from `Applicative` has exactly the right signature, and so we'll take
    //! this for our neutral element.
    //!
    //! We are now ready to formulate the `Monad` laws using this composition
    //! operator. For a `Monad` `M` and functions @f$ f : A \to M(B) @f$,
    //! @f$ g : B \to M(C) @f$ and @f$ h : C \to M(D) @f$, the following
    //! must be satisfied:
    //! @code
    //!     // associativity
    //!     monadic_compose(h, monadic_compose(g, f)) == monadic_compose(monadic_compose(h, g), f)
    //!
    //!     // right identity
    //!     monadic_compose(f, lift<M(A)>) == f
    //!
    //!     // left identity
    //!     monadic_compose(lift<M(B)>, f) == f
    //! @endcode
    //!
    //! which is to say that `M` along with monadic composition is a Monoid
    //! where the neutral element is `lift`.
    //!
    //!
    //! Refined concepts
    //! ----------------
    //! 1. `Functor`
    //! 2. `Applicative` (free implementation of `ap`)\n
    //! When the minimal complete definition for `Monad` and `Functor` are
    //! both satisfied, it is possible to implement `ap` by setting
    //! @code
    //!     ap(fs, xs) = chain(fs, [](auto f) {
    //!         return transform(xs, f);
    //!     })
    //! @endcode
    //!
    //!
    //! Concrete models
    //! ---------------
    //! `hana::lazy`, `hana::optional`, `hana::tuple`
    //!
    //!
    //! [1]: https://byorgey.wordpress.com/2009/01/12/abstraction-intuition-and-the-monad-tutorial-fallacy/
    //! [2]: https://wiki.haskell.org/Typeclassopedia#Monad
    template <typename M>
    struct Monad;
BOOST_HANA_NAMESPACE_END

#endif // !BOOST_HANA_FWD_CONCEPT_MONAD_HPP


/*
 fcs.cpp

 Copyright (c) 2014, 2015, 2016 Terumasa Tadano

 This file is distributed under the terms of the MIT license.
 Please see the file 'LICENCE.txt' in the root directory
 or http://opensource.org/licenses/mit-license.php for information.
*/

#include "fcs.h"
#include "constants.h"
#include "constraint.h"
#include "error.h"
#include "cluster.h"
#include "mathfunctions.h"
#include "memory.h"
#include "rref.h"
#include "symmetry.h"
#include "timer.h"
#include <iostream>
#include <iomanip>
#include <limits>
#include <cstddef>
#include <string>
#include <cmath>
#include "../external/combination.hpp"
#include <unordered_set>
#include <boost/algorithm/string/case_conv.hpp>

#if defined(_WIN32) || defined(_WIN64)
#undef min
#undef max
#endif

using namespace ALM_NS;

Fcs::Fcs()
{
    set_default_variables();
};

Fcs::~Fcs()
{
    deallocate_variables();
};

void Fcs::init(const Cluster *cluster,
               const Symmetry *symmetry,
               const Cell &supercell,
               const int verbosity,
               Timer *timer)
{
    int i;
    const auto maxorder = cluster->get_maxorder();

    timer->start_clock("fcs");

    if (verbosity > 0) {
        std::cout << " FORCE CONSTANT" << std::endl;
        std::cout << " ==============" << std::endl << std::endl;
        std::cout << "  Symmetry is handled in ";
        if (preferred_basis == "Lattice") {
            std::cout << "crystallographic (fractional) coordinates.";
        } else {
            std::cout << "Cartesian coordinates.";
        }
        std::cout << std::endl;
    }

    if (fc_table) {
        deallocate(fc_table);
    }
    allocate(fc_table, maxorder);

    if (nequiv) {
        deallocate(nequiv);
    }
    allocate(nequiv, maxorder);

    if (fc_zeros) {
        deallocate(fc_zeros);
    }
    allocate(fc_zeros, maxorder);

    // Generate force constants using the information of interacting atom pairs
    for (i = 0; i < maxorder; ++i) {
        generate_force_constant_table(i,
                                      supercell.number_of_atoms,
                                      cluster->get_cluster_list(i),
                                      symmetry,
                                      preferred_basis,
                                      fc_table[i],
                                      nequiv[i],
                                      fc_zeros[i],
                                      store_zeros);
    }

    set_basis_conversion_matrix(supercell);

    if (verbosity > 0) {
        std::cout << std::endl;
        for (i = 0; i < maxorder; ++i) {
            std::cout << "  Number of " << std::setw(9)
                << cluster->get_ordername(i)
                << " FCs : " << nequiv[i].size();
            std::cout << std::endl;
        }
        std::cout << std::endl;

        timer->print_elapsed();
        std::cout << " -------------------------------------------------------------------" << std::endl;
        std::cout << std::endl;
    }

    timer->stop_clock("fcs");
}

void Fcs::set_default_variables()
{
    nequiv = nullptr;
    fc_table = nullptr;
    fc_zeros = nullptr;
    fc_cart = nullptr;
    store_zeros = true;

    // preferred_basis = "Cartesian";
    preferred_basis = "Lattice";
}

void Fcs::deallocate_variables()
{
    if (nequiv) {
        deallocate(nequiv);
    }
    if (fc_table) {
        deallocate(fc_table);
    }
    if (fc_zeros) {
        deallocate(fc_zeros);
    }
    if (fc_cart) {
        deallocate(fc_cart);
    }
}

void Fcs::generate_force_constant_table(const int order,
                                        const size_t nat,
                                        const std::set<IntList> &pairs,
                                        const Symmetry *symm_in,
                                        const std::string basis,
                                        std::vector<FcProperty> &fc_vec,
                                        std::vector<size_t> &ndup,
                                        std::vector<FcProperty> &fc_zeros_out,
                                        const bool store_zeros_in) const
{
    size_t i, j;
    int i1, i2;
    int i_prim;
    int *atmn, *atmn_mapped;
    int *ind, *ind_mapped;
    int *ind_tmp, *ind_mapped_tmp;
    int nxyz;
    unsigned int isym;

    double c_tmp;

    int **xyzcomponent;

    const auto nsym = symm_in->get_SymmData().size();
    bool is_zero;
    bool *is_searched;
    int **map_sym;
    double ***rotation;
    const bool use_compatible = true;

    if (order < 0) return;

    allocate(rotation, nsym, 3, 3);
    allocate(map_sym, nat, nsym);
    int nsym_in_use = 0;

    get_available_symmop(nat,
                         symm_in,
                         basis,
                         nsym_in_use,
                         map_sym,
                         rotation,
                         use_compatible);

    allocate(atmn, order + 2);
    allocate(atmn_mapped, order + 2);
    allocate(ind, order + 2);
    allocate(ind_mapped, order + 2);
    allocate(ind_tmp, order);
    allocate(ind_mapped_tmp, order + 2);
    allocate(is_searched, 3 * nat);

    fc_vec.clear();
    ndup.clear();
    fc_zeros_out.clear();
    size_t nmother = 0;

    nxyz = static_cast<int>(std::pow(3.0, order + 2));

    allocate(xyzcomponent, nxyz, order + 2);
    get_xyzcomponent(order + 2, xyzcomponent);

    std::unordered_set<IntList> list_found;

    for (const auto &pair : pairs) {

        for (i = 0; i < order + 2; ++i) atmn[i] = pair.iarray[i];

        for (i1 = 0; i1 < nxyz; ++i1) {
            for (i = 0; i < order + 2; ++i) ind[i] = 3 * atmn[i] + xyzcomponent[i1][i];

            if (!is_ascending(order + 2, ind)) continue;

            i_prim = get_minimum_index_in_primitive(order + 2, ind, nat,
                                                    symm_in->get_nat_prim(),
                                                    symm_in->get_map_p2s());
            std::swap(ind[0], ind[i_prim]);
            sort_tail(order + 2, ind);

            is_zero = false;

            if (list_found.find(IntList(order + 2, ind)) != list_found.end()) continue; // Already exits!

            // Search symmetrically-dependent parameter set

            size_t ndeps = 0;

            for (isym = 0; isym < nsym_in_use; ++isym) {

                for (i = 0; i < order + 2; ++i) atmn_mapped[i] = map_sym[atmn[i]][isym];

                if (!is_inprim(order + 2,
                               atmn_mapped,
                               symm_in->get_nat_prim(),
                               symm_in->get_map_p2s()))
                    continue;

                for (i2 = 0; i2 < nxyz; ++i2) {

                    c_tmp = coef_sym(order + 2,
                                     rotation[isym],
                                     xyzcomponent[i1],
                                     xyzcomponent[i2]);

                    if (std::abs(c_tmp) > eps12) {
                        for (i = 0; i < order + 2; ++i)
                            ind_mapped[i] = 3 * atmn_mapped[i] + xyzcomponent[i2][i];

                        i_prim = get_minimum_index_in_primitive(order + 2,
                                                                ind_mapped,
                                                                nat,
                                                                symm_in->get_nat_prim(),
                                                                symm_in->get_map_p2s());
                        std::swap(ind_mapped[0], ind_mapped[i_prim]);
                        sort_tail(order + 2, ind_mapped);

                        if (!is_zero) {
                            bool zeroflag = true;
                            for (i = 0; i < order + 2; ++i) {
                                zeroflag = zeroflag & (ind[i] == ind_mapped[i]);
                            }
                            zeroflag = zeroflag & (std::abs(c_tmp + 1.0) < eps8);
                            is_zero = zeroflag;
                        }

                        // Add to found list (set) and fcset (vector) if the created is new one.

                        if (list_found.find(IntList(order + 2, ind_mapped)) == list_found.end()) {
                            list_found.insert(IntList(order + 2, ind_mapped));

                            fc_vec.emplace_back(FcProperty(order + 2,
                                                           c_tmp,
                                                           ind_mapped,
                                                           nmother));
                            ++ndeps;

                            // Add equivalent interaction list (permutation) if there are two or more indices
                            // which belong to the primitive cell.
                            // This procedure is necessary for constructing a sensing matrix.

                            for (i = 0; i < 3 * nat; ++i) is_searched[i] = false;
                            is_searched[ind_mapped[0]] = true;
                            for (i = 1; i < order + 2; ++i) {
                                if ((!is_searched[ind_mapped[i]]) && is_inprim(ind_mapped[i],
                                                                               symm_in->get_nat_prim(),
                                                                               symm_in->get_map_p2s())) {

                                    for (j = 0; j < order + 2; ++j) ind_mapped_tmp[j] = ind_mapped[j];
                                    std::swap(ind_mapped_tmp[0], ind_mapped_tmp[i]);
                                    sort_tail(order + 2, ind_mapped_tmp);
                                    fc_vec.emplace_back(FcProperty(order + 2,
                                                                   c_tmp,
                                                                   ind_mapped_tmp,
                                                                   nmother));

                                    ++ndeps;

                                    is_searched[ind_mapped[i]] = true;
                                }
                            }

                        }
                    }
                }
            } // close symmetry loop

            if (is_zero) {
                if (store_zeros_in) {
                    for (auto it = fc_vec.rbegin(); it != fc_vec.rbegin() + ndeps; ++it) {
                        (*it).mother = std::numeric_limits<size_t>::max();
                        fc_zeros_out.push_back(*it);
                    }
                }
                for (i = 0; i < ndeps; ++i) fc_vec.pop_back();
            } else {
                ndup.push_back(ndeps);
                ++nmother;
            }

        } // close xyz component loop
    }     // close atom number loop (iterator)

    deallocate(xyzcomponent);
    list_found.clear();
    deallocate(atmn);
    deallocate(atmn_mapped);
    deallocate(ind);
    deallocate(ind_mapped);
    deallocate(ind_tmp);
    deallocate(ind_mapped_tmp);
    deallocate(is_searched);
    deallocate(rotation);
    deallocate(map_sym);

    // sort fc_vec

    if (!ndup.empty()) {
        std::sort(fc_vec.begin(), fc_vec.begin() + ndup[0]);
        auto nbegin = ndup[0];
        for (size_t mm = 1; mm < ndup.size(); ++mm) {
            const auto nend = nbegin + ndup[mm];
            std::sort(fc_vec.begin() + nbegin, fc_vec.begin() + nend);
            nbegin += ndup[mm];
        }
    }
}

void Fcs::get_constraint_symmetry(const size_t nat,
                                  const Symmetry *symmetry,
                                  const int order,
                                  const std::string basis,
                                  const std::vector<FcProperty> &fc_table_in,
                                  const size_t nparams,
                                  const double tolerance,
                                  ConstraintSparseForm &const_out,
                                  const bool do_rref) const
{
    // Create constraint matrices arising from the crystal symmetry.
    // Necessary for hexagonal systems.

    int i;
    // int j;
    unsigned int isym;
    int ixyz;
    int *index_tmp;
    int **xyzcomponent;
    int nsym_in_use;
    std::unordered_set<FcProperty> list_found;

    typedef std::vector<ConstraintDoubleElement> ConstEntry;
    std::vector<ConstEntry> constraint_all;
    ConstEntry const_tmp;

    int **map_sym;
    double ***rotation;
    const auto nsym = symmetry->get_SymmData().size();
    const auto natmin = symmetry->get_nat_prim();
    const auto nfcs = fc_table_in.size();
    const auto use_compatible = false;

    if (order < 0 || nparams == 0) return;

    const auto nxyz = static_cast<int>(std::pow(static_cast<double>(3), order + 2));

    allocate(rotation, nsym, 3, 3);
    allocate(map_sym, nat, nsym);

    get_available_symmop(nat,
                         symmetry,
                         basis,
                         nsym_in_use,
                         map_sym,
                         rotation,
                         use_compatible);

    if (nsym_in_use == 0) {
        deallocate(rotation);
        deallocate(map_sym);
        return;
    }

    const_out.clear();

    allocate(index_tmp, order + 2);
    allocate(xyzcomponent, nxyz, order + 2);
    get_xyzcomponent(order + 2, xyzcomponent);

    // Generate temporary list of parameters
    list_found.clear();
    for (const auto &p : fc_table_in) {
        for (i = 0; i < order + 2; ++i) index_tmp[i] = p.elems[i];
        list_found.insert(FcProperty(order + 2, p.sign,
                                     index_tmp, p.mother));
    }

#ifdef _OPENMP
#pragma omp parallel
#endif
    {
        int j;
        int i_prim;
        int loc_nonzero;
        int *ind;
        int *atm_index, *atm_index_symm;
        int *xyz_index;
        double c_tmp;
        // double maxabs;

        std::unordered_set<FcProperty>::iterator iter_found;
        std::vector<double> const_now_omp;
        std::vector<std::vector<double>> const_omp;

        ConstEntry const_tmp_omp;
        std::vector<ConstEntry> constraint_list_omp;

        allocate(ind, order + 2);
        allocate(atm_index, order + 2);
        allocate(atm_index_symm, order + 2);
        allocate(xyz_index, order + 2);

        const_omp.clear();
        const_now_omp.resize(nparams);

#ifdef _OPENMP
#pragma omp for private(i, isym, ixyz), schedule(static)
#endif
        for (long ii = 0; ii < nfcs; ++ii) {

            for (i = 0; i < order + 2; ++i) {
                atm_index[i] = fc_table_in[ii].elems[i] / 3;
                xyz_index[i] = fc_table_in[ii].elems[i] % 3;
            }

            for (isym = 0; isym < nsym_in_use; ++isym) {

                for (i = 0; i < order + 2; ++i)
                    atm_index_symm[i] = map_sym[atm_index[i]][isym];
                if (!is_inprim(order + 2, atm_index_symm, natmin, symmetry->get_map_p2s())) continue;

                for (i = 0; i < nparams; ++i) const_now_omp[i] = 0.0;

                const_now_omp[fc_table_in[ii].mother] = -fc_table_in[ii].sign;

                for (ixyz = 0; ixyz < nxyz; ++ixyz) {
                    for (i = 0; i < order + 2; ++i)
                        ind[i] = 3 * atm_index_symm[i] + xyzcomponent[ixyz][i];

                    i_prim = get_minimum_index_in_primitive(order + 2, ind, nat, natmin, symmetry->get_map_p2s());
                    std::swap(ind[0], ind[i_prim]);
                    sort_tail(order + 2, ind);

                    iter_found = list_found.find(FcProperty(order + 2, 1.0, ind, 1));
                    if (iter_found != list_found.end()) {
                        c_tmp = coef_sym(order + 2, rotation[isym], xyz_index, xyzcomponent[ixyz]);
                        const_now_omp[(*iter_found).mother] += (*iter_found).sign * c_tmp;
                    }
                }

                if (!is_allzero(const_now_omp, eps8, loc_nonzero)) {
                    if (const_now_omp[loc_nonzero] < 0.0) {
                        for (j = 0; j < nparams; ++j) const_now_omp[j] *= -1.0;
                    }

                    const_tmp_omp.clear();
                    for (j = 0; j < nparams; ++j) {
                        if (std::abs(const_now_omp[j]) >= eps8) {
                            const_tmp_omp.emplace_back(j, const_now_omp[j]);
                        }
                    }
                    constraint_list_omp.emplace_back(const_tmp_omp);
                }

            } // close isym loop
        }     // close ii loop

        deallocate(ind);
        deallocate(atm_index);
        deallocate(atm_index_symm);
        deallocate(xyz_index);

#pragma omp critical
        {
            for (const auto &it : constraint_list_omp) {
                constraint_all.emplace_back(it);
            }
        }
        constraint_list_omp.clear();
    } // close openmp region

    deallocate(xyzcomponent);
    deallocate(index_tmp);
    deallocate(rotation);
    deallocate(map_sym);

    std::sort(constraint_all.begin(), constraint_all.end());
    constraint_all.erase(std::unique(constraint_all.begin(),
                                     constraint_all.end()),
                         constraint_all.end());

    MapConstraintElement const_tmp2;
    auto division_factor = 1.0;
    int counter;
    const_out.clear();

    for (const auto &it : constraint_all) {
        const_tmp2.clear();
        counter = 0;
        for (const auto &it2 : it) {
            if (counter == 0) {
                division_factor = 1.0 / it2.val;
            }
            const_tmp2[it2.col] = it2.val * division_factor;
            ++counter;
        }
        const_out.emplace_back(const_tmp2);
    }
    constraint_all.clear();

    if (do_rref) rref_sparse(nparams, const_out, tolerance);
}

void Fcs::get_constraint_symmetry_in_integer(const size_t nat,
                                             const Symmetry *symmetry,
                                             const int order,
                                             const std::string basis,
                                             const std::vector<FcProperty> &fc_table_in,
                                             const size_t nparams,
                                             const double tolerance,
                                             ConstraintSparseForm &const_out,
                                             const bool do_rref) const
{
    // Create constraint matrices arising from the crystal symmetry.
    // Necessary for hexagonal systems.
    // This does exactly the same thing as get_constraint_symmetry but assumes
    // all elements of the constraint matrix are integer.
    // This version is expected to be more stable (and fast?).

    int i;
    unsigned int isym;
    int ixyz;
    int *index_tmp;
    int **xyzcomponent;
    int nsym_in_use;
    std::unordered_set<FcProperty> list_found;

    typedef std::vector<ConstraintIntegerElement> ConstEntry;
    std::vector<ConstEntry> constraint_all;
    ConstEntry const_tmp;

    int **map_sym;
    double ***rotation;
    const auto nsym = symmetry->get_SymmData().size();
    const auto natmin = symmetry->get_nat_prim();
    const auto nfcs = fc_table_in.size();
    const auto use_compatible = false;

    if (order < 0 || nparams == 0) return;

    const auto nxyz = static_cast<int>(std::pow(static_cast<double>(3), order + 2));

    allocate(rotation, nsym, 3, 3);
    allocate(map_sym, nat, nsym);

    get_available_symmop(nat,
                         symmetry,
                         basis,
                         nsym_in_use,
                         map_sym,
                         rotation,
                         use_compatible);

    if (nsym_in_use == 0) {
        deallocate(rotation);
        deallocate(map_sym);
        return;
    }

    const_out.clear();

    allocate(index_tmp, order + 2);
    allocate(xyzcomponent, nxyz, order + 2);
    get_xyzcomponent(order + 2, xyzcomponent);

    // Generate temporary list of parameters
    list_found.clear();
    for (const auto &p : fc_table_in) {
        for (i = 0; i < order + 2; ++i) index_tmp[i] = p.elems[i];
        list_found.insert(FcProperty(order + 2, p.sign,
                                     index_tmp, p.mother));
    }

#ifdef _OPENMP
#pragma omp parallel
#endif
    {
        int j;
        int i_prim;
        int loc_nonzero;
        int *ind;
        int *atm_index, *atm_index_symm;
        int *xyz_index;
        int c_tmp;

        std::unordered_set<FcProperty>::iterator iter_found;
        std::vector<int> const_now_omp;
        std::vector<std::vector<int>> const_omp;

        ConstEntry const_tmp_omp;
        std::vector<ConstEntry> constraint_list_omp;

        allocate(ind, order + 2);
        allocate(atm_index, order + 2);
        allocate(atm_index_symm, order + 2);
        allocate(xyz_index, order + 2);

        const_omp.clear();
        const_now_omp.resize(nparams);

#ifdef _OPENMP
#pragma omp for private(i, isym, ixyz), schedule(static)
#endif
        for (long ii = 0; ii < nfcs; ++ii) {

            for (i = 0; i < order + 2; ++i) {
                atm_index[i] = fc_table_in[ii].elems[i] / 3;
                xyz_index[i] = fc_table_in[ii].elems[i] % 3;
            }

            for (isym = 0; isym < nsym_in_use; ++isym) {

                for (i = 0; i < order + 2; ++i)
                    atm_index_symm[i] = map_sym[atm_index[i]][isym];
                if (!is_inprim(order + 2, atm_index_symm, natmin, symmetry->get_map_p2s())) continue;

                for (i = 0; i < nparams; ++i) const_now_omp[i] = 0;

                const_now_omp[fc_table_in[ii].mother] = -nint(fc_table_in[ii].sign);

                for (ixyz = 0; ixyz < nxyz; ++ixyz) {
                    for (i = 0; i < order + 2; ++i)
                        ind[i] = 3 * atm_index_symm[i] + xyzcomponent[ixyz][i];

                    i_prim = get_minimum_index_in_primitive(order + 2, ind, nat, natmin, symmetry->get_map_p2s());
                    std::swap(ind[0], ind[i_prim]);
                    sort_tail(order + 2, ind);

                    iter_found = list_found.find(FcProperty(order + 2, 1.0, ind, 1));
                    if (iter_found != list_found.end()) {
                        c_tmp = nint(coef_sym(order + 2, rotation[isym], xyz_index, xyzcomponent[ixyz]));
                        const_now_omp[(*iter_found).mother] += nint((*iter_found).sign) * c_tmp;
                    }
                }

                if (!is_allzero(const_now_omp, loc_nonzero)) {
                    if (const_now_omp[loc_nonzero] < 0) {
                        for (j = 0; j < nparams; ++j) const_now_omp[j] *= -1;
                    }

                    const_tmp_omp.clear();
                    for (j = 0; j < nparams; ++j) {
                        if (std::abs(const_now_omp[j]) > 0) {
                            const_tmp_omp.emplace_back(j, const_now_omp[j]);
                        }
                    }
                    constraint_list_omp.emplace_back(const_tmp_omp);
                }

            } // close isym loop
        }     // close ii loop

        deallocate(ind);
        deallocate(atm_index);
        deallocate(atm_index_symm);
        deallocate(xyz_index);

#pragma omp critical
        {
            for (const auto &it : constraint_list_omp) {
                constraint_all.emplace_back(it);
            }
        }
        constraint_list_omp.clear();
    } // close openmp region

    deallocate(xyzcomponent);
    deallocate(index_tmp);
    deallocate(rotation);
    deallocate(map_sym);

    std::sort(constraint_all.begin(), constraint_all.end());
    constraint_all.erase(std::unique(constraint_all.begin(),
                                     constraint_all.end()),
                         constraint_all.end());

    MapConstraintElement const_tmp2;
    auto division_factor = 1.0;
    int counter;
    const_out.clear();

    for (const auto &it : constraint_all) {
        const_tmp2.clear();
        counter = 0;
        for (const auto &it2 : it) {
            if (counter == 0) {
                division_factor = 1.0 / it2.val;
            }
            const_tmp2[it2.col] = it2.val * division_factor;
            ++counter;
        }
        const_out.emplace_back(const_tmp2);
    }
    constraint_all.clear();

    if (do_rref) rref_sparse(nparams, const_out, tolerance);
}

std::vector<size_t>* Fcs::get_nequiv() const
{
    return nequiv;
}

std::vector<FcProperty>* Fcs::get_fc_table() const
{
    return fc_table;
}

std::vector<ForceConstantTable>* Fcs::get_fc_cart() const
{
    return fc_cart;
}

void Fcs::set_forceconstant_basis(const std::string preferred_basis_in)
{
    preferred_basis = preferred_basis_in;
}

std::string Fcs::get_forceconstant_basis() const
{
    return preferred_basis;
}

void Fcs::set_forceconstant_cartesian(const int maxorder,
                                      double *param_in)
{
    auto ishift = 0;
    int j;

    std::vector<int> atoms_old, atoms_now;
    std::vector<int> coords_now;
    std::vector<std::vector<int>> coord_list;
    std::vector<double> fc_list;
    int **xyzcomponent;
    double prod_matrix;

    if (fc_cart) {
        deallocate(fc_cart);
    }
    allocate(fc_cart, maxorder);

    std::vector<int> elems_permutation;

    std::vector<FcProperty> fc_table_copy;

    nfc_cart_permu.resize(maxorder);
    nfc_cart_nopermu.resize(maxorder);

    for (int i = 0; i < maxorder; ++i) {

        auto nelems = i + 2;

        atoms_old.resize(nelems);
        atoms_now.resize(nelems);
        coords_now.resize(nelems);
        coord_list.clear();
        fc_list.clear();

        const auto nxyz = static_cast<int>(std::pow(3.0, nelems));
        allocate(xyzcomponent, nxyz, nelems);
        get_xyzcomponent(nelems, xyzcomponent);

        for (j = 0; j < nelems; ++j) atoms_old[j] = -1;
        int icount = 0;

        fc_table_copy.clear();
        for (const auto &it : fc_table[i]) {
            elems_permutation = it.elems;
            do {
                fc_table_copy.emplace_back(nelems,
                                           it.sign,
                                           &elems_permutation[0],
                                           it.mother);
            } while (std::next_permutation(elems_permutation.begin() + 1,
                                           elems_permutation.end()));
        }

        // Sort fc_table_copy in ascending order of atomic indices.
        std::sort(fc_table_copy.begin(),
                  fc_table_copy.end(),
                  FcProperty::compare_atom_index);

        for (const auto &it : fc_table_copy) {

            for (j = 0; j < nelems; ++j) {
                atoms_now[j] = it.elems[j] / 3;
                coords_now[j] = it.elems[j] % 3;
            }

            if (atoms_now == atoms_old) {

                coord_list.push_back(coords_now);
                fc_list.push_back(param_in[it.mother + ishift] * it.sign);

                ++icount;

            } else {

                if (icount > 0) {
                    for (auto ixyz = 0; ixyz < nxyz; ++ixyz) {

                        auto fcs_cart = 0.0;

                        for (j = 0; j < icount; ++j) {
                            prod_matrix = 1.0;
                            for (auto k = 0; k < nelems; ++k) {
                                prod_matrix *= basis_conversion_matrix(coord_list[j][k],
                                                                       xyzcomponent[ixyz][k]);
                            }
                            fcs_cart += prod_matrix * fc_list[j];
                        }

                        if (std::abs(fcs_cart) > eps12) {
                            fc_cart[i].emplace_back(nelems,
                                                    fcs_cart,
                                                    &atoms_old[0],
                                                    xyzcomponent[ixyz]);
                        }
                    }
                }
                atoms_old = atoms_now;
                coord_list.clear();
                fc_list.clear();
                coord_list.push_back(coords_now);
                fc_list.push_back(param_in[it.mother + ishift] * it.sign);
                icount = 1;
            }
        }

        if (icount > 0) {
            for (auto ixyz = 0; ixyz < nxyz; ++ixyz) {

                auto fcs_cart = 0.0;

                for (j = 0; j < icount; ++j) {
                    prod_matrix = 1.0;
                    for (auto k = 0; k < nelems; ++k) {
                        prod_matrix *= basis_conversion_matrix(coord_list[j][k],
                                                               xyzcomponent[ixyz][k]);
                    }
                    fcs_cart += prod_matrix * fc_list[j];
                }

                if (std::abs(fcs_cart) > eps12) {
                    fc_cart[i].emplace_back(nelems,
                                            fcs_cart,
                                            &atoms_old[0],
                                            xyzcomponent[ixyz]);
                }
            }
        }

        ishift += nequiv[i].size();

        deallocate(xyzcomponent);

        nfc_cart_permu[i] = fc_cart[i].size();
        nfc_cart_nopermu[i] = std::count_if(fc_cart[i].begin(),
                                            fc_cart[i].end(),
                                            [](const ForceConstantTable &obj) { return obj.is_ascending_order; });
    }
}

std::vector<size_t> Fcs::get_nfc_cart(const int permutation) const
{
    if (permutation) {
        return nfc_cart_permu;
    } else {
        return nfc_cart_nopermu;
    }
}

void Fcs::get_available_symmop(const size_t nat,
                               const Symmetry *symmetry,
                               const std::string basis,
                               int &nsym_avail,
                               int **mapping_symm,
                               double ***rotation,
                               const bool use_compatible) const
{
    // Return mapping information of atoms and the rotation matrices of symmetry operations
    // that are (compatible, incompatible) with the given lattice basis (Cartesian or Lattice).

    // use_compatible == true returns the compatible space group (for creating fc_table)
    // use_compatible == false returnes the incompatible supace group (for creating constraint)

    // int k = 0;
    // for (const auto &it : symmetry->get_SymmData()) {
    //     std::cout << "SYMM #" << ++k << '\n';
    //     std::cout << "compatibility (Cart, Latt): " << it.compatible_with_cartesian << " " << it.compatible_with_lattice << '\n';
    //     for (auto i = 0; i < 3; ++i) {
    //         for (auto j = 0; j < 3; ++j) {
    //             std::cout << std::setw(15) << it.rotation_cart[i][j];
    //         }
    //         std::cout << '\n';
    //     }
    //     std::cout << '\n';
    //     for (auto i = 0; i < 3; ++i) {
    //         for (auto j = 0; j < 3; ++j) {
    //             std::cout << std::setw(15) << it.rotation[i][j];
    //         }
    //         std::cout << '\n';
    //     }
    //     std::cout << '\n';
    // }

    int i, j;
    int counter = 0;

    nsym_avail = 0;

    if (basis == "Cartesian") {

        for (auto it = symmetry->get_SymmData().begin(); it != symmetry->get_SymmData().end(); ++it) {

            if ((*it).compatible_with_cartesian == use_compatible) {

                for (i = 0; i < 3; ++i) {
                    for (j = 0; j < 3; ++j) {
                        rotation[nsym_avail][i][j] = (*it).rotation_cart[i][j];
                    }
                }
                for (i = 0; i < nat; ++i) {
                    mapping_symm[i][nsym_avail] = symmetry->get_map_sym()[i][counter];
                }
                ++nsym_avail;
            }
            ++counter;
        }

    } else if (basis == "Lattice") {

        for (auto it = symmetry->get_SymmData().begin(); it != symmetry->get_SymmData().end(); ++it) {
            if ((*it).compatible_with_lattice == use_compatible) {
                for (i = 0; i < 3; ++i) {
                    for (j = 0; j < 3; ++j) {
                        rotation[nsym_avail][i][j]
                            = static_cast<double>((*it).rotation[i][j]);
                    }
                }
                for (i = 0; i < nat; ++i) {
                    mapping_symm[i][nsym_avail] = symmetry->get_map_sym()[i][counter];
                }
                ++nsym_avail;
            }
            ++counter;
        }

    } else {
        deallocate(rotation);
        deallocate(mapping_symm);
        exit("get_available_symmop", "Invalid basis input");
    }
}

double Fcs::coef_sym(const int n,
                     const double * const *rot,
                     const int *arr1,
                     const int *arr2) const
{
    auto tmp = 1.0;

    for (auto i = 0; i < n; ++i) {
        tmp *= rot[arr2[i]][arr1[i]];
    }
    return tmp;
}

bool Fcs::is_ascending(const int n,
                       const int *arr) const
{
    for (auto i = 0; i < n - 1; ++i) {
        if (arr[i] > arr[i + 1]) return false;
    }
    return true;
}

int Fcs::get_minimum_index_in_primitive(const int n,
                                        const int *arr,
                                        const size_t nat,
                                        const size_t natmin,
                                        const std::vector<std::vector<int>> &map_p2s) const
{
    int i, atmnum;

    std::vector<size_t> ind(n, 3 * nat);

    for (i = 0; i < n; ++i) {

        atmnum = arr[i] / 3;

        for (size_t j = 0; j < natmin; ++j) {
            if (map_p2s[j][0] == atmnum) {
                ind[i] = arr[i];
            }
        }
    }

    auto minval = ind[0];
    auto minloc = 0;

    for (i = 0; i < n; ++i) {
        if (ind[i] < minval) {
            minval = ind[i];
            minloc = i;
        }
    }

    return minloc;
}

bool Fcs::is_inprim(const int n,
                    const int *arr,
                    const size_t natmin,
                    const std::vector<std::vector<int>> &map_p2s) const
{
    for (auto i = 0; i < n; ++i) {
        for (size_t j = 0; j < natmin; ++j) {
            if (map_p2s[j][0] == arr[i]) return true;
        }
    }
    return false;
}

bool Fcs::is_inprim(const int n,
                    const size_t natmin,
                    const std::vector<std::vector<int>> &map_p2s) const
{
    const auto atmn = n / 3;

    for (size_t i = 0; i < natmin; ++i) {
        if (map_p2s[i][0] == atmn) return true;
    }

    return false;
}

void Fcs::get_xyzcomponent(const int n,
                           int **xyz) const
{
    // Return xyz component for the given order using boost algorithm library

    int i;

    std::vector<int> v(3 * n);

    for (i = 0; i < n; ++i) v[i] = 0;
    for (i = n; i < 2 * n; ++i) v[i] = 1;
    for (i = 2 * n; i < 3 * n; ++i) v[i] = 2;

    auto m = 0;

    do {
        xyz[m][0] = v[0];
        for (i = 1; i < n; ++i) xyz[m][i] = v[i];
        ++m;
    } while (boost::next_partial_permutation(v.begin(), v.begin() + n, v.end()));
}

bool Fcs::is_allzero(const std::vector<double> &vec,
                     const double tol,
                     int &loc) const
{
    loc = -1;
    const auto n = vec.size();
    for (auto i = 0; i < n; ++i) {
        if (std::abs(vec[i]) > tol) {
            loc = i;
            return false;
        }
    }
    return true;
}

bool Fcs::is_allzero(const std::vector<int> &vec,
                     int &loc) const
{
    loc = -1;
    for (auto i = 0; i < vec.size(); ++i) {
        if (std::abs(vec[i]) > 0) {
            loc = i;
            return false;
        }
    }
    return true;
}

Eigen::Matrix3d Fcs::get_basis_conversion_matrix() const
{
    return basis_conversion_matrix;
}

void Fcs::set_basis_conversion_matrix(const Cell &supercell)
{
    if (preferred_basis == "Lattice") {
        // multiply the scale factor for making the determinant of the basis_conversion_matrix
        // as one.
        const auto scale_factor = std::pow(supercell.volume, 1.0 / 3.0) / (2.0 * pi);
        for (auto i = 0; i < 3; ++i) {
            for (auto j = 0; j < 3; ++j) {
                basis_conversion_matrix(i, j)
                    = supercell.reciprocal_lattice_vector[i][j] * scale_factor;
            }
        }
    } else if (preferred_basis == "Cartesian") {
        for (auto i = 0; i < 3; ++i) {
            for (auto j = 0; j < 3; ++j) {
                if (i == j) {
                    basis_conversion_matrix(i, j) = 1.0;
                } else {
                    basis_conversion_matrix(i, j) = 0.0;

                }
            }
        }
    }
}