static char help[] = "Solve small CMEs to benchmark intranode performance.\n\n";

#include<iomanip>
#include <petscmat.h>
#include <petscvec.h>
#include <petscviewer.h>
#include <Sys.h>
#include <armadillo>
#include <cmath>
#include <sys/stat.h>
#include "FspSolverMultiSinks.h"

namespace six_species_cme {
// stoichiometric matrix of the toggle switch model
arma::Mat<PetscInt> SM{{1, -1, 0, 0, 0, 0, 0, 0, -2, 2},
                       {0, 0, 0, 0, -1, 1, -1, 1, 1, -1},
                       {0, 0, 0, 0, -1, 1, 0, 0, 0, 0},
                       {0, 0, 0, 0, 1, -1, -1, 1, 0, 0},
                       {0, 0, 0, 0, 0, 0, 1, -1, 0, 0},
                       {0, 0, 1, -1, 0, 0, 0, 0, 0, 0}};

// reaction parameters
const PetscReal Avo = 6.022140857e23, c0 = 0.043, c1 = 0.0007, c2 = 0.078, c3 = 0.0039, c4 =
    0.012e09 / (Avo), c5 = 0.4791, c6 = 0.00012e09 / (Avo), c7 = 0.8765e-11, c8 =
    0.05e09 / (Avo), c9 = 0.5, avg_cell_cyc_time = 35 * 60.0;

// propensity function
inline PetscReal transreg_propensity(const PetscInt *X, const PetscInt k) {
  switch (k) {
    case 0:return c0 * PetscReal(X[5]);
    case 1:return c1 * PetscReal(X[0]);
    case 2:return c2 * PetscReal(X[3]);
    case 3:return c3 * PetscReal(X[5]);
    case 4:return PetscReal(X[1]) * PetscReal(X[2]);
    case 5:return c5 * PetscReal(X[3]);
    case 6:return PetscReal(X[3]) * PetscReal(X[1]);
    case 7:return c7 * PetscReal(X[4]);
    case 8:return 0.5 * PetscReal(X[0]) * PetscReal(X[0] - 1);
    case 9:return c9 * PetscReal(X[1]);
    default:return 0.0;
  }
}
int propensity(const int reaction,
               const int num_species,
               const int num_states,
               const int *states,
               PetscReal *outputs,
               void *args) {
  int (*X)[6] = ( int (*)[6] ) states;
  for (int i = 0; i < num_states; ++i) {
    outputs[i] = transreg_propensity(X[i], reaction);
  }
  return 0;
}

// Function to constraint the shape of the Fsp
void lhs_constr(PetscInt num_species, PetscInt num_constrs, PetscInt num_states, PetscInt *states, int *vals,
                void *args) {
  for (int j{0}; j < num_states; ++j) {
    for (int i{0}; i < 6; ++i) {
      vals[j * num_constrs + i] = states[num_species * j + i];
    }
  }
}

arma::Row<int> rhs_constr{10, 6, 1, 2, 1, 1};
arma::Row<double> expansion_factors{0.5, 0.5, 0.5, 0.5, 0.5, 0.5};
arma::Row<int> rhs_constr_hyperrec{10, 6, 1, 2, 1, 1};
arma::Row<double> expansion_factors_hyperrec{0.2, 0.2, 0.2, 0.2, 0.2, 0.2};

// function to compute the time-dependent coefficients of the propensity functions
int t_fun(PetscReal t, int nc, double *vals, void *args) {
  arma::Row<PetscReal> u(vals, 10, false, true);
  u.fill(1.0);

  PetscReal AV = 6.022140857 * 1.0e8 * pow(2.0, t / avg_cell_cyc_time); // cell's volume
  u(4) = 0.012e09 / AV;
  u(6) = 0.00012e09 / AV;
  u(8) = 0.05e09 / AV;
  return 0;
}
}

using arma::dvec;
using arma::Col;
using arma::Row;

using std::cout;
using std::endl;

using namespace six_species_cme;
using namespace pacmensl;

void output_marginals(MPI_Comm comm, std::string model_name, PartitioningType fsp_par_type,
                      PartitioningApproach fsp_repart_approach, std::string constraint_type,
                      DiscreteDistribution &solution, arma::Row<int> constraints);

void output_performance(MPI_Comm comm,std::string &model_name,PartitioningType fsp_par_type,
                        PartitioningApproach fsp_repart_approach,std::string constraint_type,
                        ODESolverType ode_type,
                        FspSolverMultiSinks &fsp_solver);

int ParseOptions(MPI_Comm comm, PartitioningType &fsp_par_type, PartitioningApproach &fsp_repart_approach,
                 PetscBool &output_marginal, PetscBool &fsp_log_events);

int main(int argc, char *argv[]) {
  Environment my_env(&argc, &argv, help);
  PetscMPIInt ierr, myRank, num_procs;
  PetscErrorCode petsc_err;

  MPI_Comm comm;
  MPI_Comm_dup(PETSC_COMM_WORLD, &comm);
  MPI_Comm_size(comm, &num_procs);

  // Register PETSc stages for profiling
  PetscLogStage stages[2];
  petsc_err = PetscLogStageRegister("Solve with adaptive default state set shape", &stages[0]);
  CHKERRQ(petsc_err);
  petsc_err = PetscLogStageRegister("Solve with fixed default state set shape", &stages[1]);
  CHKERRQ(petsc_err);

  std::string part_type;
  std::string part_approach;

  std::string model_name = "transcr_reg_6d";
  Model model(SM, t_fun, propensity, nullptr, nullptr, {4, 6, 8});

  PetscReal t_final = 60.00 * 5;
  PetscReal fsp_tol = 1.0e-4;
  arma::Mat<PetscInt> X0 = {2, 6, 0, 2, 0, 0};
  X0 = X0.t();
  arma::Col<PetscReal> p0 = {1.0};
  arma::Mat<PetscInt> stoich_mat = SM;

  // Default options
  PartitioningType fsp_par_type = PartitioningType::GRAPH;
  PartitioningApproach fsp_repart_approach = PartitioningApproach::REPARTITION;
  ODESolverType fsp_odes_type = CVODE;
  PetscBool output_marginal = PETSC_FALSE;
  PetscBool fsp_log_events = PETSC_FALSE;

  ierr = ParseOptions(comm, fsp_par_type, fsp_repart_approach, output_marginal, fsp_log_events);
  CHKERRQ(ierr);

  FspSolverMultiSinks fsp_solver(comm, fsp_par_type, CVODE);
  fsp_solver.SetFromOptions();
  fsp_solver.SetInitialDistribution(X0, p0);
  fsp_solver.SetModel(model);
  DiscreteDistribution solution;

  petsc_err = PetscLogStagePush(stages[0]);
  CHKERRQ(petsc_err);
  // Solve using adaptive default constraints
  fsp_solver.SetInitialBounds(rhs_constr);
  fsp_solver.SetInitialBounds(rhs_constr_hyperrec);
  fsp_solver.SetExpansionFactors(expansion_factors_hyperrec);
  fsp_solver.SetFromOptions();
  fsp_solver.SetUp();
  solution = fsp_solver.Solve(t_final, fsp_tol, 0);
  std::shared_ptr<const StateSetConstrained> fss = std::static_pointer_cast<const StateSetConstrained>(fsp_solver.GetStateSet());
  arma::Row<int> final_hyperrec_constr = fss->GetShapeBounds();
  if (fsp_log_events) {
    output_performance(PETSC_COMM_WORLD, model_name, fsp_par_type, fsp_repart_approach,
                       std::string("adaptive_default"), fsp_odes_type, fsp_solver);
  }
  if (output_marginal) {
    output_marginals(PETSC_COMM_WORLD, model_name, fsp_par_type, fsp_repart_approach,
                     std::string("adaptive_default"), solution, final_hyperrec_constr);
  }
  fsp_solver.ClearState();
  PetscPrintf(comm, "\n ================ \n");

  petsc_err = PetscLogStagePop();
  CHKERRQ(petsc_err);
  petsc_err = PetscLogStagePush(stages[1]);
  CHKERRQ(petsc_err);
  // Solve using fixed default constraints
  fsp_solver.SetInitialBounds(final_hyperrec_constr);
  fsp_solver.SetUp();
  solution = fsp_solver.Solve(t_final, fsp_tol, 0);
  if (fsp_log_events) {
    output_performance(PETSC_COMM_WORLD, model_name, fsp_par_type, fsp_repart_approach,
                       std::string("fixed_default"), fsp_odes_type, fsp_solver);
  }
  if (output_marginal) {
    output_marginals(PETSC_COMM_WORLD, model_name, fsp_par_type, fsp_repart_approach,
                     std::string("fixed_default"), solution, final_hyperrec_constr);
  }
  fsp_solver.ClearState();
  PetscPrintf(comm, "\n ================ \n");

  return ierr;
}

int ParseOptions(MPI_Comm comm, PartitioningType &fsp_par_type, PartitioningApproach &fsp_repart_approach,
                 PetscBool &output_marginal, PetscBool &fsp_log_events) {
  std::string part_type;
  std::string part_approach;
  part_type = part2str(fsp_par_type);
  part_approach = partapproach2str(fsp_repart_approach);

  // Read options for fsp
  char opt[100];
  PetscBool opt_set;
  int ierr;
  ierr = PetscOptionsGetString(NULL, PETSC_NULL, "-fsp_partitioning_type", opt, 100, &opt_set);
  CHKERRQ(ierr);
  if (opt_set) {
    fsp_par_type = str2part(std::string(opt));
  }

  ierr = PetscOptionsGetString(NULL, PETSC_NULL, "-fsp_repart_approach", opt, 100, &opt_set);
  CHKERRQ(ierr);
  if (opt_set) {
    fsp_repart_approach = str2partapproach(std::string(opt));
  }

  ierr = PetscOptionsGetString(NULL, PETSC_NULL, "-fsp_output_marginal", opt, 100, &opt_set);
  CHKERRQ(ierr);
  if (opt_set) {
    if (strcmp(opt, "1") == 0 || strcmp(opt, "true") == 0) {
      output_marginal = PETSC_TRUE;
    }
  }

  ierr = PetscOptionsGetString(NULL, PETSC_NULL, "-fsp_log_events", opt, 100, &opt_set);
  CHKERRQ(ierr);
  if (opt_set) {
    if (strcmp(opt, "1") == 0 || strcmp(opt, "true") == 0) {
      fsp_log_events = PETSC_TRUE;
    }
  }
  PetscPrintf(comm, "Partitiniong option %s \n", part2str(fsp_par_type).c_str());
  PetscPrintf(comm, "Repartitoning option %s \n", partapproach2str(fsp_repart_approach).c_str());
  return 0;
}

void output_performance(MPI_Comm comm,std::string &model_name,PartitioningType fsp_par_type,
                        PartitioningApproach fsp_repart_approach,std::string constraint_type,
                        ODESolverType ode_type,
                        FspSolverMultiSinks &fsp_solver)
{
  int myRank,num_procs;
  MPI_Comm_rank(comm,&myRank);
  MPI_Comm_size(comm,&num_procs);

  std::string ode;
  if (ode_type == KRYLOV)
  {
    ode = "krylov";
  } else
  {
    ode = "cvode";
  }

  std::string part_type;
  std::string part_approach;
  part_type     = part2str(fsp_par_type);
  part_approach = partapproach2str(fsp_repart_approach);

  // Output time breakdowns
  FspSolverComponentTiming    sum_times, min_times, max_times;
  sum_times = fsp_solver.ReduceComponentTiming("sum");
  min_times = fsp_solver.ReduceComponentTiming("min");
  max_times = fsp_solver.ReduceComponentTiming("max");

  if (myRank == 0)
  {
    struct stat buffer;
    int fstat;

    std::string   filename =
                      model_name + "_time_breakdown.dat";

    fstat = stat (filename.c_str(), &buffer);

    std::ofstream file;
    file.open(filename,std::ios_base::app);

    if (fstat != 0){
      file << "ncpu, partitioner, fsp_shape, ode_solver, min_cput, max_cput, avg_cput, mat_gen_time, ode_time, state_expand_time, min_flops, max_flops, avg_flops \n";
    }

    file << num_procs << ","
         << part_type << ","
         << constraint_type << ","
         << ode << ","
         << min_times.TotalTime << ","
         << max_times.TotalTime << ","
         << sum_times.TotalTime/num_procs << ","
         << sum_times.MatrixGenerationTime/num_procs << ","
         << sum_times.ODESolveTime/num_procs << ","
         << sum_times.StatePartitioningTime/num_procs
         << min_times.TotalFlops << ","
         << max_times.TotalFlops << ","
         << sum_times.TotalFlops/num_procs << "\n"
        ;
    file.close();
  }

  FiniteProblemSolverPerfInfo perf_info   = fsp_solver.GetSolverPerfInfo();

  if (myRank == 0){
    std::string filename =
                    model_name + "_perf_info_" + std::to_string(num_procs) + "_" + part_type + "_" + part_approach + "_" +
                        constraint_type + ".dat";
    std::ofstream file;
    file.open(filename);
    file << "Model time, ODEs size, Average processor time (sec) \n";
    for (auto i{0}; i < perf_info.n_step; ++i)
    {
      file << perf_info.model_time[i] << "," << perf_info.n_eqs[i] << "," << perf_info.cpu_time[i] << "\n";
    }
    file.close();
  }
}

void output_marginals(MPI_Comm comm, std::string model_name, PartitioningType fsp_par_type,
                      PartitioningApproach fsp_repart_approach, std::string constraint_type,
                      DiscreteDistribution &solution, arma::Row<int> constraints) {
  int myRank, num_procs;
  MPI_Comm_rank(comm, &myRank);
  MPI_Comm_size(comm, &num_procs);

  std::string part_type;
  std::string part_approach;
  part_type = part2str(fsp_par_type);
  part_approach = partapproach2str(fsp_repart_approach);

  /* Compute the marginal distributions */
  std::vector<arma::Col<PetscReal>> marginals(solution.states_.n_rows);
  for (PetscInt i{0}; i < marginals.size(); ++i) {
    marginals[i] = Compute1DMarginal(solution, i);
  }

  MPI_Comm_rank(PETSC_COMM_WORLD, &myRank);
  if (myRank == 0) {
    for (PetscInt i{0}; i < marginals.size(); ++i) {
      std::string filename =
          model_name + "_marginal_" + std::to_string(i) + "_" + std::to_string(num_procs) + "_" +
              part_type + "_" + part_approach + "_" + constraint_type + ".dat";
      marginals[i].save(filename, arma::raw_ascii);
    }
    std::string filename =
        model_name + "_constraint_bounds_" + std::to_string(num_procs) + "_" + part_type + "_" + part_approach +
            "_" + constraint_type + ".dat";
    constraints.save(filename, arma::raw_ascii);
  }
}


/*
 * lorenz_reconstruct.cpp
 * Date: 2016-02-26
 * Author: Karsten Ahnert (karsten.ahnert@gmx.de)
 * Copyright: Karsten Ahnert
 *
 * Distributed under the Boost Software License, Version 1.0.
 * (See accompanying file LICENSE_1_0.txt or
 * copy at http://www.boost.org/LICENSE_1_0.txt)
 */

#include "serialize.hpp"

#include <boost/numeric/odeint/stepper/runge_kutta4.hpp>
#include <boost/numeric/odeint/integrate/integrate_const.hpp>

#include <iostream>
#include <fstream>
#include <iterator>

struct lorenz_reconstructed
{
    lorenz_reconstructed( dynsys::individual_type individual , dynsys::norm_type xnorm , dynsys::norm_type ynorm )
    : m_individual( std::move( individual ) ) , m_xnorm( std::move( xnorm ) ) , m_ynorm( std::move( ynorm ) )
    {}

    void operator()( dynsys::state_type x , dynsys::state_type& dxdt , double t ) const
    {
        denormalize( x , m_xnorm );
        dxdt[0] = m_individual[0].root()->eval( x );
        dxdt[1] = m_individual[1].root()->eval( x );
        dxdt[2] = m_individual[2].root()->eval( x );
        denormalize( dxdt , m_ynorm );
    }

    static void denormalize( dynsys::state_type& x , dynsys::norm_type const& norm )
    {
        assert( x.size() == norm.size() );
        for( size_t i=0 ; i<x.size() ; ++i )
            x[i] = ( x[i] - norm[i].first ) / norm[i].second;
    }

    static void normalize( dynsys::state_type& x , dynsys::norm_type const& norm )
    {
        assert( x.size() == norm.size() );
        for( size_t i=0 ; i<x.size() ; ++i )
            x[i] = ( x[i] + norm[i].first ) * norm[i].second;
    }

    dynsys::individual_type m_individual;
    dynsys::norm_type m_xnorm;
    dynsys::norm_type m_ynorm;
};

int main( int argc , char** argv )
{
    if( argc != 2 )
    {
        std::cerr << "usage: " << argv[0] << " winner-file" << "\n";
        return -1;
    }

    std::ifstream fin { argv[1] };

    std::string json_string { std::istream_iterator< char >{ fin } , std::istream_iterator< char >{ } };
    auto winner = dynsys::deserialize_winner( json_string );

    std::cout << "Expressions" << std::endl;
    std::cout << gpcxx::polish_string( winner.trees[0] ) << std::endl;
    std::cout << gpcxx::polish_string( winner.trees[1] ) << std::endl;
    std::cout << gpcxx::polish_string( winner.trees[2] ) << std::endl;
    std::cout << std::endl;

    std::cout << "Normalization" << std::endl;
    for( size_t i=0 ; i<dynsys::dim ; ++i )
        std::cout << winner.xnorm[i].first << " " << winner.xnorm[i].second << " " << winner.ynorm[i].first << " " << winner.ynorm[i].second << "\n";
    std::cout << std::endl;

    auto sys = lorenz_reconstructed { winner.trees , winner.xnorm , winner.ynorm };

    using stepper_type = boost::numeric::odeint::runge_kutta4< dynsys::state_type > ;

    dynsys::state_type x {{ 10.0 , 10.0 , 10.0 }};
    stepper_type stepper;
    std::ofstream fout { "reconstruct.dat" };
    boost::numeric::odeint::integrate_const( stepper , sys , x , 0.0 , 100.0 , 0.1 , [&fout]( auto const& x , auto t ) {
        fout << x[0] << " " << x[1] << " " << x[2] << "\n"; } );

    std::ofstream fout2 { "functions.dat" };
    for( double x = -20.0 ; x <= 20.0 ; x += 2.0 )
    {
        for( double y = -20.0 ; y <= 20.0 ; y += 2.0 )
        {
            for( double z = -20.0 ; z <= 20.0 ; z += 2.0 )
            {
                dynsys::state_type xx {{ x , y , z }};
                dynsys::state_type f;
                sys( xx , f , 0.0 );
                fout2 << x << " " << y << " " << z << " " << f[0] << " " << f[1] << " " << f[2] << "\n";
            }
        }
    }

    return 0;
}


#include <iostream>
#include <cstdlib>
#include <utility>

#include <boost/numeric/ublas/vector.hpp>
#include <boost/numeric/ublas/matrix_sparse.hpp>

#include <amgcl/amg.hpp>
#include <amgcl/make_solver.hpp>
#include <amgcl/backend/builtin.hpp>
#include <amgcl/adapter/ublas.hpp>
#include <amgcl/coarsening/smoothed_aggregation.hpp>
#include <amgcl/relaxation/spai0.hpp>
#include <amgcl/solver/bicgstabl.hpp>
#include <amgcl/profiler.hpp>

#include "sample_problem.hpp"

typedef boost::numeric::ublas::compressed_matrix<
    double, boost::numeric::ublas::row_major
    > ublas_matrix;

typedef boost::numeric::ublas::vector<double> ublas_vector;

namespace amgcl {
    profiler<> prof;
}

int main(int argc, char *argv[]) {
    using amgcl::prof;

    std::vector<int>    ptr;
    std::vector<int>    col;
    std::vector<double> val;
    std::vector<double> rhs;

    prof.tic("assemble");
    int m = argc > 1 ? atoi(argv[1]) : 128;
    int n = sample_problem(m, val, col, ptr, rhs);

    // Create ublas matrix with the data.
    ublas_matrix A(n, n);
    A.reserve(ptr[n]);

    for(int i = 0; i < n; ++i)
        for(int j = ptr[i], e = ptr[i+1]; j < e; ++j)
            A.push_back(i, col[j], val[j]);
    prof.toc("assemble");

    prof.tic("build");
    amgcl::make_solver<
        amgcl::amg<
            amgcl::backend::builtin<double>,
            amgcl::coarsening::smoothed_aggregation,
            amgcl::relaxation::spai0
            >,
        amgcl::solver::bicgstabl<
            amgcl::backend::builtin<double>
            >
        > solve( amgcl::backend::map(A) );
    prof.toc("build");

    std::cout << solve.precond() << std::endl;

    ublas_vector x(n, 0);

    prof.tic("solve");
    size_t iters;
    double resid;
    std::tie(iters, resid) = solve(rhs, x);
    prof.toc("solve");

    std::cout << "Iterations: " << iters << std::endl
              << "Error:      " << resid << std::endl
              << std::endl << prof << std::endl;
}


#include "interpolate_bilinearly.h"
#include "matrix.h"
#include "round_point.h"
#include <boost/multi_array.hpp>
#include <iostream>
#include <vector>

Point interpolate_point_bilinearly(Point p1,
                                   const boost::multi_array<double, 2> *xdisp,
                                   const boost::multi_array<double, 2> *ydisp,
                                   const unsigned int lx,
                                   const unsigned int ly)
{
  const double intp_x =
    interpolate_bilinearly(p1.x(), p1.y(), xdisp, 'x', lx, ly);
  const double intp_y =
    interpolate_bilinearly(p1.x(), p1.y(), ydisp, 'y', lx, ly);
  return Point(p1.x() + intp_x, p1.y() + intp_y);
}

void InsetState::project()
{
  // Calculate displacement from proj array
  boost::multi_array<double, 2> xdisp(boost::extents[lx_][ly_]);
  boost::multi_array<double, 2> ydisp(boost::extents[lx_][ly_]);
  for (unsigned int i = 0; i < lx_; ++i) {
    for (unsigned int j=0; j<ly_; ++j) {
      xdisp[i][j] = proj_[i][j].x - i - 0.5;
      ydisp[i][j] = proj_[i][j].y - j - 0.5;
    }
  }

  // Cumulative projection
  for (unsigned int i = 0; i < lx_; ++i) {
    for (unsigned int j = 0; j < ly_; ++j) {

      // TODO: Should the interpolation be made on the basis of triangulation?
      // Calculate displacement for cumulative graticule coordinates
      const double graticule_intp_x = interpolate_bilinearly(
        cum_proj_[i][j].x,
        cum_proj_[i][j].y,
        &xdisp, 'x', lx_, ly_);
      const double graticule_intp_y = interpolate_bilinearly(
        cum_proj_[i][j].x,
        cum_proj_[i][j].y,
        &ydisp, 'y', lx_, ly_);

      // Update cumulative graticule coordinates
      cum_proj_[i][j].x += graticule_intp_x;
      cum_proj_[i][j].y += graticule_intp_y;
    }
  }

  // Specialise/curry interpolate_point_bilinearly such that it only requires
  // one argument (Point p1).
  std::function<Point(Point)> lambda =
    [&xdisp, &ydisp, lx = lx_, ly = ly_](Point p1) {
      return interpolate_point_bilinearly(p1, &xdisp, &ydisp, lx, ly);
    };

  // Apply "lambda" to all points
  transform_points(lambda);
  return;
}

// In chosen_diag() and transformed_triangle(), the input x-coordinates can
// only be 0, lx, or 0.5, 1.5, ..., lx-0.5. A similar rule applies to the
// y-coordinates.
void InsetState::exit_if_not_on_grid_or_edge(const Point p1) const
{
  if ((p1.x() != 0.0 && p1.x() != lx_ && p1.x() - int(p1.x()) != 0.5) ||
      (p1.y() != 0.0 && p1.y() != ly_ && p1.y() - int(p1.y()) != 0.5)) {
    std::cerr << "Error: Invalid input coordinate in triangulation\n"
              << "\tpt = ("
              << p1.x()
              << ", "
              << p1.y()
              << ")"
              << std::endl;
    exit(1);
  }
  return;
}

Point InsetState::projected_point(const Point p1)
{
  exit_if_not_on_grid_or_edge(p1);
  const unsigned int proj_x = std::min(
    static_cast<unsigned int>(lx_) - 1,
    static_cast<unsigned int>(p1.x()));
  const unsigned int proj_y = std::min(
    static_cast<unsigned int>(ly_) - 1,
    static_cast<unsigned int>(p1.y()));
  return Point((p1.x() == 0.0 || p1.x() == lx_) ?
               p1.x() :
               proj_[proj_x][proj_y].x,
               (p1.y() == 0.0 || p1.y() == ly_) ?
               p1.y() :
               proj_[proj_x][proj_y].y);
}

// TODO: chosen_diag() seems to be more naturally thought of as a boolean
// than an integer.

// For a graticule cell with corners stored in the XYPoint array v, determine
// whether the diagonal from v[0] to v[2] is inside the graticule cell. If
// yes, return 0. Otherwise, if the diagonal from v[1] to v[3] is inside the
// graticule cell, return 1. If neither of the two diagonals is inside the
// graticule cell, then the cell's topology is invalid; thus, we exit with an
// error message.
int InsetState::chosen_diag(const Point v[4], unsigned int *num_concave)
{
  // The input v[i].x can only be 0, lx, or 0.5, 1.5, ..., lx-0.5. A similar
  // rule applies to the y-coordinates.
  for (unsigned int i = 0; i < 4; ++i) {
    exit_if_not_on_grid_or_edge(v[i]);
  }

  // Transform the coordinates in v to the corresponding coordinates on the
  // projected grid. If the x-coordinate is 0 or lx, we keep the input. The
  // input v[i].x can only be 0, lx, or 0.5, 1.5, ..., lx-0.5. A similar rule
  // applies to the y-coordinates. This condition is checked in
  // projected_point().
  Point tv[4];
  for (unsigned int i = 0; i < 4; ++i) {
    tv[i] = projected_point(v[i]);
  }

  // Get the two possible midpoints
  const Point midpoint0(
    (tv[0].x() + tv[2].x()) / 2,
    (tv[0].y() + tv[2].y()) / 2);
  const Point midpoint1(
    (tv[1].x() + tv[3].x()) / 2,
    (tv[1].y() + tv[3].y()) / 2);

  // Get the transformed graticule cell as a polygon
  Polygon trans_graticule;
  for (unsigned int i = 0; i < 4; ++i) {
    trans_graticule.push_back(tv[i]);
  }

  // Check if graticule cell is concave
  if (!trans_graticule.is_convex()) {
    *num_concave += 1;
  }
  if (trans_graticule.bounded_side(midpoint0) == CGAL::ON_BOUNDED_SIDE) {
    return 0;
  }
  if (trans_graticule.bounded_side(midpoint1) == CGAL::ON_BOUNDED_SIDE) {
    return 1;
  }
  std::cerr << "Invalid graticule cell! At\n";
  std::cerr << "(" << tv[0].x() << ", " << tv[0].y() << ")\n";
  std::cerr << "(" << tv[1].x() << ", " << tv[1].y() << ")\n";
  std::cerr << "(" << tv[2].x() << ", " << tv[2].y() << ")\n";
  std::cerr << "(" << tv[3].x() << ", " << tv[3].y() << ")\n";
  std::cerr << "Original: \n";
  std::cerr << "(" << v[0].x() << ", " << v[0].y() << ")\n";
  std::cerr << "(" << v[1].x() << ", " << v[1].y() << ")\n";
  std::cerr << "(" << v[2].x() << ", " << v[2].y() << ")\n";
  std::cerr << "(" << v[3].x() << ", " << v[3].y() << ")\n";
  std::cerr << "i: "
            << static_cast<unsigned int>(v[0].x())
            << ", j: "
            << static_cast<unsigned int>(v[0].y())
            << std::endl;
  exit(1);
}

void InsetState::fill_graticule_diagonals()
{
  // Initialize array if running for the first time
  if (graticule_diagonals_.shape()[0] != lx_ ||
      graticule_diagonals_.shape()[1] != ly_) {
    graticule_diagonals_.resize(boost::extents[lx_ - 1][ly_ - 1]);
  }
  unsigned int n_concave = 0;  // Count concave graticule cells
  for (unsigned int i = 0; i < lx_ - 1; ++i) {
    for (unsigned int j = 0; j < ly_ - 1; ++j) {
      Point v[4];
      v[0] = Point(double(i) + 0.5, double(j) + 0.5);
      v[1] = Point(double(i) + 1.5, double(j) + 0.5);
      v[2] = Point(double(i) + 1.5, double(j) + 1.5);
      v[3] = Point(double(i) + 0.5, double(j) + 1.5);
      graticule_diagonals_[i][j] = chosen_diag(v, &n_concave);
    }
  }
  std::cerr << "Number of concave graticule cells: "
            << n_concave
            << std::endl;
  return;
}

std::array<Point, 3> InsetState::transformed_triangle(const std::array<Point, 3>
                                                      tri)
{
  std::array<Point, 3> transf_tri;
  for (unsigned int i = 0; i < 3; ++i) {
    exit_if_not_on_grid_or_edge(tri[i]);
    const auto transf_pt = projected_point(tri[i]);
    transf_tri[i] = transf_pt;
  }
  return transf_tri;
}

// Determine if a point `pt` is on the boundary of a triangle by using cross
// products to find areas spanned by pt and each triangle edge. Idea from:
// https://stackoverflow.com/questions/7050186/find-if-point-lies-on-line-segment
// This function is needed because, sometimes,
// `triangle.bounded_side(Point(x, y)) == CGAL::ON_BOUNDARY` does not return
// `true` even if the point is on the boundary.
bool is_on_triangle_boundary(const Point pt, const Polygon triangle)
{
  for (unsigned int i = 0; i < triangle.size(); ++i) {
    const auto t1 = triangle[i];
    const auto t2 = triangle[(i == triangle.size() - 1) ? 0 : i + 1];
    const double area = (t1.x() - pt.x()) * (t2.y() - pt.y()) -
                        (t2.x() - pt.x()) * (t1.y() - pt.y());
    if (almost_equal(area, 0.0)) {
      return true;
    }
  }
  return false;
}

// Get the untransformed coordinates of the triangle in which the point `pt`
// is located. After transformation, this triangle must be entirely inside
// the transformed graticule cell.
std::array<Point, 3> InsetState::untransformed_triangle(const Point pt)
{
  if (pt.x() < 0 || pt.x() > lx_ || pt.y() < 0 || pt.y() > ly_) {
    CGAL::set_pretty_mode(std::cerr);
    std::cerr << "ERROR: coordinate outside bounding box in "
              << __func__
              << "().\npt = "
              << pt
              << std::endl;
    exit(1);
  }

  // Get original graticule coordinates
  Point v[4];
  v[0] = Point(std::max(0.0, floor(pt.x() + 0.5) - 0.5),
               std::max(0.0, floor(pt.y() + 0.5) - 0.5));
  v[1] = Point(std::min(static_cast<double>(lx_), floor(pt.x() + 0.5) + 0.5),
               v[0].y());
  v[2] = Point(v[1].x(),
               std::min(static_cast<double>(ly_), floor(pt.y() + 0.5) + 0.5));
  v[3] = Point(v[0].x(),
               v[2].y());

  // TODO: diag SEEMS TO BE MORE NATURALLY THOUGHT OF AS bool INSTEAD OF int.
  // Assuming that the transformed graticule does not have self-intersections,
  // at least one of the diagonals must be completely inside the graticule.
  // We use that diagonal to split the graticule into two triangles.
  int diag;
  if (v[0].x() == 0.0 || v[0].y() == 0.0 || v[2].x() == lx_ || v[2].y() == ly_) {

    // Case where the graticule is on the edge of the grid.
    // We calculate the chosen diagonal, as graticule_diagonals does not store
    // the diagonals for edge grid cells.
    unsigned int n_concave = 0;
    diag = chosen_diag(v, &n_concave);
  } else {

    // Case where the graticule is not on the edge of the grid. We can find the
    // already computed chosen diagonal in graticule_diagonals_.
    const unsigned int x = static_cast<unsigned int>(v[0].x());
    const unsigned int y = static_cast<unsigned int>(v[0].y());
    diag = graticule_diagonals_[x][y];
  }

  // Get the two possible triangles
  Polygon triangle1;
  Polygon triangle2;
  if (diag == 0) {
    triangle1.push_back(v[0]);
    triangle1.push_back(v[1]);
    triangle1.push_back(v[2]);
    triangle2.push_back(v[0]);
    triangle2.push_back(v[2]);
    triangle2.push_back(v[3]);
  } else {
    triangle1.push_back(v[0]);
    triangle1.push_back(v[1]);
    triangle1.push_back(v[3]);
    triangle2.push_back(v[1]);
    triangle2.push_back(v[2]);
    triangle2.push_back(v[3]);
  }

  // Determine which untransformed triangle the given point is in. If the
  // point is in neither, an error is raised.
  std::array<Point, 3> triangle_coordinates;
  if ((triangle1.bounded_side(pt) == CGAL::ON_BOUNDED_SIDE) ||
      is_on_triangle_boundary(pt, triangle1)) {
    for (unsigned int i = 0; i < triangle1.size(); ++i) {
      triangle_coordinates[i] = triangle1[i];
    }
  } else if ((triangle2.bounded_side(pt) == CGAL::ON_BOUNDED_SIDE) ||
             is_on_triangle_boundary(pt, triangle2)) {
    for (unsigned int i = 0; i < triangle2.size(); ++i) {
      triangle_coordinates[i] = triangle2[i];
    }
  } else {
    std::cerr << "Point not in graticule cell!\n";
    std::cerr << "Point coordinates:\n";
    std::cerr << "(" << pt.x() << ", " << pt.y() << ")\n";
    std::cerr << "Original graticule cell:\n";
    std::cerr << "(" << v[0].x() << ", " << v[0].y() << ")\n";
    std::cerr << "(" << v[1].x() << ", " << v[1].y() << ")\n";
    std::cerr << "(" << v[2].x() << ", " << v[2].y() << ")\n";
    std::cerr << "(" << v[3].x() << ", " << v[3].y() << ")\n";
    std::cerr << "Chosen diagonal: " << diag << "\n";
    exit(1);
  }
  return triangle_coordinates;
}

Point affine_trans(const std::array<Point, 3> tri,
                   const std::array<Point, 3> org_tri,
                   const Point pt)
{
  // For each point, we make the following transformation. Suppose we find
  // that, before the cartogram transformation, a point (x, y) is in the
  // triangle (a, b, c). We want to find its position in the projected
  // triangle (p, q, r). We locally approximate the cartogram transformation
  // by an affine transformation T such that T(a) = p, T(b) = q and T(c) = r.
  // We can think of T as a 3x3 matrix
  //    -----------
  //   |t11 t12 t13|
  //   |t21 t22 t23|  such that
  //   | 0   0   1 |
  //    -----------
  //    -----------   ----------     ----------
  //   |t11 t12 t13| | a1 b1 c1 |   | p1 q1 r1 |
  //   |t21 t22 t23| | a2 b2 c2 | = | p2 q2 r2 | or TA = P.
  //   | 0   0   1 | | 1  1  1  |   |  1  1  1 |
  //    -----------   ----------     ----------
  // Hence, T = PA^{-1}.
  //                              -----------------------
  //                             |b2-c2 c1-b1 b1*c2-b2*c1|
  // We have A^{-1} = (1/det(A)) |c2-a2 a1-c1 a2*c1-a1*c2|. By multiplying
  //                             |a2-b2 b1-a1 a1*b2-a2*b1|
  //                              -----------------------
  // PA^{-1} we obtain t11, t12, t13, t21, t22, and t23. If the original
  // coordinates are (x, y) on the unprojected map, then the transformed
  // coordinates are:
  // post.x = t11*x + t12*y + t13, post.y = t21*x + t22*y + t23.
  const Point pre(pt.x(),pt.y());

  // Old triangle (a, b, c) expressed as matrix A
  const Matrix abc_mA(org_tri[0], org_tri[1], org_tri[2]);

  // New triangle (p, q, r) expressed as matrix P
  const Matrix pqr_mP(tri[0], tri[1], tri[2]);

  // Transformation matrix T
  const auto mT = pqr_mP.multiplied_with(abc_mA.inverse());

  // Transformed point
  return mT.transformed_point(pre);
}

Point InsetState::projected_point_with_triangulation(const Point pt)
{
  // Get the untransformed triangle the point is in
  const auto old_triangle = untransformed_triangle(pt);

  // Get the coordinates of the transformed triangle
  const auto new_triangle = transformed_triangle(old_triangle);

  // Get the transformed point and return it
  const auto transformed_pt = affine_trans(new_triangle, old_triangle, pt);
  return rounded_point(transformed_pt, lx_, ly_);
}

void InsetState::project_with_triangulation()
{

  // Store reference to current object and call member function
  // projected_point_with_triangulation
  // https://www.nextptr.com/tutorial/ta1430524603/
  // capture-this-in-lambda-expression-timeline-of-change
  std::function<Point(Point)> lambda =
    [&](Point p1) {
      return projected_point_with_triangulation(p1);
    };

  // Transforming all points based on triangulation
  transform_points(lambda);

  // Cumulative projection
  for (unsigned int i = 0; i < lx_; ++i) {
    for (unsigned int j = 0; j < ly_; ++j) {
      const Point old_cum_proj(cum_proj_[i][j].x, cum_proj_[i][j].y);
      const auto new_cum_proj_pt =
        projected_point_with_triangulation(old_cum_proj);
      cum_proj_[i][j].x = new_cum_proj_pt.x();
      cum_proj_[i][j].y = new_cum_proj_pt.y();
    }
  }
  return;
}


// Boost.Geometry

// Copyright (c) 2017-2018 Oracle and/or its affiliates.

// Contributed and/or modified by Vissarion Fysikopoulos, on behalf of Oracle

// Use, modification and distribution is subject to the Boost Software License,
// Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)

#ifndef BOOST_GEOMETRY_FORMULAS_MERIDIAN_INVERSE_HPP
#define BOOST_GEOMETRY_FORMULAS_MERIDIAN_INVERSE_HPP

#include <boost/math/constants/constants.hpp>

#include <boost/geometry/core/radius.hpp>

#include <boost/geometry/util/condition.hpp>
#include <boost/geometry/util/math.hpp>
#include <boost/geometry/util/normalize_spheroidal_coordinates.hpp>

#include <boost/geometry/formulas/flattening.hpp>
#include <boost/geometry/formulas/meridian_segment.hpp>

namespace boost { namespace geometry { namespace formula
{

/*!
\brief Compute the arc length of an ellipse.
*/

template <typename CT, unsigned int Order = 1>
class meridian_inverse
{

public :

    struct result
    {
        result()
            : distance(0)
            , meridian(false)
        {}

        CT distance;
        bool meridian;
    };

    template <typename T>
    static bool meridian_not_crossing_pole(T lat1, T lat2, CT diff)
    {
        CT half_pi = math::pi<CT>()/CT(2);
        return math::equals(diff, CT(0)) ||
                    (math::equals(lat2, half_pi) && math::equals(lat1, -half_pi));
    }

    static bool meridian_crossing_pole(CT diff)
    {
        return math::equals(math::abs(diff), math::pi<CT>());
    }

    template <typename T, typename Spheroid>
    static CT meridian_not_crossing_pole_dist(T lat1, T lat2, Spheroid const& spheroid)
    {
        return math::abs(apply(lat2, spheroid) - apply(lat1, spheroid));
    }

    template <typename T, typename Spheroid>
    static CT meridian_crossing_pole_dist(T lat1, T lat2, Spheroid const& spheroid)
    {
        CT c0 = 0;
        CT half_pi = math::pi<CT>()/CT(2);
        CT lat_sign = 1;
        if (lat1+lat2 < c0)
        {
            lat_sign = CT(-1);
        }
        return math::abs(lat_sign * CT(2) * apply(half_pi, spheroid)
                         - apply(lat1, spheroid) - apply(lat2, spheroid));
    }

    template <typename T, typename Spheroid>
    static result apply(T lon1, T lat1, T lon2, T lat2, Spheroid const& spheroid)
    {
        result res;

        CT diff = geometry::math::longitude_distance_signed<geometry::radian>(lon1, lon2);

        if (lat1 > lat2)
        {
            std::swap(lat1, lat2);
        }

        if ( meridian_not_crossing_pole(lat1, lat2, diff) )
        {
            res.distance = meridian_not_crossing_pole_dist(lat1, lat2, spheroid);
            res.meridian = true;
        }
        else if ( meridian_crossing_pole(diff) )
        {
            res.distance = meridian_crossing_pole_dist(lat1, lat2, spheroid);
            res.meridian = true;
        }
        return res;
    }

    // Distance computation on meridians using series approximations
    // to elliptic integrals. Formula to compute distance from lattitude 0 to lat
    // https://en.wikipedia.org/wiki/Meridian_arc
    // latitudes are assumed to be in radians and in [-pi/2,pi/2]
    template <typename T, typename Spheroid>
    static CT apply(T lat, Spheroid const& spheroid)
    {
        CT const a = get_radius<0>(spheroid);
        CT const f = formula::flattening<CT>(spheroid);
        CT n = f / (CT(2) - f);
        CT M = a/(1+n);
        CT C0 = 1;

        if (Order == 0)
        {
           return M * C0 * lat;
        }

        CT C2 = -1.5 * n;

        if (Order == 1)
        {
            return M * (C0 * lat + C2 * sin(2*lat));
        }

        CT n2 = n * n;
        C0 += .25 * n2;
        CT C4 = 0.9375 * n2;

        if (Order == 2)
        {
            return M * (C0 * lat + C2 * sin(2*lat) + C4 * sin(4*lat));
        }

        CT n3 = n2 * n;
        C2 += 0.1875 * n3;
        CT C6 = -0.729166667 * n3;

        if (Order == 3)
        {
            return M * (C0 * lat + C2 * sin(2*lat) + C4 * sin(4*lat)
                      + C6 * sin(6*lat));
        }

        CT n4 = n2 * n2;
        C4 -= 0.234375 * n4;
        CT C8 = 0.615234375 * n4;

        if (Order == 4)
        {
            return M * (C0 * lat + C2 * sin(2*lat) + C4 * sin(4*lat)
                      + C6 * sin(6*lat) + C8 * sin(8*lat));
        }

        CT n5 = n4 * n;
        C6 += 0.227864583 * n5;
        CT C10 = -0.54140625 * n5;

        // Order 5 or higher
        return M * (C0 * lat + C2 * sin(2*lat) + C4 * sin(4*lat)
                  + C6 * sin(6*lat) + C8 * sin(8*lat) + C10 * sin(10*lat));

    }
};

}}} // namespace boost::geometry::formula

#endif // BOOST_GEOMETRY_FORMULAS_MERIDIAN_INVERSE_HPP


#include <limits>
#include <ctime>
#include <fstream>
#include <cmath>
#include <random>
#include <boost/math/distributions/binomial.hpp>
#include <boost/math/distributions/normal.hpp>
#include <boost/math/special_functions/digamma.hpp>

#define LNINVSQRT2PI -0.918938533
#define INVSQRT2PI 0.3989422804
#define LN2 0.69314718056
#define PEAKWIDTH 0.325
#define MAX_EM_ITERATIONS 500
#define EM_CONVERGE 1e-3
//this is used for the minimum logValue as well - to distinguish it from an undef value which is 0
#define MIN_NOISE 1e-8
 //lookup tables
static double logx[32768]; //log base2
static double lnx[32768];  //natural logs

void init_logx();
template <class T> int fitLogNormEM(T *smooth, int nBins,vector<T> &peakBins,vector<T> &peakValues, vector<T> &peakMeans);
using namespace std;
using namespace boost::math;

template <class T> class MedianRecord{
	public:
	 vector<T> means0;
	 vector<T> means1;
	 T median0;
	 T median1;
		MedianRecord(){
   blank();
	 }
  void blank(){
			const T null=std::numeric_limits<T>::quiet_NaN();
			means0.clear();
			means1.clear();
			median0=null;
		 median1=null;
		}

	 void addValues(T mean0, T mean1){
			if(!mean0 || !mean1 || isnan(mean0) || isnan(mean1)) return;
			means0.push_back(mean0);
			means1.push_back(mean1);
		}
		void calculate_medians()	{
			if( means0.size() &&  means1.size() ){
    median0=median(means0.begin(),means0.end());
	   median1=median(means1.begin(),means1.end());
			}
		}
		private:
  template <typename Iterator>  T median(Iterator begin, Iterator end) {
	 Iterator middle = begin + (end - begin)/2;
	 std::nth_element(begin, middle, end);
  if ((end - begin) % 2) return *middle;
		Iterator lower_middle = std::max_element(begin, middle);
		return (*middle + *lower_middle) / 2.0;
 }
};

template <class T> class DeconRecord{
	public:
	T means0 [501];
	T means1 [501];
	T stdevs0 [501];
	T stdevs1 [501];
	T a0[501];
	T a1[501];
	T noiseFractions[501];
	T meansEst0[501];
	T meansEst1[501];
	int dataSizes[501];
	T loglikelihoods[501];
	bool flipped[501];
	bool converged[501];
	DeconRecord(){
		blank();
	}
	void blank(){
		const T null=std::numeric_limits<T>::quiet_NaN();
		fill(means0,means0+501,null);
		fill(means1,means1+501,null);
		fill(stdevs0,stdevs0+501,null);
		fill(stdevs1,stdevs1+501,null);
		fill(a0,a0+501,null);
		fill(a1,a1+501,null);
		fill(loglikelihoods,loglikelihoods+501,null);
		fill(meansEst0,meansEst0+501,null);
		fill(meansEst1,meansEst1+501,null);
		fill(dataSizes,dataSizes+501,0);
		fill(noiseFractions,noiseFractions+501,0);
		fill(flipped,flipped+501,0);
		fill(converged,converged+501,0);
	}
	void transfer(int i,T *meanVars){
		meansEst0[i]=meanVars[0];
		meansEst1[i]=meanVars[1];
	}

	void print_binary(FILE *fp){
		//prints out a short binary file with just the fields
		for (int i=11;i<501;i++){
			fwrite(means0+i,sizeof(T),1,fp);
			fwrite(means1+i,sizeof(T),1,fp);
		}
	}
	void print_binary(FILE *fp,int colorStart,int colorEnd){
		//prints out a short binary file with just the fields
		for (int i=colorStart;i<=colorEnd;i++){
			fwrite(means0+i,sizeof(T),1,fp);
			fwrite(means1+i,sizeof(T),1,fp);
		}
	}
	void print_text(FILE *fp,int verbosity){
		if(verbosity <0){
			print_binary(fp);
			return;
  }
		string headers[14]={"Analyte","Mean1","Mean2","StDev1","StDev2","a1","a2","LogLike","DataSize","NoiseFraction","Converged","Flip","CMean1","CMean2"};
		int nfields=3;
		if(verbosity ==1) nfields=8;
		else if (verbosity >1) nfields=14;
		for(int i=0;i<nfields-1;i++){
			fprintf(fp,"%s\t",headers[i].c_str());
		}
		fprintf(fp,"%s\n",headers[nfields-1].c_str());
		for (int i=1;i<501;i++){
			if(verbosity==0) fprintf(fp,"%3d\t%f\t%f\n",i,means0[i],means1[i]);
			else if(verbosity==1)	fprintf(fp,"%3d\t%f\t%f\t%f\t%f\t%f\t%f\t%e\n",i,means0[i],means1[i],stdevs0[i],stdevs1[i],a0[i],a1[i],loglikelihoods);
   else{
				if((means0[i] > means1 [i] && 	meansEst0[i] < meansEst1[i]) ||  (means0[i] < means1 [i] && 	meansEst0[i] > meansEst1[i])){
					flipped[i]=1;
				}
				fprintf(fp,"%3d\t%f\t%f\t%f\t%f\t%f\t%f\t%e\t%d\t%f\t%s\t%s\t%f\t%f\n",i,means0[i],means1[i],stdevs0[i],stdevs1[i],a0[i],a1[i],loglikelihoods[i],dataSizes[i],noiseFractions[i],"false\0true"+6*(int)converged[i],"false\0true"+6*(int)flipped[i],meansEst0[i],meansEst1[i]);
			}
		}
	}
	void print_text(FILE *fp,int colorStart,int colorEnd,int verbosity){
		if(verbosity <0){
			print_binary(fp,colorStart,colorEnd);
			return;
  }
		string headers[14]={"Analyte","Mean1","Mean2","StDev1","StDev2","a1","a2","LogLike","DataSize","NoiseFraction","Converged","Flip","CMean1","CMean2"};
		int nfields=3;
		if(verbosity ==1) nfields=8;
		else if (verbosity >1) nfields=14;
		for(int i=0;i<nfields-1;i++){
			fprintf(fp,"%s\t",headers[i].c_str());
		}
		fprintf(fp,"%s\n",headers[nfields-1].c_str());
		for (int i=colorStart;i<=colorEnd;i++){
			if(verbosity==0) fprintf(fp,"%3d\t%f\t%f\n",i,means0[i],means1[i]);
			else if(verbosity==1)	fprintf(fp,"%3d\t%f\t%f\t%f\t%f\t%f\t%f\t%e\n",i,means0[i],means1[i],stdevs0[i],stdevs1[i],a0[i],a1[i],loglikelihoods);
   else{
				if((means0[i] > means1 [i] && 	meansEst0[i] < meansEst1[i]) ||  (means0[i] < means1 [i] && 	meansEst0[i] > meansEst1[i])){
					flipped[i]=1;
				}
				//something wrong with format string near space <-fix
				fprintf(fp,"%3d\t%f\t%f\t%f\t%f\t%f\t%f\t%e\t%d\t%f\t%s\t%s \t%f\t%f\n",i,means0[i],means1[i],stdevs0[i],stdevs1[i],a0[i],a1[i],loglikelihoods[i],dataSizes[i],noiseFractions[i],"false\0true"+6*(int)converged[i],"false\0true"+6*(int)flipped[i],meansEst0[i],meansEst1[i]);
			}
		}
	}
};
template <class T> class EM_GMM{
	public:
	 //these are for lognormals usually expressed using base 2 - we change to natural logs so that normalization works
	 T var0,var1,var_est,var_lower,var_upper,mean0,mean1,meanEst0,meanEst1,mean_lower,mean_upper,a0,a1,p_est,p_upper,p_sigma;
	 T *data;
	 int nBins;
	 int nPoints;
	 int maxValue;
	 int minValue;
	 T loglikelihood;
	 bool converged;
	 bool normed; //normalized data?

	 EM_GMM(T *_data,bool _normed,int _nBins,int _nPoints, int _minValue,int _maxValue,T _mus[2],T _vars[2],T _a, T _p_est, T _p_sigma,T _p_upper, T _sigma_lower, T _sigma_est, T _sigma_upper){
			nPoints=_nPoints;
			mean0=_mus[0]*LN2;
			mean1=_mus[1]*LN2;
			meanEst0=mean0;
			meanEst1=mean1;
			var0=_vars[0]*LN2*LN2;
			var1=_vars[1]*LN2*LN2;;
			var_est=_sigma_est*_sigma_est*LN2*LN2;
			var_lower=_sigma_lower*_sigma_lower*LN2*LN2;
			var_upper=_sigma_upper*_sigma_upper*LN2*LN2;
			a0=_a;
			a1=1.0-a0;
			data=_data;
			nBins=_nBins;
			minValue=_minValue;
			mean_lower=lnx[minValue];
			maxValue=_maxValue;
			mean_upper=lnx[maxValue];
			p_est=_p_est;
   p_upper=_p_upper;
   p_sigma=_p_sigma;
   loglikelihood=0;
   normed=_normed;
			//hard estimate for p distorts the distribution
			//with large number of points (i.e. composite of many wells) the spread of possible p values needs incresased as using a binomial correction is the same as a hard limit
			//we assume that the mixture value is not exactly _p since we have estimates from .625 - .700 and there is probably some mixture error in addition to sampling variation which would be binomial
			//for lower numbers (i.e. individual wells) we use the estimate of mixture error to determine the beta distribution and add binomial correction
			//for higher numbers the data swamps the prior if we do this - so we explicitly calculate beta to limit the possible values

			//when given limiting based on distribution
			//calculate alpha and beta of beta distribution
			//alpha=((1-_p)/(_pErr*_pErr) -1/_p)*_p*_p
			//beta=alpha*(1/_p-1)
			//update is now (alpha+k)/(alpha+beta+n) instead of k/n
			//use this formula to derive beta-binomial correction to loglikelihood

			//will not use k/n as estimate of partition ratio - instead calculate an alpha and beta so that the
			//so that the maximum value if the entire sample partitioned 1 way the value equal to the limit
   //alpha =-(N*p_est*p_upper-N*p_est)/(p_upper-p_est)
   //beta=((N*p_est-N)*p_upper-N*p_est+N)/(p_upper-p_est)

   //for betabinomial loglikelihood for y successes in n trials
   //lgamma(n+a+b)-lgamma(y+alpha) -lgamma(n-y+beta) +(alpha+y-1)log(p)+(b+n-y-1)*log(1-p)

   //NB check and flip a1 a0 if a1>a0 - check - define alarger asmaller
	 }
	 void reset(T _mean0,T _mean1, T _var0,T _var1, T _a){
			mean0=_mean0*LN2;;
			mean1=_mean1*LN2;;
			var0=_var0*LN2*LN2;
			var1=_var1*LN2*LN2;
			a0=_a;
			a1=1.0-a0;
			loglikelihood=0;
		}
	 int seed_mclust(T input_p){
   //use m-clust to provide initial points
   //very similar to optimize function but with different estimates
   const int maxIterations=MAX_EM_ITERATIONS;
   const double converge=EM_CONVERGE;
   bool done=0;
   int nIterations=0;
   const int last=(nBins-1<maxValue)? nBins-1 : maxValue;
   T df=3.0,kp=0.01,svar=var_est;

  	//T alpha1=((1.0-p_est)/(p_sigma*p_sigma) -1.0/p_est)*p_est*p_est;
			//T beta1=alpha1*(1.0/(p_est-1));

   //fit mean stdevs to normals norma a*invsqrtpi/sigma * exp((-(x-mean)**2)/(2*var*var))
   //however the  invsqrtpi/sigma needs to be in natural log scale to properly normalize

   //adjust ln
   vector <int> map;
   vector <T> mylnx; //mapped copy of lnx
   vector <T> mydata;
   for (int j=minValue;j<=last; j++){
			//check for underflow
			 if(data[j] && !isnan(data[j])){
					map.push_back(j);
					mylnx.push_back(lnx[j]);
					mydata.push_back(data[j]);
				}
			}

   while(!done){
    double term0=log(a0)+LNINVSQRT2PI-log(sqrt(var0));
    double term1=log(a1)+LNINVSQRT2PI-log(sqrt(var1));
    double inv2Var0=1.0/(2.0*var0);
    double inv2Var1=1.0/(2.0*var1);

  //calculate weighted percentage that points in j come from distribution 1 or 2
  //then calculate stdevs and mean for expectation maximization
   double L=0,sum0=0,sumsq0=0,sum1=0,sumsq1=0,counts0=0,counts1=0;
   const int setSize=map.size();
   for (int m=0;m<setSize; m++){
				const int j=map[m];
			//check for underflow
			 	T logLike0=term0-inv2Var0*(mylnx[m]-mean0)*(mylnx[m]-mean0);
			 	T logLike1=term1-inv2Var1*(mylnx[m]-mean1)*(mylnx[m]-mean1);

			 	//check for underflows
			 	if((!logLike1 || std::isnan(logLike1)) && logLike0 && !std::isnan(logLike0)){
		     L+=mydata[m]*logLike0;
			 		 counts0+=mydata[m];
			 		 sum0+=mylnx[m]*mydata[m];
			 		 sumsq0+=mylnx[m]*mylnx[m]*mydata[m];
			 	}
			 	else if	((!logLike0 || std::isnan(logLike0)) && logLike1 && !std::isnan(logLike1)){
		     L+=mydata[m]*logLike1;
			 		 counts1+=mydata[m];
			 		 sum1+=mylnx[m]*mydata[m];
			 		 sumsq1+=mylnx[m]*mylnx[m]*mydata[m];
			 	}
			 	else{
			 		const T logRatio=logLike0-logLike1;
			 	 if(logRatio > 15){
		     L+=mydata[m]*logLike0;
			 		 counts0+=mydata[m];
			 		 sum0+=mylnx[m]*mydata[m];
			 		 sumsq0+=mylnx[m]*mylnx[m]*mydata[m];
						}
						else if (logRatio < -15){
		     L+=mydata[m]*logLike1;
			 		 counts1+=mydata[m];
			 		 sum1+=mylnx[m]*mydata[m];
			 		 sumsq1+=mylnx[m]*mylnx[m]*mydata[m];
						}
						else{
			 		 const T ratio=exp(logRatio);
			 		 const T r0=ratio/(ratio+1.0);
			 		 const T r1=1.0-r0;

					  T nPoints0=mydata[m]*r0;
				 	 T nPoints1=mydata[m]*r1;

				   counts0+=nPoints0;
				   counts1+=nPoints1;
				   L+=nPoints0*logLike0+nPoints1*logLike1;
				 	 sum0+=nPoints0*mylnx[m];
					  sumsq0+=nPoints0*mylnx[m]*mylnx[m];
				 	 sum1+=nPoints1*mylnx[m];
					  sumsq1+=nPoints1*mylnx[m]*mylnx[m];
					 }
					}
			}
		 //calculate estimate of mean
		 T mean0MLE=(sum0+kp*meanEst0)/(kp+counts0);
   T mean1MLE=(sum1+kp*meanEst1)/(kp+counts1);

   mean0MLE= (mean0MLE > mean_upper)? mean_upper : mean0MLE;
   mean0MLE= (mean0MLE < mean_lower)? mean_lower: mean0MLE;
   mean1MLE= (mean1MLE > mean_upper)? mean_upper: mean1MLE;
   mean1MLE= (mean1MLE < mean_lower)? mean_lower : mean1MLE;

   //estimate a0 a1
   //check if estimated a1 > a0 - if so switch means and var
   T a0MLE,a1MLE;
   if(counts1) a0MLE = counts0/(counts0+counts1);
   else a0MLE=1;

   if(a0MLE < .5){
			//swap terms
			 swap(mean0,mean1);
			 swap(var0,var1);
			 swap(mean0MLE,mean1MLE);
			 swap(counts1,counts0);
			 swap(a0,a1);
			 a0MLE=1.0-a0;
		 }
		 //find unormalized counts
		 T ucounts,ucounts0,ucounts1;
		 if(normed){
				T totalCounts=counts0+counts1;
				ucounts=(T) nPoints * totalCounts;
				ucounts0=(T) nPoints * counts0;
				ucounts1=(T) nPoints * counts1;
			}
		 else{
				ucounts=counts0+counts1;
				ucounts0=counts0;
				ucounts1=counts1;
			}

   a0MLE=(a0MLE > MIXTURE_RATIO + 2*MIXTURE_STD)?MIXTURE_RATIO + 2*MIXTURE_STD :a0MLE;
   a0MLE=(a0MLE < MIXTURE_RATIO - 2*MIXTURE_STD)?MIXTURE_RATIO - 2*MIXTURE_STD :a0MLE;
   a1MLE=1.0-a0MLE;
   T rawVar0MLE= sumsq0-mean0MLE*mean0MLE*counts0/(counts0+6.0);
   T rawVar1MLE= sumsq1-counts1-mean1MLE*mean1MLE*counts1/(counts1+6.0);

   if(isnan(rawVar0MLE)){
  		rawVar0MLE=(var_est+(kp*counts0)*(meanEst0-mean0MLE)*(meanEst0-mean0MLE))/(counts0+kp);
			}
			else{
				rawVar0MLE+=(var_est+(kp*counts0)*(meanEst0-mean0MLE)*(meanEst0-mean0MLE))/(counts0+kp);
			}
   if(isnan(rawVar1MLE)){
  		rawVar1MLE=(var_est+(kp*counts1)*(meanEst1-mean1MLE)*(meanEst1-mean1MLE))/(counts1+kp);
			}
			else{
				rawVar1MLE+=(var_est+(kp*counts1)*(meanEst1-mean1MLE)*(meanEst1-mean1MLE))/(counts1+kp);
			}

   T var0MLE=rawVar0MLE;
   T var1MLE=rawVar1MLE;

  //enforce limits
   var0MLE=(var0MLE > var_upper)? var_upper : var0MLE;
   var0MLE=(var0MLE < var_lower)? var_lower : var0MLE;
   var1MLE=(var1MLE > var_upper)? var_upper : var1MLE;
   var1MLE=(var1MLE < var_lower)? var_lower : var1MLE;

   if(fabs(mean0MLE-mean0) > converge || fabs(mean1MLE-mean1) > converge){
	   mean0=mean0MLE;mean1=mean1MLE;
	   var0=var0MLE;var1=var1MLE;
	   a0=a0MLE;a1=a1MLE;
		 }
		 else{
				loglikelihood=L;
				converged=1;
    var0=rawVar0MLE;var1=rawVar1MLE;
			 var0=(var0 > var_upper*2.0)? var_upper*2.0 : var0;
    var0=(var0 < var_lower/2.0)? var_lower/2.0 : var0;
    var1=(var1 > var_upper*2.0)? var_upper*2.0 : var1;
    var1=(var1< var_lower/2.0)? var_lower/2.0 : var1;
    a1=1.0-a0;
     mean0=mean0MLE;mean1=mean1MLE;
    return(1);
			}
				//adjustment for binomial - seems to give worse results - enough to soft constrain p
				//loglikelihood=L+(ucounts0+alpha1-1)*log(p_est)+(ucounts1+beta1-1)*log(1.0-p_est);				return(1);

		 if(++nIterations > maxIterations){
				loglikelihood=L;
				var0=rawVar0MLE;var1=rawVar1MLE;

			 var0=(var0 > var_upper*2.0)? var_upper*2.0 : var0;
    var0=(var0 < var_lower/2.0)? var_lower/2.0 : var0;
    var1=(var1 > var_upper*2.0)? var_upper*2.0 : var1;
    var1=(var1< var_lower/2.0)? var_lower/2.0 : var1;
    a1=1.0-a0;
				mean0=mean0MLE;mean1=mean1MLE;
				return(0);
			}
		}
	}
  int optimize(T input_p){
   //optimize mean and use common variance
    //return loglikelihood after convergence with betabinomial term
   const int maxIterations=MAX_EM_ITERATIONS;
   const double converge=EM_CONVERGE;
   bool done=0;
   int nIterations=0;
   const int last=(nBins-1<maxValue)? nBins-1 : maxValue;

  	//T alpha1=((1.0-p_est)/(p_sigma*p_sigma) -1.0/p_est)*p_est*p_est;
			//T beta1=alpha1*(1.0/(p_est-1));

   //fit mean stdevs to normals norma a*invsqrtpi/sigma * exp((-(x-mean)**2)/(2*var*var))
   //however the  invsqrtpi/sigma needs to be in natural log scale to properly normalize

   //adjust ln
   vector <int> map;
   vector <T> mylnx; //mapped copy of lnx
   vector <T> mydata;
   for (int j=minValue;j<=last; j++){
			//check for underflow
			 if(data[j] && !isnan(data[j])){
					map.push_back(j);
					mylnx.push_back(lnx[j]);
					mydata.push_back(data[j]);
				}
			}
   while(!done){
    double term0=log(a0)+LNINVSQRT2PI-log(sqrt(var0));
    double term1=log(a1)+LNINVSQRT2PI-log(sqrt(var1));
    double inv2Var0=1.0/(2.0*var0);
    double inv2Var1=1.0/(2.0*var1);

  //calculate weighted percentage that points in j come from distribution 1 or 2
  //then calculate stdevs and mean for expectation maximization
   double L=0,sum0=0,sumsq0=0,sum1=0,sumsq1=0,counts0=0,counts1=0;
   const int setSize=map.size();
   for (int m=0;m<setSize; m++){
			//check for underflow
			 	T logLike0=term0-inv2Var0*(mylnx[m]-mean0)*(mylnx[m]-mean0);
			 	T logLike1=term1-inv2Var1*(mylnx[m]-mean1)*(mylnx[m]-mean1);

			 	//check for underflows
			 	if((!logLike1 || std::isnan(logLike1)) && logLike0 && !std::isnan(logLike0)){
		     L+=mydata[m]*logLike0;
			 		 counts0+=mydata[m];
			 		 sum0+=mylnx[m]*mydata[m];
			 		 sumsq0+=mylnx[m]*mylnx[m]*mydata[m];
			 	}
			 	else if	((!logLike0 || std::isnan(logLike0)) && logLike1 && !std::isnan(logLike1)){
		     L+=mydata[m]*logLike1;
			 		 counts1+=mydata[m];
			 		 sum1+=mylnx[m]*mydata[m];
			 		 sumsq1+=mylnx[m]*mylnx[m]*mydata[m];
			 	}
			 	else{
			 		const T logRatio=logLike0-logLike1;
			 	 if(logRatio > 15){
		     L+=mydata[m]*logLike0;
			 		 counts0+=mydata[m];
			 		 sum0+=mylnx[m]*mydata[m];
			 		 sumsq0+=mylnx[m]*mylnx[m]*mydata[m];
						}
						else if (logRatio < -15){
		     L+=mydata[m]*logLike1;
			 		 counts1+=mydata[m];
			 		 sum1+=mylnx[m]*mydata[m];
			 		 sumsq1+=mylnx[m]*mylnx[m]*mydata[m];
						}
						else{
			 		 const T ratio=exp(logRatio);
			 		 const T r0=ratio/(ratio+1.0);
			 		 const T r1=1.0-r0;

					  T nPoints0=mydata[m]*r0;
				 	 T nPoints1=mydata[m]*r1;
				   counts0+=nPoints0;
				   counts1+=nPoints1;
				   L+=nPoints0*logLike0+nPoints1*logLike1;
				 	 sum0+=nPoints0*mylnx[m];
					  sumsq0+=nPoints0*mylnx[m]*mylnx[m];
				 	 sum1+=nPoints1*mylnx[m];
					  sumsq1+=nPoints1*mylnx[m]*mylnx[m];
					 }
					}
			}
		 //calculate estimate of mean
		 T mean0MLE=sum0/(counts0);
   T mean1MLE=sum1/(counts1);

   mean0MLE= (mean0MLE > mean_upper)? mean_upper : mean0MLE;
   mean0MLE= (mean0MLE < mean_lower)? mean_lower: mean0MLE;
   mean1MLE= (mean1MLE > mean_upper)? mean_upper: mean1MLE;
   mean1MLE= (mean1MLE < mean_lower)? mean_lower : mean1MLE;

   //estimate a0 a1
   //check if estimated a1 > a0 - if so switch means and var
   T a0MLE,a1MLE;
   if(counts1) a0MLE = counts0/(counts0+counts1);
   else a0MLE=1;

   if(a0MLE < .5){
			//swap terms
			 swap(mean0,mean1);
			 swap(var0,var1);
			 swap(mean0MLE,mean1MLE);
			 swap(counts1,counts0);
			 swap(a0,a1);
			 a0MLE=1.0-a0;
		 }
		 //find unormalized counts
		 T ucounts,ucounts0,ucounts1;
		 if(normed){
				T totalCounts=counts0+counts1;
				ucounts=(T) nPoints * totalCounts;
				ucounts0=(T) nPoints * counts0;
				ucounts1=(T) nPoints * counts1;
			}
		 else{
				ucounts=counts0+counts1;
				ucounts0=counts0;
				ucounts1=counts1;
			}

   a0MLE=(a0MLE > MIXTURE_RATIO + 2*MIXTURE_STD)?MIXTURE_RATIO + 2*MIXTURE_STD :a0MLE;
   a0MLE=(a0MLE < MIXTURE_RATIO - 2*MIXTURE_STD)?MIXTURE_RATIO - 2*MIXTURE_STD :a0MLE;
   a1MLE=1.0-a0MLE;
   T rawVar0MLE=(sumsq0/(counts0)-mean0MLE*mean0MLE)*.6;
   T rawVar1MLE=(sumsq1/(counts1)-mean1MLE*mean1MLE)*.6;
   if(isnan( rawVar0MLE))rawVar0MLE=var_lower/2.0;
   if(isnan( rawVar1MLE))rawVar1MLE=var_lower/2.0;
   T var0MLE=rawVar0MLE;
   T var1MLE=rawVar1MLE;

  //enforce limits
   var0MLE=(var0MLE > var_upper)? var_upper : var0MLE;
   var0MLE=(var0MLE < var_lower)? var_lower : var0MLE;
   var1MLE=(var1MLE > var_upper)? var_upper : var1MLE;
   var1MLE=(var1MLE < var_lower)? var_lower : var1MLE;

   if(fabs(mean0MLE-mean0) > converge || fabs(mean1MLE-mean1) > converge){
	   mean0=mean0MLE;mean1=mean1MLE;
	   var0=var0MLE;var1=var1MLE;
	   a0=a0MLE;a1=a1MLE;
		 }
		 else{
				loglikelihood=L;
				converged=1;
    var0=rawVar0MLE;var1=rawVar1MLE;
			 var0=(var0 > var_upper*2.0)? var_upper*2.0 : var0;
    var0=(var0 < var_lower/2.0)? var_lower/2.0 : var0;
    var1=(var1 > var_upper*2.0)? var_upper*2.0 : var1;
    var1=(var1< var_lower/2.0)? var_lower/2.0 : var1;
    a1=1.0-a0;
     mean0=mean0MLE;mean1=mean1MLE;
    //fprintf(stderr,"%f %f %f %f %f %f\n",mean0MLE*INVLOG2,mean1MLE*INVLOG2,var0MLE*INVLOG2*INVLOG2,var1MLE*INVLOG2*INVLOG2,a0MLE,a1MLE);

    return(1);
			}
				//adjustment for binomial - seems to give worse results - enough to soft constrain p
				//loglikelihood=L+(ucounts0+alpha1-1)*log(p_est)+(ucounts1+beta1-1)*log(1.0-p_est);				return(1);

		 if(++nIterations > maxIterations){
				loglikelihood=L;
				var0=rawVar0MLE;var1=rawVar1MLE;

			 var0=(var0 > var_upper*2.0)? var_upper*2.0 : var0;
    var0=(var0 < var_lower/2.0)? var_lower/2.0 : var0;
    var1=(var1 > var_upper*2.0)? var_upper*2.0 : var1;
    var1=(var1< var_lower/2.0)? var_lower/2.0 : var1;
    a1=1.0-a0;
				mean0=mean0MLE;mean1=mean1MLE;
			 //fprintf(stderr,"%f %f %f %f %f %f\n",mean0MLE*INVLOG2,mean1MLE*INVLOG2,var0MLE*INVLOG2*INVLOG2,var1MLE*INVLOG2*INVLOG2,a0MLE,a1MLE);

				return(0);
			}
		}
	}
};

void init_logx(){
	logx[0]=0;
	lnx[0]=0;
 for (int i=1;i<32768;i++){
		lnx[i]=log((double)i);
  logx[i]=lnx[i]*INVLOG2;
	}
}
template <class T> void transferData(int color,DeconRecord<T> &deconRecord,EM_GMM<T> &EM){

	deconRecord.means0[color]=(EM.mean0*INVLOG2 < MIN_NOISE)? MIN_NOISE : EM.mean0*INVLOG2;
	deconRecord.means1[color]=(EM.mean1*INVLOG2 < MIN_NOISE)? MIN_NOISE : EM.mean1*INVLOG2;
	deconRecord.meansEst0[color]=EM.meanEst0*INVLOG2;
	deconRecord.meansEst1[color]=EM.meanEst1*INVLOG2;
	deconRecord.stdevs0[color]=sqrt(EM.var0)*INVLOG2;
	deconRecord.stdevs1[color]=sqrt(EM.var1)*INVLOG2;
	deconRecord.a0[color]=EM.a0;
	deconRecord.a1[color]=EM.a1;
	deconRecord.loglikelihoods[color]=-EM.loglikelihood;
	deconRecord.dataSizes[color]=EM.nPoints;
	deconRecord.converged[color]=EM.converged;
	if((EM.meanEst0-EM.meanEst1)*(EM.mean0-EM.mean1) <0) deconRecord.flipped[color]=1;
	else	 deconRecord.flipped[color]=0;
}

template <typename T1, typename T2> class wellRecord{
	//qnorm is done by well
	//these are then combined
	public:
	 vector <T1> values[NCOLORS1OFF]; //original values in int16
	 vector <string> groupNames;
	 vector <T2> offsets[NCOLORS1OFF];
	 vector <int> nBeads;
	 vector <int> groupOffsets; //the well where the group starts
	 int nTotalWells;

	 int minValue; //minimum value for peak
	 int maxValue; //maximum value for peak
	 float peakGridSize; //gridSize for peakSearch
	 float smoothHalfWindowSize; //half window size for smoothing
	 wellRecord(string list,float minLogValue, float maxLogValue,float _smoothHalfWindowSize,float _peakGridSize){
	  int16_t header[2];
		 int16_t v;
		 nTotalWells=0;
		 minValue=pow(2.0,minLogValue); //round down bin
		 maxValue=pow(2.0,maxLogValue)+1;
		 if(maxValue > 32767){
				maxValue=32767;
			}
			smoothHalfWindowSize=_smoothHalfWindowSize;
			peakGridSize=_peakGridSize;
		 FILE *fp=fopen(list.c_str(),"r");
		 //can have very long lines
		 fseek(fp,0,SEEK_END);
		 int size=ftell(fp)+1;
		 fseek(fp,0,SEEK_SET);
		 char *line=new char[size];
		 //cerr << "allocated " << size << " bytes"<<endl;
		 if(!line){
				cerr << "unable to allocate " << size << " bytes" <<endl;
				exit(0);
			}
		 int n=0;
		 for(int i=1;i<NCOLORS1OFF;i++){
		 	offsets[i].push_back(0);
		 }
		 groupOffsets.push_back(0);
   while(fgets(line,size,fp) != NULL){
				//format is name\tfileName1\tfileName2...fileNamen\n
				vector <string> fileNames;
				string groupName;
				const char delimit[]=" \t\r\n\v\f";
				char *pch = strtok (line,delimit);
			 groupName=(pch);
			 pch = strtok (NULL,delimit);
			 //start extracting list of fileNames

			 //check for case when there is only one element per line in which case the groupName is also the file
			 if(pch == NULL){
					fileNames.push_back(groupName);
				}
				else{
				 while (pch != NULL){
      fileNames.push_back(pch);
      pch = strtok (NULL,delimit);
				 }
				}
				//now extract the information from the filenames
				int nFilesRead=0;
				for (int k=0;k<fileNames.size();k++){
				//	fprintf(stderr,"opening %s\n",fileNames[k].c_str());
			  FILE *fp1=fopen(fileNames[k].c_str(),"r");
			  if(fp1 == NULL){
			   fprintf(stderr,"unable to open %s\n",fileNames[k].c_str());
			   continue;
			  }
			  while(fread(&header,2*sizeof(int16_t),1,fp1)){
			 	 const int id=header[0];
      for(int k=0;k<header[1];k++){
		     if(fread(&v,sizeof(int16_t),1,fp1)){
				  	 if(v>=0){values[id].push_back(v);}
				 	 }
			   }
			  }
		   fclose(fp1);
		   //save sizes to offset
		   int totalRead=0;
		   for(int i=1;i<NCOLORS1OFF;i++){
			  	offsets[i].push_back(values[i].size());
			  	totalRead+=values[i].size()-offsets[i][nBeads.size()];
			  }
		   nBeads.push_back(totalRead);
					nFilesRead++;
			 }
			 if(nFilesRead){
				 groupOffsets.push_back(groupOffsets[groupOffsets.size()-1]+nFilesRead);
				 groupNames.push_back(groupName);
				 nTotalWells+=nFilesRead;
				}
		 }
		 fprintf(stderr,"closing file\n");
		 fclose(fp);
		 delete[] line;
	 }
  wellRecord(const wellRecord &A): values(A.values),groupNames(A.groupNames),groupOffsets(A.groupOffsets),offsets(A.offsets),maxValue(A.maxValue),minValue(A.minValue),smoothHalfWindowSize(A.smoothHalfWindowSize),peakGridSize(A.peakGridSize),nTotalWells(A.nTotalWells),nBeads(A.nBeads){}

	void dumpEnsemble(string outputFile){
		cerr << "dumping density to " << outputFile <<endl;
		FILE *fp=fopen(outputFile.c_str(),"w");
		int counts[32768];
		fill(counts,counts+32768,0);
		//the 0th slice can be used to store errors in the old PERL script - keep it in case this is to be implemented later in C++
		fwrite(counts,sizeof(int),32768,fp);
		for(int id=1;id<NCOLORS1OFF;id++){
			fill(counts,counts+32768,0);
			for(int k=0;k<values[id].size();k++){
				counts[values[id][k]]++;
			}
			fwrite(counts,sizeof(int),32768,fp);
		}
		fclose(fp);
	}
	void dumpWells(string outputDir){
		cerr << "dumping density to directory " << outputDir <<endl;
		for(int n=0;n<nTotalWells;n++){
			ostringstream nstream;
			nstream << "." << n;

			string outputFile=outputDir+"/"+groupNames[n]+nstream.str()+".den";
	 	FILE *fp;
	 	if(fp=fopen(outputFile.c_str(),"w")){
	 	 int counts[32768];
	 	 fill(counts,counts+32768,0);
		  //the 0th slice can be used to store errors in the old PERL script - keep it in case this is to be implemented later in C++
    fwrite(counts,sizeof(int),32768,fp);
		  for(int id=1;id<NCOLORS1OFF;id++){
					fill(counts,counts+32768,0);
			  for(int k=offsets[id][n];k<offsets[id][n+1];k++){
			  	counts[values[id][k]]++;
			  }
		  	fwrite(counts,sizeof(int),32768,fp);
		  }
		  fclose(fp);
	  }
		}
	}
	void dumpSingleWell(string outputFile,int slice){
		cerr << "dumping density of well " << slice << " to file " << outputFile <<endl;
	 FILE *fp;
	 if(fp=fopen(outputFile.c_str(),"w")){
	  int counts[32768];
	  fill(counts,counts+32768,0);
		 //the 0th slice can be used to store errors in the old PERL script - keep it in case this is to be implemented later in C++
   fwrite(counts,sizeof(int),32768,fp);
		 for(int id=1;id<NCOLORS1OFF;id++){
				fill(counts,counts+32768,0);
			 for(int k=offsets[id][slice];k<offsets[id][slice+1];k++){
			  counts[values[id][k]]++;
			 }
		  fwrite(counts,sizeof(int),32768,fp);
		 }
		 fclose(fp);
	 }
	}
	void dumpDensity(string outputFile,float densitySmoothHalfWindowSize){
	 FILE *fp;
	 if(fp=fopen(outputFile.c_str(),"w")){
	 	int counts[32768];
	 	fill(counts,counts+32768,0);
	 	float smooth[32768];
	 	fill(smooth,smooth+32768,0);
   fwrite(smooth,1,sizeof(float)*32768,fp);//0 well - keep it for errors
		 int upper[32768];
		 int lower [32768];
		 memset (lower,0,32768*sizeof(int));
		 memset (upper,0,32768*sizeof(int));
		 const float upperConstant=pow(2,densitySmoothHalfWindowSize);
		 const float lowerConstant=1.0/upperConstant;
		 for (int i=0;i<32768;i++){
		 	lower[i]= (int) (((float)i)*lowerConstant+.5);
    upper[i]= (int) (((float)i)*upperConstant+.5);
    if(upper[i] > 32767)upper[i]=32767;
	  }
		 for (int i=1; i<=NCOLORS; i++){
			 fill(smooth,smooth+32768,0);
			 int nBins=logWindowSmooth(lower,upper,smooth,&(values[i][0]),values[i].size());
		 	fwrite(smooth,1,32768*sizeof(float),fp);
			}
		 fclose(fp);
		}
	}
template<class Ta, class Tb> int logWindowSmooth(int *lower,int *upper,Ta *smooth,Tb *colorValues,int nValues){
		int maxBin=0;
		int counts[32768];
		fill(counts,counts+32768,0);
		double totalWeight=0;
		for(int i=0;i<nValues;i++){
		 counts[colorValues[i]]++;
	 }
		for(int i=1;i<32768;i++){
		 if(counts[i])maxBin=i;
	 }
		for(int i=1;i<=maxBin;i++){
   int sum=0;
   //const int limit=(upper[i]<maxBin)? maxBin:upper[i];
	 	for(int k=lower[i];k<=upper[i];k++){
				sum+=counts[k];
			}
			smooth[i]=(Ta)sum/(Ta)(upper[i]-lower[i]+1);
			//fprintf(stderr,"%d %d %d %e\n",i,lower[i],upper[i],smooth[i]);
		 totalWeight+=smooth[i];
		}
		double invTotalWeight=1.0/totalWeight;
		for(int i=0;i<=maxBin;i++){
		 smooth[i]*=invTotalWeight;
	 }
		return(maxBin+1);
	}
	template <class T> double dampenLowNoise(T *smooth,int nBins){
		//version for a single file in wells
	 double lowNoiseFraction=0;
  for(int i=0;i<minValue;i++){
  	const double factor= exp(-0.3*(minValue-i));
  	lowNoiseFraction +=(1.0-factor)*smooth[i];
  	smooth[i]*=factor;
		}
  //renormalize
  double normFactor=1.0/(1.0-lowNoiseFraction);
  for(int i=minValue;i<nBins;i++)
   smooth[i]*=normFactor;
	 return (lowNoiseFraction);
	}
	template <class T> void findPeaks(T *smooth, int nBins, T gridSize,vector<int> &peakBins, vector<T> &peakValues, vector<T> &peakMeans){
		//finds peaks using a interval (grid) search
		//uses equal log intervals and logSmoothed data
		//an interval contains a peak if the previous interval and and the following interval have lower mean values
		//the bin with the maxCounts in that interval is the location of a peak
		//peaks are then sorted by mean values (max values is more vulnerable to spikes)
		//a good value for grid size is roughly the std deviation of the expected peaks

  vector<T> maxValues,meanValues;
  vector<int>maxBins;
  if(nBins <4){
			//edge case - return the middle bin
			peakBins.resize(1);
		 peakMeans.resize(1);
		 peakValues.resize(1);
			peakBins[0]=nBins/2;
			peakMeans[0]=smooth[nBins/2];
			peakValues[0]=smooth[nBins/2];
			return;
		}

	 int minBin=pow(2,minValue);int maxBin=(pow(2,maxValue) < nBins)?pow(2,maxValue) : nBins;
	 T gridFactor=pow(2,gridSize);

	 int i=minBin;
		while(i<nBins){
			int upperBin=(int) ((T)i*gridFactor + 0.5);
	  if (upperBin >nBins-1) upperBin=nBins-1;
		 T maxValue=smooth[i];
	  int maxBin=i;
   double sum=0;
   for (int k=i+1;k<=upperBin;k++){
		 	if(smooth[k] > maxValue){
				 maxValue=smooth[k];
				 maxBin=k;
			 }
 		 sum+=smooth[k];
		 }
		 maxValues.push_back(maxValue);
		 maxBins.push_back(maxBin);
		 //meanValues.push_back((T)sum/(T)(upperBin-i+1));
		 meanValues.push_back((T)sum);
		 i=upperBin+1;
		}
		vector <T> myPeakValues,myPeakBins,myPeakMeans;
	 for  (int i=2;i<maxBins.size()-2;i++){
			if(maxBins[i-2] > minValue && maxBins[i+1] < maxValue){ //-2 necessary otherwise there is an edge effect
 		 if(meanValues[i] > meanValues[i-1] && meanValues[i] > meanValues[i+1]){
		  	myPeakBins.push_back(maxBins[i]);
			  myPeakValues.push_back(maxValues[i]);
			  myPeakMeans.push_back(meanValues[i]);
		  }
			}
		}
		//check for empty case - no peaks - seed with middle
		if(!myPeakBins.size()){
			for  (int i=maxBins.size()/3;i<maxBins.size();i+=maxBins.size()/2){
			 myPeakBins.push_back(maxBins[i]);
			 myPeakValues.push_back(maxValues[i]);
			 myPeakMeans.push_back(meanValues[i]);
			}
		}

		vector<int> index;
		index.resize(myPeakMeans.size());
		peakBins.resize(1);
		peakMeans.resize(1);
		peakValues.resize(1);
		sort_by_scores(myPeakMeans.size(),myPeakMeans.data(),index.data(),0);

		peakBins[0]=myPeakBins[index[0]];
		peakValues[0]=myPeakValues[index[0]];
		peakMeans[0]=myPeakMeans[index[0]];
	 for (int i=1;i<index.size();i++){
			const int s=index[i];
			if(myPeakMeans[s] < 0.2*myPeakMeans[0])break;
		 peakBins.push_back(myPeakBins[s]);
		 peakValues.push_back(myPeakValues[s]);
		 peakMeans.push_back(myPeakMeans[s]);
		}
	}
	template <class T> void findPeaks(int *counts, int nBins, T gridSize,vector<int> &peakBins, vector<T> &peakValues, vector<T> &peakMeans){
		//finds peaks using a interval (grid) search
		//uses equal log intervals
		//an interval contains a peak if the previous interval and and the following interval have lower mean values
		//the bin with the maxCounts in that interval is the location of a peak
		//peaks are then sorted by mean values (max values is more vulnerable to spikes)
		//a good value for grid size is roughly the std deviation of the expected peaks

  vector<T> maxValues,meanValues;
  vector<int>maxBins;
  if(nBins <4){
			//edge case - return the middle bin
			peakBins.resize(1);
		 peakMeans.resize(1);
		 peakValues.resize(1);
			peakBins[0]=nBins/2;
			peakMeans[0]=counts[nBins/2];
			peakValues[0]=counts[nBins/2];
			return;
		}

	 int minBin=pow(2,minValue);int maxBin=(pow(2,maxValue) < nBins)?pow(2,maxValue) : nBins;
	 T gridFactor=pow(2,gridSize);

	 int i=minBin;
		while(i<nBins){
			int upperBin=(int) ((T)i*gridFactor + 0.5);
	  if (upperBin >nBins-1) upperBin=nBins-1;
		 T maxValue=counts[i];
	  int maxBin=i;
   double sum=0;
   for (int k=i+1;k<=upperBin;k++){
		 	if(counts[k] > maxValue){
				 maxValue=counts[k];
				 maxBin=k;
			 }
 		 sum+=counts[k];
		 }
		 maxValues.push_back(maxValue);
		 maxBins.push_back(maxBin);
		 //meanValues.push_back((T)sum/(T)(upperBin-i+1));
		 meanValues.push_back((T)sum);
		 i=upperBin+1;
		}
		vector <T> myPeakValues,myPeakBins,myPeakMeans;
	 for  (int i=2;i<maxBins.size()-2;i++){
			if(maxBins[i-2] > minValue && maxBins[i+1] < maxValue){ //-2 necessary otherwise there is an edge effect
 		 if(meanValues[i] > meanValues[i-1] && meanValues[i] > meanValues[i+1]){
		  	myPeakBins.push_back(maxBins[i]);
			  myPeakValues.push_back(maxValues[i]);
			  myPeakMeans.push_back(meanValues[i]);
		  }
			}
		}
		//check for empty case - no peaks - seed with middle
		if(!myPeakBins.size()){
			for  (int i=maxBins.size()/3;i<maxBins.size();i+=maxBins.size()/2){
			 myPeakBins.push_back(maxBins[i]);
			 myPeakValues.push_back(maxValues[i]);
			 myPeakMeans.push_back(meanValues[i]);
			}
		}

		vector<int> index;
		index.resize(myPeakMeans.size());
		peakBins.resize(1);
		peakMeans.resize(1);
		peakValues.resize(1);
		sort_by_scores(myPeakMeans.size(),myPeakMeans.data(),index.data(),0);

		peakBins[0]=myPeakBins[index[0]];
		peakValues[0]=myPeakValues[index[0]];
		peakMeans[0]=myPeakMeans[index[0]];
	 for (int i=1;i<index.size();i++){
			const int s=index[i];
			if(myPeakMeans[s] < 0.2*myPeakMeans[0])break;
		 peakBins.push_back(myPeakBins[s]);
		 peakValues.push_back(myPeakValues[s]);
		 peakMeans.push_back(myPeakMeans[s]);
		}
	}
	template <class T> int deconvolute(DeconRecord<T> &deconRecord,FILE *fpRaw,FILE *fpSmooth){
	 if(!logx[2]){init_logx();}
		T smooth[32768];
		int counts[32768];
		fill(smooth,smooth+32768,0);
		fill(counts,counts+32768,0);
		if(fpSmooth) fwrite(smooth,1,sizeof(T)*32768,fpSmooth);
	 if(fpRaw) fwrite(counts,1,sizeof(T)*32768,fpRaw);
		int upper[32768];
		int lower [32768];
		memset (lower,0,32768*sizeof(int));
		memset (upper,0,32768*sizeof(int));
		const T upperConstant=pow(2,smoothHalfWindowSize);
		const T lowerConstant=1.0/upperConstant;
		for (int i=0;i<32768;i++){
			lower[i]= (int) (((T)i)*lowerConstant+.5);
   upper[i]= (int) (((T)i)*upperConstant+.5);
   if(upper[i] > 32767)upper[i]=32767;
	 }
		for (int i=1; i<=NCOLORS; i++){
			fill(smooth,smooth+32768,0);
			vector<T> peakValues,peakMeans;
			vector<int>peakBins;
			int nBins=logWindowSmooth(lower,upper,smooth,&(values[i][0]),values[i].size());
			double lowNoiseFraction=dampenLowNoise(smooth,nBins);
			if(fpSmooth)fwrite(smooth,1,32768*sizeof(T),fpSmooth);
			fill(counts,counts+32768,0);
			for(int k=0;k<values[i].size();k++)
			 counts[values[i][k]]++;

			if(fpRaw){
    fwrite(counts,1,32768*sizeof(T),fpRaw);
			}
   findPeaks<T>(smooth,nBins,0.2,peakBins,peakValues,peakMeans);
   if(i >10){
				T nPoints=(double)values[i].size()-lowNoiseFraction*(double)values[i].size();
			 for(int k=0;k<32768;k++){
	 			smooth[k]=counts[k];
 			}
    fitLogNormEM(smooth,0,nBins,nPoints,peakBins,deconRecord,i,1,0);
    deconRecord.noiseFractions[i]=lowNoiseFraction;
			}
		}
	}
	template <class T> int deconvolute(int nFold,int n,int nGroups,vector<DeconRecord<T>> &deconRecords,vector<MedianRecord<T>>&medianRecords,FILE *fpRaw){
		return deconvolute_peakSeed(nFold,n,nGroups,deconRecords,medianRecords,fpRaw);
		//return deconvolute_peakSeed(n,nGroups,deconRecords[0],fpRaw);
	}
 template <class T> int deconvolute_peakSeed(int n,int nGroups,DeconRecord<T> &deconRecord,FILE *fpRaw){
	 if(!logx[2]){init_logx();}
		int counts[32768];
		fill(counts,counts+32768,0);
	 if (fpRaw) fwrite(counts,1,sizeof(T)*32768,fpRaw);
	 int finalGroupOffset= (n+nGroups >= groupOffsets.size())? groupOffsets[groupOffsets.size()-1] : groupOffsets[n+nGroups];
	 //fprintf(stderr,"%d start %d final offset %d\n",n,groupOffsets[n],finalGroupOffset);
		for (int i=1; i<=NCOLORS; i++){
			const int start=offsets[i][groupOffsets[n]];
			const int finish= offsets[i][finalGroupOffset];
			int nBins=0;
			T nPoints=finish-start;
			if(nPoints >= MINBEADS){
			 vector<int>peakBins;
			 peakBins.resize(2);

			 fill(counts,counts+32768,0);
			 for(int k=start;k<finish;k++){
					if(values[i][k]>nBins) nBins=values[i][k]+1;
			  counts[values[i][k]]++;
				}
			 if(fpRaw) fwrite(counts,1,32768*sizeof(int),fpRaw);
    peakBins[0]=pow(2,8.1);
    peakBins[1]=pow(2,7.9);
    if(i >10){
					T tcounts[32768];
					fill(tcounts,tcounts+32768,0);
					int k=0;
					while (k<minValue){
						nPoints-=counts[k];
						//tcounts[k]=counts[k];
						k++;
					}
					while (k<32768){
	 			 tcounts[k]=counts[k];
	 			 k++;
 			 }
     fitLogNormEM(tcounts,0,nBins,nPoints,peakBins,deconRecord,i,1,1);
     deconRecord.noiseFractions[i]=1.0-(nPoints/(T)(finish-start));
				}
			}
			else{
			 deconRecord.dataSizes[i]=finish-start;
				if(fpRaw){
		   fill(counts,counts+32768,0);
		   for(int k=start;k<finish;k++)
			   counts[values[i][k]]++;
	    fwrite(counts,1,sizeof(int)*32768,fpRaw);
			 }
			}
		}
		return(1);
	}
	template <class T> int deconvolute_peakSeed(int nFold,int n,int nGroups,vector<DeconRecord<T>> &deconRecords,vector<MedianRecord<T>>&medianRecords,FILE *fpRaw){
  //this overload takes 1/10 slices and finds the median value
	 if(!logx[2]){init_logx();}
		int counts[32768];
	 if (fpRaw) fwrite(counts,1,sizeof(T)*32768,fpRaw);
	 int finalGroupOffset= (n+nGroups >= groupOffsets.size())? groupOffsets[groupOffsets.size()-1] : groupOffsets[n+nGroups];
	 //fprintf(stderr,"%d start %d final offset %d\n",n,groupOffsets[n],finalGroupOffset);
		for (int i=1; i<=NCOLORS; i++){
			const int start=offsets[i][groupOffsets[n]];
			const int finish= offsets[i][finalGroupOffset];
			if(i >10){
			 int nBins=0;
			 int totalPoints=finish-start;
			 if(totalPoints >= MINBEADS){
			  vector<int>peakBins(2);
			  vector<unsigned int>foldIndices=getFolds(totalPoints,nFold);;
			  for(int f=0;f<nFold;f++){
						int nPoints=0;
						fill(counts,counts+32768,0);
						unsigned int j=0;
		  	 for(int k=start;k<finish;k++){
							if(nFold >1 && foldIndices[j++] == f ) continue;
		  			if(values[i][k]>nBins) nBins=values[i][k]+1;
		 	   counts[values[i][k]]++;
		 	   nPoints++;
		 	 	}
		 	  if(fpRaw) fwrite(counts,1,32768*sizeof(int),fpRaw);
      peakBins[0]=pow(2,8.1);
      peakBins[1]=pow(2,7.9);

			 		T tcounts[32768];
			 		fill(tcounts,tcounts+32768,0);
			 		int k=0;
			 		while (k<minValue){
				 		nPoints-=counts[k];
			 			//tcounts[k]=counts[k];
				 		k++;
				 	}
				 	while (k<32768){
	 		 	 tcounts[k]=counts[k];
	 		 	 k++;
 			  }
      fitLogNormEM(tcounts,0,nBins,nPoints,peakBins,deconRecords[f],i,1,1);
      medianRecords[i].addValues(deconRecords[f].means0[i], deconRecords[f].means1[i]);
      deconRecords[f].noiseFractions[i]=1.0-(nPoints/(T)(totalPoints));
				 }
				}
			}
			else{
				//write a blank Deconrecord with nPoints
				for(int f=0;f<nFold;f++){
				 deconRecords[f].dataSizes[i]=finish-start;
				}
				if(fpRaw){
		   fill(counts,counts+32768,0);
		   for(int k=start;k<finish;k++)
			   counts[values[i][k]]++;
	    fwrite(counts,1,sizeof(int)*32768,fpRaw);
			 }
			}
		}
		return(1);
	}
 template <class T> int fitLogNormEM(T *smooth,bool normed, int nBins,int nPoints,vector<int> &peakBins,DeconRecord<T> &deconRecord,int color,bool isMode,bool seed){
  const T s0=PEAKWIDTH;
  const T s1=PEAKWIDTH;
  T sigmasqs[2]={s0*s0,s1*s1};
  vector<T> mus;
  mus.resize(peakBins.size());
  if(isMode){
   for(int i=0;i<peakBins.size();i++)
    mus[i]=logx[peakBins[i]]-s0*s0;
		}
		else{
	  for(int i=0;i<peakBins.size();i++)
    mus[i]=logx[peakBins[i]];
		}
  //starting for search is both lognormals at the estimated medians

  //keep record of initial mode estimate
		deconRecord.meansEst0[color]=logx[peakBins[0]];
		deconRecord.meansEst1[color]=logx[peakBins[1]];
		T input_mus[2]={mus[0]+ (T).05,mus[0]-(T).05};
		EM_GMM <float> EM (smooth,normed,nBins,nPoints,minValue,maxValue,input_mus,sigmasqs,.5, MIXTURE_RATIO, 0.15,0.7,.275,PEAKWIDTH,.355);
		if (seed){
			EM.reset(mus[0],mus[1],sigmasqs[0],sigmasqs[1],MIXTURE_RATIO);
			EM.seed_mclust(0.5);
			mus.resize(2);
			if(isnan(EM.mean0)){
				mus[0]=EM.mean1*INVLOG2+.05;
			 mus[1]=EM.mean1*INVLOG2-.05;
			}
			else if(isnan(EM.mean1)){
			 mus[0]=EM.mean0*INVLOG2+.05;
			 mus[1]=EM.mean0*INVLOG2-.05;
			}
			else{
				mus[0]=EM.mean0*INVLOG2+.05;
			 mus[1]=EM.mean1*INVLOG2-.05;
			}
			EM.reset(mus[0]+.05,mus[1]-.05,sigmasqs[0],sigmasqs[1],MIXTURE_RATIO);
		}

  int converged=EM.optimize(MIXTURE_RATIO);
  transferData(color,deconRecord,EM);
  double best_likelihood=EM.loglikelihood;

  for(int j=0;j<mus.size()-1;j++){
			for (int k=j+1;k<mus.size();k++){
				EM.reset(mus[j],mus[k],sigmasqs[0],sigmasqs[1],MIXTURE_RATIO);
				int converged=EM.optimize(MIXTURE_RATIO);
				double likelihood=EM.loglikelihood;
				if(likelihood < best_likelihood){
					best_likelihood=likelihood;
				 transferData(color,deconRecord,EM);
				 deconRecord.meansEst0[color]=mus[j];deconRecord.meansEst1[color]=mus[k];
				}
				EM.reset(mus[k],mus[j],sigmasqs[1],sigmasqs[0],MIXTURE_RATIO);
				converged=EM.optimize(MIXTURE_RATIO);
				likelihood=EM.loglikelihood;
				if(likelihood < best_likelihood){
					best_likelihood=likelihood;
				 transferData(color,deconRecord,EM);
				 deconRecord.meansEst0[color]=mus[j];deconRecord.meansEst1[color]=mus[k];
				}
			}
		}
 }
	vector<unsigned int> getFolds(unsigned int size, unsigned int nFold){
		//for assigning nfolds
		vector<unsigned int> shuffledIndices(size);
  std::random_device rd;
  std::mt19937 rng(rd());
		for(unsigned int i=0;i<size;i++){
			shuffledIndices[i]=i%nFold;
		}
		for	(unsigned int i=0;i<size;i++){
			std::uniform_int_distribution<int> uni(i,size-1);
			unsigned int j=uni(rng);
 	 swap(shuffledIndices[i],shuffledIndices[j]);
		}
		return shuffledIndices;
	}
};


//
// Created by Sylvain  on 5/30/20.
//

#include "SatOps/MagneticField.h"
#include "SatOps/constants.h"
#include "SatOps/ReferenceFrame.h"
#include <cmath>
#include <fstream>
#include <sstream>
#include <stdexcept>
#include <string>
#include <boost/math/special_functions/factorials.hpp>

MagneticField::MagneticField(Model model, unsigned int max_degree, const std::filesystem::path& file, double year)
    :
    PotentialField(max_degree),
    m_model(model)
{
    // Allocates and initializes vectors containing the coefficients
    unsigned int num_entries = (max_degree + 1) * (max_degree + 2) / 2;
    m_c_coeffs = std::vector(num_entries, 0.0);
    m_s_coeffs = std::vector(num_entries, 0.0);

    switch (m_model) {
        case Model::IGRF13:
            if (m_max_degree > 13)
                throw std::invalid_argument("MagneticField: The degree of expansion is out of range (max 13).");
            if (year < 1900. || year > 2025.)
                throw std::invalid_argument("MagneticField: The year is out of range. The model is valide from 1900 to 2025.");

            loadIGRF13Coeffs(file, year);
            m_radius = constants::IGRF13_RADIUS;
            m_scale_factor = 1E-9 * constants::IGRF13_RADIUS * constants::IGRF13_RADIUS;
            break;
    }
}

// The coefficients of the model are computed for the initial epoch of the simulation
void MagneticField::loadIGRF13Coeffs(const std::filesystem::path& path, double year) {

    // Gets reference epoch of the model and difference with desired time
    int T0 = static_cast<int>(year) - static_cast<int>(std::floor(year)) % 5;
    double dt = year - T0;
    int year_col = (T0 - 1900) / 5 + 3;

    // Opens file containing the geomagnetic coefficients and populates arrays
    std::ifstream data_file(path);
    if (data_file.is_open()) {

        // Skips first four lines (header)
        std::string str;
        for (int i = 0; i < 4; ++i) {
            std::getline(data_file, str);
        }

        // Reads required lines
        while (std::getline(data_file, str)) {
            std::istringstream iss(str);
            std::string value;
            std::string coeff;
            int col = 0, degree = 0, order = 0;
            double main_field_coeff = 0., secular_variation_coeff = 0.;

            while (iss >> value) {
                if (col == 0)
                    coeff = value;
                else if (col == 1)
                    degree = stoi(value);
                else if (col == 2)
                    order = stoi(value);
                else if (col == year_col)
                    main_field_coeff = stod(value);
                else if (col == 28) // if last column, secular variation is directly given
                    secular_variation_coeff = stod(value);
                else if (col == year_col + 1)
                    secular_variation_coeff = (stod(value) - main_field_coeff) / 5.0;

                ++col;
            }

            if (degree > m_max_degree)
                break;

            int i = getIndex(degree, order);
            double kron_delta = (0 == order) ? 1. : 0.;
            double unnormalization_factor = std::sqrt(2.0 * boost::math::factorial<double>(degree-order) / boost::math::factorial<double>(degree+order) - kron_delta);
            if (coeff == "g") {
                m_c_coeffs[i] = unnormalization_factor * (main_field_coeff + dt * secular_variation_coeff);
            } else if (coeff == "h") {
                m_s_coeffs[i] = unnormalization_factor * (main_field_coeff + dt * secular_variation_coeff);
            }
        }
        data_file.close();
    } else {
        throw std::invalid_argument("MagneticField: The file containing the magnetic coefficients could not be opened.");
    }
}

Vector3d MagneticField::getMagneticField(const Vector3d& itrf93_position, Matrix3d* jacobian) {
    double x = itrf93_position[0];
    double y = itrf93_position[1];
    double z = itrf93_position[2];
    double r = itrf93_position.norm();

    std::array<double, 12> sums{};
    sums.fill(0.0);
    sums[0] = m_radius / (r*r) * (3. * (m_c_coeffs[getIndex(1,1)] * x + m_s_coeffs[getIndex(1,1)] * y) + 2. * m_c_coeffs[getIndex(1,0)] * z);
    sums[1] = m_radius / r * m_c_coeffs[getIndex(1,0)];
    sums[2] = m_radius / r * m_c_coeffs[getIndex(1,1)];
    sums[3] = m_radius / r * m_s_coeffs[getIndex(1,1)];
    sums[4] = 2.0;

    Vector3d mag_field = computeGradient(x, y, z, sums, jacobian);
    mag_field.setFrame(ReferenceFrame::ITRF93);
    return mag_field;
}


// Boost.Geometry (aka GGL, Generic Geometry Library)

// Copyright (c) 2007-2012 Barend Gehrels, Amsterdam, the Netherlands.

// This file was modified by Oracle on 2014.
// Modifications copyright (c) 2014, Oracle and/or its affiliates.

// Contributed and/or modified by Adam Wulkiewicz, on behalf of Oracle

// Use, modification and distribution is subject to the Boost Software License,
// Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)

#ifndef BOOST_GEOMETRY_ALGORITHMS_DETAIL_COURSE_HPP
#define BOOST_GEOMETRY_ALGORITHMS_DETAIL_COURSE_HPP

#include <boost/geometry/core/cs.hpp>
#include <boost/geometry/core/access.hpp>
#include <boost/geometry/core/radian_access.hpp>
#include <boost/geometry/util/math.hpp>

namespace boost { namespace geometry
{

#ifndef DOXYGEN_NO_DETAIL
namespace detail
{

/// Calculate course (bearing) between two points.
template <typename ReturnType, typename Point1, typename Point2>
inline ReturnType course(Point1 const& p1, Point2 const& p2)
{
    // http://williams.best.vwh.net/avform.htm#Crs
    ReturnType dlon = get_as_radian<0>(p2) - get_as_radian<0>(p1);
    ReturnType cos_p2lat = cos(get_as_radian<1>(p2));

    // An optimization which should kick in often for Boxes
    //if ( math::equals(dlon, ReturnType(0)) )
    //if ( get<0>(p1) == get<0>(p2) )
    //{
    //    return - sin(get_as_radian<1>(p1)) * cos_p2lat);
    //}

    // "An alternative formula, not requiring the pre-computation of d"
    // In the formula below dlon is used as "d"
    return atan2(sin(dlon) * cos_p2lat,
        cos(get_as_radian<1>(p1)) * sin(get_as_radian<1>(p2))
        - sin(get_as_radian<1>(p1)) * cos_p2lat * cos(dlon));
}

} // namespace detail
#endif // DOXYGEN_NO_DETAIL

}} // namespace boost::geometry

#endif // BOOST_GEOMETRY_ALGORITHMS_DETAIL_COURSE_HPP


/**
 * \file PedalToneStackFilter.hxx
 */

#include <boost/math/tools/polynomial.hpp>

#include <ATK/EQ/PedalToneStackFilter.h>

namespace ATK
{
  template<typename DataType>
  SD1ToneCoefficients<DataType>::SD1ToneCoefficients(gsl::index nb_channels)
  :TypedBaseFilter<DataType>(nb_channels, nb_channels)
  {
  }

  template<typename DataType>
  void SD1ToneCoefficients<DataType>::setup()
  {
    Parent::setup();

    coefficients_in.assign(in_order+1, 0);
    coefficients_out.assign(out_order, 0);

    CoeffDataType tempm[2] = {static_cast<CoeffDataType>(-2) * input_sampling_rate, static_cast<CoeffDataType>(2) * input_sampling_rate};
    CoeffDataType tempp[2] = {static_cast<CoeffDataType>(1), static_cast<CoeffDataType>(1)};
    boost::math::tools::polynomial<CoeffDataType> poly1(tempm, 1);
    boost::math::tools::polynomial<CoeffDataType> poly2(tempp, 1);

    boost::math::tools::polynomial<CoeffDataType> b;
    boost::math::tools::polynomial<CoeffDataType> a;

    b += poly2 * poly2;
    b += poly2 * poly1 * (C2*R3+R4*C3+alpha*(1-alpha)*R2*C2+alpha*C2*R4);
    b += poly1 * poly1 * (C3*R4*(R3*C2+alpha*(1-alpha)*R2*C2));

    a += poly2 * poly2;
    a += poly2 * poly1 * (C2*R3+R1*C1+alpha*(1-alpha)*R2*C2+(1-alpha)*C2*R1);
    a += poly1 * poly1 * (C1*R1*(R3*C2+alpha*(1-alpha)*R2*C2));

    for(gsl::index i = 0; i < in_order + 1; ++i)
    {
      coefficients_in[i] = b[i] / a[out_order];
    }
    for(gsl::index i = 0; i < out_order; ++i)
    {
      coefficients_out[i] = -a[i] / a[out_order];
    }
  }

  template<typename DataType_>
  void SD1ToneCoefficients<DataType_>::set_tone(CoeffDataType alpha)
  {
    if(alpha < 0 || alpha > 1)
    {
      throw std::out_of_range("Tone is outside the interval [0,1]");
    }
    this->alpha = alpha;

    setup();
  }

  template<typename DataType_>
  typename SD1ToneCoefficients<DataType_>::CoeffDataType SD1ToneCoefficients<DataType_>::get_tone() const
  {
    return alpha;
  }

  template<typename DataType>
  TS9ToneCoefficients<DataType>::TS9ToneCoefficients(gsl::index nb_channels)
  :TypedBaseFilter<DataType>(nb_channels, nb_channels)
  {
  }

  template<typename DataType>
  void TS9ToneCoefficients<DataType>::setup()
  {
    Parent::setup();

    coefficients_in.assign(in_order+1, 0);
    coefficients_out.assign(out_order, 0);

    CoeffDataType tempm[2] = {static_cast<CoeffDataType>(-2) * input_sampling_rate, static_cast<CoeffDataType>(2) * input_sampling_rate};
    CoeffDataType tempp[2] = {static_cast<CoeffDataType>(1), static_cast<CoeffDataType>(1)};
    boost::math::tools::polynomial<CoeffDataType> poly1(tempm, 1);
    boost::math::tools::polynomial<CoeffDataType> poly2(tempp, 1);

    boost::math::tools::polynomial<CoeffDataType> b;
    boost::math::tools::polynomial<CoeffDataType> a;

    b += poly2 * poly2 * R2;
    b += poly2 * poly1 * (alpha * C2 * R2 * R3 + alpha * (1-alpha) * C2 * P * R2 + R2 * R4 * C2);

    a += poly2 * poly2 * (R2 + R1);
    a += poly2 * poly1 * ((1-alpha) * C2 * (alpha * P * R2 + R1 * alpha * P + R1 * R2) + R4 * C2 * (R2 + R1) + R1 * C1 * R2);
    a += poly1 * poly1 * (C2 * R4 * C1 * R2 * R1 + (1-alpha) * C2 * R1 * P * C1 * R2);

    for(gsl::index i = 0; i < in_order + 1; ++i)
    {
      coefficients_in[i] = b[i] / a[out_order];
    }
    for(gsl::index i = 0; i < out_order; ++i)
    {
      coefficients_out[i] = -a[i] / a[out_order];
    }
  }

  template<typename DataType_>
  void TS9ToneCoefficients<DataType_>::set_tone(CoeffDataType alpha)
  {
    if(alpha < 0 || alpha > 1)
    {
      throw std::out_of_range("Tone is outside the interval [0,1]");
    }
    this->alpha = alpha;

    setup();
  }

  template<typename DataType_>
  typename TS9ToneCoefficients<DataType_>::CoeffDataType TS9ToneCoefficients<DataType_>::get_tone() const
  {
    return alpha;
  }
}


// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "SphereWeights.h"

#include <algorithm>
#include <cfloat>
#include <numeric>

#include <Eigen/Sparse>
#include <Eigen/SparseQR>

#include <igl/harmonic.h>

#include <OpenGP/Image/Image.h>
#include <OpenGP/SphereMesh/helpers.h>

using namespace OpenGP;
using Vec2i = Eigen::Vector2i;
using Vec3i = Eigen::Vector3i;
using Vec4i = Eigen::Vector4i;

void calc_weights(const SphereMesh &smesh, SurfaceMesh &mesh) {

    auto vpoint = smesh.get_vertex_property<Vec4>("v:point");

    auto distribute = [&](Vec3 p, SphereMesh::Edge edge) -> Vec2 {
        Vec4 s0 = vpoint[smesh.vertex(edge, 0)];
        Vec4 s1 = vpoint[smesh.vertex(edge, 1)];

        Vec3 c0 = s0.head<3>();
        Vec4 a = s1 - s0;
        Vec3 d = a.head<3>();
        float l = d.norm();
        Vec3 an = d / l;
        Vec3 d2 = p - c0;
        Vec3 p_proj = d2 - an * d2.dot(an);
        float beta = asin(a(3) / l);
        Vec3 offset = an * p_proj.norm() * tan(beta);
        float t = an.dot(p + p_proj + offset - c0) / l;
        t = fmax(fmin(t, 1), 0);
        return Vec2(1 - t, t);
    };

    int n_pills = smesh.n_edges();
    int n_verts = mesh.n_vertices();

    Eigen::MatrixXf P(n_verts, n_pills);
    P.setZero();

    float lambda = 0.1;

    for (int i = 0; i < n_verts; ++i) {

        auto vert_i = SurfaceMesh::Vertex(i);

        for (int j = 0; j < smesh.n_edges(); ++j) {

            auto edge_j = SphereMesh::Edge(j);

            Vec4 s0 = vpoint[smesh.vertex(edge_j, 0)];
            Vec4 s1 = vpoint[smesh.vertex(edge_j, 1)];

            float sdf;
            pill_project(mesh.position(vert_i), s0, s1, &sdf);

            P(i, j) = 1.f / (1e-6 + std::pow(std::max(sdf, 0.f), 2.f));
        }

        float sum = P.row(i).sum();
        P.row(i) /= sum;
    }

    P *= lambda;

    Eigen::Matrix<double, -1, -1> V(n_verts, 3);
    Eigen::Matrix<int, -1, -1> F(mesh.n_faces(), 3);

    for (int i = 0; i < n_verts; ++i) {
        V.row(i) =
            mesh.position(SurfaceMesh::Vertex(i)).transpose().cast<double>();
    }

    for (int i = 0; i < mesh.n_faces(); ++i) {
        int j = 0;
        for (auto vert : mesh.vertices(SurfaceMesh::Face(i)))
            F(i, j++) = vert.idx();
    }

    Eigen::SparseMatrix<double> Ld(n_verts, n_verts);

    igl::harmonic(V, F, 1, Ld);

    for (int i = 0; i < n_verts; ++i) {
        Ld.coeffRef(i, i) += lambda;
    }

    Eigen::SparseMatrix<float> L = Ld.cast<float>();

    Eigen::SparseLU<Eigen::SparseMatrix<float>> solver(L);

    Eigen::MatrixXf weights(n_verts, n_pills);
    for (int i = 0; i < n_pills; ++i) {
        weights.col(i) = solver.solve(P.col(i));
    }

    auto weights_prop =
        mesh.add_vertex_property<std::vector<float>>("v:skinweight");
    auto bone_ids_prop = mesh.add_vertex_property<std::vector<int>>("v:boneid");

    for (int i = 0; i < n_verts; ++i) {
        Vec3 p = mesh.position(SurfaceMesh::Vertex(i));
        Eigen::VectorXf row = weights.row(i).transpose();
        std::vector<float> per_cap_weights;
        for (int j = 0; j < n_pills; ++j) {
            Vec2 cap_weights = distribute(p, SphereMesh::Edge(j)) * row[j];
            per_cap_weights.push_back(cap_weights[0]);
            per_cap_weights.push_back(cap_weights[1]);
        }
        std::vector<int> inds(2 * n_pills);
        std::iota(inds.begin(), inds.end(), 0);
        std::sort(inds.begin(), inds.end(), [&](int i, int j) {
            return per_cap_weights[i] > per_cap_weights[j];
        });

        std::vector<float> w;
        std::vector<int> wi;
        for (int j = 0; j < inds.size(); ++j) {
            w.push_back(per_cap_weights[inds[j]]);
            wi.push_back(inds[j]);
            if (w.back() / w.front() < 0.01)
                break;
        }

        float sum = std::accumulate(w.begin(), w.end(), 0.f);
        std::transform(w.begin(), w.end(), w.begin(),
                       [sum](float x) { return x / sum; });

        SurfaceMesh::Vertex vert(i);
        weights_prop[vert] = w;
        bone_ids_prop[vert] = wi;

    }

}