from __future__ import division, absolute_import, print_function
from past.builtins import xrange

import numpy as np
import esutil
import time

import matplotlib.pyplot as plt

from .fgcmUtilities import objFlagDict
from .fgcmUtilities import obsFlagDict

from .sharedNumpyMemManager import SharedNumpyMemManager as snmm

class FgcmStars(object):
    """
    Class to describe the stars and observations of the stars.  Note that
     after initialization you must call loadStarsFromFits() or loadStars()
     to load the star information.  This allows an external caller to clear
     out memory after it has been copied to the shared memory buffers.

    parameters
    ----------
    fgcmConfig: FgcmConfig

    Config variables
    ----------------
    minObsPerBand: int
       Minumum number of observations per band to be "good"
    sedFitBandFudgeFactors: float array
       Fudge factors for computing fnuprime for the fit bands
    sedExtraBandFudgeFactors: float array
       Fudge factors for computing fnuprime for the extra bands
    starColorCuts: list
       List that contains lists of [bandIndex0, bandIndex1, minColor, maxColor]
    sigma0Phot: float
       Floor on photometric error to add to every observation
    reserveFraction: float
       Fraction of stars to hold in reserve
    mapLongitudeRef: float
       Reference longitude for plotting maps of stars
    mapNSide: int
       Healpix nside of map plotting.
    superStarSubCCD: bool
       Use sub-ccd info to make superstar flats?
    obsFile: string, only if using fits mode
       Star observation file
    indexFile: string, only if using fits mode
       Star index file
    """

    def __init__(self,fgcmConfig):

        self.fgcmLog = fgcmConfig.fgcmLog

        self.fgcmLog.info('Initializing stars.')

        self.obsFile = fgcmConfig.obsFile
        self.indexFile = fgcmConfig.indexFile

        self.bands = fgcmConfig.bands
        self.nBands = len(fgcmConfig.bands)
        self.nCCD = fgcmConfig.nCCD
        self.minObsPerBand = fgcmConfig.minObsPerBand
        self.fitBands = fgcmConfig.fitBands
        self.nFitBands = len(fgcmConfig.fitBands)
        self.extraBands = fgcmConfig.extraBands
        self.sedFitBandFudgeFactors = fgcmConfig.sedFitBandFudgeFactors
        self.sedExtraBandFudgeFactors = fgcmConfig.sedExtraBandFudgeFactors
        self.starColorCuts = fgcmConfig.starColorCuts
        self.sigma0Phot = fgcmConfig.sigma0Phot
        self.ccdStartIndex = fgcmConfig.ccdStartIndex
        self.plotPath = fgcmConfig.plotPath
        self.outfileBaseWithCycle = fgcmConfig.outfileBaseWithCycle
        self.expField = fgcmConfig.expField
        self.ccdField = fgcmConfig.ccdField
        self.reserveFraction = fgcmConfig.reserveFraction
        self.modelMagErrors = fgcmConfig.modelMagErrors

        self.inFlagStarFile = fgcmConfig.inFlagStarFile

        self.mapLongitudeRef = fgcmConfig.mapLongitudeRef
        self.mapNSide = fgcmConfig.mapNSide

        self.lambdaStdBand = fgcmConfig.lambdaStdBand

        self.bandRequiredFlag = fgcmConfig.bandRequiredFlag
        self.bandRequiredIndex = np.where(self.bandRequiredFlag)[0]
        self.bandExtraFlag = fgcmConfig.bandExtraFlag
        self.bandExtraIndex = np.where(self.bandExtraFlag)[0]

        self.lutFilterNames = fgcmConfig.lutFilterNames
        self.filterToBand = fgcmConfig.filterToBand

        self.superStarSubCCD = fgcmConfig.superStarSubCCD

        #self.expArray = fgcmPars.expArray

        #self._loadStars(fgcmPars)

        self.magStdComputed = False
        self.allMagStdComputed = False
        self.sedSlopeComputed = False

        #if (computeNobs):
        #    allExps = np.arange(fgcmConfig.expRange[0],fgcmConfig.expRange[1],dtype='i4')
        #    self.fgcmLog.info('Checking stars with full possible range of exp numbers')
            #self.selectStarsMinObs(goodExps=allExps,doPlots=False)
        #    allExpsIndex = np.arange(fgcmPars.expArray.size)
        #    self.selectStarsMinObsExpIndex(allExpsIndex)

        self.magConstant = 2.5/np.log(10)

        self.hasXY = False

    def loadStarsFromFits(self,fgcmPars,computeNobs=True):
        """
        Load stars from fits files.

        parameters
        ----------
        fgcmPars: FgcmParameters
        computeNobs: bool, default=True
           Compute number of observations of each star/band

        Config variables
        ----------------
        indexFile: string
           Star index file
        obsFile: string
           Star observation file
        inFlagStarFile: string, optional
           Flagged star file
        """

        import fitsio

        # read in the observation indices...
        startTime = time.time()
        self.fgcmLog.info('Reading in observation indices...')
        index = fitsio.read(self.indexFile, ext='INDEX')
        self.fgcmLog.info('Done reading in %d observation indices in %.1f seconds.' %
                         (index.size, time.time() - startTime))

        # read in obsfile and cut
        startTime = time.time()
        self.fgcmLog.info('Reading in star observations...')
        obs = fitsio.read(self.obsFile, ext=1)
        # cut down to those that are indexed
        obs = obs[index['OBSINDEX']]
        self.fgcmLog.info('Done reading in %d observations in %.1f seconds.' %
                         (obs.size, time.time() - startTime))

        # and positions...
        startTime = time.time()
        self.fgcmLog.info('Reading in star positions...')
        pos = fitsio.read(self.indexFile, ext='POS')
        self.fgcmLog.info('Done reading in %d unique star positions in %.1f secondds.' %
                         (pos.size, time.time() - startTime))

        #obsBand = np.core.defchararray.strip(obs['BAND'][:])
        obsFilterName = np.core.defchararray.strip(obs['FILTERNAME'][:])

        if (self.inFlagStarFile is not None):
            self.fgcmLog.info('Reading in list of previous flagged stars from %s' %
                             (self.inFlagStarFile))

            inFlagStars = fitsio.read(self.inFlagStarFile, ext=1)

            flagID = inFlagStars['OBJID']
            flagFlag = inFlagStars['OBJFLAG']
        else:
            flagID = None
            flagFlag = None

        # FIXME: add support to x/y from fits files
        if ('X' in obs.dtype.names and 'Y' in obs.dtype.names):
            self.fgcmLog.info('Found X/Y in input observations')
            obsX = obs['X']
            obsY = obs['Y']
        else:
            obsX = None
            obsY = None

        # process
        self.loadStars(fgcmPars,
                       obs[self.expField],
                       obs[self.ccdField],
                       obs['RA'],
                       obs['DEC'],
                       obs['MAG'],
                       obs['MAGERR'],
                       obsFilterName,
                       pos['FGCM_ID'],
                       pos['RA'],
                       pos['DEC'],
                       pos['OBSARRINDEX'],
                       pos['NOBS'],
                       obsX=obsX,
                       obsY=obsY,
                       flagID=flagID,
                       flagFlag=flagFlag,
                       computeNobs=computeNobs)

        # and clear memory
        index = None
        obs = None
        pos = None

    def loadStars(self, fgcmPars,
                  obsExp, obsCCD, obsRA, obsDec, obsMag, obsMagErr, obsFilterName,
                  objID, objRA, objDec, objObsIndex, objNobs, obsX=None, obsY=None,
                  flagID=None, flagFlag=None, computeNobs=True):
        """
        Load stars from arrays

        parameters
        ----------
        fgcmPars: fgcmParameters
        obsExp: int array
           Exposure number (or equivalent) for each observation
        obsCCD: int array
           CCD number (or equivalent) for each observation
        obsRA: double array
           RA for each observation (degrees)
        obsDec: double array
           Dec for each observation (degrees)
        obsMag: float array
           Raw ADU magnitude for each observation
        obsMagErr: float array
           Raw ADU magnitude error for each observation
        obsFilterName: string array
           Filter name for each observation
        objID: int array
           Unique ID number for each object
        objRA: double array
           RA for each object (degrees)
        objDec: double array
           Dec for each object (degrees)
        objObsIndex: int array
           For each object, where in the obs table to look
        objNobs: int array
           number of observations of this object (all bands)
        obsX: float array, optional
           x position for each observation
        obsY: float array, optional
           y position for each observation
        flagID: int array, optional
           ID of each object that is flagged from previous cycle
        flagFlag: int array, optional
           Flag value from previous cycle
        computeNobs: bool, default=True
           Compute number of good observations of each object?
        """

        # FIXME: check that these are all the same length!

        self.obsIndexHandle = snmm.createArray(obsRA.size, dtype='i4')
        snmm.getArray(self.obsIndexHandle)[:] = np.arange(obsRA.size)

        # need to stuff into shared memory objects.
        #  nStarObs: total number of observations of all starus
        self.nStarObs = obsRA.size

        #  obsExp: exposure number of individual observation (pointed by obsIndex)
        self.obsExpHandle = snmm.createArray(self.nStarObs,dtype='i4')
        #  obsExpIndex: exposure index
        self.obsExpIndexHandle = snmm.createArray(self.nStarObs,dtype='i4')
        #  obsCCD: ccd number of individual observation
        self.obsCCDHandle = snmm.createArray(self.nStarObs,dtype='i2')
        #  obsBandIndex: band index of individual observation
        self.obsBandIndexHandle = snmm.createArray(self.nStarObs,dtype='i2')
        #  obsLUTFilterIndex: filter index in LUT of individual observation
        self.obsLUTFilterIndexHandle = snmm.createArray(self.nStarObs,dtype='i2')
        #  obsFlag: individual bad observation
        self.obsFlagHandle = snmm.createArray(self.nStarObs,dtype='i2')
        #  obsRA: RA of individual observation
        self.obsRAHandle = snmm.createArray(self.nStarObs,dtype='f8')
        #  obsDec: Declination of individual observation
        self.obsDecHandle = snmm.createArray(self.nStarObs,dtype='f8')
        #  obsSecZenith: secant(zenith) of individual observation
        self.obsSecZenithHandle = snmm.createArray(self.nStarObs,dtype='f8')
        #  obsMagADU: log raw ADU counts of individual observation
        ## FIXME: need to know default zeropoint?
        self.obsMagADUHandle = snmm.createArray(self.nStarObs,dtype='f4')
        #  obsMagADUErr: raw ADU counts error of individual observation
        self.obsMagADUErrHandle = snmm.createArray(self.nStarObs,dtype='f4')
        #  obsMagADUModelErr: modeled ADU counts error of individual observation
        self.obsMagADUModelErrHandle = snmm.createArray(self.nStarObs,dtype='f4')
        #  obsSuperStarApplied: SuperStar correction that was applied
        self.obsSuperStarAppliedHandle = snmm.createArray(self.nStarObs,dtype='f4')
        #  obsMagStd: corrected (to standard passband) mag of individual observation
        self.obsMagStdHandle = snmm.createArray(self.nStarObs,dtype='f4',syncAccess=True)
        if (obsX is not None and obsY is not None):
            self.hasXY = True

            #  obsX: x position on the CCD of the given observation
            self.obsXHandle = snmm.createArray(self.nStarObs,dtype='f4')
            #  obsY: y position on the CCD of the given observation
            self.obsYHandle = snmm.createArray(self.nStarObs,dtype='f4')
        else:
            # hasXY = False
            if self.superStarSubCCD:
                raise ValueError("Input stars do not have x/y but superStarSubCCD is set.")

        snmm.getArray(self.obsExpHandle)[:] = obsExp
        snmm.getArray(self.obsCCDHandle)[:] = obsCCD
        snmm.getArray(self.obsRAHandle)[:] = obsRA
        snmm.getArray(self.obsDecHandle)[:] = obsDec
        snmm.getArray(self.obsMagADUHandle)[:] = obsMag
        snmm.getArray(self.obsMagADUErrHandle)[:] = obsMagErr
        snmm.getArray(self.obsMagStdHandle)[:] = obsMag   # same as raw at first
        snmm.getArray(self.obsSuperStarAppliedHandle)[:] = 0.0
        if self.hasXY:
            snmm.getArray(self.obsXHandle)[:] = obsX
            snmm.getArray(self.obsYHandle)[:] = obsY

        self.fgcmLog.info('Applying sigma0Phot = %.4f to mag errs' %
                         (self.sigma0Phot))

        obsMagADUErr = snmm.getArray(self.obsMagADUErrHandle)

        obsFlag = snmm.getArray(self.obsFlagHandle)
        bad, = np.where(obsMagADUErr <= 0.0)
        obsFlag[bad] |= obsFlagDict['BAD_ERROR']
        if (bad.size > 0):
            self.fgcmLog.info('Flagging %d observations with bad errors.' %
                             (bad.size))

        obsMagADUErr[:] = np.sqrt(obsMagADUErr[:]**2. + self.sigma0Phot**2.)

        # Initially, we set the model error to the observed error
        obsMagADUModelErr = snmm.getArray(self.obsMagADUModelErrHandle)
        obsMagADUModelErr[:] = obsMagADUErr[:]

        startTime = time.time()
        self.fgcmLog.info('Matching observations to exposure table.')
        obsExpIndex = snmm.getArray(self.obsExpIndexHandle)
        obsExpIndex[:] = -1
        a,b=esutil.numpy_util.match(fgcmPars.expArray,
                                    snmm.getArray(self.obsExpHandle)[:])
        obsExpIndex[b] = a
        self.fgcmLog.info('Observations matched in %.1f seconds.' %
                         (time.time() - startTime))

        bad, = np.where(obsExpIndex < 0)
        obsFlag[bad] |= obsFlagDict['NO_EXPOSURE']

        if (bad.size > 0):
            self.fgcmLog.info('Flagging %d observations with no associated exposure.' %
                             (bad.size))

        # match bands and filters to indices
        startTime = time.time()
        self.fgcmLog.info('Matching observations to bands.')

        #for i in xrange(self.nBands):
        #    use, = np.where(obsBand == self.bands[i])
        #    if (use.size == 0):
        #        raise ValueError("No observations in band %s!" % (self.bands[i]))
        #    snmm.getArray(self.obsBandIndexHandle)[use] = i

        # new version for multifilter support
        # First, we have the filterNames
        for filterIndex,filterName in enumerate(self.lutFilterNames):
            #try:
            #    bandIndex, = np.where(self.filterToBand[filterName] == self.bands)
            #except:
            #    self.fgcmLog.info('WARNING: observations with filter %s not in config' % (filterName))
            #    bandIndex = -1
            try:
                bandIndex = self.bands.index(self.filterToBand[filterName])
            except:
                self.fgcmLog.info('WARNING: observations with filter %s not in config' % (filterName))
                bandIndex = -1

            # obsFilterName is an array from fits/numpy.  filterName needs to be encoded to match
            use, = np.where(obsFilterName == filterName.encode('utf-8'))
            if use.size == 0:
                self.fgcmLog.info('WARNING: no observations in filter %s' % (filterName))
            else:
                snmm.getArray(self.obsLUTFilterIndexHandle)[use] = filterIndex
                snmm.getArray(self.obsBandIndexHandle)[use] = bandIndex

        self.fgcmLog.info('Observations matched in %.1f seconds.' %
                         (time.time() - startTime))

        #obs=None

        #startTime=time.time()
        #self.fgcmLog.info('Reading in star positions...')
        #pos=fitsio.read(self.indexFile,ext='POS')
        #self.fgcmLog.info('Done reading in %d unique star positions in %.1f secondds.' %
        #                 (pos.size,time.time()-startTime))

        #  nStars: total number of unique stars
        #self.nStars = pos.size
        self.nStars = objID.size

        #  objID: unique object ID
        self.objIDHandle = snmm.createArray(self.nStars,dtype='i4')
        #  objRA: mean RA for object
        self.objRAHandle = snmm.createArray(self.nStars,dtype='f8')
        #  objDec: mean Declination for object
        self.objDecHandle = snmm.createArray(self.nStars,dtype='f8')
        #  objObsIndex: for each object, the first
        self.objObsIndexHandle = snmm.createArray(self.nStars,dtype='i4')
        #  objNobs: number of observations of this object (all bands)
        self.objNobsHandle = snmm.createArray(self.nStars,dtype='i4')
        #  objNGoodObsHandle: number of good observations, per band
        self.objNGoodObsHandle = snmm.createArray((self.nStars,self.nBands),dtype='i4')

        #snmm.getArray(self.objIDHandle)[:] = pos['FGCM_ID'][:]
        #snmm.getArray(self.objRAHandle)[:] = pos['RA'][:]
        #snmm.getArray(self.objDecHandle)[:] = pos['DEC'][:]
        snmm.getArray(self.objIDHandle)[:] = objID
        snmm.getArray(self.objRAHandle)[:] = objRA
        snmm.getArray(self.objDecHandle)[:] = objDec

        #try:
            # new field name
        #    snmm.getArray(self.objObsIndexHandle)[:] = pos['OBSARRINDEX'][:]
        #except:
            # old field name
        #    snmm.getArray(self.objObsIndexHandle)[:] = pos['OBSINDEX'][:]
        #snmm.getArray(self.objNobsHandle)[:] = pos['NOBS'][:]
        snmm.getArray(self.objObsIndexHandle)[:] = objObsIndex
        snmm.getArray(self.objNobsHandle)[:] = objNobs

        #  minObjID: minimum object ID
        self.minObjID = np.min(snmm.getArray(self.objIDHandle))
        #  maxObjID: maximum object ID
        self.maxObjID = np.max(snmm.getArray(self.objIDHandle))

        #  obsObjIDIndex: object ID Index of each observation
        #    (to get objID, then objID[obsObjIDIndex]

        startTime = time.time()
        self.fgcmLog.info('Indexing star observations...')
        self.obsObjIDIndexHandle = snmm.createArray(self.nStarObs,dtype='i4')
        obsObjIDIndex = snmm.getArray(self.obsObjIDIndexHandle)
        objID = snmm.getArray(self.objIDHandle)
        obsIndex = snmm.getArray(self.obsIndexHandle)
        objObsIndex = snmm.getArray(self.objObsIndexHandle)
        objNobs = snmm.getArray(self.objNobsHandle)
        ## FIXME: check if this extra obsIndex reference is necessary or not.
        ##   probably extraneous.
        for i in xrange(self.nStars):
            obsObjIDIndex[obsIndex[objObsIndex[i]:objObsIndex[i]+objNobs[i]]] = i
        self.fgcmLog.info('Done indexing in %.1f seconds.' %
                         (time.time() - startTime))

        #pos=None
        obsObjIDIndex = None
        objID = None
        obsIndex = None
        objObsIndex = None
        objNobs = None

        # and create a objFlag which flags bad stars as they fall out...

        self.objFlagHandle = snmm.createArray(self.nStars,dtype='i2')

        # and read in the previous bad stars if available
        #if (self.inBadStarFile is not None):
        #   self.fgcmLog.info('Reading in list of previous bad stars from %s' %
        #                     (self.inBadStarFile))

        #    objID = snmm.getArray(self.objIDHandle)
        #    objFlag = snmm.getArray(self.objFlagHandle)

        #    inBadStars = fitsio.read(self.inBadStarFile,ext=1)

        #    a,b=esutil.numpy_util.match(inBadStars['OBJID'],
        #                                objID)

        #    self.fgcmLog.info('Flagging %d stars as bad.' %
        #                     (a.size))

        #    objFlag[b] = inBadStars['OBJFLAG'][a]
        if (flagID is not None):
            # the objFlag contains information on RESERVED stars
            objID = snmm.getArray(self.objIDHandle)
            objFlag = snmm.getArray(self.objFlagHandle)

            a,b=esutil.numpy_util.match(flagID, objID)

            test,=np.where((flagFlag[a] & objFlagDict['VARIABLE']) > 0)
            self.fgcmLog.info('Flagging %d stars as variable from previous cycles.' %
                             (test.size))
            test,=np.where((flagFlag[a] & objFlagDict['RESERVED']) > 0)
            self.fgcmLog.info('Flagging %d stars as reserved from previous cycles.' %
                             (test.size))

            objFlag[b] = flagFlag[a]
        else:
            # we want to reserve stars, if necessary
            if self.reserveFraction > 0.0:
                objFlag = snmm.getArray(self.objFlagHandle)

                nReserve = int(self.reserveFraction * objFlag.size)
                reserve = np.random.choice(objFlag.size,
                                           size=nReserve,
                                           replace=False)

                self.fgcmLog.info('Reserving %d stars from the fit.' % (nReserve))
                objFlag[reserve] |= objFlagDict['RESERVED']

        # And we need to record the mean mag, error, SED slopes...

        #  objMagStdMean: mean standard magnitude of each object, per band
        self.objMagStdMeanHandle = snmm.createArray((self.nStars,self.nBands),dtype='f4',
                                                    syncAccess=True)
        #  objMagStdMeanErr: error on the mean standard mag of each object, per band
        self.objMagStdMeanErrHandle = snmm.createArray((self.nStars,self.nBands),dtype='f4')
        #  objSEDSlope: linearized approx. of SED slope of each object, per band
        self.objSEDSlopeHandle = snmm.createArray((self.nStars,self.nBands),dtype='f4',
                                                  syncAccess=True)
        #  objMagStdMeanNoChrom: mean std mag of each object, no chromatic correction, per band
        self.objMagStdMeanNoChromHandle = snmm.createArray((self.nStars,self.nBands),dtype='f4')

        # note: if this takes too long it can be moved to the star computation,
        #       but it seems pretty damn fast (which may raise the question of
        #       why it needs to be precomputed...)
        # compute secZenith for every observation

        startTime=time.time()
        self.fgcmLog.info('Computing secZenith for each star observation...')
        objRARad = np.radians(snmm.getArray(self.objRAHandle))
        objDecRad = np.radians(snmm.getArray(self.objDecHandle))
        ## FIXME: deal with this at some point...
        hi,=np.where(objRARad > np.pi)
        objRARad[hi] -= 2*np.pi
        obsExpIndex = snmm.getArray(self.obsExpIndexHandle)
        obsObjIDIndex = snmm.getArray(self.obsObjIDIndexHandle)
        obsIndex = snmm.getArray(self.obsIndexHandle)

        obsHARad = (fgcmPars.expTelHA[obsExpIndex] +
                    fgcmPars.expTelRA[obsExpIndex] -
                    objRARad[obsObjIDIndex])
        tempSecZenith = 1./(np.sin(objDecRad[obsObjIDIndex]) * fgcmPars.sinLatitude +
                            np.cos(objDecRad[obsObjIDIndex]) * fgcmPars.cosLatitude *
                            np.cos(obsHARad))

        bad,=np.where(obsFlag != 0)
        tempSecZenith[bad] = 1.0  # filler here, but these stars aren't used
        snmm.getArray(self.obsSecZenithHandle)[:] = tempSecZenith
        self.fgcmLog.info('Computed secZenith in %.1f seconds.' %
                         (time.time() - startTime))

        if (computeNobs):
            self.fgcmLog.info('Checking stars with all exposure numbers')
            allExpsIndex = np.arange(fgcmPars.expArray.size)
            self.selectStarsMinObsExpIndex(allExpsIndex)

    def selectStarsMinObsExpIndex(self, goodExpsIndex, temporary=False,
                                  minObsPerBand=None):
        """
        Select stars that have at least the minimum number of observations per band,
         using a list of good exposures

        parameters
        ----------
        goodExpsIndex: int array
           Array of good (photometric) exposure indices
        temporary: bool, default=False
           Only flag bad objects temporarily
        minObsPerBand: int
           Specify the min obs per band, or use self.minObsPerBand
        """

        if (minObsPerBand is None):
            minObsPerBand = self.minObsPerBand

        # Given a list of good exposures, which stars have at least minObs observations
        #  in each required band?

        obsExpIndex = snmm.getArray(self.obsExpIndexHandle)
        obsBandIndex = snmm.getArray(self.obsBandIndexHandle)
        obsObjIDIndex = snmm.getArray(self.obsObjIDIndexHandle)
        objNGoodObs = snmm.getArray(self.objNGoodObsHandle)
        obsFlag = snmm.getArray(self.obsFlagHandle)
        objFlag = snmm.getArray(self.objFlagHandle)

        self.fgcmLog.info('Selecting good stars from %d exposures.' %
                         (goodExpsIndex.size))
        _,goodObs=esutil.numpy_util.match(goodExpsIndex,obsExpIndex)

        # Filter out bad (previously flagged) individual observations
        gd, = np.where(obsFlag[goodObs] == 0)
        goodObs = goodObs[gd]

        # count all the good observations
        objNGoodObs[:,:] = 0
        np.add.at(objNGoodObs,
                  (obsObjIDIndex[goodObs],
                   obsBandIndex[goodObs]),
                  1)

        # and find the minimum of all the required bands
        minObs = objNGoodObs[:,self.bandRequiredIndex].min(axis=1)

        # reset too few obs flag if it's already set
        if not temporary:
            objFlag &= ~objFlagDict['TOO_FEW_OBS']

        # choose the bad objects with too few observations
        bad,=np.where(minObs < minObsPerBand)

        if (not temporary) :
            objFlag[bad] |= objFlagDict['TOO_FEW_OBS']

            self.fgcmLog.info('Flagging %d of %d stars with TOO_FEW_OBS' % (bad.size,self.nStars))
        else:
            objFlag[bad] |= objFlagDict['TEMPORARY_BAD_STAR']

            self.fgcmLog.info('Flagging %d of %d stars with TEMPORARY_BAD_STAR' % (bad.size,self.nStars))

    def selectStarsMinObsExpAndCCD(self, goodExps, goodCCDs, minObsPerBand=None):
        """
        Select stars that have at least the minimum number of observations per band,
         using a list of good exposures and ccds.

        parameters
        ----------
        goodExps: int array
           Array of good (photometric) exposure numbers
        goodCCDs: int array
           Array of good (photometric) ccd numbers
        minObsPerBand: int
           Specify the min obs per band, or use self.minObsPerBand
        """

        if (minObsPerBand is None):
            minObsPerBand = self.minObsPerBand

        if (goodExps.size != goodCCDs.size) :
            raise ValueError("Length of goodExps and goodCCDs must be the same")

        obsExp = snmm.getArray(self.obsExpHandle)
        obsExpIndex = snmm.getArray(self.obsExpIndexHandle)
        obsCCD = snmm.getArray(self.obsCCDHandle)
        obsBandIndex = snmm.getArray(self.obsBandIndexHandle)
        obsObjIDIndex = snmm.getArray(self.obsObjIDIndexHandle)
        objNGoodObs = snmm.getArray(self.objNGoodObsHandle)
        obsFlag = snmm.getArray(self.obsFlagHandle)
        objFlag = snmm.getArray(self.objFlagHandle)

        self.fgcmLog.info( 'Selecting good stars from %d exposure/ccd pairs.' %
                         (goodExps.size))

        # hash together exposure and ccd and match this
        obsHash = obsExp * (self.nCCD + self.ccdStartIndex) + obsCCD
        goodHash = goodExps * (self.nCCD + self.ccdStartIndex) + goodCCDs

        _,goodObs = esutil.numpy_util.match(goodHash, obsHash)

        # Filter out bad (previously flagged) individual observations
        gd, = np.where(obsFlag[goodObs] == 0)
        goodObs = goodObs[gd]

        # count all the good observations
        objNGoodObs[:,:] = 0
        np.add.at(objNGoodObs,
                  (obsObjIDIndex[goodObs],
                   obsBandIndex[goodObs]),
                  1)

                # and find the minimum of all the required bands
        minObs = objNGoodObs[:,self.bandRequiredIndex].min(axis=1)

        # reset too few obs flag if it's already set
        objFlag &= ~objFlagDict['TOO_FEW_OBS']

        # choose the bad objects with too few observations
        bad,=np.where(minObs < minObsPerBand)

        objFlag[bad] |= objFlagDict['TOO_FEW_OBS']
        self.fgcmLog.info('Flagging %d of %d stars with TOO_FEW_OBS' % (bad.size,self.nStars))

    def plotStarMap(self,mapType='initial'):
        """
        Plot star map.

        parameters
        ----------
        mapType: string, default='initial'
           A key for labeling the map.
        """

        import healpy as hp
        try:
            from .fgcmPlotmaps import plot_hpxmap
        except:
            self.fgcmLog.info("Map plotting not available.  Sorry!")
            return

        goodStars,=np.where(snmm.getArray(self.objFlagHandle)[:] == 0.0)

        theta = (90.0-snmm.getArray(self.objDecHandle)[goodStars])*np.pi/180.
        phi = snmm.getArray(self.objRAHandle)[goodStars]*np.pi/180.

        ipring = hp.ang2pix(self.mapNSide,theta,phi)

        densMap = esutil.stat.histogram(ipring,min=0,max=12*self.mapNSide*self.mapNSide-1)
        densMap = densMap.astype(np.float32)

        bad,=np.where(densMap == 0)
        densMap[bad] = hp.UNSEEN

        raStarRot = snmm.getArray(self.objRAHandle)[goodStars]
        hi,=np.where(raStarRot > 180.0)
        raStarRot[hi] -= 360.0

        decStar = snmm.getArray(self.objDecHandle)[goodStars]

        fig,ax = plot_hpxmap(densMap,
                             raRange=[np.min(raStarRot),np.max(raStarRot)],
                             decRange=[np.min(decStar),np.max(decStar)],
                             lonRef = self.mapLongitudeRef)

        fig.savefig('%s/%s_%sGoodStars.png' % (self.plotPath, self.outfileBaseWithCycle,
                                               mapType))
        plt.close(fig)

    def computeObjectSEDSlopes(self,objIndicesIn):
        """
        Compute fnuprime (object SED slopes) for a list of objects.
        Output is saved in objSEDSlope.

        parameters
        ----------
        objIndicesIn: int array
           Array of object indices to do computation
        """

        if self.nBands < 3:
            # cannot compute SED slopes ... just leave at 0
            return

        # work on multiple indices

        objMagStdMean = snmm.getArray(self.objMagStdMeanHandle)
        objSEDSlope = snmm.getArray(self.objSEDSlopeHandle)

        objMagStdMeanLock = snmm.getArrayBase(self.objMagStdMeanHandle).get_lock()
        objSEDSlopeLock = snmm.getArrayBase(self.objSEDSlopeHandle).get_lock()

        # select out good ones
        # NOTE: assumes that the required bands are sequential.
        #  in fact, this whole thing does.
        ## FIXME: require required bands to be explicitly sequential

        ## NOTE: this check is probably redundant, since we already have
        #   a list of good stars in most cases.

        # protect access to copy to local
        objMagStdMeanLock.acquire()

        objMagStdMeanOI = objMagStdMean[objIndicesIn,:]

        # release access
        objMagStdMeanLock.release()

        # and make a temporary local copy of the SED
        objSEDSlopeOI = np.zeros((objIndicesIn.size,self.nBands),dtype='f4')

        maxMag = np.max(objMagStdMeanOI[:,self.bandRequiredIndex.min():
                                              self.bandRequiredIndex.max()+1],axis=1)

        goodIndicesOI,=np.where(maxMag < 90.0)

        # can this be non-looped?
        S=np.zeros((goodIndicesOI.size,self.nBands-1),dtype='f8')
        for i in xrange(self.nBands-1):
            S[:,i] = (-1/self.magConstant) * (objMagStdMeanOI[goodIndicesOI,i+1] -
                                              objMagStdMeanOI[goodIndicesOI,i]) / (
                (self.lambdaStdBand[i+1] - self.lambdaStdBand[i]))

        ## FIXME: will have to handle u band "extra"

        tempIndex=self.bandRequiredIndex[0]
        objSEDSlopeOI[goodIndicesOI, tempIndex] = (
            S[:, tempIndex] + self.sedFitBandFudgeFactors[0] * (
                S[:, tempIndex+1] + S[:, tempIndex]))

        # and the middle ones...
        #  these are straight averages
        for tempIndex in self.bandRequiredIndex[1:-1]:
            objSEDSlopeOI[goodIndicesOI,tempIndex] = (
                self.sedFitBandFudgeFactors[tempIndex] * (
                    S[:,tempIndex-1] + S[:,tempIndex]) / 2.0)

        # and the last one
        tempIndex = self.bandRequiredIndex[-1]
        objSEDSlopeOI[goodIndicesOI,tempIndex] = (
            S[:,tempIndex-1] + self.sedFitBandFudgeFactors[-1] * (
                (self.lambdaStdBand[tempIndex] - self.lambdaStdBand[tempIndex-1]) /
                (self.lambdaStdBand[tempIndex] - self.lambdaStdBand[tempIndex-2])) *
            (S[:,tempIndex-1] - S[:,tempIndex-2]))

        # and the extra bands, only redward now
        #tempIndex = self.bandRequiredIndex[-1]
        #for i in xrange(len(self.bandExtraIndex)):
        #    extraIndex=self.bandExtraIndex[i]
        #    use,=np.where(objMagStdMeanOI[goodIndicesOI,extraIndex] < 90.0)
        #    objSEDSlopeOI[goodIndicesOI[use],extraIndex] = (
        #        S[use,tempIndex-1] + self.sedExtraBandFudgeFactors[i] * (
        #            (self.lambdaStd[tempIndex] - self.lambdaStd[tempIndex-1]) /
        #            (self.lambdaStd[tempIndex] - self.lambdaStd[tempIndex-2])) *
        #        (S[use,tempIndex-1] - S[use,tempIndex-2]))
        for i in xrange(len(self.bandExtraIndex)):
            extraIndex=self.bandExtraIndex[i]
            use,=np.where(objMagStdMeanOI[goodIndicesOI,extraIndex] < 90.0)
            objSEDSlopeOI[goodIndicesOI[use],extraIndex] = (
                S[use,extraIndex-1] + self.sedExtraBandFudgeFactors[i] * (
                    (self.lambdaStdBand[extraIndex] - self.lambdaStdBand[extraIndex-1]) /
                    (self.lambdaStdBand[extraIndex] - self.lambdaStdBand[extraIndex-2])) *
                (S[use,extraIndex-1] - S[use,extraIndex-2]))

        # and save the values, protected
        objSEDSlopeLock.acquire()

        objSEDSlope[objIndicesIn,:] = objSEDSlopeOI

        objSEDSlopeLock.release()

    def computeObjectSEDSlopesLUT(self, objIndicesIn, fgcmLUT):
        """
        Compute fnuprime (object SED slopes) for a list of objects, from the SED fit
          in the look-up table.  Experimental.
        Output is saved in objSEDSlope.

        parameters
        ----------
        objIndicesIn: int array
           Array of object indices to do computation
        fgcmLUT: FgcmLUT
        """

        objMagStdMean = snmm.getArray(self.objMagStdMeanHandle)
        objSEDSlope = snmm.getArray(self.objSEDSlopeHandle)

        objMagStdMeanLock = snmm.getArrayBase(self.objMagStdMeanHandle).get_lock()
        objSEDSlopeLock = snmm.getArrayBase(self.objSEDSlopeHandle).get_lock()

        # protect access to copy to local
        objMagStdMeanLock.acquire()

        objMagStdMeanOI = objMagStdMean[objIndicesIn,:]

        # release access
        objMagStdMeanLock.release()

        # and make a temporary local copy of the SED
        #objSEDSlopeOI = np.zeros((objIndicesIn.size,self.nBands),dtype='f4')

        # compute SED color...
        ## FIXME: make this configurable
        objSEDColorOI = objMagStdMeanOI[:,0] - objMagStdMeanOI[:,2]

        # do the look-up
        objSEDSlopeOI = fgcmLUT.computeSEDSlopes(objSEDColorOI)

        # and save the values, protected
        objSEDSlopeLock.acquire()

        objSEDSlope[objIndicesIn,:] = objSEDSlopeOI

        objSEDSlopeLock.release()

    def performColorCuts(self):
        """
        Make the color cuts that are specified in the config.
        """

        if (not self.magStdComputed):
            raise ValueError("Must compute magStd before performing color cuts")

        objMagStdMean = snmm.getArray(self.objMagStdMeanHandle)
        objFlag = snmm.getArray(self.objFlagHandle)

        for cCut in self.starColorCuts:
            thisColor = objMagStdMean[:,cCut[0]] - objMagStdMean[:,cCut[1]]
            bad,=np.where((thisColor < cCut[2]) |
                          (thisColor > cCut[3]))
            objFlag[bad] |= objFlagDict['BAD_COLOR']

            self.fgcmLog.info('Flag %d stars of %d with BAD_COLOR' % (bad.size,self.nStars))

    def applySuperStarFlat(self,fgcmPars):
        """
        Apply superStarFlat to raw magnitudes.

        parameters
        ----------
        fgcmPars: FgcmParameters
        """

        self.fgcmLog.info('Applying SuperStarFlat to raw magnitudes')

        obsMagADU = snmm.getArray(self.obsMagADUHandle)
        obsSuperStarApplied = snmm.getArray(self.obsSuperStarAppliedHandle)
        obsExpIndex = snmm.getArray(self.obsExpIndexHandle)
        obsCCDIndex = snmm.getArray(self.obsCCDHandle) - self.ccdStartIndex

        # two different tracks, if x/y available or not.

        if self.hasXY:
            # new style

            from .fgcmUtilities import poly2dFunc

            obsX = snmm.getArray(self.obsXHandle)
            obsY = snmm.getArray(self.obsYHandle)

            epochFilterHash = (fgcmPars.expEpochIndex[obsExpIndex]*
                               (fgcmPars.nLUTFilter+1)*(fgcmPars.nCCD+1) +
                               fgcmPars.expLUTFilterIndex[obsExpIndex]*
                               (fgcmPars.nCCD+1) +
                               obsCCDIndex)

            h, rev = esutil.stat.histogram(epochFilterHash, rev=True)

            for i in xrange(h.size):
                if h[i] == 0: continue

                i1a = rev[rev[i]:rev[i+1]]

                # get the indices for this epoch/filter/ccd
                epInd = fgcmPars.expEpochIndex[obsExpIndex[i1a[0]]]
                fiInd = fgcmPars.expLUTFilterIndex[obsExpIndex[i1a[0]]]
                cInd = obsCCDIndex[i1a[0]]

                obsSuperStarApplied[i1a] = poly2dFunc(np.vstack((obsX[i1a],
                                                        obsY[i1a])),
                                                       *fgcmPars.parSuperStarFlat[epInd, fiInd, cInd, :])
        else:
            # old style

            obsSuperStarApplied[:] = fgcmPars.expCCDSuperStar[obsExpIndex,
                                                              obsCCDIndex]

        # And finally apply the superstar correction
        obsMagADU[:] += obsSuperStarApplied[:]

    def applyApertureCorrection(self,fgcmPars):
        """
        Apply aperture corrections to raw magnitudes.

        parameters
        ----------
        fgcmPars: FgcmParameters
        """

        self.fgcmLog.info('Applying ApertureCorrections to raw magnitudes')

        obsExpIndex = snmm.getArray(self.obsExpIndexHandle)

        obsMagADU = snmm.getArray(self.obsMagADUHandle)

        # Note that EXP^gray = < <mstd>_j - mstd_ij >
        #  when we have seeing loss, that makes mstd_ij larger and EXP^gray smaller
        #  So the slope of aperCorr is negative.
        #  If we add aperCorr to each of mstd_ij, then we get a smaller (brighter)
        #  magnitude.  And this will bring mstd_ij closer to <mstd>_j

        obsMagADU[:] += fgcmPars.expApertureCorrection[obsExpIndex]

    def computeModelMagErrors(self, fgcmPars):
        """
        Compute model magnitude errors.

        parameters
        ----------
        fgcmPars: FgcmParameters
        """

        if (fgcmPars.compModelErrFwhmPivot[0] <= 0.0) :
            self.fgcmLog.info('No model for mag errors, so mag errors are unchanged.')
            return

        if not self.modelMagErrors:
            self.fgcmLog.info('Model magnitude errors are turned off.')
            return

        if not self.magStdComputed:
            raise RuntimeError("Must run FgcmChisq to compute magStd before computeModelMagErrors")

        self.fgcmLog.info('Computing model magnitude errors for photometric observations')

        objFlag = snmm.getArray(self.objFlagHandle)
        objNGoodObs = snmm.getArray(self.objNGoodObsHandle)
        objMagStdMean = snmm.getArray(self.objMagStdMeanHandle)

        obsObjIDIndex = snmm.getArray(self.obsObjIDIndexHandle)
        obsFlag = snmm.getArray(self.obsFlagHandle)
        obsExpIndex = snmm.getArray(self.obsExpIndexHandle)
        obsBandIndex = snmm.getArray(self.obsBandIndexHandle)
        obsMagADU = snmm.getArray(self.obsMagADUHandle)
        obsMagADUErr = snmm.getArray(self.obsMagADUErrHandle)
        obsMagADUModelErr = snmm.getArray(self.obsMagADUModelErrHandle)
        obsMagStd = snmm.getArray(self.obsMagStdHandle)

        obsExptime = fgcmPars.expExptime[obsExpIndex]
        obsFwhm = fgcmPars.expFwhm[obsExpIndex]
        obsSkyBrightness = fgcmPars.expSkyBrightness[obsExpIndex]

        # we will compute all stars that are possibly good, including reserved
        resMask = 255 & ~objFlagDict['RESERVED']
        goodStars, = np.where((objFlag & resMask) == 0)

        goodStarsSub, goodObs = esutil.numpy_util.match(goodStars,
                                                        obsObjIDIndex,
                                                        presorted=True)

        # Do we want to allow more selection of exposures here?
        gd, = np.where((obsFlag[goodObs] == 0) &
                       (fgcmPars.expFlag[obsExpIndex[goodObs]] == 0))
        goodObs = goodObs[gd]
        goodStarsSub = goodStarsSub[gd]

        # loop over bands
        for bandIndex in xrange(fgcmPars.nBands):
            use, = np.where((obsBandIndex[goodObs] == bandIndex) &
                            (objNGoodObs[obsObjIDIndex[goodObs], bandIndex] > self.minObsPerBand))
            pars = fgcmPars.compModelErrPars[:, bandIndex]
            fwhmPivot = fgcmPars.compModelErrFwhmPivot[bandIndex]
            skyPivot = fgcmPars.compModelErrSkyPivot[bandIndex]
            exptimePivot = fgcmPars.compModelErrExptimePivot[bandIndex]

            obsMagADUMeanGOu = (objMagStdMean[obsObjIDIndex[goodObs[use]], bandIndex] -
                                (obsMagStd[goodObs[use]] - obsMagADU[goodObs[use]]) -
                                2.5 * np.log10(obsExptime[goodObs[use]] / exptimePivot))

            modErr = 10.**(pars[0] + pars[1] * obsMagADUMeanGOu + pars[2] * obsMagADUMeanGOu**2. +
                           pars[3] * np.log10(obsFwhm[goodObs[use]] / fwhmPivot) +
                           pars[4] * np.log10(obsSkyBrightness[goodObs[use]] / skyPivot) +
                           pars[5] * obsMagADUMeanGOu * np.log10(obsFwhm[goodObs[use]] / fwhmPivot) +
                           pars[6] * obsMagADUMeanGOu * np.log10(obsSkyBrightness[goodObs[use]] / skyPivot))

            obsMagADUModelErr[goodObs[use]] = np.sqrt(modErr**2. + self.sigma0Phot**2.)

            # debug bit...
            """
            plt.set_cmap('viridis')
            fig = plt.figure(1, figsize=(8,6))
            fig.clf()
            ax = fig.add_subplot(111)

            ax.hexbin(obsMagADUErr[goodObs[use]], obsMagADUModelErr[goodObs[use]], bins='log')
            ax.plot([0., 0.08], [0., 0.08], 'r--')
            ax.set_title('band = %s, %d' % (self.bands[bandIndex], use.size))
            ax.set_xlabel('Observed error')
            ax.set_ylabel('Model Error')

            fig.savefig('temp_%s.png' % (self.bands[bandIndex]))
            plt.close(fig)
            """
    def saveFlagStarIndices(self,flagStarFile):
        """
        Save flagged stars to fits.

        parameters
        ----------
        flagStarFile: string
           Filename to output.
        """

        import fitsio

        flagObjStruct = self.getFlagStarIndices()

        self.fgcmLog.info('Saving %d flagged star indices to %s' %
                         (flagObjStruct.size,flagStarFile))

        # set clobber == True?
        fitsio.write(flagStarFile,flagObjStruct,clobber=True)

    def getFlagStarIndices(self):
        """
        Retrieve flagged star indices.
        """

        objID = snmm.getArray(self.objIDHandle)
        objFlag = snmm.getArray(self.objFlagHandle)

        # we only store VARIABLE and RESERVED stars
        # everything else should be recomputed based on the good exposures, calibrations, etc
        flagMask = (objFlagDict['VARIABLE'] |
                    objFlagDict['RESERVED'])

        flagged,=np.where((objFlag & flagMask) > 0)

        flagObjStruct = np.zeros(flagged.size,dtype=[('OBJID',objID.dtype),
                                                     ('OBJFLAG',objFlag.dtype)])
        flagObjStruct['OBJID'] = objID[flagged]
        flagObjStruct['OBJFLAG'] = objFlag[flagged]

        return flagObjStruct

    def saveStdStars(self, starFile, fgcmPars):
        """
        Save standard stars.  Note that this does not fill in holes.

        parameters
        ----------
        starFile: string
           Output star file
        fgcmPars: FgcmParameters
        """

        import fitsio

        self.fgcmLog.info( 'Saving standard stars to %s' % (starFile))

        objID = snmm.getArray(self.objIDHandle)
        objFlag = snmm.getArray(self.objFlagHandle)
        objRA = snmm.getArray(self.objRAHandle)
        objDec = snmm.getArray(self.objDecHandle)
        objNGoodObs = snmm.getArray(self.objNGoodObsHandle)
        objMagStdMean = snmm.getArray(self.objMagStdMeanHandle)
        objMagStdMeanErr = snmm.getArray(self.objMagStdMeanErrHandle)

        # reset TEMPORARY_BAD_STAR
        #objFlag &= ~objFlagDict['TEMPORARY_BAD_STAR']

        # only take photometric exposures...
        #goodExpsIndex, = np.where(fgcmPars.expFlag == 0)

        # this doesn't work because we'd have to recompute all the mags
        # this is more honest about what stars are actually well measured

        #self.selectStarsMinObsExpIndex(goodExpsIndex, minObsPerBand=1, temporary=True)

        rejectMask = (objFlagDict['BAD_COLOR'] | objFlagDict['VARIABLE'] |
                      objFlagDict['TOO_FEW_OBS'])

        goodStars, = np.where((objFlag & rejectMask) == 0)

        outCat = np.zeros(goodStars.size, dtype=[('FGCM_ID', 'i8'),
                                                 ('RA', 'f8'),
                                                 ('DEC', 'f8'),
                                                 ('NGOOD', 'i4', self.bands.size),
                                                 ('MAG_STD', 'f4', self.bands.size),
                                                 ('MAGERR_STD', 'f4', self.bands.size)])

        outCat['FGCM_ID'] = objID[goodStars]
        outCat['RA'] = objRA[goodStars]
        outCat['DEC'] = objDec[goodStars]
        outCat['NGOOD'] = objNGoodObs[goodStars, :]
        outCat['MAG_STD'][:, :] = objMagStdMean[goodStars, :]
        outCat['MAGERR_STD'][:, :] = objMagStdMeanErr[goodStars, :]

        # reset TEMPORARY_BAD_STAR
        #objFlag &= ~objFlagDict['TEMPORARY_BAD_STAR']

        fitsio.write(starFile, outCat, clobber=True)

    def __getstate__(self):
        # Don't try to pickle the logger.

        state = self.__dict__.copy()
        del state['fgcmLog']
        return state


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Oct  3 07:47:36 2019

@author: bressler
"""

import SBCcode as sbc
import numpy as np
import scipy
import matplotlib.pyplot as plt
from collections import Counter
import runlistscatalogue as rlc
from PICOcode.REFPROP.SeitzModel import SeitzModel
from os import listdir
from os.path import isfile,join

#LED_INDUCED_DEAD_TIME_FRACTION= 0.1289

def mult(runs,mergedfilename,Qbins,FeedbackTransducer, lightBins, binsname, LED_INDUCED_DEAD_TIME_FRACTION):
    if FeedbackTransducer == "PT4":
        pt = 3
    elif FeedbackTransducer == "PT6":
        pt = 5
    spectrumFile = open('/nashome/b/bressler/sbcoutput/'+mergedfilename+"%s_coincident_spectrum.txt"%binsname,'w')
    separator = '\t'
    spectrumFile.write("lightBin\t"+separator.join(["Q=%.1f-%.1fkeV"%(Qbins[i],
                                                                      Qbins[i+1]) for i in range(len(Qbins)-1)])+"\n")
    spectrumFile_multiples = open('/nashome/b/bressler/sbcoutput/'+mergedfilename+"%s_coincident_spectrum_multibub.txt"%binsname,'w')
    spectrumFile_multiples.write("lightBin\t"+separator.join(["Q=%.1f-%.1fkeV"%(Qbins[i],
                                                                      Qbins[i+1]) for i in range(len(Qbins)-1)])+"\n")
    fractionFile = open('/nashome/b/bressler/sbcoutput/'+mergedfilename+"%s_coincident_spectrum_fraction.txt"%binsname,'w')
    fractionFile.write("lightBin\t"+separator.join(["Q=%.1f-%.1fkeV"%(Qbins[i],
                                                                      Qbins[i+1]) for i in range(len(Qbins)-1)])+"\n")

    #with open("/nashome/b/bressler/sbcoutput/%s_merged_oldPMT.txt"%mergedfilename,"r") as fin:
    with open("/nashome/b/bressler/sbcoutput/%s_merged.txt"%mergedfilename,"r") as fin:
        data = fin.readlines()

    headers = data[0].split()
    runind = headers.index("run")
    eventind = headers.index("event")
    xind = headers.index("x")
    yind = headers.index("y")
    zind = headers.index("z")
    lagind = headers.index("lag")
    spectind = headers.index("PMTphe")
    blockedind = headers.index('isBlocked')
    nbubind = headers.index('nbub')
    x=[]
    y=[]
    z=[]
    evid = []
    lag = []
    spect = []
    isblocked = []
    nbub = []
    i=0
    for line in data:
        if i>0:
            split_line = line.split()
            evid.append(split_line[runind]+"-"+split_line[eventind])
            isblocked.append(float(split_line[blockedind]))
            x.append(float(split_line[xind]))
            y.append(float(split_line[yind]))
            z.append(float(split_line[zind]))
            lag.append(float(split_line[lagind]))
            spect.append(float(split_line[spectind]))
            nbub.append(float(split_line[nbubind]))

        i+=1

    counts = []
    nbubs = []
    didntpass = 0
    bubInfo = []
    eventcount = np.zeros(87)
    spectra = [[] for i in range(87)]
    twobub_spectra = [[] for i in range(87)]
    threebub_spectra = [[] for i in range(87)]
    LT = np.zeros(87)
    LT_by_Q = np.zeros(len(Qbins)-1)
    expand_times = [[] for i in range(87)]
    totbub = 0
    onebubcount = np.zeros(87)
    twobubcount = np.zeros(87)
    threebubcount = np.zeros(87)
    setpoints = []
    elt = 0
    temps = []

    for run in runs:
        print(run)
        z_low = -4
        z_high = 0
        preq = 1
        runrawpath = '/bluearc/storage/SBC-17-data/'+run
        events = [evnt for evnt in listdir(runrawpath) if not isfile(join(runrawpath,evnt))]
        Nevents = len(events)
        runreconpath = "/pnfs/coupp/persistent/grid_output/SBC-17/output/%s/"%run
        historyfilename = runreconpath+"HistoryAnalysis_%s.bin"%run
        history = sbc.DataHandling.ReadBinary.ReadBlock(historyfilename)
        getbubfile = "/coupp/data/home/coupp/HumanGetBub_output_SBC-17/HumanGetBub_%s.bin"%run
        c = sbc.DataHandling.ReadBinary.ReadBlock(getbubfile)

        eventn = c["ev"]
        count = Counter(eventn)
        edges = history["PressureEdge"][pt]
        centersp = [(edges[i]+edges[i+1])/2 for i in range(len(edges)-1)]
        #centers = Qvals
        e0 = sbc.DataHandling.GetSBCEvent.GetEvent(runrawpath,0,"slowDAQ")
        T=np.mean(e0["slowDAQ"]["T1"])
        temps.append(T)
        sm = SeitzModel(list(edges),T,'xenon')
        edges_by_Q = sm.Q
        centers = [(edges_by_Q[i] + edges_by_Q[i+1])/2 for i in range(len(edges_by_Q)-1)]

        print("T=%f C"%T)

        for c in count:
            N = count[c]
            counts.append(N)
            """
            if N%2 != 0 and N != 1:
                print("lines isn't even")
                print(c)
                print(N)
            elif N%2 == 0:
                n = N/2
                nbubs.append(n)
                """
        for eventn in range(Nevents):
            try:
                indices_back = 30
                event_ID = run+"-"+str(eventn)
                mergedind = evid.index(event_ID)
                bl = isblocked[mergedind]
                light = spect[mergedind]
                ev_z = z[mergedind]
                if np.isnan(light):
                    light = 0
                ev_z = z[mergedind]
                #onebub = not np.isnan(runposreco["z"][0][int(eventn)])
                n_from_merged = nbub[mergedind]
                n = 0
                N = count[eventn]
                if N%2 == 0:
                    n = N/2
                    nbubs.append(n)

                if n_from_merged != n:
                    print('n and merged n dont match')
                    print("n here: %d"%n)
                    print("n from merged: %d"%n_from_merged)
                e = sbc.DataHandling.GetSBCEvent.GetEvent(runrawpath,eventn,"slowDAQ","event")
                t = e["slowDAQ"]["elapsed_time"]
                tcenters = [(t[i+1] + t[i])/2 for i in range(len(t)-1)]
                trig_time = t[list(e["slowDAQ"]["TriggerOut"]).index(1.0)-indices_back]
                pslope = np.diff(e["slowDAQ"][FeedbackTransducer])
                expstartind = list(pslope).index(min(list(pslope)))
                exp_start = tcenters[expstartind]
                trig_pressure = e["slowDAQ"][FeedbackTransducer][list(e["slowDAQ"]["TriggerOut"]).index(1.0)-indices_back]
                T = np.mean(e["slowDAQ"]["T1"])
                SM = SeitzModel(trig_pressure,T,'xenon')
                Q = (SM.Q)
                #print(Q[0])
                pset = e["event"]["Pset"]
                ev_lt = e["event"]['livetime']
                elt += ev_lt
                if pset not in setpoints:
                    setpoints.append(pset)
                    #print(pset)
                #print(len(history["PressureBins"][eventn]))
                times = history["PressureBins"][eventn][pt][:]
                """
                plt.figure()
                plt.plot([trig_time,trig_time],[0,200],label="Bubble time")
                plt.plot([exp_start,exp_start],[0,200],label="Expansion start")
                plt.plot([0,max(t)],[trig_pressure,trig_pressure],label="Bubble pressure")
                plt.plot(t,e["slowDAQ"]["PT6"],label="PT6")
                plt.plot(tcenters,pslope,label="Pressure slope")
                plt.xlabel("time",fontsize=18)
                plt.ylabel("signal",fontsize=18)
                plt.legend(fontsize=18)
                plt.show
                """
                if n == 1 and ev_lt > 20:
                    totbub += 1
                    for i in range(len(centers)):
                        if trig_time > exp_start:
                            #print(trig_pressure)
                            #print(pset)
                            if (trig_pressure >= edges[i]
                                and trig_pressure <= edges[i+1]) and abs(trig_pressure-pset)<preq/2:
                                onebubcount[i] += 1
                                eventcount[i] += 1
                                if not bl:
                                    spectra[i].append(light)
                                    #if light < 1:
                                        #print("%s-%d is passing single not blocked and light < 1"%(run, eventn))
                                bubInfo.append([n,Q,bl,light,ev_z])
                                #print("added %s-%d to bubInfo"%(run,eventn))
                                expand_times[i].append(ev_lt)
                            if abs(centersp[i]-pset)<preq/2:
                                LT[i] += times[i]*(1-LED_INDUCED_DEAD_TIME_FRACTION)
                                for j in range(len(Qbins)-1):
                                    if Q >= Qbins[j] and Q <= Qbins[j+1]:
                                        LT_by_Q[j] += times[i]*(1-LED_INDUCED_DEAD_TIME_FRACTION)
                    if Q>1.5 and Q<2 and abs(trig_pressure-pset)<preq/2 and trig_time > exp_start:
                        print(event_ID)
                        print(Q)

                elif n == 2 and ev_lt > 20:
                    totbub += 1
                    for i in range(len(centers)):
                        if trig_time > exp_start:
                            if (trig_pressure >= edges[i]
                                and trig_pressure <= edges[i+1]) and abs(trig_pressure-pset)<preq/2:
                                eventcount[i] += 1
                                twobubcount[i] += 1
                                expand_times[i].append(ev_lt)
                                bubInfo.append([n,Q,bl,light,ev_z])
                                if not bl:
                                    twobub_spectra[i].append(light)
                                print("%s-%d multibubble event: %d bubbles, %.2f phe"%(run, eventn, n, light))
                            if abs(centersp[i]-pset)<preq/2:
                                LT[i] += times[i]*(1-LED_INDUCED_DEAD_TIME_FRACTION)
                                for j in range(len(Qbins)-1):
                                    if Q >= Qbins[j] and Q <= Qbins[j+1]:
                                        LT_by_Q[j] += times[i]*(1-LED_INDUCED_DEAD_TIME_FRACTION)

                elif n >= 3 and ev_lt > 20:
                    for i in range(len(centers)):
                        if trig_time > exp_start:
                            if (trig_pressure >= edges[i]
                                and trig_pressure <= edges[i+1]) and abs(trig_pressure-pset)<preq/2:
                                eventcount[i] += 1
                                threebubcount[i] += 1
                                expand_times[i].append(ev_lt)
                                bubInfo.append([n,Q,bl,light,ev_z])
                                if not bl:
                                    threebub_spectra[i].append(light)
                                print("%s-%d multibubble event: %d bubbles, %.2f phe"%(run, eventn, n, light))
                            if abs(centersp[i]-pset)<preq/2:
                                LT[i] += times[i]*(1-LED_INDUCED_DEAD_TIME_FRACTION)
                                for j in range(len(Qbins)-1):
                                    if Q >= Qbins[j] and Q <= Qbins[j+1]:
                                        LT_by_Q[j] += times[i]*(1-LED_INDUCED_DEAD_TIME_FRACTION)

                else:
                    if n>0:
                        didntpass += 1
                    for i in range(len(centers)):
                        if trig_time > exp_start and ev_lt > 20:
                            if (trig_pressure >= edges[i]
                                and trig_pressure <= edges[i+1]) and abs(trig_pressure-pset)<preq/2:
                                eventcount[i] += 1
                                expand_times[i].append(ev_lt)
                                bubInfo.append([n,Q,bl,light,ev_z])
                            if abs(centersp[i]-pset)<preq/2:
                                LT[i] += times[i]*(1-LED_INDUCED_DEAD_TIME_FRACTION)
                                for j in range(len(Qbins)-1):
                                    if Q >= Qbins[j] and Q <= Qbins[j+1]:
                                        LT_by_Q[j] += times[i]*(1-LED_INDUCED_DEAD_TIME_FRACTION)

                #if (ev_z > z_low and ev_z < z_high):

            except Exception as x:
                print(x)
                #break

    blockedEvents = [x for x in bubInfo if x[2] == 1]
    print("fraction of LED-blocked bubbles: "+str(len(blockedEvents)/len([x for x in bubInfo if x[0]>0])))
    print("Number of single-bubble zero phe non-blocked events:" + str(len([x for x in bubInfo if x[0]==1 and x[2] == 0 and (x[3]>0 and x[3] <1)])))
    print("Number of single-bubble one phe non-blocked events:" + str(len([x for x in bubInfo if x[0]==1 and x[2] == 0 and (x[3]>1 and x[3] <2)])))
    print("Number of single-bubble two phe non-blocked events:" + str(len([x for x in bubInfo if x[0]==1 and x[2] == 0 and (x[3]>2 and x[3] <3)])))

    rateList1 = [onebubcount[i]/LT[i] for i in range(len(onebubcount))]
    rateErrList1h = [(0.5+np.sqrt(onebubcount[i]+0.25))/LT[i] for i in range(len(onebubcount))]
    rateErrList1l = [(-0.5+np.sqrt(onebubcount[i]+0.25))/LT[i] for i in range(len(onebubcount))]
    standard_error1 = [np.sqrt(onebubcount[i])/LT[i] for i in range(len(onebubcount))]

    rateList2 = [twobubcount[i]/LT[i] for i in range(len(twobubcount))]
    rateErrList2h = [(0.5+np.sqrt(twobubcount[i]+0.25))/LT[i] for i in range(len(twobubcount))]
    rateErrList2l = [(-0.5+np.sqrt(twobubcount[i]+0.25))/LT[i] for i in range(len(twobubcount))]

    rateList3 = [threebubcount[i]/LT[i] for i in range(len(threebubcount))]
    rateErrList3h = [(0.5+np.sqrt(threebubcount[i]+0.25))/LT[i] for i in range(len(threebubcount))]
    rateErrList3l = [(-0.5+np.sqrt(threebubcount[i]+0.25))/LT[i] for i in range(len(threebubcount))]

    rateListE = [eventcount[i]/LT[i] for i in range(len(eventcount))]
    rateErrListEh = [(0.5+np.sqrt(eventcount[i]+0.25))/LT[i] for i in range(len(eventcount))]
    rateErrListEl = [(-0.5+np.sqrt(eventcount[i]+0.25))/LT[i] for i in range(len(eventcount))]
    """
    plt.figure()
    q = [bub[1] for bub in bubInfo]
    plt.hist(q,int(np.ceil(np.sqrt(len(q)))))
    plt.xlabel('Seitz Threshold')
    plt.ylabel('Count')
    plt.show()

    #Threshold:
    plt.figure()
    plt.errorbar(centers,rateList1,[rateErrList1l,rateErrList1h],fmt='ro',label='Single Bubbles')
    plt.errorbar(centers,rateList2,[rateErrList2l,rateErrList2h],fmt='go',label='Double Bubbles')
    plt.errorbar(centers,rateList3,[rateErrList3l,rateErrList3h],fmt='bo',label='Triple Bubbles')
    plt.errorbar(centers,rateListE,[rateErrListEl,rateErrListEh],fmt='ko',label='All Events')
    #plt.yscale('log')
    plt.ylim([1e-4,0.1])
    plt.xlabel('Seitz Threshold [keV]',fontsize=18)
    plt.ylabel('Rate [Hz]', fontsize=18)
    plt.legend(fontsize=18)
    plt.grid()
    plt.show

    #single bubbles
    def line(x,m,b):
        return m*x + b

    QForFit = []
    RForFit = []
    errForFit = []
    for i in range(len(rateList1)):
        if (not np.isnan(rateList1[i])) and (not np.isinf(rateList1[i])) and rateList1[i] != 0:
            QForFit.append(centers[i])
            RForFit.append(rateList1[i])
            errForFit.append(standard_error1[i])

    popt,pcov = scipy.optimize.curve_fit(line,QForFit,RForFit,p0=[0.005,0.01],sigma=errForFit)
    plt.figure()
    plt.errorbar(QForFit,RForFit,yerr=errForFit,fmt="ro")
    xfine = np.linspace(min(centers),3,50)
    y1 = line(xfine,popt[0]+pcov[0,0]**0.5, popt[1]-pcov[1,1]**0.5)
    y2 = line(xfine,popt[0]-pcov[0,0]**0.5, popt[1]+pcov[1,1]**0.5)
    plt.plot(xfine,line(xfine,popt[0],popt[1]),'k-')
    plt.plot(xfine,y1,'k--')
    plt.plot(xfine,y2,'k--')
    plt.plot([0,5],[0,0],'b-')
    plt.fill_between(xfine,y1,y2,facecolor='gray',alpha=0.1)
    plt.ylabel('Rate [Hz]',fontsize=18)
    plt.xlabel('Seitz Threshold [keV]',fontsize=18)
    plt.title('Crosses x-axis at %f'%(-popt[1]/popt[0]))
    plt.show
    """

    totbub = len(nbubs)
    print("total number of bubbles: %d"%totbub)
    print("total number of events: %d" %len(counts))
    print("didn't pass: %d"%didntpass)
    m = max(nbubs)
    for i in range(len(nbubs)):
        if nbubs[i]>3:
            nbubs[i] = 3

    binedges = [0.5,1.5,2.5,3.5] #bins for bubble multiplicity
    bincenters = [1,2,3]
    plt.figure()
    ns, _ = np.histogram(nbubs,binedges)
    print("bubble counts: ")
    print(ns)
    plt.errorbar(bincenters,ns,np.sqrt(ns),fmt='o')
    plt.yscale('log')
    plt.title("Max Nbub: %d"%int(m),fontsize=20)
    plt.xlabel("Nbub",fontsize=18)
    plt.ylabel("counts",fontsize=18)
    plt.xlim(0.5,3.5)
    plt.grid()
    plt.show

    f = [ns[i]/totbub for i in range(len(ns))]
    ferrh = [(0.5+np.sqrt(ns[i]+0.25))/totbub for i in range(len(ns))]
    ferrl = [(-0.5+np.sqrt(ns[i]+0.25))/totbub for i in range(len(ns))]

    plt.figure()
    plt.errorbar(bincenters,f,[ferrl,ferrh],fmt='o')
    plt.xlabel("Nbub",fontsize=18)
    plt.ylabel("fraction of bubbles",fontsize=18)
    plt.xlim([0.5,3.5])
    plt.grid()
    plt.show

    """
    #Pressure
    plt.figure()
    plt.errorbar(centersp,rateList1,[rateErrList1l,rateErrList1h],fmt='ro',label='Single Bubbles')
    plt.errorbar(centersp,rateList2,[rateErrList2l,rateErrList2h],fmt='go',label='Double Bubbles')
    plt.errorbar(centersp,rateList3,[rateErrList3l,rateErrList3h],fmt='bo',label='Triple Bubbles')
    plt.errorbar(centersp,rateListE,[rateErrListEl,rateErrListEh],fmt='ko',label='All Events')
    plt.yscale('log')
    plt.xlabel('Pressure [psia]',fontsize=18)
    plt.ylabel('Rate [Hz]', fontsize=18)
    plt.legend(fontsize=18)
    plt.grid()
    plt.show

    plt.figure()
    hy = []
    hx = []
    for i in range(len(expand_times)):
        if len(expand_times[i])>0:
            for j in range(len(expand_times[i])):
                hx.append(centersp[i])
                hy.append(expand_times[i][j])

    plt.hist2d(hx,hy,bins=(len(centers),30),cmap='magma')
    plt.colorbar()
    plt.xlabel('PT6 [psia]',fontsize=18)
    plt.ylabel('Total time since expansion [s]',fontsize=18)
    plt.show

    hy = []
    hx = []
    for i in range(len(expand_times)):
        if len(spectra[i])>0:
            for j in range(len(spectra[i])):
                hx.append(centersp[i])
                hy.append(spectra[i][j])
    plt.figure()
    plt.hist2d(hx,hy,bins=(40,int(max(hy))),cmap='Greys')
    plt.colorbar()
    plt.xlabel('PT6 [psia]',fontsize=18)
    plt.ylabel('Light collected [phe]',fontsize=18)
    plt.yscale('symlog',linthreshy = 0.9)
    plt.show

    xedges = np.arange(88)
    #yedges = np.arange(int(max(hy))+1)
    bins = [10**i+0.5 for i in range(5)] # light bins
    #bins = np.arange(0.5,1+np.ceil(max(spect)))
    bins = np.insert(bins,0,0.5)
    bins=np.insert(bins,0,-0.5)
    bins=np.insert(bins,0,-1.5)
    H, xedges, yedges = np.histogram2d(hx,hy,bins=(xedges,bins))
    for i in range(len(xedges)-1):
        for j in range(len(H[i,:])):
            #print(LT[i])
            H[i,j] = H[i,j]/LT[i]
    print("max rate: %f"%np.amax(H))

    fig = plt.figure()
    ax = fig.add_subplot(1,1,1)
    X,Y = np.meshgrid(xedges,yedges)

    im=ax.pcolormesh(X,Y,H.T,cmap="magma")
    plt.yscale('symlog',linthreshy = 0.9)
    fig.colorbar(im,ax=ax)
    plt.show()
    """
    hy = []
    hx = []
    hy_multi = []
    hx_multi = []
    for bubble in bubInfo:
        if bubble[0]==1 and bubble[2] == 0:
        #if bubble[0]==1:
            hx.append(bubble[1]) # Seitz threshold
            hy.append(bubble[3]) # phe
        elif (bubble[0] == 2 or bubble[0] == 3):
            hx_multi.append(bubble[1])
            hy_multi.append(bubble[3])
    plt.figure()
    plt.hist2d(hx,hy,bins=(40,int(max(hy))),cmap='Greys')
    plt.colorbar()
    plt.xlabel('Q [keV]',fontsize=18)
    plt.ylabel('Light collected [phe]',fontsize=18)
    plt.yscale('symlog',linthreshy = 0.9)
    plt.show

    #print(temps)

    xedges = edges
    Qedges = np.zeros(np.shape(xedges))
    for i in range(1,len(xedges)):
        SM = SeitzModel(float(xedges[i]),np.mean(temps),'xenon')
        if SM is not None:
            Qedges[i]=SM.Q

    OArateQ = [(Qbins[i]+Qbins[i+1])/2 for i in range(len(Qbins)-1)]
    OArate_multi = []
    OArateErr_multi = []
    OArate_singles = []
    OArateErr_singles = []
    Qedges = Qbins
    #print(Qbins)
    #yedges = np.arange(int(max(hy))+1)

    bins = lightBins # light bins
    #bins = np.arange(1,int(1+np.ceil(max(spect))))
    bins = np.insert(bins,0,0.5)
    bins=np.insert(bins,0,-0.5)
    #bins=np.insert(bins,0,-1.5)

    #bins = [-0.5, 2**9+0.5, 2**13+0.5]
    binc=[(bins[i+1]+bins[i])/2 for i in range(len(bins)-1)]
    binwidths = [bins[i+1]-bins[i] for i in range(len(bins)-1)]
    print(binwidths)
    H, Qedges, yedges = np.histogram2d(hx,hy,bins=(Qedges,bins)) # using hx and hy defined a few lines above in the previous plot section
    H_multi, Qedges, yedges_multi = np.histogram2d(hx_multi, hy_multi, bins=(Qedges, bins))
    multi_ns = H_multi.copy()
    ns = H.copy()
    errorbararray_upper = np.zeros_like(H)
    errorbararray_lower = np.zeros_like(H)
    errorbararray_multi_upper = np.zeros_like(H_multi)
    errorbararray_multi_lower = np.zeros_like(H_multi)
    print("total number of events in H: %d"%np.sum(H))
    print("total number of multiples in H_multi: %d"%np.sum(H_multi))
    for i in range(len(LT_by_Q)):
        OArate_singles.append(sum(H[i,:])/LT_by_Q[i])
        OArateErr_singles.append(np.sqrt(sum(H[i,:]))/LT_by_Q[i])
        for j in range(len(H[i,:])):
            H[i,j] = H[i,j]/LT_by_Q[i]
            if ns[i,j] > 10:
                errorbararray_upper[i,j]= np.sqrt(ns[i,j])/LT_by_Q[i]
                errorbararray_lower[i,j]= np.sqrt(ns[i,j])/LT_by_Q[i]
            else:
                errorbararray_upper[i,j]= (0.5+np.sqrt(ns[i,j]+0.25))/LT_by_Q[i]
                errorbararray_lower[i,j]= (-0.5+np.sqrt(ns[i,j]+0.25))/LT_by_Q[i]
            #print(H[i,j])
            #print(errorbararray[i,j])
            if np.isnan(H[i,j]):
                H[i,j]=0
                errorbararray_upper[i,j] = 0
                errorbararray_lower[i,j] = 0
        OArate_multi.append(sum(H_multi[i,:])/LT_by_Q[i])
        OArateErr_multi.append(np.sqrt(sum(H_multi[i,:]))/LT_by_Q[i])
        for j in range(len(H_multi[i,:])):
            H_multi[i,j] = H_multi[i,j]/LT_by_Q[i]
            if multi_ns[i,j] > 10:
               errorbararray_multi_upper[i,j]= np.sqrt(multi_ns[i,j])/LT_by_Q[i]
               errorbararray_multi_lower[i,j]= np.sqrt(multi_ns[i,j])/LT_by_Q[i]
            else:
               errorbararray_multi_upper[i,j]= (0.5+np.sqrt(multi_ns[i,j] + 0.25))/LT_by_Q[i]
               errorbararray_multi_lower[i,j]= (-0.5+np.sqrt(multi_ns[i,j] + 0.25))/LT_by_Q[i]
            #print(H[i,j])
            #print(errorbararray[i,j])
            if np.isnan(H_multi[i,j]):
                H_multi[i,j]=0
                errorbararray_multi_upper[i,j] = 0
                errorbararray_multi_lower[i,j] = 0
    #print(sum(H))
    #print(OArate)
    print("singles:")
    print(ns)
    print("multiples:")
    print(multi_ns)

    plt.figure()
    plt.errorbar(OArateQ,OArate_singles,OArateErr_singles,marker='v',linestyle='')
    plt.grid()
    plt.xlabel('Seitz Threshold [keV]')
    plt.ylabel('Rate [Hz]')
    plt.show()

    print("max rate: %f"%np.amax(H))
    print("number in onebubcount: %f"%np.sum(onebubcount))
    print(np.sum([rateList1[i] * LT[i] for i in range(len(rateList1)) if not np.isnan(rateList1[i])]))
    fig = plt.figure()
    ax = fig.add_subplot(1,2,1)
    ax.title.set_text('Rates [Hz/bin]')
    X,Y = np.meshgrid(Qedges,yedges)

    im=ax.pcolormesh(X,Y,H.T,cmap="Greys")
    plt.yscale('symlog',linthreshy = 0.9)
    plt.ylabel("Collected light [phe]")
    plt.xlabel("Seitz Threshold [keV]")
    fig.colorbar(im,ax=ax)

    ax2 = fig.add_subplot(1,2,2)
    ax2.title.set_text('Number of bubbles in each bin')
    im2=ax2.pcolormesh(X,Y,ns.T,cmap="Blues")
    plt.yscale('symlog',linthreshy = 0.9)
    plt.ylabel("Collected light [phe]")
    plt.xlabel("Seitz Threshold [keV]")
    fig.colorbar(im2,ax=ax2)
    #plt.show()

    fig = plt.figure()
    ax3 = fig.add_subplot(1,1,1)
    ax3.title.set_text("Rates per Scintillation: fraction")
    for k in range(len(Qedges)-1):
        print("%f-%f keV: %f bubbles in %f seconds"%(Qedges[k],Qedges[k+1],sum(ns[k,:]),LT_by_Q[k]))
        if LT_by_Q[k]>100:
            ax3.errorbar(binc,H[k,:]/OArate_singles[k],[(errorbararray_upper[k,:]/OArate_singles[k]),(errorbararray_lower[k,:]/OArate_singles[k])],marker='.',markersize=7,
                         linestyle='none',label="%f-%f keV"%(Qedges[k],Qedges[k+1]))
    for l in range(len(bins)-1):
        fractionFile.write("%.1f-%.1fphe\t"%(bins[l],bins[l+1]))
        for k in range(len(Qedges)-1):
            fractionFile.write("%f,%f,%f\t"%(H[k,l]/OArate_singles[k], errorbararray_upper[k,l]/OArate_singles[k], errorbararray_lower[k,l]/OArate_singles[k]))
        fractionFile.write('\n')
    plt.xscale('symlog',linthreshx=0.5)
    #plt.yscale('symlog',linthreshy=0.0001)
    plt.legend(fontsize=15)
    plt.xlabel("collected light [phe]")
    plt.ylabel("Fraction")
    #plt.ylim([0,0.1])
    plt.grid(which='both',axis='both')
    plt.show()

    fig = plt.figure()
    ax3 = fig.add_subplot(1,2,1)
    #ax3.title.set_text("Rates per Scintillation")
    ax4 = fig.add_subplot(1,2,2)
    for k in range(len(Qedges)-1):
        print("%f-%f keV: %f seconds"%(Qedges[k],Qedges[k+1],LT_by_Q[k]))
        if LT_by_Q[k]>100:
            ax3.errorbar(binc,H[k,:],[errorbararray_upper[k,:], errorbararray_lower[k,:]],marker='.',markersize=7,
                         linestyle='none',label="%f-%f keV"%(Qedges[k],Qedges[k+1]))
            Hnew = []
            errorbarnew_upper = []
            errorbarnew_lower = []
            for l in range(len(binwidths)):
                Hnew.append(np.divide(H[k,l],binwidths[l]))
                errorbarnew_upper.append(np.divide(errorbararray_upper[k,l],binwidths[l]))
                errorbarnew_lower.append(np.divide(errorbararray_lower[k,l],binwidths[l]))
            ax4.errorbar(binc, Hnew, [errorbarnew_upper, errorbarnew_lower],
                         marker='.', markersize=7, linestyle='none',
                         label='%f-%f keV'%(Qedges[k],Qedges[k+1]))
    for l in range(len(bins)-1):
        spectrumFile.write("%.1f-%.1fphe\t"%(bins[l],bins[l+1]))
        for k in range(len(Qedges)-1):
            spectrumFile.write("%f,%f,%f\t"%(H[k,l],errorbararray_upper[k,l], errorbararray_lower[k,l]))
        spectrumFile.write('\n')
    ax3.set_xscale('symlog',linthreshx=0.5)
    ax4.set_xscale('symlog',linthreshx=0.5)
    ax3.set_yscale('log')
    ax4.set_yscale('log')
    #plt.yscale('symlog',linthreshy=0.0001)
    ax3.legend(fontsize=15)
    ax4.legend(fontsize=15)
    plt.xlabel("collected light [phe]")
    ax3.set_ylabel("Rate per bin [Hz]")
    ax4.set_ylabel("rate per phe [Hz/phe]")
    #plt.ylim([0,0.1])
    ax3.grid(which='both',axis='both')
    ax4.grid(which='both',axis='both')
    plt.show()

    fig = plt.figure()
    ax3 = fig.add_subplot(1,2,1)
    ax3.title.set_text("Rates per Scintillation, multiples")
    ax4 = fig.add_subplot(1,2,2)
    for k in range(len(Qedges)-1):
        print("%f-%f keV: %f seconds"%(Qedges[k],Qedges[k+1],LT_by_Q[k]))
        if LT_by_Q[k]>100:
            ax3.errorbar(binc,H_multi[k,:],[errorbararray_multi_upper[k,:], errorbararray_multi_upper[k,:]],marker='.',markersize=7,
                         linestyle='none',label="%f-%f keV"%(Qedges[k],Qedges[k+1]))
            Hnew_multi = []
            errorbarnew_multi_upper = []
            errorbarnew_multi_lower = []
            for l in range(len(binwidths)):
                Hnew_multi.append(np.divide(H_multi[k,l],binwidths[l]))
                errorbarnew_multi_upper.append(np.divide(errorbararray_multi_upper[k,l],binwidths[l]))
                errorbarnew_multi_lower.append(np.divide(errorbararray_multi_lower[k,l],binwidths[l]))
            ax4.errorbar(binc, Hnew_multi, [errorbarnew_multi_upper, errorbarnew_multi_lower],
                         marker='.', markersize=7, linestyle='none',
                         label='%f-%f keV'%(Qedges[k],Qedges[k+1]))
    for l in range(len(bins)-1):
        spectrumFile_multiples.write("%.1f-%.1fphe\t"%(bins[l],bins[l+1]))
        for k in range(len(Qedges)-1):
            spectrumFile_multiples.write("%f,%f,%f\t"%(H_multi[k,l],errorbararray_multi_upper[k,l], errorbararray_multi_lower[k,l]))
        spectrumFile_multiples.write('\n')
    ax3.set_xscale('symlog',linthreshx=0.5)
    ax4.set_xscale('symlog',linthreshx=0.5)
    ax3.set_yscale('log')
    ax4.set_yscale('log')
    #plt.yscale('symlog',linthreshy=0.0001)
    ax3.legend(fontsize=15)
    ax4.legend(fontsize=15)
    plt.xlabel("collected light [phe]")
    ax3.set_ylabel("Rate per bin [Hz]")
    ax4.set_ylabel("rate per phe [Hz/phe]")
    #plt.ylim([0,0.1])
    ax3.grid(which='both',axis='both')
    ax4.grid(which='both',axis='both')
    plt.show()

    plt.figure()
    plt.scatter([x[1] for x in bubInfo],[x[3] for x in bubInfo])
    plt.yscale('symlog',linthreshy=0.5)
    plt.xlabel('Seitz Threshold [keV]')
    plt.ylabel('collected light [phe]')
    plt.show

    fig,ax1 = plt.subplots()
    ax2 = ax1.twinx()
    ax1.set_xlabel('Seitz Threshold (keV)',fontsize=18)
    ax1.set_ylabel('Live time (s)',fontsize=18)
    ax2.set_ylabel('Number of bubbles',fontsize=18)
    ax1.scatter(centers,LT,20,'r')
    ax2.scatter(centers,onebubcount,20,'b')
    plt.show

    spectrumFile.close()
    spectrumFile_multiples.close()
    fractionFile.close()

def main():
    runs = rlc.cfJuneminus50CCombined
    #runs=['20170628_9']
    mult(runs,'cfJuneminus50CCombined',[0,1,1.5,2,2.5,3],"PT6", [(2**i)+0.5 for i in range(14)], 'powerof2bins', 0.1289)
if __name__=="__main__":
    main()