import numpy as np
import psr_utils as pu
import matplotlib.pyplot as plt
import pyslalib.slalib as slalib
from argparse import ArgumentParser
from frequencyoptimizer import PulsarNoise,TelescopeNoise,GalacticNoise
from frequencyoptimizer import FrequencyOptimizer

NCHAN = 16
NSTEPS = 16

Tcmb = 3.0

usage = """%(prog)s [options]

Receiver/telescope parameters can be specified via a text file using
the -r/--rx-specs argument.  The text file should have
space-delineated columns of

Frequency(GHz) Trx(K) Gain(K/Jy) FractionalGainError

frequencyoptimizer.py will interpolate over these values at the
frequencies of interest.  If nothing is specified for -r/--rx-specs,
then frequency-independent receiver/telescope parameters will be taken
from command line arguments with defaults as specified below.

Pulsar parameters can be specified from a python dictionary or from
command line options with defaults as specified below.  If using a
python dictionary, then the dictionary file should contain one and
only one dictionary of dictionaries, with the top-level keys being
pulsar names, and the second-level keys/values being

name - Pulsar name
period - Pulsar period (ms)
DM - Dispersion measure (pc/cc)
flux_1GHz - Pulsar flux at 1 GHz (mJy)
spec_index - Pulsar spectral index (S_nu \propto nu^\alpha)
W50 - Pulse profile FWHM (us)
Weff - Effective pulsar profile width (us)
Uscale - Pulse profile scaling parameter
rms_J1 - Single pulse RMS due to jitter (us)
scat_ts - Scattering timescale at 1 GHz (us)
scat_ts_var - Scattering timescale variability at 1 GHz (us)
diss_ts - Diffractive interstellar scintillation timescale (s)

The script make_dict.py can be used to convert a space-delineated text
file with the above columns into a suitable dictionary.  See
psr_info.py and psr_info.txt for examples."""

parser = ArgumentParser(description="Predict pulsar TOA precision", usage=usage)
rx_group = parser.add_argument_group(title="Receiver specs")
pulsar_group = parser.add_argument_group(title="Pulsar parameters")
rx_group.add_argument("-r", "--rx-specs",
                      help="File containing receiver performance specs")
rx_group.add_argument("-L", "--low-freq", type=float, default=1.0,
                      help="Low frequency (GHz; default=%(default)s)")
rx_group.add_argument("-H", "--high-freq", type=float, default=2.0,
                      help="High frequency (GHz; default=%(default)s)")
rx_group.add_argument("-T", "--trx", dest="Trx", type=float, default=20,
                      help="Receiver temperature (K; default=%(default)s)")
rx_group.add_argument("-G", "--gain", type=float, default=2.0,
                      help="Telescope gain (K/Jy; default=%(default)s)")
rx_group.add_argument("-e", "--epsilon", type=float, default=0.01,
                      help="Fractional gain instability (default=%(default)s)")
parser.add_argument("-t", "--tobs", type=float, default=1800.0,
                    help="Observing time (s; default=%(default)s)")

pulsar_group.add_argument("-d", "--psr-dict",
                          help="Python dictionary containing pulsar parameters")

pulsar_group.add_argument("-n", "--name", default="Fake",
                          help="Pulsar name (default=%(default)s)")
pulsar_group.add_argument("-P", "--period", type=float, default=3.0,
                          help="Pulsar period (ms; default=%(default)s)")
pulsar_group.add_argument("-D", "--dm", type=float, default=30.0,
                          help="Pulsar DM (pc/cc; default=%(default)s)")
pulsar_group.add_argument("-F", "--flux", dest="flux_1GHz", type=float,
                          default=10.0,
                          help=("Pulsar flux density at 1 GHz (mJy; "
                                "default=%(default)s)"))
pulsar_group.add_argument("-a", "--alpha", dest="spec_index", type=float,
                          default=-1.7,
                          help="Pulsar spectral index (default=%(default)s)")
pulsar_group.add_argument("-w", "--width", dest="W50", type=float,
                          default=300.0,
                          help="Pulsar FWHM (us; default=%(default)s)")
pulsar_group.add_argument("--weff", dest="Weff", type=float,
                          help="Pulsar effective width (us; default=1.2 x W50)")
pulsar_group.add_argument("-U", "--uscale", type=float, default=10.0,
                          help=("Pulsar profile scaling factor"))
pulsar_group.add_argument("-j", "--jitter", dest="rms_J1", type=float,
                          default=100.0,
                          help=("Pulsar single-pulse jitter RMS (us; "
                                "default=%(default)s)"))
pulsar_group.add_argument("-s", "--scat-ts", type=float, default=0.01,
                          help=("Pulsar scattering timescale at 1 GHz "
                                "(us; default=%(default)s)"))
pulsar_group.add_argument("-S", "--scat-ts-var", type=float, default=0.05,
                          help=("Pulsar scattering timescale variability at "
                                "1 GHz (us; default=%(default)s)"))
pulsar_group.add_argument("-i", "--diss-ts", type=float, default=1000.0,
                          help=("Pulsar diffractive ISS timescale at 1 GHz (s; "
                                "default=%(default)s"))
args = parser.parse_args()

if args.rx_specs is not None:
    interpolate = True
    rx_freq,Trx,gain,epsilon = np.loadtxt(args.rx_specs,unpack=True)
    low_freq = np.min(rx_freq)
    high_freq = np.max(rx_freq)
    freqs = np.logspace(np.log10(low_freq),np.log10(high_freq),NCHAN)
else:
    interpolate = False
    Trx = args.Trx
    gain = args.gain
    epsilon = args.epsilon
    low_freq = args.low_freq
    high_freq = args.high_freq
    freqs = np.logspace(np.log10(low_freq),np.log10(high_freq),NCHAN)
    rx_freq = freqs.copy()

if args.psr_dict is not None:
    import importlib
    psr_dict = importlib.import_module(args.psr_dict.rstrip(".py"))
    for a in dir(psr_dict):
        if not a.startswith("__"):
            psrs = psr_dict.__getattribute__(a)
            break
else:
    psrs = {args.name: {
        "name": args.name,
        "period": args.period,
        "DM": args.dm,
        "flux_1GHz": args.flux_1GHz,
        "spec_index": args.spec_index,
        "W50": args.W50,
        "Weff": 1.2*args.W50 if args.Weff is None else args.Weff,
        "uscale": args.uscale,
        "rms_J1": args.rms_J1,
        "scat_ts": args.scat_ts,
        "scat_ts_var": args.scat_ts_var,
        "diss_ts": args.diss_ts }}
sigmas = []
telescope_noise = TelescopeNoise(gain=gain,T_const=Trx+Tcmb,epsilon=epsilon,
                                 T=args.tobs,rx_nu=rx_freq,
                                 interpolate=interpolate)
galactic_noise = GalacticNoise()
for name,psr in psrs.items():
    if psr["W50"] is None and psr["Weff"] is not None:
        psr["W50"] = 0.66*psr["Weff"]
    if psr["scat_ts"] is None and psr["DM"] is not None:
        psr["scat_ts"] = 1000*pu.pulse_broadening(psr["DM"],1000.0)
    if None not in psr.values():
        pulsar_noise = PulsarNoise(
            name,alpha=-psr["spec_index"],I_0=psr["flux_1GHz"],DM=psr["DM"],
            taud=psr["scat_ts"]*1.5**4.4,tauvar=psr["scat_ts_var"]*1.5**4.4*0.5,
            dtd=psr["diss_ts"],P=psr["period"],Uscale=psr["uscale"],
            sigma_Js=np.zeros(NCHAN)+psr["rms_J1"] / \
            np.sqrt(1000*args.tobs/psr["period"]),
            Weffs=np.zeros(NCHAN)+psr["Weff"],W50s=np.zeros(NCHAN)+psr["W50"])
        frequency_optimizer = FrequencyOptimizer(
            pulsar_noise,galactic_noise,telescope_noise,
            numin=low_freq,numax=high_freq,nchan=NCHAN,log=True,nsteps=NSTEPS,
            frac_bw=False,full_bandwidth=False,masks=None)
        sigma = frequency_optimizer.calc_single(freqs)
        sigmas.append(sigma)
        print("%-10s   %.3f"%(name,sigma))

sigma_mean = np.mean(sigmas)
sigma_median = np.median(sigmas)
sigma_std = np.std(sigmas)

print("Mean sigma = %.3f"%sigma_mean)
print("Median sigma = %.3f"%sigma_median)
print("STD sigma = %.3f"%sigma_std)


from __future__ import print_function
from astropy.io import fits, ascii as asc
from astropy.convolution import Gaussian2DKernel
from scipy.interpolate import interp1d

import numpy as np
import os
import glob
import sys
#import fsps

from ..utils.smoothing import smooth
from ..filters import get_filter

#import some bits for the instruments
from ..detectors import CCD250
from ..telescopes import VLT

__all__ = ["MAVIS_Imager"]

class ImagingInstrument:
    """
    Generic instrument class with some generic instrument routines.
    """
    def __init__(self):
        self.small_num = 1e-70
        self.source_obs = None
        self.sky_obs = None

        self.transmission = None
        self.wavelength = None
        self.pivot = None

    def _EE_gaussian(self, seeing, binning=1, **kwargs):
        sigma_pix = seeing / self.pix_scale / 2.355 #relies on the v-band seeing here

        wave_temp = np.linspace(self.inst_wavelength.min(), self.inst_wavelength.max(), 50)
        sig_temp = sigma_pix * (wave_temp/.500)**(-1./5.)
        ee_temp = np.zeros_like(wave_temp)
        for ii, tsig in enumerate(sig_temp):
            ee_temp[ii] = Gaussian2DKernel(tsig, x_size=binning, y_size=binning,
                                            mode='oversample', factor=100).array.sum()

        ee_array = interp1d(wave_temp, ee_temp, kind='quadratic')(self.inst_wavelength)

        return ee_array, binning**2

    def _set_filter(self, filt):
        #fetch filter transmission curve from FSPS
        #this sets the wavelength grid, so no rebinning needed

        #pull information for this filter
        fobj = get_filter(filt)
        fwl, ftrans = fobj.transmission

        ftrans = np.maximum(ftrans, 0.)
        trans_interp = np.asarray(np.interp(self.inst_wavelength, fwl/1e4,
                                  ftrans, left=0., right=0.), dtype=np.float)

        #normalize transmission
        ttrans = np.trapz(np.copy(trans_interp), self.inst_wavelength)
        if ttrans < self.small_num: ttrans = 1.
        ntrans = np.maximum(trans_interp / ttrans, 0.0)

        #stupid, but re-normalize to peak of 1 (since all other throughput terms
        #are included in the instrument throughput
        self.trans_norm = np.copy(ntrans)/ntrans.max()
        self.transmission = ntrans
        self.pivot = fobj.lambda_eff
        return

    def _patch_nan(self, signal):
        nans = np.isnan(signal)
        interpolator = interp1d(self.inst_wavelength[~nans], signal[~nans], bounds_error='extrapolate')
        signal[nans] = interpolator(self.inst_wavelength[nans])
        return signal

    def make_sky_spectrum(self, sky, source, source_wave):
        sky_wave, sky_emm, sky_trans = sky()

        match_res_sky = np.interp(sky_wave, source_wave, source.res_pix,
                                 left=source.res_pix[0], right=source.res_pix[-1])
        offset_res_sky = np.sqrt(np.clip((match_res_sky*source.step/sky.step)**2 -
                                         sky.res_pix**2, 1e-10, None))
        conv_emm = smooth(sky_emm, offset_res_sky)
        conv_trans = smooth(sky_trans, offset_res_sky)

        #resample onto output grid
        sky_emm_resampled = np.interp(self.inst_wavelength, sky_wave, conv_emm)
        sky_trans_resampled = np.clip(np.interp(self.inst_wavelength,
                                                     sky_wave, conv_trans),0,1)

        return sky_emm_resampled, sky_trans_resampled

    def calc_sn(self, source, sky=None, dit=3600.,
                ndit=None, sn=None, seeing=1., binning=1, band='johnson_v', strehl=None):

        #generate source spectrum
        source_wave, source_phot = source()

        #resample onto outputpixel grid
        source_resampled = np.interp(self.inst_wavelength, source_wave, source_phot)

        #if a sky object is also supplied, convolve it to match the instrument properties
        if sky is not None:
            sky_emm_resampled, sky_trans_resampled = self.make_sky_spectrum(sky, source, source_wave)
        else:
            sky_trans_resampled = np.ones(len(self.inst_wavelength))
            sky_emm_resampled = np.zeros(len(self.inst_wavelength))

        #store transmission spectrum
        self.sky_trans = np.copy(sky_trans_resampled)

        #estimate the ensquared energy and pixel area
        self.obs_ee, self.obs_area = self._ee(seeing, binning=binning, strehl=strehl)

        #total source spectrum
        if source.norm_sb:
            self.cfact = sky_trans_resampled*dit*self.total_throughput*self.step*\
                         self.telescope.area*self.pix_scale**2 * self.obs_area
        else:
            #get ensquared energy and area in pixels
            self.cfact = sky_trans_resampled*dit*self.total_throughput*\
                         self.step*self.telescope.area*self.obs_ee

        source_obs = np.copy(source_resampled)*self.cfact #photons

        #sky is always done correctly-ish.
        sky_obs = np.copy(sky_emm_resampled)*dit*self.total_throughput*self.step*\
                  self.telescope.area*self.pix_scale**2 * self.obs_area #total area, photons#/um

        ##set filter
        for iband in band:
            self._set_filter(iband)

            #integrate transmission for total counts
            self.source_obs = self._patch_nan(source_obs)*self.trans_norm
            self.sky_obs = self._patch_nan(sky_obs)*self.trans_norm

            #total noise calculation
            self.dark_noise = self.obs_area*self.detector.dark*dit
            self.read_noise = self.obs_area*self.detector.rn**2
            self.sky_noise = np.nansum(self.sky_obs)
            self.obj_noise = np.nansum(self.source_obs)
            self.noise = self.obj_noise + self.sky_noise + self.read_noise + self.dark_noise #per dit

            if sn is not None and ndit is None: #provided S/N, work out ndit to reach target S/N
                indit = np.int(np.ceil(np.sqrt(self.noise)*sn/self.source_obs.sum()))
                print("NDIT={2} to reach S/N={0} with DIT={1} in {3}".format(sn, dit, indit, iband))
            elif sn is None and ndit is not None:
                isn = np.sqrt(ndit)*self.source_obs.sum() / np.sqrt(self.noise)
                print("S/N={0:4.2f} at with NDIT={1} and DIT={2} in {3}".format(isn, ndit, dit, iband))

        return

    def get_mag_limit(self, sn=None, sky=None, dit=3600.,
                      ndit=None, seeing=1., binning=1, band='johnson_v', strehl=None,
                      norm='point'):

        #if a sky object is also supplied, convolve it to match the instrument properties
        if sky is not None:
            sky_wave, sky_emm, sky_trans = sky()

            #resample onto output grid
            sky_emm_resampled = np.interp(self.inst_wavelength, sky_wave, sky_emm)
            sky_trans_resampled = np.clip(np.interp(self.inst_wavelength,
                                                         sky_wave, sky_trans),0,1)
        else:
            sky_trans_resampled = np.ones(len(self.inst_wavelength))
            sky_emm_resampled = np.zeros(len(self.inst_wavelength))

        #store transmission spectrum
        self.sky_trans = np.copy(sky_trans_resampled)

        #estimate the ensquared energy and pixel area
        self.obs_ee, self.obs_area = self._ee(seeing, binning=binning, strehl=strehl)

        #total source spectrum
        if norm == 'point':
            #get ensquared energy and area in pixels
            self.cfact = sky_trans_resampled*dit*self.total_throughput*\
                         self.step*self.telescope.area*self.obs_ee
        else:
            self.cfact = sky_trans_resampled*dit*self.total_throughput*self.step*\
                         self.telescope.area*self.pix_scale**2 * self.obs_area

        #sky is always done correctly-ish.
        sky_obs = np.copy(sky_emm_resampled)*dit*self.total_throughput*self.step*\
                  self.telescope.area*self.pix_scale**2 * self.obs_area #total area, photons#/um

        ##set filter
        store_limit = []
        store_pivot = []
        for iband in band:
            self._set_filter(iband)

            #integrate transmission for total counts
            self.sky_obs = self._patch_nan(sky_obs)*self.trans_norm

            #total noise calculation
            self.dark_noise = self.obs_area*self.detector.dark*dit
            self.read_noise = self.obs_area*self.detector.rn**2
            self.sky_noise = np.nansum(self.sky_obs)
            self.noise = self.sky_noise + self.read_noise + self.dark_noise #per dit

            #quadratic terms for solution
            a = ndit / sn**2
            c = self.noise

            source_obs = 0.5*(1. + np.sqrt(1. + 4*a*c)) * sn**2 / ndit #total counts

            #print(source_obs, self.pivot)
            store_limit.append(-2.5*np.log10(source_obs * self.pivot * 6.626196e-27 / 100**2 / np.nansum(self.cfact*self.trans_norm) / 1e4)-48.6)
            store_pivot.append(self.pivot/1e4)

        return store_pivot, store_limit

class MAVIS_Imager(ImagingInstrument):
    """
    A MAVIS-like instrument.

    Assumes that the general properties of the MAVIS spectrograph
    are MUSE-like, but with an added throughput hit from the AO system
    and more elaborate PSF model.
    """

    def __init__(self, pix_scale=0.00736, jitter=5, detector=None, telescope=None):
        #check for reasonable jitter values
        if jitter not in [5,10,20,30,40]:
            raise ValueError('Input jitter must be one of 5, 10, 20, 30, or 40 (mas)')

        #initialize the model base
        ImagingInstrument.__init__(self)

        #set the pixel scale
        self.pix_scale = pix_scale

        #wavelength business
        self.step = 1. / 1e4 #microns
        self.wmin, self.wmax = 3700./1e4, 10100./1e4
        self.inst_wavelength = np.arange(self.wmin, self.wmax, self.step)

        #initialize the provided detector object
        if detector is None:
            self.detector = CCD250()
        else:
            self.detector = detector()

        #get detector QE
        self.qe = self.detector.qe_interp(self.inst_wavelength)

        #initialize the telescope
        if telescope is None:
            self.telescope = VLT()
        else:
            self.telescope = telescope()

        #if the instrument throughput curves are already included, this should be set to 1
        self.telescope_throughput = np.interp(self.inst_wavelength, self.telescope.telescope_wave, self.telescope.telescope_eff)

        #compute the combined throughput - including filter transmission
        self.total_throughput = self.telescope_throughput * self.qe * 0.98**2

        #patch in low throughput at the notch
        self.notch = (self.inst_wavelength > 0.580) & (self.inst_wavelength < 0.597)

        #get path for bundled package files
        bfile_dir = os.path.join(os.path.dirname(sys.modules['mavisetc'].__file__), 'data')

        #fold in AOM throughput
        data = np.array(asc.read(os.path.join(bfile_dir, 'mavis/mavis_AOM_throughput.csv')))
        twave = np.array(data['col1'])
        ttpt = np.array(data['col2'])

        self.ao_throughput = np.interp(self.inst_wavelength, np.array(twave), np.array(ttpt),
                                left=ttpt[0], right=ttpt[-1])
        self.total_throughput *= self.ao_throughput

        #pre-load EE profiles
        ee_files = glob.glob(os.path.join(bfile_dir, 'mavis/PSF_{0}mas*EEProfile.dat'.format(jitter)))
        wave = []
        ee_interp = []
        for ii, ee_file in enumerate(ee_files):
            twave = os.path.split(ee_file)[1].split('_')[2][:-2]
            wave.append(float(twave)/1e3)
            with open(ee_file, 'r') as file:
                tr, tee = [], []
                for line in file:
                    temp = line.strip().split(',')
                    tr.append(float(temp[0])/self.pix_scale) #in pixels
                    tee.append(float(temp[1]))
                ee_interp.append(interp1d(tr, tee, bounds_error=False, fill_value='extrapolate'))
        self._ee_profile_wave = np.array(wave)
        self._ee_profile_interp = ee_interp

        self._ee = self._EE_lookup

    def _EE_lookup(self, seeing, binning=1, **kwargs):
        """
        A quick and dirty lookup/interpolation function to generate ensquared
        energy profiles based on simulations of the MAVIS PSF.
        """
        #Seeing isn't relevant for MAVIS at the moment, so it only depends on the binning
        iwave = np.argsort(self._ee_profile_wave)

        wave_out = np.zeros(len(iwave), dtype=np.float)
        ee_out = np.zeros(len(iwave), dtype=np.float)
        for ii, idx in enumerate(iwave): #sorted arguments?
            wave_out[ii] = self._ee_profile_wave[idx]
            ee_out[ii] = self._ee_profile_interp[idx](binning/2.)

        ee_interped = interp1d(wave_out, ee_out, fill_value='extrapolate')(self.inst_wavelength)
        return ee_interped, binning**2


""" Defines the OplessModel class"""
#***************************************************************************************************
# Copyright 2015, 2019 National Technology & Engineering Solutions of Sandia, LLC (NTESS).
# Under the terms of Contract DE-NA0003525 with NTESS, the U.S. Government retains certain rights
# in this software.
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License.  You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0 or in the LICENSE file in the root pyGSTi directory.
#***************************************************************************************************

import numpy as _np
import collections as _collections

from .model import Model as _Model
from .evaltree import EvalTree as _EvalTree
from .labeldicts import OutcomeLabelDict as _OutcomeLabelDict
from .circuit import Circuit as _Circuit
from .polynomial import Polynomial as _Polynomial
from ..tools import slicetools as _slct

from .opcalc import compact_deriv as _compact_deriv, float_product as prod, \
    safe_bulk_eval_compact_polys as _safe_bulk_eval_compact_polys

class OplessModelTree(_EvalTree):
    def __init__(self, circuit_list, lookup, outcome_lookup, cache=None):
        _EvalTree.__init__(self, circuit_list)
        self.element_indices = lookup
        self.outcomes = outcome_lookup
        self.num_final_strs = len(circuit_list)  # circuits
        max_el_index = -1
        for elIndices in lookup.values():
            max_i = elIndices.stop - 1 if isinstance(elIndices, slice) else max(elIndices)
            max_el_index = max(max_el_index, max_i)
        self.num_final_els = max_el_index + 1
        self.cache = cache

class OplessModel(_Model):
    """
    TODO docstring
    """

    def __init__(self, state_space_labels):
        """
        Creates a new Model.  Rarely used except from derived classes
        `__init__` functions.

        Parameters
        ----------
        state_space_labels : StateSpaceLabels or list or tuple
            The decomposition (with labels) of (pure) state-space this model
            acts upon.  Regardless of whether the model contains operators or
            superoperators, this argument describes the Hilbert space dimension
            and imposed structure.  If a list or tuple is given, it must be
            of a from that can be passed to `StateSpaceLabels.__init__`.
        """
        _Model.__init__(self, state_space_labels)

        #Setting things the rest of pyGSTi expects but probably shouldn't...
        self.simtype = "opless"
        self.basis = None
        self.dim = 0

    def get_dimension(self):
        return self.dim

    def get_num_outcomes(self, circuit):  # needed for sparse data detection
        raise NotImplementedError("Derived classes should implement this!")

    def probs(self, circuit, clipTo=None, cache=None):
        """
        Construct a dictionary containing the probabilities of every spam label
        given a operation sequence.

        Parameters
        ----------
        circuit : Circuit or tuple of operation labels
          The sequence of operation labels specifying the operation sequence.

        clipTo : 2-tuple, optional
           (min,max) to clip probabilities to if not None.

        Returns
        -------
        probs : dictionary
            A dictionary such that
            probs[SL] = pr(SL,circuit,clipTo)
            for each spam label (string) SL.
        """
        raise NotImplementedError("Derived classes should implement this!")

    def dprobs(self, circuit, returnPr=False, clipTo=None):
        """
        Construct a dictionary containing the probability derivatives of every
        spam label for a given operation sequence.

        Parameters
        ----------
        circuit : Circuit or tuple of operation labels
          The sequence of operation labels specifying the operation sequence.

        returnPr : bool, optional
          when set to True, additionally return the probabilities.

        clipTo : 2-tuple, optional
           (min,max) to clip returned probability to if not None.
           Only relevant when returnPr == True.

        Returns
        -------
        dprobs : dictionary
            A dictionary such that
            dprobs[SL] = dpr(SL,circuit,gates,G0,SPAM,SP0,returnPr,clipTo)
            for each spam label (string) SL.
        """
        eps = 1e-7
        orig_pvec = self.to_vector()
        Np = self.num_params()
        probs0 = self.probs(circuit, clipTo, None)

        deriv = {k: _np.empty(Np, 'd') for k in probs0.keys()}
        for i in range(Np):
            p_plus_dp = orig_pvec.copy()
            p_plus_dp[i] += eps
            self.from_vector(p_plus_dp)
            probs1 = self.probs(circuit, clipTo, None)
            for k, p0 in probs0.items():
                deriv[k][i] = (probs1[k] - p0) / eps
        self.from_vector(orig_pvec)

        if returnPr:
            return {k: (p0, deriv[k]) for k in probs0.keys()}
        else:
            return deriv

    def bulk_evaltree_from_resources(self, circuit_list, comm=None, memLimit=None,
                                     distributeMethod="default", subcalls=[],
                                     dataset=None, verbosity=0):
        #TODO: choose these based on resources, and enable split trees
        minSubtrees = 0
        numSubtreeComms = 1
        maxTreeSize = None
        evTree = self.bulk_evaltree(circuit_list, minSubtrees, maxTreeSize,
                                    numSubtreeComms, dataset, verbosity)
        return evTree, 0, 0, evTree.element_indices, evTree.outcomes

    def bulk_evaltree(self, circuit_list, minSubtrees=None, maxTreeSize=None,
                      numSubtreeComms=1, dataset=None, verbosity=0):
        raise NotImplementedError("Derived classes should implement this!")

    def bulk_probs(self, circuit_list, clipTo=None, check=False,
                   comm=None, memLimit=None, dataset=None, smartc=None):
        evalTree, _, _, elIndices, outcomes = self.bulk_evaltree_from_resources(circuit_list, comm, memLimit, "default",
                                                                                [], dataset)
        vp = _np.empty(evalTree.num_final_elements(), 'd')
        self.bulk_fill_probs(vp, evalTree, clipTo, check, comm)

        ret = _collections.OrderedDict()
        for i, opstr in enumerate(evalTree):
            elInds = _slct.indices(elIndices[i]) \
                if isinstance(elIndices[i], slice) else elIndices[i]
            ret[opstr] = _OutcomeLabelDict(
                [(outLbl, vp[ei]) for ei, outLbl in zip(elInds, outcomes[i])])
        return ret

    def bulk_dprobs(self, circuit_list, returnPr=False, clipTo=None,
                    check=False, comm=None, wrtBlockSize=None, dataset=None):
        memLimit = None
        evalTree, _, _, elIndices, outcomes = self.bulk_evaltree_from_resources(circuit_list, comm, memLimit,
                                                                                "default", [], dataset)
        nElements = evalTree.num_final_elements()
        nDerivCols = self.num_params()

        vdp = _np.empty((nElements, nDerivCols), 'd')
        vp = _np.empty(nElements, 'd') if returnPr else None

        self.bulk_fill_dprobs(vdp, evalTree,
                              vp, clipTo, check, comm,
                              None, wrtBlockSize)

        ret = _collections.OrderedDict()
        for i, opstr in enumerate(evalTree):
            elInds = _slct.indices(elIndices[i]) \
                if isinstance(elIndices[i], slice) else elIndices[i]
            if returnPr:
                ret[opstr] = _OutcomeLabelDict(
                    [(outLbl, (vdp[ei], vp[ei])) for ei, outLbl in zip(elInds, outcomes[i])])
            else:
                ret[opstr] = _OutcomeLabelDict(
                    [(outLbl, vdp[ei]) for ei, outLbl in zip(elInds, outcomes[i])])
        return ret

    def bulk_fill_probs(self, mxToFill, evalTree, clipTo=None, check=False, comm=None):
        if False and evalTree.cache:  # TEST (disabled)
            cpolys = evalTree.cache
            ps = _safe_bulk_eval_compact_polys(cpolys[0], cpolys[1], self._paramvec, (evalTree.num_final_elements(),))
            assert(_np.linalg.norm(_np.imag(ps)) < 1e-6)
            ps = _np.real(ps)
            if clipTo is not None: ps = _np.clip(ps, clipTo[0], clipTo[1])
            mxToFill[:] = ps
        else:
            for i, c in enumerate(evalTree):
                cache = evalTree.cache[i] if evalTree.cache else None
                probs = self.probs(c, clipTo, cache)
                elInds = _slct.indices(evalTree.element_indices[i]) \
                    if isinstance(evalTree.element_indices[i], slice) else evalTree.element_indices[i]
                for k, outcome in zip(elInds, evalTree.outcomes[i]):
                    mxToFill[k] = probs[outcome]

    def bulk_fill_dprobs(self, mxToFill, evalTree, prMxToFill=None, clipTo=None,
                         check=False, comm=None, wrtBlockSize=None,
                         profiler=None, gatherMemLimit=None):

        Np = self.num_params()
        p = self.to_vector()

        if False and evalTree.cache:  # TEST (disabled)
            cpolys = evalTree.cache
            if prMxToFill is not None:
                ps = _safe_bulk_eval_compact_polys(cpolys[0], cpolys[1], p, (evalTree.num_final_elements(),))
                assert(_np.linalg.norm(_np.imag(ps)) < 1e-6)
                ps = _np.real(ps)
                if clipTo is not None: ps = _np.clip(ps, clipTo[0], clipTo[1])
                prMxToFill[:] = ps
            dpolys = _compact_deriv(cpolys[0], cpolys[1], list(range(Np)))
            dps = _safe_bulk_eval_compact_polys(dpolys[0], dpolys[1], p, (evalTree.num_final_elements(), Np))
            mxToFill[:, :] = dps
        else:
            # eps = 1e-6
            for i, c in enumerate(evalTree):
                cache = evalTree.cache[i] if evalTree.cache else None
                probs0 = self.probs(c, clipTo, cache)
                dprobs0 = self.dprobs(c, False, clipTo, cache)
                elInds = _slct.indices(evalTree.element_indices[i]) \
                    if isinstance(evalTree.element_indices[i], slice) else evalTree.element_indices[i]
                for k, outcome in zip(elInds, evalTree.outcomes[i]):
                    if prMxToFill is not None:
                        prMxToFill[k] = probs0[outcome]
                    mxToFill[k, :] = dprobs0[outcome]

                    #Do this to fill mxToFill instead of calling dprobs above as it's a little faster for finite diff?
                    #for j in range(Np):
                    #    p_plus_dp = p.copy()
                    #    p_plus_dp[j] += eps
                    #    self.from_vector(p_plus_dp)
                    #    probs1 = self.probs(c,clipTo,cache)
                    #    mxToFill[k,j] = (probs1[outcome]-probs0[outcome]) / eps
                    #self.from_vector(p)

    def __str__(self):
        raise "Derived classes should implement OplessModel.__str__ !!"

class SuccessFailModel(OplessModel):
    def __init__(self, state_space_labels, use_cache=False):
        OplessModel.__init__(self, state_space_labels)
        self.use_cache = use_cache

    def get_num_outcomes(self, circuit):  # needed for sparse data detection
        return 2

    def _success_prob(self, circuit, cache):
        raise NotImplementedError("Derived classes should implement this!")

    def _success_dprob(self, circuit, cache):
        raise NotImplementedError("Derived classes should implement this!")

    #FUTURE?: def _fill_circuit_probs(self, array_to_fill, outcomes, circuit, clipTo):
    def probs(self, circuit, clipTo=None, cache=None):
        """
        Construct a dictionary containing the probabilities of every spam label
        given a operation sequence.

        Parameters
        ----------
        circuit : Circuit or tuple of operation labels
          The sequence of operation labels specifying the operation sequence.

        clipTo : 2-tuple, optional
           (min,max) to clip probabilities to if not None.

        Returns
        -------
        probs : dictionary
            A dictionary such that
            probs[outcome] = pr(outcome,circuit,clipTo).
        """
        sp = self._success_prob(circuit, cache)
        if clipTo is not None: sp = _np.clip(sp, clipTo[0], clipTo[1])
        return _OutcomeLabelDict([('success', sp), ('fail', 1 - sp)])

    def dprobs(self, circuit, returnPr=False, clipTo=None, cache=None):
        """
        Construct a dictionary containing the probability derivatives of every
        spam label for a given operation sequence.

        Parameters
        ----------
        circuit : Circuit or tuple of operation labels
          The sequence of operation labels specifying the operation sequence.

        returnPr : bool, optional
          when set to True, additionally return the probabilities.

        clipTo : 2-tuple, optional
           (min,max) to clip returned probability to if not None.
           Only relevant when returnPr == True.

        Returns
        -------
        dprobs : dictionary
            A dictionary such that
            dprobs[SL] = dpr(SL,circuit,gates,G0,SPAM,SP0,returnPr,clipTo)
            for each spam label (string) SL.
        """
        try:
            dsp = self._success_dprob(circuit, cache)
        except NotImplementedError:
            return OplessModel.dprobs(self, circuit, returnPr, clipTo)

        if returnPr:
            sp = self._success_prob(circuit, cache)
            if clipTo is not None: sp = _np.clip(sp, clipTo[0], clipTo[1])
            return {('success',): (sp, dsp), ('fail',): (1 - sp, -dsp)}
        else:
            return {('success',): dsp, ('fail',): -dsp}

    def poly_probs(self, circuit):
        """
        Same as probs(...) but return polynomials.
        """
        sp = self._success_prob_poly(circuit)
        return _OutcomeLabelDict([('success', sp), ('fail', _Polynomial({(): 1.0}) - sp)])

    def simplify_circuits(self, circuits, dataset=None):
        rawdict = None  # TODO - is this needed?
        lookup = {i: slice(2 * i, 2 * i + 2, 1) for i in range(len(circuits))}
        outcome_lookup = {i: (('success',), ('fail',)) for i in range(len(circuits))}

        return rawdict, lookup, outcome_lookup, 2 * len(circuits)

    def bulk_evaltree(self, circuit_list, minSubtrees=None, maxTreeSize=None,
                      numSubtreeComms=1, dataset=None, verbosity=0):
        lookup = {i: slice(2 * i, 2 * i + 2, 1) for i in range(len(circuit_list))}
        outcome_lookup = {i: (('success',), ('fail',)) for i in range(len(circuit_list))}

        if self.use_cache == "poly":
            #Do precomputation here
            polys = []
            for i, circuit in enumerate(circuit_list):
                print("Generating probs for circuit %d of %d" % (i + 1, len(circuit_list)))
                probs = self.poly_probs(circuit)
                polys.append(probs['success'])
                polys.append(probs['fail'])
            compact_polys = compact_poly_list(polys)
            cache = compact_polys
        elif self.use_cache is True:
            cache = [self._circuit_cache(circuit) for circuit in circuit_list]
        else:
            cache = None

        return OplessModelTree(circuit_list, lookup, outcome_lookup, cache)

#TODO: move this to polynomial.py??

def compact_poly_list(list_of_polys):
    """Create a single vtape,ctape pair from a list of normal Polynomals """
    tapes = [p.compact() for p in list_of_polys]
    vtape = _np.concatenate([t[0] for t in tapes])
    ctape = _np.concatenate([t[1] for t in tapes])
    return vtape, ctape

class ErrorRatesModel(SuccessFailModel):

    def __init__(self, error_rates, nQubits, state_space_labels=None, alias_dict={}, idlename='Gi'):
        """
        todo
        """
        if state_space_labels is None:
            state_space_labels = ['Q%d' % i for i in range(nQubits)]
        else:
            assert(len(state_space_labels) == nQubits)

        SuccessFailModel.__init__(self, state_space_labels, use_cache=True)

        gate_error_rate_keys = (list(error_rates['gates'].keys()))
        readout_error_rate_keys = (list(error_rates['readout'].keys()))

        # if gate_error_rate_keys[0] in state_space_labels:
        #     self._gateind = True
        # else:
        #     self._gateind = False

        self._idlename = idlename
        self._alias_dict = alias_dict.copy()
        self._gate_error_rate_indices = {k: i for i, k in enumerate(gate_error_rate_keys)}
        self._readout_error_rate_indices = {k: i + len(gate_error_rate_keys)
                                            for i, k in enumerate(readout_error_rate_keys)}
        self._paramvec = _np.concatenate(
            (_np.array([_np.sqrt(error_rates['gates'][k]) for k in gate_error_rate_keys], 'd'),
             _np.array([_np.sqrt(error_rates['readout'][k]) for k in readout_error_rate_keys], 'd'))
        )

    def __str__(self):
        s = "Error Rates model with error rates: \n" + \
            "\n".join(["%s = %g" % (k, self._paramvec[i]**2) for k, i in self._gate_error_rate_indices.items()]) + \
            "\n" + \
            "\n".join(["%s = %g" % (k, self._paramvec[i]**2) for k, i in self._readout_error_rate_indices.items()])
        return s

    def to_dict(self):
        error_rate_dict = {'gates': {}, 'readout': {}}
        error_rate_dict['gates'] = {k: self._paramvec[i]**2 for k, i in self._gate_error_rate_indices.items()}
        error_rate_dict['readout'] = {k: self._paramvec[i]**2 for k, i in self._readout_error_rate_indices.items()}
        asdict = {'error_rates': error_rate_dict, 'alias_dict': self._alias_dict.copy()}
        return asdict

    def _circuit_cache(self, circuit):
        if not isinstance(circuit, _Circuit):
            circuit = _Circuit.fromtup(circuit)

        depth = circuit.depth()
        width = circuit.width()
        g_inds = self._gate_error_rate_indices
        r_inds = self._readout_error_rate_indices

        # if self._gateind:
        #     inds_to_mult_by_layer = []
        #     for i in range(depth):

        #         layer = circuit.get_layer(i)
        #         inds_to_mult = []
        #         usedQs = []

        #         for gate in layer:
        #             if len(gate.qubits) > 1:
        #                 usedQs += list(gate.qubits)
        #                 inds_to_mult.append(g_inds[frozenset(gate.qubits)])

        #         for q in circuit.line_labels:
        #             if q not in usedQs:
        #                 inds_to_mult.append(g_inds[q])

        #         inds_to_mult_by_layer.append(_np.array(inds_to_mult, int))

        # else:
        layers_with_idles = [circuit.get_layer_with_idles(i, idleGateName=self._idlename) for i in range(depth)]
        inds_to_mult_by_layer = [_np.array([g_inds[self._alias_dict.get(str(gate), str(gate))] for gate in layer], int)
                                 for layer in layers_with_idles]

        # Bit-flip readout error as a pre-measurement depolarizing channel.
        inds_to_mult = [r_inds[q] for q in circuit.line_labels]
        inds_to_mult_by_layer.append(_np.array(inds_to_mult, int))

        # The scaling constant such that lambda = 1 - alpha * epsilon where lambda is the diagonal of a depolarizing
        # channel with entanglement infidelity of epsilon.
        alpha = 4**width / (4**width - 1)

        return (width, depth, alpha, 1 / 2**width, inds_to_mult_by_layer)

class TwirledLayersModel(ErrorRatesModel):

    def __init__(self, error_rates, nQubits, state_space_labels=None, alias_dict={}, idlename='Gi'):
        """
        todo
        """
        ErrorRatesModel.__init__(self, error_rates, nQubits, state_space_labels=state_space_labels,
                                 alias_dict=alias_dict, idlename=idlename)

    def _success_prob(self, circuit, cache):
        """
        todo
        """
        pvec = self._paramvec**2
        if cache is None:
            cache = self._circuit_cache(circuit)

        width, depth, alpha, one_over_2_width, inds_to_mult_by_layer = cache
        # The success probability for all the operations (the entanglment fidelity for the gates)
        sp = 1.0 - pvec

        # The depolarizing constant for the full sequence of twirled layers.
        lambda_all_layers = 1.0
        for inds_to_mult in inds_to_mult_by_layer[:-1]:
            lambda_all_layers *= 1 - alpha * (1 - prod(sp[inds_to_mult]))
        # lambda_all_layers = prod([(1 - alpha * (1 - prod(sp[inds_to_mult])))
        #                           for inds_to_mult in inds_to_mult_by_layer[:-1]])

        # The readout success probability.
        successprob_readout = prod(sp[inds_to_mult_by_layer[-1]])
        # THe success probability of the circuit.
        successprob_circuit = lambda_all_layers * (successprob_readout - one_over_2_width) + one_over_2_width

        return successprob_circuit

    def _success_dprob(self, circuit, cache):
        pvec = self._paramvec**2
        dpvec_dparams = 2 * self._paramvec

        if cache is None:
            cache = self._circuit_cache(circuit)

        # p = product_layers(1 - alpha * (1 - prod_[inds4layer](1 - param))) * \
        #     (prod_[inds4LASTlayer](1 - param) - 1 / 2**width)
        # Note: indices cannot be repeated in a layer, i.e. either a given index appears one or zero times in inds4layer

        width, depth, alpha, one_over_2_width, inds_to_mult_by_layer = cache
        sp = 1.0 - pvec
        deriv = _np.zeros(len(pvec), 'd')

        nLayers = len(inds_to_mult_by_layer)
        lambda_per_layer = _np.empty(nLayers, 'd')
        for i, inds_to_mult in enumerate(inds_to_mult_by_layer[:-1]):
            lambda_per_layer[i] = 1 - alpha * (1 - prod(sp[inds_to_mult]))

        successprob_readout = prod(sp[inds_to_mult_by_layer[-1]])
        lambda_per_layer[nLayers - 1] = successprob_readout - one_over_2_width
        lambda_all_layers = prod(lambda_per_layer)  # includes readout factor as last layer

        #All layers except last
        for i, inds_to_mult in enumerate(inds_to_mult_by_layer[:-1]):
            lambda_all_but_current_layer = lambda_all_layers / lambda_per_layer[i]
            # for each such ind, when we take deriv wrt this index, we need to differentiate this layer, etc.
            for ind in inds_to_mult:
                deriv[ind] += lambda_all_but_current_layer * alpha * \
                    (prod(sp[inds_to_mult]) / sp[ind]) * -1.0  # what if sp[ind] == 0?

        #Last layer
        lambda_all_but_current_layer = lambda_all_layers / lambda_per_layer[-1]
        for ind in inds_to_mult_by_layer[-1]:
            deriv[ind] += lambda_all_but_current_layer * (successprob_readout / sp[ind]) * -1.0  # what if sp[ind] == 0?

        return deriv * dpvec_dparams

class TwirledGatesModel(ErrorRatesModel):

    def __init__(self, error_rates, nQubits, state_space_labels=None, alias_dict={}, idlename='Gi'):
        """
        todo
        """
        ErrorRatesModel.__init__(self, error_rates, nQubits, state_space_labels=state_space_labels,
                                 alias_dict=alias_dict, idlename=idlename)

    def _circuit_cache(self, circuit):
        width, depth, alpha, one_over_2_width, inds_to_mult_by_layer = super()._circuit_cache(circuit)
        all_inds_to_mult = _np.concatenate(inds_to_mult_by_layer[:-1])
        readout_inds_to_mult = inds_to_mult_by_layer[-1]
        all_inds_to_mult_cnt = _np.zeros(self.num_params(), int)
        for i in all_inds_to_mult:
            all_inds_to_mult_cnt[i] += 1
        return width, depth, alpha, one_over_2_width, all_inds_to_mult, readout_inds_to_mult, all_inds_to_mult_cnt

    def _success_prob(self, circuit, cache):
        """
        todo
        """
        pvec = self._paramvec**2
        if cache is None:
            cache = self._circuit_cache(circuit)

        width, depth, alpha, one_over_2_width, all_inds_to_mult, readout_inds_to_mult, all_inds_to_mult_cnt = cache
        # The success probability for all the operations (the entanglment fidelity for the gates)
        sp = 1.0 - pvec

        # The 'lambda' for all gates (+ readout, which isn't used).
        lambda_ops = 1.0 - alpha * pvec

        # The depolarizing constant for the full sequence of twirled gates.
        lambda_all_layers = prod(lambda_ops[all_inds_to_mult])
        # The readout success probability.
        successprob_readout = prod(sp[readout_inds_to_mult])
        # THe success probability of the circuit.
        successprob_circuit = lambda_all_layers * (successprob_readout - one_over_2_width) + one_over_2_width

        return successprob_circuit

    def _success_dprob(self, circuit, cache):
        """
        todo
        """
        pvec = self._paramvec**2
        dpvec_dparams = 2 * self._paramvec

        if cache is None:
            cache = self._circuit_cache(circuit)

        width, depth, alpha, one_over_2_width, all_inds_to_mult, readout_inds_to_mult, all_inds_to_mult_cnt = cache
        sp = 1.0 - pvec
        lambda_ops = 1.0 - alpha * pvec
        deriv = _np.zeros(len(pvec), 'd')

        # The depolarizing constant for the full sequence of twirled gates.
        lambda_all_layers = prod(lambda_ops[all_inds_to_mult])
        for i, n in enumerate(all_inds_to_mult_cnt):
            deriv[i] = n * lambda_all_layers / lambda_ops[i] * -alpha  # -alpha = d(lambda_ops/dparam)

        # The readout success probability.
        readout_deriv = _np.zeros(len(pvec), 'd')
        successprob_readout = prod(sp[readout_inds_to_mult])
        for ind in readout_inds_to_mult:
            readout_deriv[ind] = (successprob_readout / sp[ind]) * -1.0  # what if sp[ind] == 0?

        # The success probability of the circuit.
        #successprob_circuit = lambda_all_layers * (successprob_readout - one_over_2_width) + one_over_2_width

        # product rule
        return (deriv * (successprob_readout - one_over_2_width) + lambda_all_layers * readout_deriv) * dpvec_dparams

class AnyErrorCausesFailureModel(ErrorRatesModel):

    def __init__(self, error_rates, nQubits, state_space_labels=None, alias_dict={}, idlename='Gi'):
        """
        todo
        """
        ErrorRatesModel.__init__(self, error_rates, nQubits, state_space_labels=state_space_labels,
                                 alias_dict=alias_dict, idlename=idlename)

    def _circuit_cache(self, circuit):
        width, depth, alpha, one_over_2_width, inds_to_mult_by_layer = super()._circuit_cache(circuit)
        all_inds_to_mult = _np.concatenate(inds_to_mult_by_layer)
        all_inds_to_mult_cnt = _np.zeros(self.num_params(), int)
        for i in all_inds_to_mult:
            all_inds_to_mult_cnt[i] += 1
        return all_inds_to_mult, all_inds_to_mult_cnt

    def _success_prob(self, circuit, cache):
        """
        todo
        """
        pvec = self._paramvec**2

        if cache is None:
            cache = self._circuit_cache(circuit)

        all_inds_to_mult, all_inds_to_mult_cnt = cache
        # The success probability for all the operations (the entanglment fidelity for the gates)
        sp = 1.0 - pvec

        # The probability that every operation succeeds.
        successprob_circuit = prod(sp[all_inds_to_mult])

        return successprob_circuit

    def _success_dprob(self, circuit, cache):
        """
        todo
        """
        pvec = self._paramvec**2
        dpvec_dparams = 2 * self._paramvec

        if cache is None:
            cache = self._circuit_cache(circuit)

        all_inds_to_mult, all_inds_to_mult_cnt = cache
        sp = 1.0 - pvec
        successprob_circuit = prod(sp[all_inds_to_mult])
        deriv = _np.zeros(len(pvec), 'd')
        for i, n in enumerate(all_inds_to_mult_cnt):
            deriv[i] = n * successprob_circuit / sp[i] * -1.0

        return deriv * dpvec_dparams

class AnyErrorCausesRandomOutputModel(ErrorRatesModel):

    def __init__(self, error_rates, nQubits, state_space_labels=None, alias_dict={}, idlename='Gi'):
        """
        todo
        """
        ErrorRatesModel.__init__(self, error_rates, nQubits, state_space_labels=state_space_labels,
                                 alias_dict=alias_dict, idlename=idlename)

    def _circuit_cache(self, circuit):
        width, depth, alpha, one_over_2_width, inds_to_mult_by_layer = super()._circuit_cache(circuit)
        all_inds_to_mult = _np.concatenate(inds_to_mult_by_layer)
        all_inds_to_mult_cnt = _np.zeros(self.num_params(), int)
        for i in all_inds_to_mult:
            all_inds_to_mult_cnt[i] += 1
        return one_over_2_width, all_inds_to_mult, all_inds_to_mult_cnt

    def _success_prob(self, circuit, cache):
        """
        todo
        """
        pvec = self._paramvec**2
        if cache is None:
            cache = self._circuit_cache(circuit)

        one_over_2_width, all_inds_to_mult, all_inds_to_mult_cnt = cache
        # The success probability for all the operations (the entanglment fidelity for the gates)
        sp = 1.0 - pvec

        # The probability that every operation succeeds.
        successprob_all_ops = prod(sp[all_inds_to_mult])
        # The circuit succeeds if all ops succeed, and has a random outcome otherwise.
        successprob_circuit = successprob_all_ops + (1 - successprob_all_ops) * one_over_2_width

        return successprob_circuit

    def _success_dprob(self, circuit, cache):
        """
        todo
        """
        pvec = self._paramvec**2
        dpvec_dparams = 2 * self._paramvec

        if cache is None:
            cache = self._circuit_cache(circuit)

        one_over_2_width, all_inds_to_mult, all_inds_to_mult_cnt = cache
        sp = 1.0 - pvec

        successprob_all_ops = prod(sp[all_inds_to_mult])
        deriv = _np.zeros(len(pvec), 'd')
        for i, n in enumerate(all_inds_to_mult_cnt):
            deriv[i] = n * successprob_all_ops / sp[i] * -1.0

        # The circuit succeeds if all ops succeed, and has a random outcome otherwise.
        # successprob_circuit = successprob_all_ops + (1 - successprob_all_ops) / 2**width
        # = const + (1-1/2**width)*successprobs_all_ops
        deriv *= (1.0 - one_over_2_width)
        return deriv * dpvec_dparams

    # def ORIGINAL_success_prob(self, circuit, cache):
    #     """
    #     todo
    #     """
    #     if not isinstance(circuit, _Circuit):
    #         circuit = _Circuit.fromtup(circuit)

    #     depth = circuit.depth()
    #     width = circuit.width()
    #     pvec = self._paramvec
    #     g_inds = self._gate_error_rate_indices
    #     r_inds = self._readout_error_rate_indices

    #     if self.model_type in ('FE', 'FiE+U'):

    #         twoQgates = []
    #         for i in range(depth):
    #             layer = circuit.get_layer(i)
    #             twoQgates += [q.qubits for q in layer if len(q.qubits) > 1]

    #         sp = 1
    #         oneqs = {q: depth for q in circuit.line_labels}

    #         for qs in twoQgates:
    #             sp = sp * (1 - pvec[g_inds[frozenset(qs)]])
    #             oneqs[qs[0]] += -1
    #             oneqs[qs[1]] += -1

    #         sp = sp * _np.prod([(1 - pvec[g_inds[q]])**oneqs[q]
    #                             * (1 - pvec[r_inds[q]]) for q in circuit.line_labels])

    #         if self.model_type == 'FiE+U':
    #             sp = sp + (1 - sp) * (1 / 2**width)

    #         return sp

    #     if self.model_type == 'GlobalDep':

    #         p = 1
    #         for i in range(depth):

    #             layer = circuit.get_layer(i)
    #             sp_layer = 1
    #             usedQs = []

    #             for gate in layer:
    #                 if len(gate.qubits) > 1:
    #                     usedQs += list(gate.qubits)
    #                     sp_layer = sp_layer * (1 - pvec[g_inds[frozenset(gate.qubits)]])

    #             for q in circuit.line_labels:
    #                 if q not in usedQs:
    #                     sp_layer = sp_layer * (1 - pvec[g_inds[q]])

    #             p_layer = 1 - 4**width * (1 - sp_layer) / (4**width - 1)
    #             p = p * p_layer

    #         # Bit-flip readout error as a pre-measurement depolarizing channel.
    #         sp_layer = _np.prod([(1 - 3 * pvec[r_inds[q]] / 2) for q in circuit.line_labels])
    #         p_layer = 1 - 4**width * (1 - sp_layer) / (4**width - 1)
    #         p = p * p_layer
    #         sp = p + (1 - p) * (1 / 2**width)

    #         return sp