# -*- coding: utf-8 -*-

"""Functions and classes for reading and writing ADIPLS binary output.  Many
return or contain what I call ``cs`` arrays.  These are defined in Section 8.2 of
the `ADIPLS documentation`_.  They are structured arrays containing
various scalar results from the frequency calculation.

    .. _ADIPLS documentation: https://sourceforge.net/p/mesa/code/HEAD/tree/trunk/adipls/adipack.c/notes/adiab.prg.c.pdf
"""
import numpy as np
import warnings
from .constants import G_DEFAULT
from .utils import integrate, complement, regularize
from .utils import AdiabaticStellarModel

def read_one_cs(f):
    """Utility function to parse one ``cs`` array from a binary file
    handle ``f``."""
    cs = np.fromfile(f, dtype=cs_floats, count=1)
    cs = cs.astype(cs_dtypes, copy=False)
    return cs

def load_pointwise_data(filename, ncols):
    """Utility function for common structure of ADIPLS data that has a
    value at each point in a stellar model. e.g. eigenfunction or
    kernel files.

    Parameters
    ----------
    filename: str
        Name of the file to be read.
    ncols: int
        Number of columns in the data.

    Returns
    -------
    css: structured array
        The ``cs`` arrays for each mode.
    data: list of arrays
        The point-wise data arrays for each mode.

    """
    css = []
    data = []

    with open(filename, 'rb') as f:
        while True:
            if not f.read(4): break
            css.append(read_one_cs(f))
            nnw = np.fromfile(f, dtype='i', count=1)[0]
            row = np.fromfile(f, dtype='d', count=ncols*nnw).reshape((-1, ncols))
            data.append(row)
            f.read(4)

    return np.squeeze(css), np.squeeze(data)

def load_agsm(filename, return_object=True):
    """Reads an ADIPLS grand summary file and returns a structured array.

    If `return_object` is `True`, instead returns an
    :py:class:`ADIPLSGrandSummary` object.  This is the
    default behaviour as of v0.0.12.  The old behaviour will be
    dropped completely from v0.1.0.

    Parameters
    ----------
    filename: str
        Name of the grand summary file, usually starting or ending
        with ``agsm``.

    Returns
    -------
    css: structured array
        The ``cs`` arrays for each mode.

    """

    css = []

    with open(filename, 'rb') as f:
        while True:
            if not f.read(4): break
            css.append(read_one_cs(f))
            f.read(4)

    if return_object:
        return ADIPLSGrandSummary(np.squeeze(css))
    else:
        warnings.warn("From tomso 0.1.0+, `adipls.load_agsm` will only "
                      "return an `ADIPLSGrandSummary` object: use "
                      "`return_object=True` to mimic future behaviour",
                      FutureWarning)
        return np.squeeze(css)

def load_amde(filename, nfmode=1, return_object=True):
    """Reads an ADIPLS eigenfunction file written with the specified value
    of ``nfmode`` in the input file (either 1, 2 or 3).

    If `return_object` is `True`, instead returns an
    :py:class:`ADIPLSEigenfunctions` object.  This is the default
    behaviour as of v0.0.12.  The old behaviour will be dropped
    completely from v0.1.0.

    Parameters
    ----------
    filename: str
        Name of the eigenfunction file, usually starting or ending
        with ``amde``.
    nfmode: int, optional
        ADIPLS's ``nfmode`` parameter, which determines the format of
        the eigenfunction data.  See Section 8.4 of the `ADIPLS
        documentation`_ for details of the output.  Note that for
        ``nfmode=2`` or ``3``, the fractional radius is returned as an
        extra (third) component.

    Returns
    -------
    css: structured array
        The ``cs`` arrays for each mode.
    eigs: list of arrays
        The eigenfunction arrays for each mode.  Each array has seven
        columns: the fractional radius :math:`x` and six columns of
        ADIPLS's :math:`y` matrix.  The first four columns of
        :math:`y` are defined by equation (2.5) of the documentation
        and the last two by equations (4.4) and (4.6).
    x: array
        Fractional radius co-ordinate ``x`` of the eigenfunctions.  If
        ``nfmode=1``, the radial co-ordinate is also stored in
        ``eigs[...,0]`` but it's returned anyway so that the returned
        data always has the same structure.

    """

    if nfmode == 1:
        css, data = load_pointwise_data(filename, 7)
        x = data[0,:,0]
    elif nfmode == 2 or nfmode == 3:
        # thanks to Vincent Boening for this
        ncols = 2
        css = []
        data = []
        with open(filename, 'rb') as f:
            f.read(4)
            nnw = np.fromfile(f, dtype='i', count=1)[0]
            x = np.fromfile(f, dtype='d', count=nnw)
            f.read(4)

            while True:
                if not f.read(4): break
                css.append(read_one_cs(f))
                row = np.fromfile(f, dtype='d', count=ncols*nnw).reshape((-1, ncols))
                data.append(row)
                f.read(4)
    else:
        raise ValueError('nfmode must be 1, 2 or 3 but got %i' % nfmode)

    if return_object:
        return ADIPLSEigenfunctions(np.squeeze(css), np.squeeze(data), x=x, nfmode=nfmode)
    else:
        warnings.warn("From tomso 0.1.0+, `adipls.load_amde` will only "
                      "return an `ADIPLSEigenfunctions` object: use "
                      "`return_object=True` to mimic future behaviour",
                      FutureWarning)
        return np.squeeze(css), np.squeeze(data), x

def load_amdl(filename, return_nmod=False, live_dangerously=False,
              return_object=True, G=G_DEFAULT):
    """Reads an ADIPLS model file.  See Section 5 of the `ADIPLS
    documentation`_ for details.

    If `return_object` is `True`, instead returns an
    :py:class:`ADIPLSStellarModel` object.  This is the default
    behaviour as of v0.0.12.  The old behaviour will be dropped
    completely from v0.1.0.

    Parameters
    ----------
    filename: str
        Name of the model file, usually starting or ending with ``amdl``.
    return_nmod: bool, optional
        If `True`, return the ``nmod`` parameter in the file.
    live_dangerously: bool, optional
        If `True`, load the file even if it looks like it might be
        too large for an AMDL file (i.e. has more than a million points).
    G: float, optional
        Value for the gravitational constant, in cgs units.  If not
        given (which is the default behaviour), we use the module-wise
        default value.

    Returns
    -------
    D: 1-d NumPy array
        Global data, as defined by eq. (5.2) of the ADIPLS
        documentation.
    A: 2-d NumPy array
        Point-wise data, as defined by eq. (5.1) of the ADIPLS
        documentation.
    nmod: int, optional
        The model number.  I'm not sure what it's used for but it
        doesn't seem to matter.  Only returned if `return_nmod=True`.

    """

    with open(filename, 'rb') as f:
        f.read(4)
        nmod = np.fromfile(f, dtype='i', count=1)[0]
        nn = np.fromfile(f, dtype='i', count=1)[0]
        if not live_dangerously and nn > 1000000:
            raise IOError("Model appears to have %i points; "
                          "it probably isn't an AMDL file. "
                          "If you're sure that it is, try again "
                          "with live_dangerously=True" % nn)

        D = np.fromfile(f, dtype='d', count=8)
        A = np.fromfile(f, dtype='d', count=6*nn).reshape((-1,6))
        f.read(4)
        # check that this is the end of the file

    if return_object:
        return ADIPLSStellarModel(D, A, nmod=nmod, G=G)
    else:
        warnings.warn("From tomso 0.1.0+, `adipls.load_amdl` will only "
                      "return an `ADIPLSStellarModel` object: use "
                      "`return_object=True` to mimic future behaviour",
                      FutureWarning)
        if return_nmod:
            return D, A, int(nmod)
        else:
            return D, A

def load_rkr(filename, return_object=True):
    """Reads an ADIPLS rotational kernel file.

    If `return_object` is `True`, instead returns an
    :py:class:`ADIPLSRotationKenerls` object.  This is the default
    behaviour as of v0.0.12.  The old behaviour will be dropped
    completely from v0.1.0.

    Parameters
    ----------
    filename: str
        Name of the kernel file, usually starting or ending with
        ``rkr``.

    Returns
    -------
    css: structured array
        The ``cs`` arrays for each mode.
    rkrs: list of arrays
        The kernel arrays for each mode.  Each array has two columns:
        the fractional radius :math:`x` and the kernel :math:`K(x)`.

    """

    if return_object:
        return ADIPLSRotationKernels(*load_pointwise_data(filename, 2))
    else:
        warnings.warn("From tomso 0.1.0+, `adipls.load_rkr` will only "
                      "return an `ADIPLSRotationKernels` object: use "
                      "`return_object=True` to mimic future behaviour",
                      FutureWarning)
    return load_pointwise_data(filename, 2)

def save_amdl(filename, D, A, nmod=0):
    """Writes an ADIPLS model file, given data in the same form as
    returned by :py:meth:`load_amdl`.  See Section 5 of the `ADIPLS
    documentation`_ for details.

    Parameters
    ----------
    filename: str
        Name of the model file, usually starting or ending with amdl.
    D: 1-d NumPy array
        Global data, as defined by eq. (5.2) of the `ADIPLS
        documentation`_.
    A: 2-d NumPy array
        Point-wise data, as defined by eq. (5.1) of the `ADIPLS
        documentation`_.
    nmod: int, optional
        The model number.  I'm not sure what it's used for but it
        doesn't seem to matter.

    """
    nn = len(A)
    length = np.array(8*(1+8+6*nn), dtype=np.int32)
    with open(filename, 'wb') as f:
        length.tofile(f)
        np.array((nmod,), dtype=np.int32).tofile(f)
        np.array((nn,), dtype=np.int32).tofile(f)
        D.tofile(f)
        A.tofile(f)
        length.tofile(f)

def amdl_get(key_or_keys, D, A, G=G_DEFAULT):
    """Retrieves physical properties of an AMDL model from the ``D`` and
    ``A`` arrays.

    Parameters
    ----------
    keys: list of strs
        A desired variable or a list of desired variables.  Current
        options are:

        - ``M``: total mass (float)
        - ``R``: photospheric radius (float)
        - ``P_c``: central pressure (float)
        - ``rho_c``: central density (float)
        - ``r``: radius (array)
        - ``x``: fractional radius (array)
        - ``m``: mass co-ordinate (array)
        - ``q``: fractional mass co-ordinate (array)
        - ``g``: gravity (array)
        - ``rho``: density (array)
        - ``P``: pressure (array)
        - ``AA``: Ledoux discriminant (array)
        - ``Hp``: pressure scale height (array)
        - ``G1``: first adiabatic index (array)
        - ``cs2``: sound speed squared (array)
        - ``cs``: sound speed (array)
        - ``tau``: acoustic depth

        For example, if ``D`` and ``A`` have been returned from
        :py:meth:`load_amdl`, you could use

        >>> M, m = adipls.amdl_get(['M', 'm'], D, A)

        to get the total mass and mass co-ordinate.  If you only want
        one variable, you don't need to use a list.  The return type
        is just the one corresponding float or array.  So, to get a
        single variable you could use either

        >>> x, = adipls.amdl_get(['x'], D, A)

        or

        >>> x = adipls.amdl_get('x', D, A)

    D: 1-d array
        Global data, as defined by eq. (5.2) of the `ADIPLS
        documentation`_ and returned by :py:meth:`load_amdl`.
    A: 2-d array
        Point-wise data, as defined by eq. (5.1) of the `ADIPLS
        documentation`_ and returned by :py:meth:`load_amdl`.

    Returns
    -------
    output: list of floats and arrays
        A list returning the floats or arrays in the order requested
        by the parameter ``keys``.

    """
    M, R, P_c, rho_c = D[:4]
    x = A[:,0]                                      # fractional radius
    q = A[:,1]*x**3                                 # fractional mass
    m = q*M                                         # mass
    r = x*R                                         # radius
    G1 = A[:,3]                                     # first adiabatic index
    AA = A[:,4]                                     # Ledoux disciminant

    # we can safely ignore 0/0s here
    with np.errstate(invalid='ignore'):
        g = G*m/r**2
        rho = A[:,5]*m/r**3/4./np.pi                # density
        rho[x==0] = rho_c
        P = G*m*rho/G1/r/A[:,2]                     # pressure
        P[x==0] = P_c

    Hp = P/(rho*g)                                  # pressure scale height
    cs2 = G1*P/rho                                  # sound speed squared
    cs = np.sqrt(cs2)                               # sound speed
    tau = -integrate(1./cs[::-1], r[::-1])[::-1]    # acoustic depth

    if type(key_or_keys) == str:
        keys = [key_or_keys]
        just_one = True
    else:
        keys = key_or_keys
        just_one = False

    output = []
    for key in keys:
        if key == 'M': output.append(M)
        elif key == 'R': output.append(R)
        elif key == 'P_c': output.append(P_c)
        elif key == 'rho_c': output.append(rho_c)
        elif key == 'r': output.append(r)
        elif key == 'x': output.append(x)
        elif key == 'm': output.append(m)
        elif key == 'q': output.append(q)
        elif key == 'g': output.append(g)
        elif key == 'rho': output.append(rho)
        elif key == 'P': output.append(P)
        elif key == 'AA': output.append(AA)
        elif key == 'Hp': output.append(Hp)
        elif key == 'G1': output.append(G1)
        elif key == 'cs2': output.append(cs2)
        elif key == 'cs': output.append(cs)
        elif key == 'tau': output.append(tau)
        else: raise ValueError('%s is not a valid key for adipls.amdl_get' % key)

    if just_one:
        assert(len(output) == 1)
        return output[0]
    else:
        return output

def kernels(cs, eig, D, A, G=G_DEFAULT, alpha=None):
    """Returns the density and squared sound speed kernels.  I have tried
    to make this as notationally similar to Gough & Thompson (1991) as
    possible.  The kernels are normalized to have unit integrals over
    the radius *r*.

    Parameters
    ----------
    cs: structured array
        The ``cs`` array for the mode.
    eig: np.array, shape(N,7)
        Eigenfunction data for the mode, as returned by
        :py:meth:`load_amde`.
    D: 1-d array
        Global data, as defined by eq. (5.2) of the `ADIPLS
        documentation`_ and returned by :py:meth:`load_amdl`.
    A: 2-d array
        Point-wise data, as defined by eq. (5.1) of the `ADIPLS
        documentation`_ and returned by :py:meth:`load_amdl`.
    G: float, optional
        Value for the gravitational constant, in cgs units.  If not
        given (which is the default behaviour), we use the module-wise
        default value.
    alpha: float, optional
        Coefficient of the complementary function.  If ``None``, computed
        as in Michael Thompson's kernel code.

    Returns
    -------
    K_cs2: np.array, length N
        The sound speed squared structure kernel.
    K_rho: np.array, length N
        The density structure kernel.

    """

    l = cs['l']
    M, R, P_c, rho_c = D[:4]                # mass and radius from FGONG
    y = eig.T
    sigma2 = cs['sigma2']
    omega = np.sqrt(sigma2*G*M/R**3)        # convert to angular frequency
    L2 = l*(l+1)

    x = A[:,0]
    r = x*R                                 # radial co-ordinate
    m = A[:,1]*x**3*M                       # mass co-ordinate
    g = G*m/r**2                            # gravity
    g[x==0] = 0.
    rho = A[:,1]*A[:,5]*M/4./np.pi/R**3     # density
    rho[x==0] = rho_c
    G1 = A[:,3]                             # first adiabatic index
    P = G*m*rho/G1/r/A[:,2]                 # pressure
    P[x==0] = P_c
    cs2 = G1*P/rho                          # square of the sound speed
    A1 = A[:,1]
    A2 = A[:,2]
    Vg = A2[:]
    drho_dr = -(A[:,4]+A[:,2])*rho/r        # density gradient
    drho_dr[x==0] = 0.

    xi_r = y[1]*R

    if l == 0:
        xi_h = 0.*xi_r  # radial modes have zero horizontal component
        chi = Vg/x*(y[1]-sigma2/A1/x*y[2])
        dxi_r_dr = chi - 2.*y[1]/x
        dPhi_dr = -4.*np.pi*G*rho*xi_r
        Phi = -complement(dPhi_dr, r)  # but actually you don't even need it
    elif l > 0:
        xi_h = y[2]*R/L2
        eta = L2*A1/sigma2
        chi = Vg/x*(y[1]-y[2]/eta-y[3])
        dxi_r_dr = chi - 2.*y[1]/x + y[2]/x
        dPhi_dr = -g/x*(y[3] + y[4]) - y[3]*R*(4.*np.pi*G*rho - 2.*g/r)
        Phi = -g*R*y[3]
    else:
        raise ValueError('l must be non-negative')

    chi[x==0] = 0.
    dxi_r_dr[x==0] = 0.
    dPhi_dr[x==0] = 0.
    Phi_r = Phi/r
    Phi_r[x==0] = 0.

    S = np.trapz((xi_r**2 + L2*xi_h**2)*rho*r**2, r)

    K_cs2 = rho*cs2*chi**2*r**2  # c.f. equation (60)
    K_cs2 = K_cs2/S/omega**2/2.

    # following InversionKit (103)
    K_rho = cs2*chi**2 - omega**2*(xi_r**2+L2*xi_h**2) \
        - 2.*g*xi_r*(chi - dxi_r_dr) \
        + 4.*np.pi*G*rho*xi_r**2 \
        - 4.*np.pi*G*complement((2.*rho*chi+xi_r*drho_dr)*xi_r, r) \
        + 2.*(xi_r*dPhi_dr + L2*xi_h*Phi_r)
    K_rho = K_rho*rho*r**2/2./S/omega**2

    comp = rho*r**2
    if alpha is None:
        alpha = np.trapz(K_rho*comp, r)/np.trapz(comp*comp, r)

    K_rho = K_rho - alpha*comp

    return K_cs2, K_rho

def fgong_to_amdl(glob, var, G=G_DEFAULT):
    """Converts FGONG data (in the form of `glob` and `var`, as returned
    by :py:meth:`~tomso.fgong.load_fgong`) into ADIPLS binary data,
    which can be saved using :py:meth:`save_amdl`.

    The output should be identical (to within a few times machine
    error) to the output of ``fgong-amdl.d`` tool distributed with
    ADIPLS.

    Parameters
    ----------
    glob: NumPy array
        The scalar (or global) variables for the stellar model
    var: NumPy array
        The point-wise variables for the stellar model. i.e. things
        that vary through the star like temperature, density, etc.
    G: float, optional
        Value for the gravitational constant, in cgs units.  If not
        given (which is the default behaviour), we use the module-wise
        default value.

    Returns
    -------
    D: 1-d array
        Global data, as defined by eq. (5.2) of the `ADIPLS
        documentation`_.
    A: 2-d array
        Point-wise data, as defined by eq. (5.1) of the `ADIPLS
        documentation`_.

    """
    warnings.warn("From tomso 0.1.0+, FGONG must be converted to "
                  "AMDL by loading an `FGONG` object and using its "
                  "`to_amdl` function.  To mimic the future behaviour, "
                  "load an FGONG object by using `fgong.load_fgong` "
                  "with `return_object=True`.",
                  FutureWarning)

    M, R = glob[:2]
    r, P, rho, G1, AA = var[::-1,[0,3,4,9,14]].T
    m = np.exp(var[::-1,1])*M

    ioff = (0 if r[0] < 1e6 else 1)
    nn = len(var) + ioff

    # convert profile
    A = np.zeros((nn, 6))

    # we can safely ignore division by 0 here
    with np.errstate(divide='ignore', invalid='ignore'):
        A[ioff:,0] = r/R
        A[ioff:,1] = m/M/(r/R)**3
        A[ioff:,2] = G*m*rho/(G1*P*r)
        A[ioff:,3] = G1
        A[ioff:,4] = AA
        A[ioff:,5] = 4.*np.pi*rho*r**3/m

    A[0,0] = 0.
    A[0,1] = 4.*np.pi/3.*rho[0]*R**3/M
    A[0,2] = 0.
    A[0,3] = G1[0]
    A[0,4] = 0.
    A[0,5] = 3.

    # convert header
    D = np.zeros(8)
    D[0] = M
    D[1] = R
    D[2] = P[0]
    D[3] = rho[0]

    # second derivatives at centre are given
    if glob[10] < 0.:
        D[4] = -glob[10]/G1[0]
        D[5] = -glob[11]
    else:
        D[4] = 4.*np.pi/3.*G*(rho[0]*R)**2/(P[0]*G1[0])
        # D[5] = np.nanmax((A[1:,4]/A[1:,0]**2)[A[1:,0]<0.05])
        # D[5] = np.max((D[5], 0.))+D[4]
        D[5] = D[4]

    D[6] = -1.
    D[7] = 0.

    if A[-1,4] <= 10.:
        # chop off outermost point
        A = A[:-1]
        nn -= 1

    return D, A

def amdl_to_fgong(D, A, G=G_DEFAULT):
    """Converts ADIPLS binary data (in the form of `D` and `A`, as
    returned by :py:meth:`load_amdl`) into FGONG data,
    which can be saved using :py:meth:`~tomso.fgong.save_fgong`.

    Designed to be the inverse of the ``fgong-amdl.d`` tool
    distributed with ADIPLS, modulo the fact that various thermal
    variables (e.g. temperature and luminosity) are not stored in the
    AMDL format, and that the innermost point may have been truncated.
    It's impossible to tell when reversing the process, so we assume
    not.

    The output should be identical (to within a few times machine
    error) of going from FGONG to AMDL and back again.

    Parameters
    ----------
    D: 1-d array
        Global data, as defined by eq. (5.2) of the `ADIPLS
        documentation`_.
    A: 2-d array
        Point-wise data, as defined by eq. (5.1) of the `ADIPLS
        documentation`_.
    G: float, optional
        Value for the gravitational constant, in cgs units.  If not
        given (which is the default behaviour), we use the module-wise
        default value.

    Returns
    -------
    glob: NumPy array
        The scalar (or global) variables for the stellar model
    var: NumPy array
        The point-wise variables for the stellar model. i.e. things
        that vary through the star like temperature, density, etc.

    """
    warnings.warn("From tomso 0.1.0+, AMDL should be converted to "
                  "FGONG by loading an `ADIPLSStellarModel` object "
                  "and using its `to_fgong` function.  To mimic the "
                  "future behaviour, load an FGONG object by using "
                  "`adipls.load_amdl` with `return_object=True`.",
                  FutureWarning)
    M, R = D[:2]

    glob = np.zeros(15)
    var = np.zeros((len(A), 40))

    r = A[:,0]*R
    q = A[:,1]*A[:,0]**3
    m = q*M
    G1 = A[:,3]
    AA = A[:,4]

    # we can safely ignore division by 0 here
    with np.errstate(divide='ignore', invalid='ignore'):
        lnq = np.log(q)
        rho = A[:,5]*m/(4.*np.pi*r**3)
        P = G*m*rho/(G1*r*A[:,2])

    P[0] = D[2]
    rho[0] = D[3]

    var[::-1,0] = r
    var[::-1,1] = lnq
    var[::-1,3] = P
    var[::-1,4] = rho
    var[::-1,9] = G1
    var[::-1,14] = AA

    glob[0] = M
    glob[1] = R
    glob[10] = -D[4]*G1[0]
    glob[11] = -D[5]

    return glob, var

cs_dtypes = [('xmod','float'), ('M','float'), ('R','float'),
             ('P_c','float'), ('rho_c','float'), ('D_5','float'),
             ('D_6','float'), ('D_7','float'), ('D_8','float'),
             ('A_2(x_s)','float'), ('A_5(x_s)','float'),
             ('x_1','float'), ('sigma2_Omega','float'),
             ('x_f','float'), ('fctsbc','int'), ('fcttbc','int'),
             ('lambda','float'), ('l','int'), ('n','int'),
             ('sigma2','float'), ('sigma2_c','float'),
             ('y_1,max','float'), ('x_max', 'float'), ('E','float'),
             ('Pi_E','float'), ('Pi_V','float'), ('nu_V','float'),
             ('ddsig','float'), ('ddsol','float'),
             ('y_1(x_s)','float'), ('y_2(x_s)','float'),
             ('y_3(x_s)','float'), ('y_4(x_s)','float'),
             ('z_1,max','float'), ('xhat_max','float'),
             ('beta_nl','float'), ('nu_Ri','float'), ('m','int')]

for i in range(len(cs_dtypes), 50):
    cs_dtypes.append(('col%i' % i, 'float'))

cs_floats = [(k, 'float') for (k,v) in cs_dtypes]

class ADIPLSStellarModel(AdiabaticStellarModel):
    """A class that contains and allows one to manipulate the data in a
    stellar model stored in ADIPLS's internal binary model format.
    See Section 5 of the `ADIPLS documentation`_ for details.

    This will usually be provided from a file by using
    :py:meth:`load_amdl` but an object can be constructed from any
    similarly structured arrays.

    The main attributes are the **D** and **A** arrays, which follow
    the definitions in the ADIPLS documentation.  The data in these
    arrays can be accessed via the attributes with more
    physically-meaningful names (e.g. the radius is
    ``ADIPLSStellarModel.r``).

    Some of these values can also be set via the attributes if doing
    so is unambiguous. For example, the fractional radius **x** is not a
    member of the **var** array but setting **x** will assign the actual
    radius **r**, which is the first column of **var**.  Values that are
    settable are indicated in the list of parameters.

    Parameters
    ----------
    D: 1-d array
        Global data, as defined by eq. (5.2) of the ADIPLS
        documentation.
    A: 2-d array
        Point-wise data, as defined by eq. (5.1) of the ADIPLS
        documentation.
    nmod: int, optional
        The model number.  I'm not sure what it's used for but it
        doesn't seem to matter.
    G: float, optional
        Value for the gravitational constant, in cgs units.  If not
        given (which is the default behaviour), we use the module-wise
        default value.

    Attributes
    ----------
    nn: int
        number of points in stellar model (i.e. number of rows in **A**)
    M: float, settable
        total mass
    R: float, settable
        photospheric radius
    P_c: float, settable
        central pressure
    rho_c: float, settable
        central density
    x: NumPy array, settable
        fractional radius co-ordinate
    q: NumPy array, settable
        fractional mass co-ordinate
    lnq: NumPy array, settable
        natural logarithm of the fractional mass co-ordinate
    Vg: NumPy array
        homology invariant *V/Gamma_1*
    Gamma_1: NumPy array, settable
        first adiabatic index, aliased by **G1**
    G1: NumPy array, settable
        first adiabatic index, alias of **Gamma_1**
    AA: NumPy array, settable
        Ledoux discriminant
    U: NumPy array
        homology invariant *dlnm/dlnr*
    V: NumPy array
        homology invariant *dlnP/dlnr*
    r: NumPy array, settable
        radius co-ordinate
    m: NumPy array, settable
        mass co-ordinate
    P: NumPy array
        pressure
    rho: NumPy array
        density
    g: NumPy array
        local gravitational acceleration
    Hp: NumPy array
        pressure scale height
    Hrho: NumPy array
        density scale height
    N2: NumPy array
        squared Brunt–Väisälä (angular) frequency
    cs2: NumPy array
        squared adiabatic sound speed
    cs: NumPy array
        adiabatic sound speed
    tau: NumPy array
        acoustic depth
    """
    def __init__(self, D, A, nmod=0, G=G_DEFAULT):
        self.D = D
        self.A = A
        self.nmod = nmod
        self.G = G

    def __len__(self):
        return len(self.A)

    def __repr__(self):
        with np.printoptions(threshold=10):
            return('ADIPLSStellarModel(\nD=\n%s,\nA=\n%s,\nG=%.15g,\nnmod=%s\n)' % (self.D, self.A, self.G, self.nmod))

    def to_file(self, filename):
        """Save the model to an ADIPLS binary stellar model file (usually
        either starting or ending with `amdl`).

        Parameters
        ----------
        filename: str
            Filename to which the data is written.
        """
        save_amdl(filename, self.D, self.A, nmod=self.nmod)

    def to_fgong(self, reverse=True, ivers=1300):
        """Convert the model to an :py:class:`~tomso.fgong.FGONG` object.

        Note that the ADIPLS binary format only has the data necessary
        to compute adiabiatic stellar oscillations, so the FGONG will
        be missing some data (e.g. temperature, luminosity).

        Parameters
        ----------
        reverse: bool, optional
            If ``True`` (the default), store the FGONG data ordered
            from the surface to the centre.  Otherwise, store the
            FGONG data ordered from the centre to the surface.
        """
        from .fgong import FGONG

        # `amdl_to_fgong` already reverses the data by default
        if reverse:
            return FGONG(*amdl_to_fgong(self.D, self.A, G=self.G),
                         ivers=ivers)
        else:
            return FGONG(*amdl_to_fgong(self.D, self.A[::-1], G=self.G),
                         ivers=ivers)

    def to_gyre(self, version=None):
        """Convert the model to an :py:class:`~tomso.gyre.GYREStellarModel`
        object.

        Note that the ADIPLS binary format only has the data necessary
        to compute adiabiatic stellar oscillations, so the GYRE
        stellar model will be missing some data (e.g. temperature,
        luminosity).

        Parameters
        ----------
        version: int, optional
            Specify GYRE format version number times 100. i.e.,
            ``version=101`` produce a file with data version 1.01.  If
            ``None`` (the default), the latest version available in
            TOMSO is used.
        """
        from .gyre import gyre_header_dtypes, gyre_data_dtypes, GYREStellarModel

        if version is None:
            version = max([k for k in gyre_header_dtypes.keys()])

        header = np.zeros(1, gyre_header_dtypes[version])
        header['M'] = self.D[0]
        header['R'] = self.D[1]
        header['L'] = 42.0
        header['version'] = version

        data = np.ones(self.nn, gyre_data_dtypes[version])
        g = GYREStellarModel(header[0], data, G=self.G)

        g.r = self.r
        g.m = self.m
        g.P = self.P
        g.rho = self.rho
        g.Gamma_1 = self.Gamma_1
        g.N2 = self.N2

        # GYRE doesn't know if it's doing adiabatic or non-adiabatic
        # modes when it reads the file, so it does some calculations
        # expecting meaningful data.  We fudge this so we don't get
        # FPEs.
        g.kappa = 42.0
        g.L_r = 42.0
        if version < 101:
            g.data['eps_tot'] = 42.0
        else:
            g.data['eps'] = 42.0

        g.data['k'] = np.arange(self.nn) + 1
        return g

    # AMDL parameters that can be derived from data
    @property
    def nn(self): return len(self.A)

    # Various properties for easier access to the data in `glob` and
    # `var`.

    @property
    def M(self): return self.D[0]

    @M.setter
    def M(self, val): self.D[0] = val

    @property
    def R(self): return self.D[1]

    @R.setter
    def R(self, val): self.D[1] = val

    @property
    def P_c(self): return self.D[2]

    @P_c.setter
    def P_c(self, val): self.D[2] = val

    @property
    def rho_c(self): return self.D[3]

    @rho_c.setter
    def rho_c(self, val): self.D[3] = val

    @property
    def x(self): return self.A[:,0]

    @x.setter
    def x(self, val): self.A[:,0] = val

    @property
    def q(self): return self.A[:,1]*self.x**3

    @q.setter
    def q(self, val): self.A[:,1] = val/self.x**3

    @property
    def Vg(self): return self.A[:,2]

    @Vg.setter
    def Vg(self, val): self.A[:,2] = val

    @property
    def Gamma_1(self): return self.A[:,3]

    @Gamma_1.setter
    def Gamma_1(self, val): self.A[:,3] = val

    @property
    def G1(self): return self.A[:,3]

    @G1.setter
    def G1(self, val): self.A[:,3] = val

    @property
    def AA(self): return self.A[:,4]

    @AA.setter
    def AA(self, val): self.A[:,4] = val

    @property
    def U(self): return self.A[:,5]

    @U.setter
    def U(self, val): self.A[:,5] = val

    @property
    def V(self): return self.Vg*self.Gamma_1

    @V.setter
    def V(self, val): self.Vg = val/self.Gamma_1

    @property
    def r(self): return self.x*self.R

    @r.setter
    def r(self, val): self.x = val/self.R

    @property
    def m(self): return self.q*self.M

    @m.setter
    def m(self, val): self.q = val/self.M

    @property
    @regularize(y0=-np.inf, x0=1e-308)
    def lnq(self): return np.log(self.q)

    @lnq.setter
    def lnq(self, val): self.q = np.exp(val)

    @property
    def P(self):
        with np.errstate(invalid='ignore'):
            val = self.G*self.m*self.rho/(self.Gamma_1*self.r*self.A[:,2])

        val[self.x==0] = self.P_c
        return val

    @property
    def rho(self):
        with np.errstate(invalid='ignore'):
            val = self.A[:,5]*self.m/self.r**3/4./np.pi

        val[self.x==0] = self.rho_c
        return val

    @property
    @regularize()
    def N2(self): return self.AA*self.g/self.r

    @property
    def tau(self):
        tau = integrate(1./self.cs[::-1], self.r[::-1])[::-1]
        return np.max(tau)-tau

class ADIPLSGrandSummary(object):
    """A class that represents the information for a set of mode
    frequencies, loaded from an ADIPLS grand summary file (often
    starting or ending with ``agsm``).  The main data is stored in the
    ``css`` attribute, which is a structured array.  This will usually
    be provided from a file by using :py:meth:`load_agsm` but an object can be
    constructed from any similarly structured array.

    A subset of the information in the ``css`` array is made available
    through attributes.

    Parameters
    ----------
    css: structured NumPy array
        The ``cs`` arrays for each mode.

    Attributes
    ----------
    G: float
        gravitational constant
    M: float
        total mass
    R: float
        photospheric radius
    l: NumPy array of ints
        angular degrees
    n: NumPy array of ints
        angular degrees
    sigma2: NumPy array of floats
        square of the dimensionless angular eigenfrequency
    sigma2_c: NumPy array of floats
        square of the dimensionless angular eigenfrequency corrected
        for the Cowling approximation
    Pi_E: NumPy array of floats
        eigenperiod, in seconds
    Pi_V: NumPy array of floats
        variational period, in seconds
    nu_Ri: NumPy array of floats
        cyclic eigenfrequency corrected using Richardson
        extrapolation, in Hz
    nu_V: NumPy array of floats
        variational cyclic frequency, in Hz
    nu_E: NumPy array of floats
        cyclic eigenfrequency, in seconds
    nu_c: NumPy array of floats
        cyclic eigenfrequency corrected for the Cowling approximation,
        in Hz
    nu: NumPy array of floats
        alias of ``nu_c``
    E: NumPy array of floats
        Normalised mode inertia (see eq. (4.3) of ADIPLS notes).  Note
        that ADIPLS's definition is smaller than GYRE's by a factor of
        4π.
    beta: NumPy array of floats
        Weight for rotation kernel (see eq. (4.7) of ADIPLS notes or
        (8.43) of JCD's oscillation notes).

    """
    def __init__(self, css):
        self.css = css

    def __len__(self):
        return len(self.css)

    def __str__(self):
        return '\n'.join([
            '%s' % type(self),
            'G    %11.6g cm³/g/s²' % self.G,
            'M    %9.3e g    %7.3f Msun' % (self.M, self.M/1.98841e33),
            'R    %9.3e cm   %7.3f Rsun' % (self.R, self.R/695.7e8)])

    def __repr__(self):
        with np.printoptions(threshold=10):
            return('ADIPLSGrandSummary(\ncss=\n%s)' % self.css)

    @property
    def G(self): return self.R**3/self.M/self.sigma2[0]*(2.*np.pi/self.Pi_E[0])**2

    @property
    def M(self): return self.css[0]['M']

    @property
    def R(self): return self.css[0]['R']

    @property
    def l(self): return self.css['l']

    @property
    def n(self): return self.css['n']

    @property
    def sigma2(self): return self.css['sigma2']

    @property
    def sigma2_c(self): return self.css['sigma2_c']

    @property
    def Pi_E(self): return self.css['Pi_E']*60.0

    @property
    def Pi_V(self): return self.css['Pi_V']*60.0

    @property
    def nu_Ri(self): return self.css['nu_Ri']/1e3

    @property
    def nu_V(self): return self.css['nu_V']/1e3

    @property
    def nu_E(self): return 1/self.Pi_E

    @property
    def nu_c(self): return np.sqrt(self.sigma2_c/self.sigma2)/self.Pi_E

    @property
    def nu(self): return self.nu_c

    @property
    def E(self): return self.css['E']

    @property
    def beta(self): return self.css['beta_nl']

    def index_ln(self, l, n):
        """Returns the index of mode with angular degree *l*
        and radial order *n*."""
        return np.where((self.l==l)&(self.n==n))[0][0]

    def index_nl(self, n, l):
        """Returns the index of mode with radial order *n*
        and angular degree *l*."""
        return self.index_ln(l, n)

class ADIPLSEigenfunctions(ADIPLSGrandSummary):
    """A class that represents the information for a set of eigenfunction
    data kernels produced by ADIPLS.  This will usually be provided
    from a file by using :py:meth:`load_amde` but an object can be
    constructed from any similarly structured array.

    Parameters
    ----------
    css: structured NumPy array
        The ``cs`` arrays for each mode.
    eigs: 3-d NumPy array
        The eigenfunction arrays for each mode.  The nth element of
        the array has the eigenfunction data for the nth mode, in
        the same order as the summary data in *css*.  The number of
        rows in the array for a given mode is the number of meshpoints
        in the model.  The number of columns is either 6 or 2,
        depending on **nfmode**.
    nfmode: int
        The output mode used by ADIPLS' when the data was stored.
    x: NumPy array, optional
        If **nfmode** is 2 or 3, the fractional radius must be
        provided separately.  If **nfmode** is 1, it will be inferred
        from the eigenfunction data if not explicitly provided.

    This class has all the attributes of
    :py:class:`ADIPLSGrandSummary` as well as the following extras.

    Attributes
    ----------
    x: NumPy array
        fractional radius co-ordinate
    eigs: list of NumPy arrays
        The nth row is the eigenfunction data for the nth mode, in
        the same order as the summary data in *css*.

    """
    # TODO: add some derived properties: xi_r, xi_h
    # TODO: add access to eigenfunctions by n and l, like rotation kernels
    def __init__(self, css, eigs, nfmode=1, x=None):
        ADIPLSGrandSummary.__init__(self, css)

        self.nfmode = nfmode
        if nfmode == 1:
            if x is None:
                self.x = eigs[0,:,0]
            else:
                self.x = x
        elif nfmode == 2 or nfmode == 3:
            self.x = x
        else:
            raise ValueError('nfmode must be 1, 2 or 3, not %i' % nfmode)

        self.eigs = eigs

    def __str__(self):
        return super(ADIPLSEigenfunctions, self).__str__()

    def __repr__(self):
        with np.printoptions(threshold=10):
            return('ADIPLSEigenfunctions(nfmode=%i,\ncss=\n%s,\neigs=\n%s)' % (self.nfmode, self.css, self.eigs))

    def eig_ln(self, l, n):
        "Load eigenfunction by *l* and *n*."
        return self.eigs[(self.l==l)&(self.n==n)][0]

    def eig_nl(self, n, l):
        "Load eigenfunction by *n* and *l*."
        return self.eig_ln(l, n)

class ADIPLSRotationKernels(ADIPLSGrandSummary):
    """A class that represents the information for a set of rotational
    kernels produced by ADIPLS.  This will usually be provided from a
    file by using :py:meth:`load_rkr` but an object can be constructed
    from any similarly structured array.

    Parameters
    ----------
    css: structured NumPy array
        The ``cs`` arrays for each mode.
    rkrs: list of arrays
        The kernel arrays for each mode.  Each array has two columns:
        the fractional radius :math:`x` and the kernel :math:`K(x)`.

    This class has all the attributes of
    :py:class:`ADIPLSGrandSummary` as well as the following extras.

    Attributes
    ----------
    x: NumPy array
        fractional radius co-ordinate
    K: list of NumPy arrays
        The nth row is the rotation kernel for the nth mode, in
        the same order as the summary data in *css*.  The mode with
        radial order *n* and angular degree *l* can be accessed by the
        functions ``K_ln(l,n)`` or ``K_nl(n,l)``.

    """
    def __init__(self, css, rkr):
        ADIPLSGrandSummary.__init__(self, css)
        self.x = rkr[0,:,0]
        self.K = rkr[:,:,1]

    def __str__(self):
        return super(ADIPLSRotationKernels, self).__str__()

    def __repr__(self):
        with np.printoptions(threshold=10):
            return('ADIPLSRotationKernels(\ncss=\n%s,\nx=%s,\nK=\n%s)' % (self.css, self.x, self.K))

    def K_ln(self, l, n):
        "Load kernel by *l* and *n*."
        return self.K[(self.l==l)&(self.n==n)][0]

    def K_nl(self, n, l):
        "Load kernel by *n* and *l*."
        return self.K_ln(l, n)


#!/usr/bin/env python

# Modified by Jakob Gelszinnis

"""

 What is the standard of this data format anyhow. If we have a load function, do we also have save function other than .pickle
 ... the issue of pickling is that it leads to very strong coupling.

"""

from __future__ import division,print_function

import os
import errno
import struct
import sys
import numpy as np
import clusterbuster.constants as cgsconst

const = cgsconst.ConstantsCGS()

def get_identi (f, end, verbose = False ) :
   bytes  = f.read( 4 )
   if len(bytes) != 4 :
      if verbose: print(' get_identi: no further valid identifier found')
      return None
   identy = ''.join( struct.unpack( end+'4c', bytes ) )
   return identy

def Loadsnap(strSn, transform=True, headerc='Mpc'):

  #====  Hoeft_radio/q_mach_machr_table.txt for spectral index
    snap = Snapshot(strSn, transform=transform, headerc=headerc)  #, radio_name = strRa
    snap.loaddata()
    snap.head['gamma'] = 5./3.  # add adiabatic expansion factor

    #==== psiFile   for psi factor; machfile for mach-numbers conversion factors
    return snap

def savesnap(strSn, savefolder, transform=True, headerc='Mpc'):

    snap = Snapshot(strSn, transform=transform, headerc=headerc)  #, radio_name = strRa
    snap.savedata(savefolder)

def load_snap(snap, verbose=False):

   if verbose:
     print(' load snapshot <%s> :' % snap.name)

   try :
      f = open( snap.name, 'rb' )
   except IOError:
      print('Fatal Error: apparently %s does not exist' % (snap.name))
      print('... I better stop now !!!')
      sys.exit()

   end = '<' # args.endian

   # check FileInit
   FileInit = struct.unpack( end+'II',  f.read( 2*4 ) )
   if FileInit != (1,2) :
      print('Fatal Error: FileInit != (1,2)')
      print('(possibly something wrong with endian)')
      print('... I better stop now !!!')
      sys.exit()

   # read header
   NumPart, Mvir, Rvir, Xc, Yc, Zc, aexpan, hubble = struct.unpack( end+'lfffffff', f.read( 8*4 ) )
   """DEVELOPMENT, allows to use data that were already transformed"""
   if snap.headerc == 'Mpc':
       Xc *= 1e3   #  Mpc -> kpc
       Yc *= 1e3   #  Mpc -> kpc
       Zc *= 1e3   #  Mpc -> kpc

   elif snap.headerc != 'kpc':
       print('Snapshot header of central coordinates: UNIT "%s" unkown!' % (snap.headerc))

   if verbose:
      print('   NumPart = %6i'         % NumPart )
      print('   Mvir    = %.3e Msun/h' % Mvir    )
      print('   Rvir    = %.3e kpc/h'  % Rvir    )
      print('   Xc      = %.3e kpc/h'  % Xc      )
      print('   Yc      = %.3e kpc/h'  % Yc      )
      print('   Zc      = %.3e kpc/h'  % Zc      )
      print('   axpan   = %.3f    '    % aexpan  )
      print('   hubble  = %.3f    '    % hubble  )
   f.read( 512 - 10*4 )

   snap.head = { 'hubble':hubble, 'aexpan':aexpan, 'Xc':Xc, 'Yc':Yc, 'Zc':Zc, 'NumPart':NumPart, 'Mvir':Mvir, 'Rvir':Rvir }

   while True:

      identi = get_identi ( f, end )
      if identi == None :
         break

      if verbose: print(' following identifer found: <%s>' % identi)

#      print('_',identi)
      if identi == 'POS ':
         snap.pos  =  np.fromfile(f, dtype=end+'f', count=3*NumPart ).reshape( (NumPart, 3 ) )
         if snap.transform:
             snap.pos[:,0] -= Xc
             snap.pos[:,1] -= Yc
             snap.pos[:,2] -= Zc
      elif identi == 'VEL ' :
         snap.vel  =  np.fromfile( f, dtype=end+'f', count=3*NumPart ).reshape( (NumPart, 3 ) )
      elif identi == 'MGAS' :
         snap.mgas =  np.fromfile( f, dtype=end+'f', count=NumPart )
      elif identi == 'ID  ' :
         snap.id   =  np.fromfile( f, dtype=end+'I', count=NumPart )
      elif identi == 'HSML' :
         snap.hsml =  np.fromfile( f, dtype=end+'f', count=NumPart )
      elif identi == 'U   ' :
         snap.u    =  np.fromfile( f, dtype=end+'f', count=NumPart )
      elif identi == 'RHO ' :
         snap.rho  =  np.fromfile( f, dtype=end+'f', count=NumPart )
      elif identi == 'ENDT' :
         snap.endt =  np.fromfile( f, dtype=end+'f', count=NumPart )
      elif identi == 'NORM' :
         snap.norm =  np.fromfile( f, dtype=end+'f', count=3*NumPart ).reshape( (NumPart, 3 ) )
      elif identi == 'UUP ' :
         snap.uup  =  np.fromfile( f, dtype=end+'f', count=NumPart )
      elif identi == 'UDOW' :
         snap.udow =  np.fromfile( f, dtype=end+'f', count=NumPart )
      elif identi == 'RUP ' :
         snap.rup  =  np.fromfile( f, dtype=end+'f', count=NumPart )
      elif identi == 'RDOW' :
         snap.rdow =  np.fromfile( f, dtype=end+'f', count=NumPart )
      elif identi == 'DIVV' :
         snap.divv =  np.fromfile( f, dtype=end+'f', count=NumPart )
      elif identi == 'MACH' :
         snap.mach =  np.fromfile( f, dtype=end+'f', count=NumPart )
      else :
         print('Identifier is not recognized'  )
         break

   f.close()
   if verbose:
      print(' load snapshot ... done')

   return

def save_snap ( snap, filename, verbose=False) :

   if verbose:
       print(' save snapshot <%s> :' % snap.name)

#   try :

   if not os.path.exists(os.path.dirname(filename)):
        try:
            os.makedirs(os.path.dirname(filename))
        except OSError as exc: # Guard against race condition
            if exc.errno != errno.EEXIST:
                raise
   fwrite = open( filename, 'w' )
   end = '<'

   fwrite.write(struct.pack( end+'II',  1,2 ))

   """ The snapshot is altered, i.e. this is smelly codeas it should just save a snapshot. In our case only not of relevance because, we either save the snapshots or create radio maps."""
   if snap.transform:
         snap.pos[:,0] += snap.head['Xc']
         snap.pos[:,1] += snap.head['Yc']
         snap.pos[:,2] += snap.head['Zc']

   if snap.headerc == 'Mpc':
       snap.head['Xc'] /= 1e3   #  Mpc -> kpc
       snap.head['Yc'] /= 1e3   #  Mpc -> kpc
       snap.head['Zc'] /= 1e3   #  Mpc -> kpc
   elif snap.headerc != 'kpc':
       print('Snapshot header of central coordinates: UNIT "%s" unkown!' % (snap.headerc))

   snap.head['NumPart'] = snap.hsml.shape[0]
   fwrite.write(struct.pack( end+'lfffffff', snap.head['NumPart'], snap.head['Mvir'], snap.head['Rvir'], snap.head['Xc'], snap.head['Yc'], snap.head['Zc'], snap.head['aexpan'], snap.head['hubble'] ) )

#   f.read( 512 - 10*4 )
   fwrite.write( '{0:0472b}'.format(0) )

   pairs  = [ ['POS ',lambda x:  x.pos],
              ['VEL ',lambda x:  x.vel],
              ['NORM',lambda x:  x.norm] ]
   for ident,lambd in pairs:
#       print(struct.pack( end+'s', ident), struct.pack( end+'c', ident[0]), struct.pack( end+'c', ident[1]), struct.pack( end+'c', ident[2]), struct.pack( end+'c', ident[3]))
#       leads to an bug, because 4 chars do not equal an string of for chars:
#       fwrite.write( struct.pack( end+'s', ident)) #end+'4c'
       try:
           towrite = lambd(snap).astype('f').tostring()
           fwrite.write( struct.pack( end+'c', ident[0]) + struct.pack( end+'c', ident[1]) + struct.pack( end+'c', ident[2]) + struct.pack( end+'c', ident[3]))
           fwrite.write( towrite  )#end+f
       except:
           print('save_snap:',ident,'not found')

   pairs  = [ ['HSML',lambda x:x.hsml],
#              ['MGAS',lambda x:x.mgas],
              ['U   ',lambda x:x.u],
              ['RHO ',lambda x:x.rho],
              ['ENDT',lambda x:x.endt],
              ['UUP ',lambda x:x.uup],
              ['UDOW',lambda x:x.udow],
              ['RUP ',lambda x:x.rup],
              ['RDOW',lambda x:x.rdow],
              ['DIVV',lambda x:x.divv],
              ['MACH',lambda x:x.mach],]
   for ident,lambd in pairs:
#       leads to an bug, because 4 chars do not equal an string of for chars:
#       fwrite.write( struct.pack( end+'s', ident)) #
       try:
           towrite = lambd(snap).astype('f').tostring()
           fwrite.write( struct.pack( end+'c', ident[0]) + struct.pack( end+'c', ident[1]) + struct.pack( end+'c', ident[2]) + struct.pack( end+'c', ident[3]))
           fwrite.write( towrite)
       except:
           print('save_snap:',ident,'not found')

   fwrite.close()

   if verbose:
      print(' saving snapshot ... done')

   return

class Snapshot:
   # A simple class for a snapshot of one resimulated Volume of MUSIC-2

   def __init__(self, name, radio_name='', transform=True, headerc='Mpc') :

      self.name       = name.replace('Additional2','') # Compability with old folder structure
      self.radio_name = radio_name
      self.head = None
      self.hsml = None
      self.pos  = None
      self.vel  = None
      self.mgas = None
      self.id   = None
      self.u    = None
      self.rho  = None
      self.endt = None
      self.norm = None
      self.uup  = None
      self.udow = None
      self.rup  = None
      self.rdow = None
      self.divv = None
      self.mach = None
      self.radi = None
      self.xray = None
      self.transform  = transform
      self.headerc    = headerc

   def loaddata(self, term=False) :
      if term: print('endian= ', '<') # args.endian
      load_snap ( self, )

   def savedata(self, savefolder, term=False):
      if term: print('endian= ', '<') # args.endian
      save_snap ( self, savefolder)

# conversion factor gadget density to proper baryon density [cm-3]
def conversion_fact_gadget_rho_to_nb ( head ) :
   fact  = 1e10 * const.solar_mass / head['hubble'] / const.proton_mass
   fact /= ( 1e3 * const.parsec / head['hubble'] )**3.
   fact /= ( head['aexpan'] )**3.
   return fact

# conversion factor gadget density to proper baryon density [cm-3] as if the snapshot was taken at z=0
def conversion_fact_gadget_rho_to_nb_z0 ( head ) :
   fact  = 1e10 * const.solar_mass / head['hubble'] / const.proton_mass
   fact /= ( 1e3 * const.parsec / head['hubble'] )**3.
   #fact /= ( head['aexpan'] )**3.
   return fact

# electron density for fully ionized medium
# ......    Y = 4nHe / ( nH + 4nHe )
# ......    ne / nb = nH + 2nHe / ( nH + 4nHe )
# ......    nb / ntot = nH + 4nHe / ( 2nH + 3nHe )    (note ne = nH + 2nHe, fully ionized)
def conversion_fact_ne_per_nb( ) :
	nHe_per_nH  = const.Y_helium / ( 4.0 * ( 1.0 - const.Y_helium ))
	ne_per_nb   = 1.0 + 2.0 * nHe_per_nH / ( 1 + 4.0 * nHe_per_nH )
	return ne_per_nb

def conversion_fact_nb_per_ntot ( ) :
	nHe_per_nH  = const.Y_helium / ( 4.0 * ( 1.0 - const.Y_helium ))
	nb_per_ntot = ( 1.0 + 4.0*nHe_per_nH ) / ( 2.0 + 3.0 * nHe_per_nH )
	return nb_per_ntot

# convert gadget specific energy density (U) to temperature [keV]
# ......   GADGET CONVERSION FROM U TO T
# ......   XH      = Zmgas(6,*)/mass_gas(*)
# ......   yHelium = (1. - XH)/(4.*XH)
# ......   mu      = (1 + 4.* yHelium)/ (1.+ yHelium + ne1gas )
# ......   temp    = GAMMA_MINUS1 * ugas * mu * 1.6726 / 1.3806 * 1.e-8 ; / BOLTZMANN  * PROTONMASS
# ......   temp    = temp * 1e10 ; UnitEnergy_in_cgs/UnitMass_in_g (to get T in Kelvin)
def conversion_fact_gadget_U_to_keV( head ) :
   nb_per_ntot = conversion_fact_nb_per_ntot( )
   fact  = const.kilometer_per_sec**2. * ( head['gamma'] - 1. ) * nb_per_ntot * const.proton_mass
   fact /= 1e3*const.electron_volt
   return fact

def comH_to_phys( head, z=None ) :

    if z is not None:
        return 1/(1+z)/head['hubble']
    else:
        return head['aexpan']/head['hubble']


# -*- coding: utf-8 -*-
"""
Created on Thu Jul  1 10:26:52 2021

@author: dlvilla

Created on Tue Apr 20 14:50:39 2021

Copyright Notice
=================

Copyright 2021 National Technology and Engineering Solutions of Sandia, LLC.
Under the terms of Contract DE-NA0003525, there is a non-exclusive license
for use of this work by or on behalf of the U.S. Government.
Export of this program may require a license from the
United States Government.

Please refer to the LICENSE.md file for a full description of the license
terms for MEWS.

The license for MEWS is the Modified BSD License and copyright information
must be replicated in any derivative works that use the source code.

@author: dlvilla

MEWS = Multi-senario Extreme Weather Simulator

METHODS   mews.weather.DOE2Weather.read_doe2_bin and
          mews.weather.DOE2Weather.write_doe2_bin

   Are translations of parts of BIN2TXT.F and TXT2BIN.F translated with permission
   from James and Jeff Hirsch and Associates (JJH&A). The license for these utilities
   must be formed with (JJH&A) before they are distributed in any other package
   besides MEWS

"""

from numpy import zeros,cumsum,arange, int64, float64,array
import numpy as np
import pandas as pd
from subprocess import Popen, PIPE
from os.path import isfile, dirname,basename, join
from os import chdir as cd
from os import remove as rm
from os import getcwd as pwd
import os
from shutil import copy as cp
from pandas import DataFrame, DatetimeIndex, DateOffset, Series
from datetime import datetime, timedelta
import warnings
import logging
import struct

class DataFormats():
    # changing this will cause significant portions of the code to fail
    header_dtype = np.dtype([('location_IWDID','a20'),
                                            ('year_IWYR', 'i4'),
                                            ('latitude_WLAT','f4'),
                                            ('longitude_WLONG','f4'),
                                            ('timezone_IWTZN','i4'),
                                            ('record_length_LRECX','i4'),
                                            ('number_days_NUMDAY','i4'),
                                            ('clearness_number_CLN_IM1','f4'),
                                            ('ground_temperature_GT_IM1','f4'),
                                            ('solar_flag_IWSOL','i4')])
    column_description = ['MONTH (1-12)',
        'DAY OF MONTH',
        'HOUR OF DAY',
        'WET BULB TEMP (DEG F)',
        'DRY BULB TEMP (DEG F)',
        'PRESSURE (INCHES OF HG)',
        'CLOUD AMOUNT (0 - 10)',
        'SNOW FLAG (1 = SNOWFALL)',
        'RAIN FLAG (1 = RAINFALL)',
        'WIND DIRECTION (0 - 15; 0=N, 1=NNE, ETC)',
        'HUMIDITY RATIO (LB H2O/LB AIR)',
        'DENSITY OF AIR (LB/CU FT)',
        'SPECIFIC ENTHALPY (BTU/LB)',
        'TOTAL HOR. SOLAR (BTU/HR-SQFT)',
        'DIR. NORMAL SOLAR (BTU/HR-SQFT)',
        'CLOUD TYPE (0 - 2)',
        'WIND SPEED (KNOTS)']

class DOE2_Weather_Error(Exception):
    def __init__(self,error_message):
        self.error_message = error_message

class DOE2Weather(object):

    def __init__(self):
        self.column_description = DataFormats.column_description

    def _rm_file(self,filenamepath):
        if isfile(filenamepath):
            try:
                rm(filenamepath)
            except:
                raise DOE2_Weather_Error("The operating system will not allow python to remove the " +
                            filenamepath + " file!")

    def bin2txt(self,binfilename,bin2txtpath):
        txtfilename = ""
        if isfile(bin2txtpath) and isfile(binfilename):
            curdir = pwd()
            try:
                cd(dirname(bin2txtpath))
                self._rm_file("WEATHER.FMT")
                self._rm_file("WEATHER.BIN")
                if not os.path.isabs(binfilename):
                    binfilename2 = os.path.join(curdir,binfilename)
                else:
                    binfilename2 = binfilename
                cp(binfilename2,"WEATHER.BIN")

                # no arguments needed
                pp = Popen(basename(bin2txtpath),stdout=PIPE, stderr=PIPE, shell=True)
                output, errors = pp.communicate()
                if not errors == b'':
                    warnings.warn("WARNING! An error was recorded by Popen.communicate but this does not mean the BIN2TXT did not work. Investigate further to verify it worked.")
                txtfilename = join(dirname(binfilename2) , basename(binfilename2).split(".")[0] + ".txt")
                cp("WEATHER.FMT",txtfilename)
                cd(curdir)
            except:
                # return to the correct directory
                try:
                    cd(curdir)
                    raise DOE2_Weather_Error("The bin to text process failed!")
                except:
                    raise DOE2_Weather_Error("The OS will not allow return to the original directory. " + curdir)
        else:
            if not isfile(bin2txtpath):
                raise DOE2_Weather_Error("doe2bin2txt.conver_bin_to_txt: the requested bin2txtpath" +
                        " executable does not exist! A valid path to the BIN2TXT.EXE" +
                        " utility must be provided.")
            else:
                raise DOE2_Weather_Error("doe2bin2txt.conver_bin_to_txt: the requested binfilename" +
                        " does not exist! A valid path to a valid DOE2 weather binary" +
                        " file must be provided.")
        return txtfilename

    def txt2bin(self,txtfilename,txt2binpath,binfilename):
        if isfile(txt2binpath) and isfile(txtfilename):
            curdir = pwd()
            try:
                change_dir = len(dirname(txt2binpath)) != 0
                if change_dir:
                    cd(dirname(txt2binpath))
                self._rm_file("WEATHER.BIN")
                if txtfilename != "WEATHER.FMT":
                   cp(txtfilename,"WEATHER.FMT")
                # no arguments needed
                p = Popen(basename(txt2binpath),stdout=PIPE, stderr=PIPE, shell=True)
                output, errors = p.communicate()
                if not errors == b'':
                    warnings.warn("The process produced an error make sure the process worked!\n\n" + str(errors) + "\n\n" + str(output))
                cp("WEATHER.BIN",binfilename)
                self._rm_file("WEATHER.BIN")
                cd(curdir)
            except:
                # return to the correct directory
                try:
                    cd(curdir)
                except:
                    raise DOE2_Weather_Error("The OS will not allow return to the original directory.\n\n " + curdir)
        else:
            if not isfile(txt2binpath):
                raise DOE2_Weather_Error("doe2bin2txt.txt2bin: the requested bin2txtpath" +
                        " executable does not exist! A valid path to the TXT2BIN.EXE" +
                        " utility must be provided.")
            else:
                raise DOE2_Weather_Error("doe2bin2txt.txt2bin: the requested txtfilename" +
                        " does not exist! A valid path to a valid DOE2 weather textfile" +
                        " file must be provided.")

    def df2bin(self,df, binfilename, use_exe=False, start_datetime=None, hour_in_file=None, txt2bin_exepath=None,
               location=None,fyear=None,latitude=None,longitude=None,timezone=None,
               iwsz=2,iftyp=3,clearness_number=None,ground_temp=None):

        """
        df2bin(df, binfilename, start_datetime, hour_in_file, txt2bin_exepath,
               location=None,fyear=None,latitude=None,longitude=None,timezone=None,
               iwsz=2,iftyp=3,clearness_number=None,ground_temp=None)

        Parameters
        ----------
            df              : pd.Dataframe: must contain the columns originally read from a
                                   DOE2 BIN weather file format (in the table below)
                                   if use_exe = False, df must have a
            binfilename     : str : path and filename that will be output with the
                                   weather signals in df.
            use_exe         : bool : optional : Default=False
                                    Set to True if using TXT2BIN is desired
                                    instead of python. Changing use_exe between
                                    reads and writes is not allowed

            All other parameters only apply if use_exe = True

            start_datetime  : datetime :start date and time to be output in the weather file
                                    typically this is Jan 1st, fyear
            hour_in_file    : int : Either 8760 or 8784
            txt2bin_exepath : str : path and filename that point to TXT2BIN.EXE
                                    DOE2 utility that can be obtained from www.doe2.com
                                    after forming a license agreement with James Hirsch
                                    and associates.
            location .. and all other inputs

        Returns
        =======

        None

        %% From TXT2BIN.FMT - this gives the exact format needed to output a
        %                     text file that TXT2BIN.EXE can process.
        %       THIS DOCUMENTS A FORMATTED WEATHER FILE (WEATHER.FMT) MADE FROM
        %       A PACKED BINARY DOE2 WEATHER FILE (WEATHER.BIN) USING WTHFMT2.EXE
        %       AND THE EXTRA FILE NEEDED (not really!) TO PACK IT WITH FMTWTH2.EXE
        %
        % on input.dat:
        %
        % Record 1       IWSZ,IFTYP
        %                FORMAT(12X,I1,17X,I1)
        %
        %       IWSZ           WORD SIZE          1 = 60-BIT, 2 = 30-BIT
        %       IFTYP          FILE TYPE          1 = OLD, 2 = NORMAL (NO SOLAR),
        %                                         3 = THE DATA HAS SOLAR
        % on weather.fmt:
        %
        % Record 1       (IWDID(I),I=1,5),IWYR,WLAT,WLONG,IWTZN,IWSOL
        %                FORMAT(5A4,I5,2F8.2,2I5)
        %
        % Record 2       (CLN(I),I=1,12)
        %                FORMAT(12F6.2)
        %
        % Record 3       (GT(I),I=1,12)
        %                FORMAT(12F6.1)
        %
        % Records 4,8763
        %                KMON, KDAY, KH, WBT, DBT, PATM, CLDAMT, ISNOW,
        %                IRAIN, IWNDDR, HUMRAT, DENSTY, ENTHAL, SOLRAD,
        %                DIRSOL, ICLDTY, WNDSPD
        %                FORMAT(3I2,2F5.0,F6.1,F5.0,2I3,I4,F7.4,F6.3,F6.1,2F7.1,I3,F5.0)
        %       IWDID          LOCATION I.D.
        %       IWYR           YEAR
        %       WLAT           LATITUDE
        %       WLONG          LONGITUDE
        %       IWTZN          TIME ZONE NUMBER
        %       IWSOL          SOLAR FLAG         IWSOL = IWSZ + (IFTYP-1)*2 - 1
        %       CLN            CLEARNESS NO.
        %       GT             GROUND TEMP.       (DEG R)
        %       KMON           MONTH              (1-12)
        %       KDAY           DAY OF MONTH
        %       KH             HOUR OF DAY
        %       WBT            WET BULB TEMP      (DEG F)
        %       DBT            DRY BULB TEMP      (DEG F)
        %       PATM           PRESSURE           (INCHES OF HG)
        %       CLDAMT         CLOUD AMOUNT       (0 - 10)
        %       ISNOW          SNOW FLAG          (1 = SNOWFALL)
        %       IRAIN          RAIN FLAG          (1 = RAINFALL)
        %       IWNDDR         WIND DIRECTION     (0 - 15; 0=N, 1=NNE, ETC)
        %       HUMRAT         HUMIDITY RATIO     (LB H2O/LB AIR)
        %       DENSTY         DENSITY OF AIR     (LB/CU FT)
        %       ENTHAL         SPECIFIC ENTHALPY  (BTU/LB)
        %       SOLRAD         TOTAL HOR. SOLAR   (BTU/HR-SQFT)
        %       DIRSOL         DIR. NORMAL SOLAR  (BTU/HR-SQFT)
        %       ICLDTY         CLOUD TYPE         (0 - 2)
        %       WNDSPD         WIND SPEED         KNOTS"""
        if hasattr(self,'use_exe'):
            if self.use_exe != use_exe:
                raise ValueError("This class does not support switching between"+
                                 " using Python and using the BIN2TXT.F and TXT2BIN.F"+
                                 " executables!")
        else:
            self.use_exe = use_exe

        if use_exe:
            cdir = pwd()
            #try:
            change_dir = len(dirname(binfilename)) != 0
            if change_dir:
                cd(dirname(binfilename))
            self._rm_file("WEATHER.FMT")
            self._rm_file("WEATHER.BIN")
            with open("INPUT.DAT",'w') as dat:
                dat.write('            {0:1.0f}                 {1:1.0f}'.format(iwsz,iftyp))
            with open("WEATHER.FMT",'w') as fmt:
                # 3 header rows
    # header_dtype = np.dtype([('location_IWDID','a20'),
    #                                         ('year_IWYR', 'i4'),
    #                                         ('latitude_WLAT','f4'),
    #                                         ('longitude_WLONG','f4'),
    #                                         ('timezone_IWTZN','i4'),
    #                                         ('record_length_LRECX','i4'),
    #                                         ('number_days_NUMDAY','i4'),
    #                                         ('clearness_number_CLN_IM1','f4'),
    #                                         ('ground_temperature_GT_IM1','f4'),
    #                                         ('solar_flag_IWSOL','i4')])

                if isinstance(df.headers[0]['location_IWDID'][0],np.bytes_):
                    location_str = df.headers[0]['location_IWDID'][0].decode('ascii')
                else:
                    location_str = df.headers[0]['location_IWDID'][0]

                row1 = '{0:20s}{1:5d}{2:8.2f}{3:8.2f}{4:5d}{5:5d}\n'.format(
                    location_str,
                    df.headers[0]['year_IWYR'][0],
                    df.headers[0]['latitude_WLAT'][0],
                    df.headers[0]['longitude_WLONG'][0],
                    df.headers[0]['timezone_IWTZN'][0],
                    df.headers[0]['solar_flag_IWSOL'][0])
                fmt.write(row1)
                clearness_number = [head['clearness_number_CLN_IM1'][0] for head in df.headers]
                ground_temp = [head['ground_temperature_GT_IM1'][0] for head in df.headers]

                fmt.write((12*'{:6.2f}'+'\n').format(*clearness_number))
                fmt.write((12*'{:6.2f}'+'\n').format(*ground_temp))

                for index, row in df.iterrows():
                    fmt.write((3*'{:2.0f}'+2*'{:5.0f}' + '{:6.1f}{:5.0f}' + 2*'{:3.0f}' +
                     '{:4.0f}{:7.4f}{:6.3f}{:6.1f}' + 2*'{:7.1f}'+'{:3.0f}{:5.0f}\n').format(
                             *row.tolist()))
            if isfile(txt2bin_exepath):
                if change_dir:
                    cp(txt2bin_exepath,".")
                else:
                    cp(txt2bin_exepath,".")
                new_exe_path = join(".",basename(txt2bin_exepath))
                self.txt2bin("WEATHER.FMT",new_exe_path,os.path.basename(binfilename))
            else:
                cd(cdir)
                raise DOE2_Weather_Error("The txt2bin.exe utility is not present at: \n\n" + txt2bin_exepath)
            cd(cdir)

        else:
            # size and type checking
            m = df[self.column_description].values

            if m.shape[1] != 17:
                raise ValueError("This function only handles dataframes with"+
                                 " 17 columns as defined for DOE-2 BIN weather files!")
            elif not hasattr(df,'headers'):
                raise ValueError("The input dataframe df must have an attribute "+
                                 "'headers' that contains a list of ")
            headers = df.headers
            DOE2Weather.write_doe2_bin(m, headers, binfilename)

    def bin2df(self,binfilename, start_datetime=None, hour_in_file=None, bin2txt_exepath=None, timezone=None, dst=None, use_exe=False):
        """ This function was originally written in matlab and is the
        "ReadDOE2BINTXTFile.m" function except that it also includes conversion
        of the BIN file into a text file

        DST is a list of the start of daylight savings and end so that
        adjustments can be made and the time stamps adjusted for daylight savings.
    % the input *.txt "filename" must come from a DOE2 bin file that has been
    %  converted to a text file. It has the following columns of information:
    % Column Number   Variable      Description         Units
    %C 1              IM2            MOMTH              (1-12)
    %C 2             ID             DAY OF MONTH
    %C 3             IH             HOUR OF DAY
    %C 4             CALC(1)        WET BULB TEMP      (DEG F)
    %C 5             CALC(2)        DRY BULB TEMP      (DEG F)
    %C 6             CALC(3)        PRESSURE           (INCHES OF HG)
    %C 7             CALC(4)        CLOUD AMOUNT       (0 - 10)
    %C 8             ISNOW          SNOW FLAG          (1 = SNOWFALL)
    %C 9             IRAIN          RAIN FLAG          (1 = RAINFALL)
    %C 10            IWNDDR         WIND DIRECTION     (0 - 15; 0=N, 1=NNE, ETC)
    %C 11            CALC(8)        HUMIDITY RATIO     (LB H2O/LB AIR)
    %C 12            CALC(9)        DENSITY OF AIR     (LB/CU FT)
    %C 13            CALC(10)       SPECIFIC ENTHALPY  (BTU/LB)
    %C 14            CALC(11)       TOTAL HOR. SOLAR   (BTU/HR-SQFT)
    %C 15            CALC(12)       DIR. NORMAL SOLAR  (BTU/HR-SQFT)
    %C 16            ICLDTY         CLOUD TYPE         (0 - 2)
    %C 17            CALC(14)       WIND SPEED         KNOTS

        """
        if (timezone is None and not dst is None) or (not timezone is None and dst is None):
            raise ValueError("The timezone and dst values must be specified together!")

        hour_in_day = 24
        if hasattr(self,'use_exe'):
            if use_exe != self.use_exe:
                raise ValueError("This class does not support switching between" +
                                 " using BIN2TXT.EXE and TXT2BIN.EXE and using" +
                                 " Python for translation! None returned as a result!")
        else:
            # set the mode of operation of the class
            self.use_exe = use_exe

        if use_exe:
            # this is the old way of doing things.
            txtname = self.bin2txt(binfilename, bin2txt_exepath)

            if len(txtname)==0:
                raise DOE2_Weather_Error("bom2df:The bin file was not successfully converted please troubleshoot!")

            num = 0
            m = zeros((hour_in_file,17))
            # this is specific to the conversion utility and how it writes out ASCII.
            EntryLength = [0, 2, 2, 2, 5, 5, 6, 5, 3, 3, 4, 7, 6, 6, 7, 7, 3, 5]
            j = 0
            i = 0
            b_lines = []

            with open(txtname,'r') as h:
                for text_line in h:
                    if num <= 2:
                        num += 1 # skip three lines
                        b_lines.append(text_line)
                    else:
                        for mm,nn in zip(cumsum(EntryLength[0:-1]),cumsum(EntryLength[1:])):
                            m[j,i] = float(text_line[mm:nn])
                            i+=1
                        j +=1
                        i = 0
        else:
            # use pure python to do this
            m, headers = self.read_doe2_bin(binfilename)

        # adjust for leap year by repeating February 28th on February 29th.
        if hour_in_file == 8784:
            # February 28th is the 59th day of the year
            # Febrary 29th is the 60th day of a year
            sid = 59 * 24

            # shift all of March1st to December 31 over 24 hours
            m[sid+24:] = m[sid:-24]
            # repeat February 28th
            m[sid:sid+24] = m[sid-24:sid]
            # reassign Feb 28th to Feb 29th -
            # day of month column
            ind = self.column_description.index("DAY OF MONTH")
            m[sid:sid+24,ind] = 29
            MDAYS = [31,29,31,30,31,30,31,31,30,31,30,31]
        else:
            MDAYS = [31,28,31,30,31,30,31,31,30,31,30,31]

        dateVec = []
        reached_hours_to_next = True
        start_datetime_was_None = False
        get_year_from_headers = False

        month = 0  # 0 = Jan
        for i in arange(m.shape[0]):

            # see whether a replacement year has been provided - if not, use what
            # is in the BIN file - if Python is used, the the header of each
            # month can have a different year if it is TMY3
            if start_datetime is None and use_exe:
                start_datetime = datetime(year=int(b_lines[0][20:25]),day=1,month=1)
                start_datetime_was_None = True
            elif start_datetime is None and reached_hours_to_next and not use_exe:
                hour_count = 0
                hours_to_next = MDAYS[month]* hour_in_day

                year = headers[month]['year_IWYR'][0]
                month += 1
                start_datetime_was_None = True
                get_year_from_headers = True
            elif not use_exe and start_datetime is None:
                hour_count += 1

            if get_year_from_headers:
                if hour_count > hours_to_next:
                    reached_hours_to_next = True
                else:
                    reached_hours_to_next = False

            # for cases with a replacement year OR use_exe where the text file
            # does not convey the year at every header
            if use_exe or not start_datetime_was_None:

                current_time = start_datetime+timedelta(hours=float(i))
                dateVec.append(datetime(current_time.year,current_time.month,
                                        current_time.day,current_time.hour,0,0))
            #
            else:
                dateVec.append(datetime(year,int(m[i,0]),int(m[i,1]),int(m[i,2])-1,0,0))

            if not dst is None:
                #Handle daylight savings correctly
                if dateVec[-1] == dst[0]:
                    dateVec[-1] = dateVec[-1] + DateOffset(hour=1)
                elif dateVec[-1] == dst[1]:
                    dateVec[-1] = dateVec[-1] - DateOffset(hour=1)

        dateTimeIn = DatetimeIndex(dateVec)
        if not timezone is None:
            try:
                dateTimeIn = dateTimeIn.tz_localize(timezone, ambiguous=True)
            except Exception as e:
                warnings.warn("The time zone localization process failed. "
                              + " This probably happened because the time zone "
                              + "input is incorrect!")
                raise e

        df = DataFrame(index=dateTimeIn,data=m,columns=self.column_description,dtype=float64)

        # add header information (different depending on use_exe)

        # suppress warnings here. - We want to add attributes to the dataframe
        # and are NOT trying to add columns
        warnings.simplefilter("ignore",category=UserWarning)

        if use_exe:
            clearness_number = [float(x) for x in b_lines[1].split(" ") if len(x)!=0]
            ground_temps = [float(x) for x in b_lines[2].split(" ") if len(x)!=0]
            headers = []
            month = 0

            for num,temp in zip(clearness_number,ground_temps):
                header_list = [(b_lines[0][0:20],
                                     np.int32(b_lines[0][20:25]),
                                     np.float32(b_lines[0][25:33]),
                                     np.float32(b_lines[0][33:41]),
                                     np.int32(b_lines[0][41:46]),
                                     np.int32(1),
                                     np.int32(MDAYS[month]),
                                     np.float32(clearness_number[month]),
                                     np.float32(ground_temps[month]),
                                     np.int32(b_lines[0][46:51]))]
                lheaders = np.array(header_list,dtype=DataFormats.header_dtype)

                headers.append(lheaders)

        df.headers = headers

        return df

    @staticmethod
    def read_doe2_bin(binfilename):
        with open(binfilename,mode='rb') as bh:
            bin_content = bh.read()
        """
        obj.read_doe2_bin(binfilename)

        Parameters
        ==========

        binfilename : str : valid path and file name to a DOE-2 *.BIN weather
                            file.

        Returns
        =======

        m : np.array(x,17) : x = length of BIN file (ussually 8760)
            The columns of this array are:

            % Column Number   Variable      Description         Units
            %C 1              IM2            MOMTH              (1-12)
            %C 2             ID             DAY OF MONTH
            %C 3             IH             HOUR OF DAY
            %C 4             CALC(1)        WET BULB TEMP      (DEG F)
            %C 5             CALC(2)        DRY BULB TEMP      (DEG F)
            %C 6             CALC(3)        PRESSURE           (INCHES OF HG)
            %C 7             CALC(4)        CLOUD AMOUNT       (0 - 10)
            %C 8             ISNOW          SNOW FLAG          (1 = SNOWFALL)
            %C 9             IRAIN          RAIN FLAG          (1 = RAINFALL)
            %C 10            IWNDDR         WIND DIRECTION     (0 - 15; 0=N, 1=NNE, ETC)
            %C 11            CALC(8)        HUMIDITY RATIO     (LB H2O/LB AIR)
            %C 12            CALC(9)        DENSITY OF AIR     (LB/CU FT)
            %C 13            CALC(10)       SPECIFIC ENTHALPY  (BTU/LB)
            %C 14            CALC(11)       TOTAL HOR. SOLAR   (BTU/HR-SQFT)
            %C 15            CALC(12)       DIR. NORMAL SOLAR  (BTU/HR-SQFT)
            %C 16            ICLDTY         CLOUD TYPE         (0 - 2)
            %C 17            CALC(14)       WIND SPEED         KNOTS

        headers : list : list of np.array with specialized dtype to capture
            the header for each month of data in the DOE-2 *.BIN file.
            The dtype spec is as follows:
                np.dtype([('location_IWDID','a20'),
                          ('year_IWYR', 'i4'),
                          ('latitude_WLAT','f4'),
                          ('longitude_WLONG','f4'),
                          ('timezone_IWTZN','i4'),
                          ('record_length_LRECX','i4'),
                          ('number_days_NUMDAY','i4'),
                          ('clearness_number_CLN_IM1','f4'),
                          ('ground_temperature_GT_IM1','f4'),
                          ('solar_flag_IWSOL','i4')])

        License Note:
        =============

        # THIS IS A TRANSLATION OF BIN2TXT.F except the TXT file is never written
        # because data for direct use in Python is desired.

        # THIS reverse engineering was approved by Jeff Hirsch on behalf of
        # James and Jeff Hirsch and Associates (JJH&A)
        # along with putting the code on GITHUB with the contingency that the
        # JJH&A license be acknowledged. The original correspondence is provided
        # below:

        # Wed 7/7/2021 4:10 PM
        Yes, you have my permission to distribute your DOE-2 weather file
        python libraries with TXT2BIN and BIN2TXT executables on Github or
        also translate the fortran source versions of those apps into python
        and then distribute on Github as long as your acknowledge the
        JJH&A license

        ________________________________________
        Jeff Hirsch
        James J. Hirsch & Associates
        Voice mail: (XXX) XXX-XXXX
        mobile: (XXX) XXX-XXXX
        -----------------------------------------------------
        From: Villa, Daniel L
        Sent: Wednesday, July 7, 2021 11:02 AM
        To: Jeff.Hirsch@DOE2.com
        Subject: Tranlate TXT2BIN.F and BIN2TXT.F into Python and distribute
                 as open source??

        Jeff,

        I have built a tool that inputs and outputs DOE-2 weather files with
        extreme events. I use the TXT2BIN.F and BIN2TXT.F executables with
        the version of DOE-2 that I have licensed with Hirsch and Associates.
        I would like to be able to distribute the python libraries with these
        executables on Github or else be able to translate the *.F files into
        python but know that the code is distributed under your license.

        Would Hirsch and Associates be willing to let me create Python
        versions of TXT2BIN.F and BIN2TXT.F and to distribute them as open
        source code on GitHUB? I understand and respect Hirsch and Associate’s
        decision if this is not allowed. Thank you.

        Daniel Villa
        Energy-Water System Integration Department 08825
        Sandia National Laboratories
        dlvilla@sandia.gov
        XXX-XXX-XXXX

        """

        # initiate constants in BIN2TXT
        MDAYS = [31,28,31,30,31,30,31,31,30,31,30,31]
        XMASK_1D = np.array([-99., -99., 15., 0., 0., 0., 0., 0., .02, -30., 0.,
                          .0, .0, .0, .0, 10., 1., 1., .1, 1., 1., 1., 1.,
                          .0001, .001, .5, 1., 1., 1., 1., 0., 0.])
        XMASK = XMASK_1D.reshape((2,16)).T

        # record length is 6200 and Fortran puts a 4 byte buffer at the beginning
        # and end of the record. making each record a total of 6208 bytes.
        #
        recl = 6200
        num_buf_bytes = 4
        tot_recl = recl + 2 * num_buf_bytes
        blocksize=148992

        header_length = 56
        num_month_in_year = 12
        num_hour_in_day = 24
        headers = []

        # read in headers
      #   DO 100 IM1=1,12
      #   READ (10) (IWDID(I),I=1,5),IWYR,WLAT,WLONG,IWTZN,LRECX,NUMDAY,
      # _          CLN(IM1),GT(IM1),IWSOL
      #   READ (10) IDUM
      #   100 CONTINUE

        for IM1 in range(num_month_in_year):
            head_start_byte = (recl + 2*num_buf_bytes) * IM1 + num_buf_bytes
            headers.append(np.frombuffer(bin_content[head_start_byte:head_start_byte+header_length+1],
                                         dtype=DataFormats.header_dtype
                                         ,count=1)
                            )
        # Read and process data
        LRECX = 0
        byte_position = num_buf_bytes
        IWTH = np.zeros(15)  # keep 1 indexing and leave the first element 0
        CALC = np.zeros(15)
        data_records = []
        IDAT30 = np.zeros(1537)
        iterIH = 0
        iter_max = 1e6
        for IM2 in range(1,num_month_in_year+1):
            IDE = MDAYS[IM2-1]
            for ID in range(1,IDE+1):
                IH = 1
                while IH <= num_hour_in_day and iterIH < iter_max:
                    IRECX = int(IM2 * 2 + (ID-1)/16 - 1)   #105
                    IDX = int(np.mod(ID-1,16) + 1)
                    comparison = int(IRECX-LRECX)
                    if comparison < 0:
                        IDIF = int(LRECX - IRECX + 1)
                        for I in range(IDIF):
                            if np.mod(byte_position,tot_recl)==0 and byte_position > num_buf_bytes:
                                byte_position = byte_position - tot_recl
                            elif byte_position > num_buf_bytes:
                                byte_position = byte_position - np.mod(byte_position,tot_recl)
                        next_record = byte_position + tot_recl
                        cur_dat = DOE2Weather._doe2_bin_data_format(bin_content,byte_position,next_record)
                        LRECX = cur_dat['record_length_LRECX']
                        byte_position = next_record
                    elif comparison == 0:
                        IDAT30[1:] = cur_dat['IDAT30']
                        IP1 = int(96*(IDX-1) + 4*IH - 3)
                        IWTH[3] = IDAT30[IP1]/65536
                        IWTH[1] = np.mod(IDAT30[IP1],65536)/256
                        IWTH[2] = np.mod(IDAT30[IP1],256)
                        IWTH[11] = IDAT30[IP1+1]/1048576
                        IWTH[12] = np.mod(IDAT30[IP1+1],1048576)/1024
                        IWTH[4] = np.mod(IDAT30[IP1+1],1024)/64
                        IWTH[5] = np.mod(IDAT30[IP1+1],64)/32
                        IWTH[6] = np.mod(IDAT30[IP1+1],32)/16
                        IWTH[7] = np.mod(IDAT30[IP1+1],16)
                        IWTH[8] = IDAT30[IP1+2]/128
                        IWTH[9] = np.mod(IDAT30[IP1+2],128)
                        IWTH[10] = IDAT30[IP1+3]/2048
                        IWTH[13] = np.mod(IDAT30[IP1+3],2048)/128
                        IWTH[14] = np.mod(IDAT30[IP1+3],128)
                        for I in range(1,15):
                            CALC[I] = float(IWTH[I])*XMASK[I-1,2-1] + XMASK[I-1,1-1]

                        ISNOW = int(CALC[5] + .01)
                        IRAIN = int(CALC[6] + .01)
                        IWNDDR = int(CALC[7] + .01)
                        ICLDTY = int(CALC[13] + .01)

                        data_records.append([IM2,ID,IH,CALC[1],CALC[2],CALC[3],
                                             CALC[4],ISNOW,IRAIN,IWNDDR,CALC[8],
                                             CALC[9],CALC[10],CALC[11],CALC[12],
                                             ICLDTY,CALC[14]])
                        IH += 1

                    elif comparison > 0:
                        next_record = byte_position + tot_recl
                        cur_dat = DOE2Weather._doe2_bin_data_format(bin_content,byte_position,next_record)
                        LRECX = cur_dat['record_length_LRECX']
                        byte_position = next_record

                    iterIH += 1
                if iterIH >= iter_max:
                    raise StopIteration("The while loop over days has gotten"
                                        +" stuck! The file being read may not"
                                        +" be the correct BIN Format for DOE-2.")
        m = np.array(data_records)
        return m, headers

    @staticmethod
    def _doe2_bin_data_format(bin_content,byte_position,next_record):
        return np.frombuffer(bin_content[byte_position:next_record+1],
                             np.dtype([('location_IWDID','a20'),
                                            ('year_IWYR', 'i4'),
                                            ('latitude_WLAT','f4'),
                                            ('longitude_WLONG','f4'),
                                            ('timezone_IWTZN','i4'),
                                            ('record_length_LRECX','i4'),
                                            ('number_days_NUMDAY','i4'),
                                            ('clearness_number_CLN_IM1','f4'),
                                            ('ground_temperature_GT_IM1','f4'),
                                            ('solar_flag_IDUM','i4'),
                                            ('IDAT30','1536i4')]),count=1)[0]
    @staticmethod
    def write_doe2_bin(m,headers,binfilename,IWSZ=1,IFTYP=2):
        """
        DOE2Weather.write_doe2_bin(m,headers,binfilename,IWSZ=1,IFTYP=2)

        Parameters
        ==========

        m : np.array(x,17) : x = length of BIN file (ussually 8760)
            The columns of this array are:

            % Column Number   Variable      Description         Units
            %C 1              IM2            MOMTH              (1-12)
            %C 2             ID             DAY OF MONTH
            %C 3             IH             HOUR OF DAY
            %C 4             CALC(1)        WET BULB TEMP      (DEG F)
            %C 5             CALC(2)        DRY BULB TEMP      (DEG F)
            %C 6             CALC(3)        PRESSURE           (INCHES OF HG)
            %C 7             CALC(4)        CLOUD AMOUNT       (0 - 10)
            %C 8             ISNOW          SNOW FLAG          (1 = SNOWFALL)
            %C 9             IRAIN          RAIN FLAG          (1 = RAINFALL)
            %C 10            IWNDDR         WIND DIRECTION     (0 - 15; 0=N, 1=NNE, ETC)
            %C 11            CALC(8)        HUMIDITY RATIO     (LB H2O/LB AIR)
            %C 12            CALC(9)        DENSITY OF AIR     (LB/CU FT)
            %C 13            CALC(10)       SPECIFIC ENTHALPY  (BTU/LB)
            %C 14            CALC(11)       TOTAL HOR. SOLAR   (BTU/HR-SQFT)
            %C 15            CALC(12)       DIR. NORMAL SOLAR  (BTU/HR-SQFT)
            %C 16            ICLDTY         CLOUD TYPE         (0 - 2)
            %C 17            CALC(14)       WIND SPEED         KNOTS

        headers : list : list of np.array with specialized dtype to capture
            the header for each month of data in the DOE-2 *.BIN file.
            The dtype spec is as follows:
                np.dtype([('location_IWDID','a20'),
                          ('year_IWYR', 'i4'),
                          ('latitude_WLAT','f4'),
                          ('longitude_WLONG','f4'),
                          ('timezone_IWTZN','i4'),
                          ('record_length_LRECX','i4'),
                          ('number_days_NUMDAY','i4'),
                          ('clearness_number_CLN_IM1','f4'),
                          ('ground_temperature_GT_IM1','f4'),
                          ('solar_flag_IWSOL','i4')])

        binfilename : str : valid path/filename for which a file will be
            written.

        Returns
        =======
        None

        License Note:
        =============

        # THIS IS A TRANSLATION OF TXT2BIN.F except the TXT file is never read
        # from because data is coming from Python.

        # THIS reverse engineering was approved by Jeff Hirsch on behalf of
        # James and Jeff Hirsch and Associates (JJH&A)
        # along with putting the code on GITHUB with the contingency that the
        # JJH&A license be acknowledged. The original correspondence is provided
        # below:

        # Wed 7/7/2021 4:10 PM
        Yes, you have my permission to distribute your DOE-2 weather file
        python libraries with TXT2BIN and BIN2TXT executables on Github or
        also translate the fortran source versions of those apps into python
        and then distribute on Github as long as your acknowledge the
        JJH&A license

        ________________________________________
        Jeff Hirsch
        James J. Hirsch & Associates
        Voice mail: (XXX) XXX-XXXX
        mobile: (XXX) XXX-XXXX
        -----------------------------------------------------
        From: Villa, Daniel L
        Sent: Wednesday, July 7, 2021 11:02 AM
        To: Jeff.Hirsch@DOE2.com
        Subject: Tranlate TXT2BIN.F and BIN2TXT.F into Python and distribute
                 as open source??

        Jeff,

        I have built a tool that inputs and outputs DOE-2 weather files with
        extreme events. I use the TXT2BIN.F and BIN2TXT.F executables with
        the version of DOE-2 that I have licensed with Hirsch and Associates.
        I would like to be able to distribute the python libraries with these
        executables on Github or else be able to translate the *.F files into
        python but know that the code is distributed under your license.

        Would Hirsch and Associates be willing to let me create Python
        versions of TXT2BIN.F and BIN2TXT.F and to distribute them as open
        source code on GitHUB? I understand and respect Hirsch and Associate’s
        decision if this is not allowed. Thank you.

        Daniel Villa
        Energy-Water System Integration Department 08825
        Sandia National Laboratories
        dlvilla@sandia.gov
        XXX-XXX-XXXX

        FORTRAN VARIABLE MEANING KEY:

            C
            C     IWSZ           WORD SIZE          1 = 60-BIT, 2 = 30-BIT
            C     IFTYP          FILE TYPE          1 = OLD, 2 = NORMAL (NO SOLAR),
            C                                       3 = THE DATA HAS SOLAR
            C     IWDID          LOCATION I.D.
            C     IWYR           YEAR
            C     WLAT           LATITUDE
            C     WLONG          LONGITUDE
            C     IWTZN          TIME ZONE NUMBER
            C     IWSOL          SOLAR FLAG         FUNCTION OF IWSZ + IFTYP
            C     CLN            CLEARNESS NO.
            C     GT             GROUND TEMP.       (DEG R)
            C     KMON           MONTH              (1-12)
            C     KDAY           DAY OF MONTH
            C     KH             HOUR OF DAY
            C     WBT            WET BULB TEMP      (DEG F)
            C     DBT            DRY BULB TEMP      (DEG F)
            C     PATM           PRESSURE           (INCHES OF HG)
            C     CLDAMT         CLOUD AMOUNT       (0 - 10)
            C     ISNOW          SNOW FLAG          (1 = SNOWFALL)
            C     IRAIN          RAIN FLAG          (1 = RAINFALL)
            C     IWNDDR         WIND DIRECTION     (0 - 15; 0=N, 1=NNE, ETC)
            C     HUMRAT         HUMIDITY RATIO     (LB H2O/LB AIR)
            C     DENSTY         DENSITY OF AIR     (LB/CU FT)
            C     ENTHAL         SPECIFIC ENTHALPY  (BTU/LB)
            C     SOLRAD         TOTAL HOR. SOLAR   (BTU/HR-SQFT)
            C     DIRSOL         DIR. NORMAL SOLAR  (BTU/HR-SQFT)
            C     ICLDTY         CLOUD TYPE         (0 - 2)
            C     WNDSPD         WIND SPEED         KNOTS

        """
        # record length is 6200 and Fortran puts a 4 byte buffer at the beginning
        # and end of the record. making each record a total of 6208 bytes.
        #
        recl = 6200
        num_buf_bytes = 4
        tot_recl = recl + 2 * num_buf_bytes
        num_month_in_year = 12
        num_hour_in_day = 24

        CLN = []; GT = []; IWYR = []; WLAT = []; WLONG = []; IWTZN = [];
        LRECX = []; NUMDAY = []; IWSOL = []; IWDID = [];

        # translate to the original syntax for clarity
        for header in headers:
            IWDID.append(header['location_IWDID'][0])
            CLN.append(header["clearness_number_CLN_IM1"][0])
            GT.append(header["ground_temperature_GT_IM1"][0])
            IWYR.append(header["year_IWYR"][0])
            WLAT.append(header["latitude_WLAT"][0])
            WLONG.append(header["longitude_WLONG"][0])
            IWTZN.append(header["timezone_IWTZN"][0])
            LRECX.append(header["record_length_LRECX"][0])  #not needed
            NUMDAY.append(header["number_days_NUMDAY"][0]) # not needed
            IWSOL.append(header["solar_flag_IWSOL"][0])  # not needed recalculated here

        MDAYS = np.zeros(num_month_in_year+1)
        MDAYS[1:] = np.array([31,28,31,30,31,30,31,31,30,31,30,31])
        IDAT = np.zeros(1536+1,dtype=np.int32)

        # record buffer. TODO- find out the exact 4 bytes FORTRAN puts here!
        IDUM = np.int32(recl)  # Fortran buffers with 4 bytes at the begin and
                            # end of each record indicating the length in bytes
                            # of the record.

        if (IWSZ == 0):
            raise ValueError("IWSZ = 0 does not work in Python because the WEATHER.FMT file does not exist!")
        else:
            IWSOL = np.int32(IWSZ + (IFTYP-1)*2 - 1)

        count = 0
        # ARRAYS are zero base! - The fortran is base 1 - watch out!
        with open(binfilename,'wb',buffering=tot_recl) as file:
            for IM in range(1,num_month_in_year+1):
                IDE = np.int32(MDAYS[IM])
                for ID in range(1,IDE+1):

                    IRECXO = np.int32(IM*2 + (ID-1)/16 - 1)
                    IDXO = np.int32(np.mod(ID-1,16) + 1)

                    for IH in range(1,num_hour_in_day+1):

                        KMON = m[count,0]      # Month
                        KDAY = m[count,1]      # Day
                        KH = m[count,2]        # Hour
                        WBT = m[count,3]       # Wet bulb temperature (F)
                        DBT = m[count,4]       # Dry bulb temperature (F)
                        PATM = m[count,5]      # Atmospheric Pressure (in Hg)
                        CLDAMT = m[count,6]    # Cloud amount (0-10)
                        ISNOW = m[count,7]     # Snow flag (1=snowing)
                        IRAIN = m[count,8]     # Rainfall flag (1=precipitation happening)
                        IWNDDR = m[count,9]    # Wind Direction (0 - 15; 0=N, 1=NNE, ETC)
                        HUMRAT = m[count,10]   # Humidity ratio (lb h2o/lb air)
                        DENSTY = m[count,11]   # Air density (lb/ft3)
                        ENTHAL = m[count,12]   # Enthalpy (BTU/LB)
                        SOLRAD = m[count,13]   # Diffuse solar radiation (BTU/HR-SQFT)
                        DIRSOL = m[count,14]   # Direct solar radiation ((BTU/HR-SQFT))
                        ICLDTY = m[count,15]   # Cloud type (0-2)
                        WNDSPD = m[count,16]   # Wind speed (knots)

                        ISOL = np.int32(SOLRAD + .5)
                        IDN = np.int32(DIRSOL + .5)
                        IWET = np.int32(WBT+99.5)
                        IDRY = np.int32(DBT+99.5)
                        IPRES = np.int32(PATM*10.-149.5)
                        ICLDAM = np.int32(CLDAMT)
                        IWNDSP = np.int32(WNDSPD+0.5)
                        IHUMRT = np.int32(HUMRAT*10000.+0.5)
                        IDENS = np.int32(DENSTY*1000.-19.5)
                        IENTH = np.int32(ENTHAL*2.0+60.5)
                        IP1 = np.int32((IDXO-1)*96 + IH*4 - 3)
                        IDAT[IP1] = np.int32(IPRES*65536 + IWET*256 + IDRY)
                        IDAT[IP1+1] = np.int32(ISOL*1048576 + IDN*1024 +
                                      ICLDAM*64 + ISNOW*32 + IRAIN*16 + IWNDDR)
                        IDAT[IP1+2] = np.int32(IHUMRT*128 + IDENS)
                        IDAT[IP1+3] = np.int32(IENTH*2048 + ICLDTY*128 + IWNDSP)

                    if ID != 16 and ID != IDE:
                        pass # keep the loop
                    else:
                        # IDUM is the padding written by Fortran that
                        #   has to be explicitly included here.
                        byte_arr = struct.pack('i',IDUM)
                        byte_arr = byte_arr + struct.pack('20s',IWDID[IM-1])

                        byte_arr = (byte_arr +
                                    struct.pack('i',IWYR[IM-1]) +
                                    struct.pack('f',WLAT[IM-1]) +
                                    struct.pack('f',WLONG[IM-1]) +
                                    struct.pack('i',IWTZN[IM-1]) +
                                    struct.pack('i',IRECXO) +
                                    struct.pack('i',IDE) +
                                    struct.pack('f',CLN[IM-1]) +
                                    struct.pack('f',GT[IM-1]) +
                                    struct.pack('i',IWSOL))
                        byte_arr = byte_arr + b"".join([struct.pack('i',i4) for i4 in IDAT[1:]])
                        byte_arr = byte_arr + struct.pack('i',IDUM)

                        file.write(byte_arr)

                        count += 1
        return # end of write_doe2_bin