#!/usr/bin/env python
"""
gapfill : Fills gaps of flux data from Eddy covariance measurements according
          to Reichstein et al. (Global Change Biology, 2005) or estimate flux
          uncertainties after Lasslop et al. (Biogeosciences, 2008).

This module was written by Matthias Cuntz while at Department of
Computational Hydrosystems, Helmholtz Centre for Environmental
Research - UFZ, Leipzig, Germany, and continued while at Institut
National de Recherche pour l'Agriculture, l'Alimentation et
l'Environnement (INRAE), Nancy, France.

Copyright (c) 2012-2020 Matthias Cuntz - mc (at) macu (dot) de
Released under the MIT License; see LICENSE file for details.

* Written Mar 2012 by Matthias Cuntz - mc (at) macu (dot) de
* Ported to Python 3, Feb 2013, Matthias Cuntz
* Input data can be ND-array, Apr 2014, Matthias Cuntz
* Bug in longestmarginalgap: was only working at time series edges, rename it
  to longgap, Apr 2014, Matthias Cuntz
* Keyword fullday, Apr 2014, Matthias Cuntz
* Input can be pandas Dataframe or numpy array(s), Apr 2020, Matthias Cuntz
* Using numpy docstring format, May 2020, Matthias Cuntz
* error estimates are undef by default, Jun 2021, Matthias Cuntz
* mean of values for error estimates, Jun 2021, Matthias Cuntz

.. moduleauthor:: Matthias Cuntz

The following functions are provided

.. autosummary::
   gapfill
"""
from __future__ import division, absolute_import, print_function
import numpy as np
import pandas as pd

__all__ = ['gapfill']

def gapfill(dfin, flag=None, date=None, timeformat='%Y-%m-%d %H:%M:%S',
            colhead=None,
            sw_dev=50., ta_dev=2.5, vpd_dev=5.,
            longgap=60, fullday=False, undef=-9999, ddof=1,
            err=False, errmean=False, verbose=0):
    """
    Fills gaps in flux data from Eddy covariance measurements with
    Marginal Distribution Sampling (MDS) according to Reichstein et al.
    (Global Change Biology, 2005).

    This means, if there is a gap in the data, look for similar meteorological
    conditions (defined as maximum possible deviations) in a certain time
    window and fill with the average of these 'similar' values.

    The routine can also do the same search for similar meteorological
    conditions for every data point and calculate its standard deviation as a
    measure of uncertainty after Lasslop et al. (Biogeosciences, 2008).

    Parameters
    ----------
    dfin : pandas.Dataframe or numpy.array
        time series of fluxes to fill as well as
        meteorological variables incoming short-wave radiation,
        air temperature, air vapour pressure deficit.

        `dfin` can be a pandas.Dataframe with the columns
        'SW_IN' (or starting with 'SW_IN') for incoming short-wave radiation [W m-2]
        'TA'    (or starting with 'TA\_') for air temperature [deg C]
        'VPD'   (or starting with 'VPD') for air vapour deficit [hPa]
        and columns with ecosystem fluxes with possible missing values (gaps).
        The index is taken as date variable.

        `dfin` can also me a numpy array with the same columns. In this case
        `colhead`, `date`, and possibly `dateformat` must be given.
    flag : pandas.Dataframe or numpy.array, optional
        flag Dataframe or array has the same shape as dfin. Non-zero values in
        `flag` will be treated as missing values in `dfin`.

        `flag` must follow the same rules as `dfin` if pandas.Dataframe.

        If `flag` is numpy array, `df.columns.values` will be used as
        column heads and the index of `dfin` will be copied to `flag`.
    date : array_like of string, optional
        1D-array_like of calendar dates in format given in `timeformat`.

        `date` must be given if `dfin` is numpy array.
    timeformat : str, optional
        Format of dates in `date`, if given (default: '%Y-%m-%d %H:%M:%S').
        See strftime documentation of Python's datetime module:
        https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior
    colhed : array_like of str, optional
        column names if `dfin` is numpy array. See `dfin` for mandatory
        column names.
    sw_dev : float, optional
        threshold for maximum deviation of global radiation (default: 50)
    ta_dev : float, optional
        threshold for maximum deviation of air Temperature (default: 2.5)
    vpd_dev : float, optional
        threshold for maximum deviation of vpd (default: 5.)
    longgap : int, optional
        avoid extraploation into a gap longer than `longgap` days (default: 60)
    fullday : bool, optional
        True: move beginning of large gap to start of next day and move end of
              large gap to end of last day (default: False)
    undef : float, optional
        values having `undef` value are treated as missing values in `dfin`
        (default: -9999)

        np.nan is not allowed (not working).
    ddof : int, optional
        Delta Degrees of Freedom. The divisor used in calculation of standard
        deviation for error estimates (`err=True`) is ``N-ddof``, where ``N``
        represents the number of elements (default: 1).
    err : bool, optional
        True: fill every data point with standard deviation instead of mean,
        i.e. used for error generation as in Lasslop et al. (Biogeosci 2008)
        (default: False)
    errmean : bool, optional
        True: also return mean value of values for error estimate
        `if err == True` (default: False)
    shape : bool or tuple, optional
        True: output have the same shape as input data if `dfin` is
        numpy array; if a tuple is given, then this tuple is used to reshape.

        False: outputs are 1D arrays if `dfin` is numpy array (default: False).
    verbose : int, optional
        Verbosity level 0-3 (default: 0). 0 is no output; 3 is very verbose.

    Returns
    -------
    pandas.Dataframe(s) or numpy array(s)
        `if not err:` filled_data, quality_class

        `if err and not errmean:` err_estimate

        `if err and errmean:` err_estimate, mean_estimate

        pandas.Dataframe(s) will be returned if `dfin` was Dataframe.

        numpy array(s) will be returned if `dfin` was numpy array.

    Notes
    -----
    If `err`, there is no error estimate if there are no meteorological
    conditions in the vicinity of the data point (first cycle of
    Reichstein et al. GCB 2005).

    Routine does not work with `undef=np.nan`.

    Reichstein et al. (2005)
        On the separation of net ecosystem exchange into assimilation and
        ecosystem respiration: review and improved algorithm
        Global Change Biology 11, 1424-1439

    Lasslop et al. (2008)
        Inﬂuences of observation errors in eddy ﬂux data on inverse model
        parameter estimation
        Biogeosciences, 5, 1311–1324

    Examples
    --------
    >>> import numpy as np
    >>> from fread import fread
    >>> from date2dec import date2dec
    >>> from dec2date import dec2date
    >>> ifile = 'test_gapfill.csv' # Tharandt 1998 = Online tool example file
    >>> undef = -9999.
    >>> # data
    >>> dat   = fread(ifile, skip=2, transpose=True)
    >>> ndat  = dat.shape[1]
    >>> head  = fread(ifile, skip=2, header=True)
    >>> head1 = head[0]
    >>> # colhead
    >>> idx   = []
    >>> for i in head1:
    ...     if i in ['NEE', 'LE', 'H', 'Rg', 'Tair', 'VPD']:
    ...         idx.append(head1.index(i))
    >>> colhead = ['FC', 'LE', 'H', 'SW_IN', 'TA', 'VPD']
    >>> # data
    >>> dfin = dat[idx,:]
    >>> # flag
    >>> flag = np.where(dfin == undef, 2, 0)
    >>> flag[0, :] = dat[head1.index('qcNEE'), :].astype(int)
    >>> flag[1, :] = dat[head1.index('qcLE'), :].astype(int)
    >>> flag[2, :] = dat[head1.index('qcH'), :].astype(int)
    >>> flag[np.where(flag==1)] = 0
    >>> # date
    >>> day_id  = head1.index('Day')
    >>> hour_id = head1.index('Hour')
    >>> ntime   = dat.shape[1]
    >>> year  = np.ones(ntime, dtype=int) * 1998
    >>> hh    = dat[hour_id, :].astype(int)
    >>> mn    = np.rint((dat[hour_id,:] - hh) * 60.).astype(int)
    >>> y0    = date2dec(yr=year[0], mo=1, dy=1, hr=hh, mi=mn)
    >>> jdate = y0 + dat[day_id, :]
    >>> adate = dec2date(jdate, eng=True)
    >>> # fill
    >>> dat_f, flag_f = gapfill(dfin, flag=flag, date=adate, colhead=colhead,
    ...                         undef=undef, verbose=0)
    >>> print('{:d} {:d} {:d} {:d} {:d} {:d}'.format(*flag_f[0, 11006:11012]))
    1 1 1 2 2 2
    >>> print('{:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f}'.format(
    ...       *dat_f[0, 11006:11012]))
    -18.68 -15.63 -19.61 -15.54 -12.40 -15.33

    >>> # 1D err
    >>> dat_std = gapfill(dfin, flag=flag, date=adate, colhead=colhead,
    ...                   undef=undef, verbose=0, err=True)
    >>> print('{:.3f} {:.3f} {:.3f} {:.3f} {:.3f} {:.3f}'.format(
    ...       *dat_std[0, 11006:11012]))
    5.372 13.118 6.477 -9999.000 -9999.000 -9999.000

    >>> dat_err = np.ones(ndat, dtype=int)*(-1)
    >>> kk      = np.where((dat_std[0, :] != undef) & (dat_f[0, :] != 0.))[0]
    >>> dat_err[kk] = np.abs(dat_std[0,kk]/dat_f[0,kk]*100.).astype(int)
    >>> print('{:d} {:d} {:d} {:d} {:d} {:d}'.format(*dat_err[11006:11012]))
    28 83 33 -1 -1 -1

    >>> # 1D err + mean
    >>> dat_std, dat_mean = gapfill(dfin, flag=flag, date=adate,
    ...                             colhead=colhead, undef=undef, verbose=0,
    ...                             err=True, errmean=True)
    >>> print('{:.3f} {:.3f} {:.3f} {:.3f} {:.3f} {:.3f}'.format(
    ...       *dat_std[0, 11006:11012]))
    5.372 13.118 6.477 -9999.000 -9999.000 -9999.000
    >>> print('{:.3f} {:.3f} {:.3f} {:.3f} {:.3f} {:.3f}'.format(
    ...       *dat_mean[0, 11006:11012]))
    -18.677 -15.633 -19.610 -9999.000 -9999.000 -9999.000

    History
    -------
    Written,  Matthias Cuntz, Mar 2012 - modified gap_filling.py
    Modified, Matthias Cuntz, Feb 2013 - ported to Python 3
              Matthias Cuntz, Apr 2014 - assert
                                       - data ND-array
                                       - longestmarginalgap was only working at
                                         beginning and end of time series
                                         renamed to longgap
                                       - fullday
              Matthias Cuntz, Apr 2020 - Input can be pandas Dataframe or
                                         numpy array(s)
              Matthias Cuntz, May 2020 - numpy docstring format
              Matthias Cuntz, Jun 2021 - prefill error estimates with undef
                                       - errmean
    """
    # Check input
    # numpy or panda
    if isinstance(dfin, (np.ndarray, np.ma.MaskedArray)):
        isnumpy = True
        istrans = False
        astr = 'colhead must be given if input is numpy.ndarray.'
        assert colhead is not None, astr
        if dfin.shape[0] == len(colhead):
            istrans = True
            df = pd.DataFrame(dfin.T, columns=colhead)
        elif dfin.shape[1] == len(colhead):
            df = pd.DataFrame(dfin, columns=colhead)
        else:
            estr = 'Length of colhead must be number of columns in input'
            estr = estr + ' array. len(colhead)=' + str(len(colhead))
            estr = estr + ' shape(input)=(' + str(dfin.shape[0])
            estr = estr + ',' + str(dfin.shape[1]) + ').'
            raise ValueError(estr)
        assert date is not None, 'date must be given if input is numpy arrary.'
        df['Datetime'] = pd.to_datetime(date, format=timeformat)
        df.set_index('Datetime', drop=True, inplace=True)
    else:
        isnumpy = False
        istrans = False
        astr = 'Input must be either numpy.ndarray or pandas.DataFrame.'
        assert isinstance(dfin, pd.core.frame.DataFrame), astr
        df = dfin.copy(deep=True)

    # Incoming flags
    if flag is not None:
        if isinstance(flag, (np.ndarray, np.ma.MaskedArray)):
            fisnumpy = True
            fistrans = False
            if flag.shape[0] == len(df):
                ff = pd.DataFrame(flag, columns=df.columns.values)
            elif flag.shape[1] == len(df):
                fistrans = True
                ff = pd.DataFrame(flag.T, columns=df.columns.values)
            else:
                estr  = 'flag must have same shape as data array. data:'
                estr += ' ({:d},{:d}); flag: ({:d},{:d})'.format(
                    dfin.shape[0], dfin.shape[1], flag.shape[0], flag.shape[1])
                raise ValueError(estr)
            ff = ff.set_index(df.index)
        else:
            fisnumpy = False
            fistrans = False
            astr = 'Flag must be either numpy.ndarray or pandas.DataFrame.'
            assert isinstance(flag, pd.core.frame.DataFrame), astr
            ff = flag.copy(deep=True)
    else:
        fisnumpy = isnumpy
        fistrans = istrans
        # flags: 0: good; 1: input flagged; 2: output flagged
        ff              = df.copy(deep=True).astype(int)
        ff[:]           = 0
        ff[df == undef] = 1
        ff[df.isna()]   = 1

    # Data and flags
    sw_id = ''
    for cc in df.columns:
        if cc.startswith('SW_IN_') or (cc == 'SW_IN'):
            sw_id = cc
            break
    ta_id = ''
    for cc in df.columns:
        if cc.startswith('TA_') or (cc == 'TA'):
            ta_id = cc
            break
    vpd_id = ''
    for cc in df.columns:
        if cc.startswith('VPD_') or (cc == 'VPD'):
            vpd_id = cc
            break
    astr = 'Global radiation with name SW or starting with SW_'
    astr = astr + ' must be in input.'
    assert sw_id,  astr
    astr = 'Air temperature with name TA or starting with TA_'
    astr = astr + ' must be in input.'
    assert ta_id,  astr
    astr = 'Vapour pressure deficit with name VPD or starting'
    astr = astr + ' with VPD_ must be in input.'
    assert vpd_id, astr

    sw      = df[sw_id].to_numpy()
    sw_flg  = ff[sw_id].to_numpy()
    ta      = df[ta_id].to_numpy()
    ta_flg  = ff[ta_id].to_numpy()
    vpd     = df[vpd_id].to_numpy()
    vpd_flg = ff[vpd_id].to_numpy()

    # dfill is filled data
    # ffill is fill flag if not err else error estimate
    dfill = df.copy(deep=True)
    if err:
        ffill = df.copy(deep=True)
    else:
        ffill    = ff.copy(deep=True)
        ffill[:] = 0

    # Times
    # number of data points per week; basic factor of the time window
    week    = pd.Timedelta('1 W') / (df.index[1] - df.index[0])
    nperday = week // 7
    hour    = df.index.hour + df.index.minute/60.
    day     = (df.index.to_julian_date()-0.5).astype(int)

    # Filling variables
    ndata = len(df)
    for hcol in df.columns:

        if hcol.startswith('SW_IN_') or (hcol == 'SW_IN'):
            continue
        if hcol.startswith('TA_')    or (hcol == 'TA'):
            continue
        if hcol.startswith('VPD_')   or (hcol == 'VPD'):
            continue

        if verbose > 0:
            if err:
                print('  Error estimate ', str(hcol))
            else:
                print('  Filling ', str(hcol))

        data  = df[hcol].to_numpy()
        dflag = ff[hcol].to_numpy()

        data_f  = dfill[hcol].to_numpy()
        dflag_f = ffill[hcol].to_numpy()

        if err:
            data_f[:]  = undef
            dflag_f[:] = undef

        # Large margins

        # Check for large margins at beginning
        largegap   = np.zeros(ndata, dtype=bool)
        firstvalid = np.amin(np.where(dflag == 0)[0])
        lastvalid  = np.amax(np.where(dflag == 0)[0])
        nn         = int(nperday*longgap)
        if firstvalid > nn:
            if verbose > 1:
                print('    Large margin at beginning: ', firstvalid)
            largegap[0:(firstvalid-nn)] = True
        if lastvalid < (ndata-nn):
            if verbose > 1:
                print('    Large margin at end: ', lastvalid-nn)
            largegap[(lastvalid+nn):] = True

        # Large gaps

        # search largegap - code from maskgroup.py
        index  = []
        length = []
        count  = 0
        for i in range(ndata):
            if i == 0:
                if dflag[i] != 0:
                    index += [i]
                    count  = 1
            if i > 0:
                if (dflag[i] != 0) and (dflag[i-1] == 0):
                    index += [i]
                    count  = 1
                elif dflag[i] != 0:
                    count += 1
                elif (dflag[i] == 0) and (dflag[i-1] != 0):
                    length += [count]
                    count = 0
                else:
                    pass
        if count > 0:
            length += [count]

        # set largegap
        for i in range(len(index)):
            if length[i] > nn:
                if verbose > 1:
                    print('    Large gap: ', index[i], ':', index[i]+length[i])
                largegap[index[i]:index[i]+length[i]] = True

        # set or unset rest of days in large gaps
        if fullday:
            for i in range(ndata-1):
                # end of large margin
                if largegap[i] and not largegap[i+1]:
                    largegap[np.where(day == day[i])[0]] = False
                # beginning of large margin
                elif not largegap[i] and largegap[i+1]:
                    largegap[np.where(day == day[i])[0]] = False
                else:
                    continue

        # Gap filling

        # flag for all meteorological conditions
        meteo_flg = (ta_flg == 0) & (vpd_flg == 0) & (sw_flg == 0)
        # flag for all meteorological conditions and data
        total_flg = meteo_flg & (dflag == 0)

        # Fill loop over all data points
        for j in range(ndata):
            if not err:
                # no reason to go further, no gap -> continue
                if (dflag[j] == 0) | largegap[j]:
                    continue
            # 3 Methods
            #   1. ta, vpd and global radiation sw;
            #   2. just global radiation sw;
            #   3. no meteorolgical conditions: take the mean of +- hour

            # for better overview: dynamic calculation of radiation threshold
            # minimum 20; maximum 50 [Wm-2] according to private correspondence
            # with Markus Reichstein
            sw_devmax = np.maximum(20., np.minimum(sw[j], sw_dev))

            # Method 1: all met conditions
            if meteo_flg[j]:
                # search for values around the met-conditions
                # in a window of time
                # (one week in the first iteration and odd weeks in the next)
                j1  = j - np.arange(1, week+1, dtype=int) + 1
                j2  = j + np.arange(1, week, dtype=int)
                jj  = np.append(j1, j2)
                win = np.unique(np.sort(np.clip(jj, 0, ndata-1)))
                # get boolean array where meteo-conditions are in a given width
                conditions = ( (np.abs(sw[win]-sw[j])   < sw_devmax) &
                               (np.abs(ta[win]-ta[j])   < ta_dev) &
                               (np.abs(vpd[win]-vpd[j]) < vpd_dev) &
                               total_flg[win] )
                num4avg = np.sum(conditions)
                # we need at least two samples with similar conditions
                if num4avg >= 2:
                    dat = np.ma.array(data[win], mask=~conditions)
                    if verbose > 2:
                        print('    m1.1: ', j, win.size, dat.mean(),
                              dat.std(ddof=ddof))
                    data_f[j] = dat.mean()
                    if err:
                        dflag_f[j] = dat.std(ddof=ddof)
                    else:
                        # assign also quality category of gap filling
                        dflag_f[j] = 1
                    continue
                else:  # --> extend time window to two weeks
                    j1  = j - np.arange(1, 2*week+1, dtype=int) + 1
                    j2  = j + np.arange(1, 2*week, dtype=int)
                    jj  = np.append(j1, j2)
                    win = np.unique(np.sort(np.clip(jj, 0, ndata-1)))
                    conditions = ( (np.abs(sw[win]  - sw[j])  < sw_devmax) &
                                   (np.abs(ta[win]  - ta[j])  < ta_dev) &
                                   (np.abs(vpd[win] - vpd[j]) < vpd_dev) &
                                   total_flg[win] )
                    num4avg = np.sum(conditions)
                    if num4avg >= 2:
                        dat = np.ma.array(data[win], mask=~conditions)
                        if verbose > 2:
                            print('    m1.2: ', j, win.size, dat.mean(),
                                  dat.std(ddof=ddof))
                        data_f[j] = dat.mean()
                        if err:
                            dflag_f[j] = dat.std(ddof=ddof)
                        else:
                            # assign also quality category of gap filling
                            dflag_f[j] = 1
                        continue

            if err:
                continue
            # if you come here, gap-filling rather than error estimate

            # If nothing is found under similar meteo within two weeks,
            # look for global radiation within one week ->

            # Method 2: just global radiation available
            if sw_flg[j] == 0:
                j1  = j - np.arange(1, week+1, dtype=int) + 1
                j2  = j + np.arange(1, week, dtype=int)
                jj  = np.append(j1, j2)
                win = np.unique(np.sort(np.clip(jj, 0, ndata-1)))
                # get boolean array where meteo-conditions are in a given width
                conditions = ( (np.abs(sw[win]-sw[j]) < sw_devmax) &
                               total_flg[win] )
                num4avg = np.sum(conditions)
                # we need at least two samples with similar conditions
                if num4avg >= 2:
                    dat = np.ma.array(data[win], mask=~conditions)
                    if verbose > 2:
                        print('    m2: ', j, win.size, dat.mean(),
                              dat.std(ddof=ddof))
                    data_f[j]  = dat.mean()
                    dflag_f[j] = 1
                    continue

            # If still nothing is found under similar sw within one week,
            # take the same hour within 1-7 days

            # Method 3: same hour
            enough = False
            for i in range(2):
                t_win = (nperday * (2*i+1))//2
                j1  = j - np.arange(1, t_win+1, dtype=int) + 1
                j2  = j + np.arange(1, t_win, dtype=int)
                jj  = np.append(j1, j2)
                win = np.unique(np.sort(np.clip(jj, 0, ndata-1)))
                conditions = ( (np.abs(hour[win]-hour[j]) < 1.1)
                               & (dflag[win] == 0) )
                num4avg = np.sum(conditions)
                if num4avg >= 2:
                    dat = np.ma.array(data[win], mask=~conditions)
                    if verbose > 2:
                        print('    m3.{:d}: '.format(i), j, win.size,
                              dat.mean(), dat.std(ddof=ddof))
                    data_f[j] = dat.mean()
                    if i == 0:
                        dflag_f[j] = 1
                    else:
                        dflag_f[j] = 2
                    break

            # sanity check
            if dflag_f[j] > 0:
                continue

            # If still nothing is found, start a new cycle
            # with increased window size
            # Method 4: same as 1 but for 3-12 weeks
            if meteo_flg[j]:
                for multi in range(3, 12):
                    j1  = j - np.arange(1, multi*week+1, dtype=int) + 1
                    j2  = j + np.arange(1, multi*week, dtype=int)
                    jj  = np.append(j1, j2)
                    win = np.unique(np.sort(np.clip(jj, 0, ndata-1)))
                    conditions = ( (np.abs(sw[win]  - sw[j])  < sw_devmax) &
                                   (np.abs(ta[win]  - ta[j])  < ta_dev) &
                                   (np.abs(vpd[win] - vpd[j]) < vpd_dev) &
                                   total_flg[win] )
                    num4avg = np.sum(conditions)
                    # we need at least two samples with similar conditions
                    if num4avg >= 2:
                        dat = np.ma.array(data[win], mask=~conditions)
                        if verbose > 2:
                            print('    m4.{:d}: '.format(multi), j, win.size,
                                  dat.mean(), dat.std(ddof=ddof))
                        data_f[j] = dat.mean()
                        # assign also quality category of gap filling
                        if multi <= 2:
                            dflag_f[j] = 1
                        elif multi > 4:
                            dflag_f[j] = 3
                        else:
                            dflag_f[j] = 2
                        break

                # Check because continue does not support
                # to jump out of two loops
                if dflag_f[j] > 0:
                    continue

            # Method 5: same as 2 but for 2-12 weeks
            if sw_flg[j] == 0:
                for multi in range(2, 12):
                    j1  = j - np.arange(1, multi*week+1, dtype=int) + 1
                    j2  = j + np.arange(1, multi*week, dtype=int)
                    jj  = np.append(j1, j2)
                    win = np.unique(np.sort(np.clip(jj, 0, ndata-1)))
                    # get boolean array where meteo-conditions are
                    # in a given width
                    conditions = ( (np.abs(sw[win] - sw[j]) < sw_devmax) &
                                   total_flg[win] )
                    num4avg = np.sum(conditions)
                    # we need at least two samples with similar conditions
                    if num4avg >= 2:
                        dat = np.ma.array(data[win], mask=~conditions)
                        if verbose > 2:
                            print('    m5.{:d}: '.format(multi), j, win.size,
                                  dat.mean(), dat.std(ddof=ddof))
                        data_f[j] = dat.mean()
                        if multi == 0:
                            dflag_f[j] = 1
                        elif multi <= 2:
                            dflag_f[j] = 2
                        else:
                            dflag_f[j] = 3
                        break

                if dflag_f[j] > 0:
                    continue

            # Method 6: same as 3 but for 3-120 days
            for i in range(3, 120):
                t_win = nperday * (2*i+1)/2
                j1  = j - np.arange(1, t_win+1, dtype=int) + 1
                j2  = j + np.arange(1, t_win, dtype=int)
                jj  = np.append(j1, j2)
                win = np.unique(np.sort(np.clip(jj, 0, ndata-1)))
                conditions = ( (np.abs(hour[win]-hour[j]) < 1.1)
                               & (dflag[win] == 0) )
                num4avg = np.sum(conditions)
                if num4avg >= 2:
                    dat = np.ma.array(data[win], mask=~conditions)
                    if verbose > 2:
                        print('    m6.{:d}: '.format(i), j, win.size,
                              dat.mean(), dat.std(ddof=ddof))
                    data_f[j]  = dat.mean()
                    dflag_f[j] = 3
                    break

        dfill[hcol] = data_f
        ffill[hcol] = dflag_f

    # Finish

    if isnumpy:
        if istrans:
            dfout = dfill.to_numpy().T
        else:
            dfout = dfill.to_numpy()
    else:
        dfout = dfill

    if fisnumpy:
        if fistrans:
            ffout = ffill.to_numpy().T
        else:
            ffout = ffill.to_numpy()
    else:
        ffout = ffill

    if err:
        if errmean:
            return ffout, dfout
        else:
            return ffout
    else:
        return dfout, ffout

if __name__ == '__main__':
    import doctest
    doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)

    # import numpy as np
    # from fread import fread
    # from date2dec import date2dec
    # from dec2date import dec2date
    # from autostring import astr
    # ifile = 'test_gapfill.csv' # Tharandt 1998 = Online tool example file
    # undef = -9999.
    # # Day Hour NEE         qcNEE  LE    qcLE  H       qcH  Rg    Tair  Tsoil  rH     VPD  Ustar
    # # --  --   umolm-2s-1  --     Wm-2  --    Wm-2    --   Wm-2  degC  degC   %      hPa  ms-1
    # # 1   0.5  -1.21       1      1.49  1     -11.77  1    0     7.4   4.19   55.27  4.6  0.72
    # dat   = fread(ifile, skip=2, transpose=True)
    # # dat = dat[:,:1000]
    # ndat  = dat.shape[1]
    # head  = fread(ifile, skip=2, header=True)
    # head1 = head[0]
    # # colhead
    # idx   = []
    # for i in head1:
    #     if i in ['NEE', 'LE', 'H', 'Rg', 'Tair', 'VPD']: idx.append(head1.index(i))
    # colhead = ['FC', 'LE', 'H', 'SW_IN', 'TA', 'VPD']
    # # data
    # dfin = dat[idx,:]
    # # flag
    # flag = np.where(dfin == undef, 2, 0)
    # flag[0,:] = dat[head1.index('qcNEE'),:].astype(int)
    # flag[1,:] = dat[head1.index('qcLE'),:].astype(int)
    # flag[2,:] = dat[head1.index('qcH'),:].astype(int)
    # flag[np.where(flag==1)] = 0
    # # date
    # day_id  = head1.index('Day')
    # hour_id = head1.index('Hour')
    # ntime   = dat.shape[1]
    # year  = np.ones(ntime, dtype=int) * 1998
    # hh    = dat[hour_id,:].astype(int)
    # mn    = np.rint((dat[hour_id,:]-hh)*60.).astype(int)
    # y0    = date2dec(yr=year[0], mo=1, dy=1, hr=hh, mi=mn)
    # jdate = y0 + dat[day_id,:]
    # adate = dec2date(jdate, eng=True)
    # # fill
    # dat_f, flag_f = gapfill(dfin, flag=flag, date=adate, colhead=colhead, undef=undef, verbose=0)
    # print(astr(flag_f[0,11006:11012],0,pp=True))
    # # ['1' '1' '1' '2' '2' '2']
    # print(astr(dat_f[0,11006:11012],3,pp=True))
    # # ['-18.678' '-15.633' '-19.610' '-15.536' '-12.402' '-15.329']

    # # 1D err
    # dat_std = gapfill(dfin, flag=flag, date=adate, colhead=colhead, undef=undef, verbose=0, err=True)
    # print(astr(dat_std[0,11006:11012],3,pp=True))
    # # ['    5.372' '   13.118' '    6.477' '-9999.000' '-9999.000' '-9999.000']

    # dat_err     = np.ones(ndat, dtype=int)*(-1)
    # kk          = np.where((dat_std[0,:] != undef) & (dat_f[0,:] != 0.))[0]
    # dat_err[kk] = np.abs(dat_std[0,kk]/dat_f[0,kk]*100.).astype(int)
    # print(astr(dat_err[11006:11012],pp=True))
    # # [' 28' ' 83' ' 33' ' -1' ' -1' ' -1']


from PyQt5 import QtCore, QtGui, QtWidgets
import res_rc
import ui as gui
import global_ as g
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas
from matplotlib.backends.backend_qt5agg import NavigationToolbar2QT as NavigationToolbar
from matplotlib.figure import Figure

import pickle
import sys
import random
import numpy as np
import argparse
import Reader
import Spectrum
import Algorithm
class Button:
    def __init__(self,button,entry):
        self.button = button
        self.entry = entry
        g.values[self.entry] = self.button.value()
        self.button.valueChanged.connect(self.valuechange)
    def valuechange(self):
        g.values[self.entry] = self.button.value()
        if(self.entry == "lb" or self.entry == "hb"):
            for i in range(len(g.canvas_list)):
                try:
                    g.canvas_list[i].change_x()
                except:
                    pass
        elif(self.entry == "w" or self.entry == "T"):
            try:
                g.Spectrum.Boltzmann_weight_IR()
                g.canvas_list[2].plot_IR_theo()
                g.Spectrum.Boltzmann_weight_VCD()
                g.canvas_list[3].plot_VCD_theo()
            except:
                pass
class Click_Button:
    def __init__(self,button,entry,args = None):
        self.button = button
        self.entry = entry
        self.button.clicked.connect(self.click)
        self.args = args
    def click(self):
        if(self.entry == "normalize_1"):
            try:
                tmp = (g.exp_ir[:,0] <= g.values["hb"]) & (g.exp_ir[:,0] >= g.values["lb"])
                g.exp_ir[:,1] = g.exp_ir[:,1]/np.max(g.exp_ir[tmp,1])
                g.canvas_list[0].plot_IR()
            except:
                pass
            try:
                tmp = (g.exp_vcd[:,0] <= g.values["hb"]) & (g.exp_vcd[:,0] >= g.values["lb"])
                g.exp_vcd[:,1] = g.exp_vcd[:,1]/np.max(np.abs(g.exp_vcd[tmp,1]))
                g.canvas_list[1].plot_VCD()
            except:
                pass
        elif(self.entry == "normalize_2"):
            try:
                tmp = (g.theo_ir[:,0] <= g.values["hb"]) & (g.theo_ir[:,0] >= g.values["lb"])
                g.theo_ir[:,1] = g.theo_ir[:,1]/np.max(g.theo_ir[tmp,1])
                g.canvas_list[2].plot_IR()
            except:
                pass
            try:
                tmp = (g.theo_vcd[:,0] <= g.values["hb"]) & (g.theo_vcd[:,0] >= g.values["lb"])
                g.theo_vcd[:,1] = g.theo_vcd[:,1]/np.max(np.abs(g.theo_vcd[tmp,1]))
                g.canvas_list[3].plot_VCD()
            except:
                pass
        elif(self.entry == "automatic"):
            try:
                tmp = (g.exp_ir[:,0] <= g.values["hb"]) & (g.exp_ir[:,0] >= g.values["lb"])
                tmp_ir = np.asarray(g.exp_ir[tmp])
                g.peak_list_x = []
                g.peak_list_y = []
                g.peak_list_VCD_y = []
                for i in range(1,len(tmp_ir)-1):
                    if(tmp_ir[i-1,1]<=tmp_ir[i,1]>=tmp_ir[i+1,1]):
                        g.peak_list_x.append(tmp_ir[i,0])
                        g.peak_list_y.append(tmp_ir[i,1])
                print(g.peak_list_x)
                g.canvas_list[0].plot_peaks()
                g.exp_peaks = np.zeros((len(g.peak_list_x),2))
                g.exp_peaks[:,0] = np.asarray(g.peak_list_x)
                g.exp_peaks[:,1] = np.asarray(g.peak_list_y)
                for peak in g.peak_list_x:
                    g.peak_list_VCD_y.append(g.exp_vcd[abs(g.exp_vcd[:,0]-peak)<10e-1,1][0])
                g.canvas_list[1].plot_peaks_VCD()
            except:
                pass
        elif(self.entry == "align"):
            try:
                del Algo
                print("del")
            except:
                pass
            Algo = Algorithm.Algorithm()
            if(g.set_VCD==False):
                g.returnvalue, g.old_freq, g.freq_new, g.inten_new = Algo.Needleman_IR()
            else:
                g.returnvalue, g.old_freq, g.freq_new, g.inten_new,g.inten_VCD_new = Algo.Needleman_IR()
            g.canvas_list[4].plot_IR_assigned()
            g.Spectrum.IR_shifted()
            if(g.set_VCD==True):
                g.Spectrum.VCD_shifted()
                p_ir,p_vcd = g.Spectrum.integrate()

                self.args.setText("Score: " + str(g.returnvalue)[0:6]+"\np_ir: " + str(p_ir)[0:4]+"\np_vcd: " + str(p_vcd)[0:4]+"\n")
            else:
                p_ir = g.Spectrum.integrate()
                self.args.setText("Score: " + str(g.returnvalue)[0:6]+"\np_ir: " + str(p_ir)[0:4]+"\n")
            g.canvas_list[5].plot_IR_shifted()
class Load_Button:
    def __init__(self,button,entry):
        self.button = button
        self.entry = entry
        self.button.clicked.connect(self.click)
    def click(self):
        if(self.entry == "experimental IR"):
            try:
                fileName, _ = QtWidgets.QFileDialog.getOpenFileName(None,"Select "+self.entry +" Spectrum","","")
                g.exp_ir = np.loadtxt(fileName,usecols=(0,1))
                g.exp_ir = g.exp_ir[g.exp_ir[:,0].argsort()]
                g.canvas_list[0].plot_IR()
                g.set_IR = True
            except:
                pass
        elif(self.entry == "experimental VCD"):
            try:
                fileName, _ = QtWidgets.QFileDialog.getOpenFileName(None,"Select "+self.entry +" Spectrum","","")
                g.exp_vcd = np.loadtxt(fileName,usecols=(0,1,))
                g.exp_vcd = g.exp_vcd[g.exp_vcd[:,0].argsort()]
                g.canvas_list[1].plot_VCD()
                g.set_VCD = True
            except:
                pass
        elif(self.entry == "energies"):
            fileName_energy, _ = QtWidgets.QFileDialog.getOpenFileName(None,"Select "+self.entry +" Energies","","")
            g.E = pickle.load(open(fileName_energy,"rb"))
        elif(self.entry == "theoretical IR"):
            fileName_IR, _ = QtWidgets.QFileDialog.getOpenFileName(None,"Select "+self.entry +" Spectrum","","")
            g.theo_ir = pickle.load(open(fileName_IR,"rb"))
            g.Spectrum.Boltzmann_weight_IR()
            g.canvas_list[2].plot_IR_theo()
        elif(self.entry == "theoretical VCD"):
            try:
                fileName_VCD, _ = QtWidgets.QFileDialog.getOpenFileName(None,"Select "+self.entry +" Spectrum","","")
                g.theo_vcd = pickle.load(open(fileName_VCD,"rb"))
                g.Spectrum.Boltzmann_weight_VCD()
                g.canvas_list[3].plot_VCD_theo()
            except:
                pass
class Canvas(FigureCanvas):
    def __init__(self, single = None, parent = None, Button = None, dpi = 100):
        height = parent.height()/100.
        width = parent.width()/100.
        fig = Figure(figsize=(width, height), dpi=dpi)
        FigureCanvas.__init__(self, fig)
        self.setParent(parent)
        self.ax = self.figure.add_subplot(111)
        #Button
    def change_x(self):
        self.ax.set_xlim(g.values["lb"],g.values["hb"])
        self.draw()
    def plot_IR(self):
        self.delete()
        self.ax.plot(g.exp_ir[:,0],g.exp_ir[:,1],color="black")
        self.ax.set_ylim(0,1.05)
        self.ax.set_xlim(g.values["lb"],g.values["hb"])
        self.draw()
    def plot_IR_shifted(self):
        self.delete()
        self.ax.plot(g.exp_ir[:,0],g.exp_ir[:,1],color="black")
        self.ax.plot(g.IR_shifted[:,0],g.IR_shifted[:,1],color="red")
        if(g.set_VCD==True):
            self.ax.plot(g.exp_vcd[:,0],g.exp_vcd[:,1],"--",color="black")
            self.ax.plot(g.VCD_shifted[:,0],g.VCD_shifted[:,1],"--",color="red")
        self.ax.set_ylim(-1.05,1.05)
        self.ax.set_xlim(g.values["lb"],g.values["hb"])
        self.draw()
    def plot_peaks(self):
        self.delete()
        self.ax.plot(g.exp_ir[:,0],g.exp_ir[:,1],color="black")
        self.ax.set_ylim(0,1.05)
        self.ax.set_xlim(g.values["lb"],g.values["hb"])
        self.ax.plot(g.peak_list_x,g.peak_list_y,"o",color="blue")
        self.draw()
    def plot_peaks_VCD(self):
        self.delete()
        self.ax.plot(g.exp_vcd[:,0],g.exp_vcd[:,1],"--",color="black")
        self.ax.plot(g.peak_list_x,g.peak_list_VCD_y,"o",color="blue")
        self.ax.set_ylim(-1.05,1.05)
        self.ax.set_xlim(g.values["lb"],g.values["hb"])
        self.draw()
    def delete(self):
        try:
            while(len(self.ax.lines)>0):
                self.ax.lines[-1].remove()
        except:
            pass
    def plot_VCD(self):
        self.delete()
        self.ax.plot(g.exp_vcd[:,0],g.exp_vcd[:,1],"--",color="black")
        self.ax.set_ylim(-1,1)
        self.ax.set_xlim(g.values["lb"],g.values["hb"])
        self.draw()
    def plot_IR_theo(self):
        self.delete()
        self.ax.plot(g.spectrum_boltzmann[:,0],g.spectrum_boltzmann[:,1],color="red") ##x_axis 0..2000
        self.ax.set_ylim(0,1)
        self.ax.set_xlim(g.values["lb"],g.values["hb"])
        tmp_ir = g.spectrum_boltzmann[g.values["lb"]:g.values["hb"]]
        g.theo_peaks_x = []
        g.theo_peaks_y = []
        for i in range(1,len(tmp_ir)-1):
            if(tmp_ir[i-1,1]<=tmp_ir[i,1]>=tmp_ir[i+1,1]):
                g.theo_peaks_x.append(tmp_ir[i,0])
                g.theo_peaks_y.append(tmp_ir[i,1])
        self.ax.plot(g.theo_peaks_x,g.theo_peaks_y,"o",color="blue")
        g.theo_peaks = np.zeros((len(g.theo_peaks_x),2))
        g.theo_peaks[:,0] = np.asarray(g.theo_peaks_x)
        g.theo_peaks[:,1] = np.asarray(g.theo_peaks_y)
        self.draw()
    def plot_IR_assigned(self):
        self.delete()
        #g.returnvalue, g.old_freq, g.freq, g.inten
        self.ax.plot(g.spectrum_boltzmann[:,0],g.spectrum_boltzmann[:,1],color="red") ##x_axis 0..2000
        self.ax.plot(g.exp_ir[:,0],g.exp_ir[:,1],color="black")
        if(g.set_VCD==True):
            self.ax.plot(g.exp_vcd[:,0],g.exp_vcd[:,1],"--",color="black")
            self.ax.plot(g.spectrum_boltzmann_vcd[:,0],g.spectrum_boltzmann_vcd[:,1],"--",color="red")
        for i in range(len(g.old_freq)):
            self.ax.plot([g.old_freq[i],g.freq_new[i]],[g.inten_new[i],g.inten_new[i]],color="blue")
        self.ax.set_ylim(-1,1)
        self.ax.set_xlim(g.values["lb"],g.values["hb"])
        tmp_ir = g.spectrum_boltzmann[g.values["lb"]:g.values["hb"]]

        self.draw()
    def plot_VCD_theo(self):
        self.delete()
        self.ax.plot(g.spectrum_boltzmann_vcd[:,0],g.spectrum_boltzmann_vcd[:,1],"--",color="red") ##x_axis 0..2000
        tmp_vcd = g.spectrum_boltzmann_vcd[g.values["lb"]:g.values["hb"]]
        self.ax.set_ylim(-1,1)
        g.peak_list_VCD_y_theo = []
        for peak in g.theo_peaks_x:
            g.peak_list_VCD_y_theo.append(tmp_vcd[np.abs(tmp_vcd[:,0]-peak)<10e-3,1][0])
        self.ax.plot(g.theo_peaks_x,g.peak_list_VCD_y_theo,"o",color="blue")
        self.ax.set_xlim(g.values["lb"],g.values["hb"])
        self.draw()
if __name__ == "__main__":
    import sys
    app = QtWidgets.QApplication(sys.argv)
    MainWindow = QtWidgets.QMainWindow()
    ui = gui.Ui_MainWindow()
    ui.setupUi(MainWindow)
    g.Spectrum = Spectrum.Spectrum()
    g.canvas_list = []
    g.canvas_list.append(Canvas(parent = ui.exp_ir_graph))
    g.canvas_list.append(Canvas(parent = ui.exp_vcd_graph))
    g.canvas_list.append(Canvas(parent = ui.theo_ir_graph))
    g.canvas_list.append(Canvas(parent = ui.theo_vcd_graph))
    g.canvas_list.append(Canvas(parent = ui.assignment_graph))
    g.canvas_list.append(Canvas(parent = ui.shifted_graph))
    g.list_buttons = []
    g.list_buttons.append(Button(ui.w,"w"))
    g.list_buttons.append(Button(ui.lb,"lb"))
    g.list_buttons.append(Button(ui.hb,"hb"))
    g.list_buttons.append(Button(ui.sigma_1,"s0"))
    g.list_buttons.append(Button(ui.sigma_2,"s1"))
    g.list_buttons.append(Button(ui.mu,"mu"))
    g.list_buttons.append(Button(ui.cutoff,"c"))
    g.list_buttons.append(Button(ui.temperature,"T"))

    g.list_buttons.append(Load_Button(ui.Load_EXP,"experimental IR"))
    g.list_buttons.append(Load_Button(ui.Load_EXP_VCD,"experimental VCD"))
    g.list_buttons.append(Load_Button(ui.load_theo_exp,"theoretical IR"))
    g.list_buttons.append(Load_Button(ui.load_theo_vcd,"theoretical VCD"))
    g.list_buttons.append(Load_Button(ui.load_theo_exp_2,"energies"))
    #g.list_buttons.append(Load_Button(ui.load_theo_vcd,"theoretical VCD"))

    g.list_buttons.append(Click_Button(ui.normalize_1,"normalize_1"))
    g.list_buttons.append(Click_Button(ui.normalize_2,"normalize_2"))
    g.list_buttons.append(Click_Button(ui.automatic,"automatic"))
    g.list_buttons.append(Click_Button(ui.align,"align",args = ui.results))

    MainWindow.show()
    sys.exit(app.exec_())


"""
Molecule generation and representation for generating theoretical spectre
"""
import json
import os
from bisect import (
    bisect_left,
    bisect_right,
)
from typing import (
    List,
    Optional,
)

import IsoSpecPy as iso
import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import typer  # pylint: disable=C0411
from molmass import Formula
from scipy.interpolate import interp1d
from scipy.spatial import (
    ckdtree,
    distance,
)

from msaris.utils.distributions_util import generate_gauss_distribution
from msaris.utils.intensities_util import (
    get_spectrum_by_close_values,
    norm,
)
from msaris.utils.molecule_utils import convert_into_formula_for_plot

class Molecule:  # pylint: disable=R0902
    """
    Molecule generation and saving for performing molecule search
    """

    def __init__(self, *, formula: str = "", delta: float = 0.5):
        self.formula = formula  # saving for using to refer
        self.brutto = self._get_brutto() if formula else None
        self.mass_out: List = []
        self.intens_out: List = []
        self.mz: np.array = np.array([])
        self.it: np.array = np.array([])
        self.bar_raw = np.array([])
        self.averaged_mass: float = 0.0
        self.delta: float = delta

    def _get_brutto(self) -> str:
        """
        Generating brutto formula from provided one

        :returns: brutto formula
        """
        f = Formula(self.formula)
        return "".join(map(lambda x: f"{x[0]}{x[1]}", f.composition()))

    def _get_bar_isotope(self):
        """
        Generating raw bar isotope pattern
        :return:
        """
        non_zero = np.argwhere(np.array(self.intens_out) > 0)
        self.bar_raw = self.intens_out.copy()
        for ind in non_zero:
            left = bisect_left(self.mass_out, self.mass_out[ind] - self.delta)
            right = bisect_right(
                self.mass_out, self.mass_out[ind] + self.delta
            )
            self.bar_raw[left:right] = self.intens_out[ind]

    def calculate(
        self, *, resolution: int = 20, ppm: int = 50, scale: bool = False
    ) -> None:
        """

        :param resolution: generating m/z and intensities for provided formula
        :return: None
        """
        scaling: np.array

        try:
            sp = iso.IsoTotalProb(formula=self.brutto, prob_to_cover=0.99999)
        except ValueError:
            raise ValueError(f"Invalid {self.formula}")

        for mass, prob in sp:
            prob *= 100.0
            self.mass_out += [mass]
            self.intens_out += [prob]

        self.mz, self.it, self.averaged_mass = generate_gauss_distribution(
            self.mass_out, self.intens_out, ppm=ppm, resolution=resolution
        )

        if scale:
            scaling = 100 / max(self.it)
        else:
            scaling = max(self.intens_out) / max(self.it)
        # scaling resulting curve
        self.it = self.it * scaling

    def plot(
        self,
        *,
        save: bool = False,
        path: str = "./",
        name: Optional[str] = None,
    ) -> None:
        """
        Plot spectra

        :param save: bool value to save image of spectra
        :param path: path to save image
        :param name: name format

        :return: None
        """
        # TODO: change to be more flexible for output params
        plt.rcParams["figure.figsize"] = (30, 30)
        # plot settings
        fig, (ax_spiketrain, ax_filtered) = plt.subplots(2, 1, sharex=True)
        ax_spiketrain.tick_params(axis="x", labelbottom=True, rotation=-90)
        ax_spiketrain.tick_params(axis="both")
        # tick parameters
        plt.xticks(
            np.arange(
                int(min(self.mass_out)) - 1, int(max(self.mass_out)) + 2, 1.0
            ),
            rotation=-90,
        )
        markerline, stemlines, baseline = ax_spiketrain.stem(
            self.mass_out,
            self.intens_out,
            use_line_collection="True",
            linefmt="grey",
            markerfmt="D",
            basefmt="k-",
            bottom=0,
        )
        markerline.set_markerfacecolor("none")
        plt.setp(stemlines, "linewidth", 0.9)
        plt.setp(markerline, "linewidth", 0.8)
        plt.setp(baseline, "linewidth", 0.9)
        ax_spiketrain.set_title("Original spike train from IsoSpec data")
        ax_spiketrain.set_ylabel("Relative intensity, %")
        ax_spiketrain.set_xlabel("Mass, Da")

        ax_filtered.plot(self.mz, self.it, color="blue", lw=1.2)
        # axes labels
        ax_filtered.set_title("Gaussian-filtered predicted spectra")
        ax_filtered.set_ylabel("Relative intensity, %")
        ax_filtered.set_xlabel("Mass, Da")
        plt.rcParams.update({"font.size": 30})

        if save:
            name = f"{path}{name}.png" if name else f"{path}{self.formula}.png"
            fig.savefig(name, dpi=300, format="png", bbox_inches="tight")

        plt.show()
        plt.close()

    def to_dict(self) -> dict:
        """
        Present result in dict format
        :return: dictionary of the main parameters
        """
        return {
            "formula": self.formula,
            "brutto": self.brutto,
            "mz": self.mz.tolist(),
            "it": self.it.tolist(),
            "mass_out": self.mass_out,
            "intens_out": self.intens_out,
            "averaged_mass": self.averaged_mass,
        }

    def to_json(self, path: str = "./", name: Optional[str] = None) -> None:
        """
        Saves the molecule's to json

        :param path: string default save to place where executed
        :param name: redifine name default is formula with .mol format
        :return: None
        """

        if not os.path.isdir(path):
            os.makedirs(path)

        name = f"{self.formula}.json" if name is None else f"{name}.json"
        if not path.endswith("/"):
            path = f"{path}/"

        with open(f"{path}{name}", "w") as outfile:
            json.dump(self.to_dict(), outfile)

        typer.echo(f"✨ JSON with was created: {os.path.abspath(path)}{name} ✨")

    def read_dict_data(self, data: dict) -> None:
        """
        Gets Molecule from dictionary representation of molecule

        :param data: data in dictionary format
        :return: None
        """
        for field, value in data.items():
            if field in ("mz", "it"):
                value = np.array(value)
            setattr(self, field, value)

    def load(self, file_path: str) -> None:
        """
        Load file in JSON format

        :param: Path to load data
        :return: None
        """
        with open(file_path, "r") as file:
            self.read_dict_data(json.load(file))

    def __str__(self) -> str:
        return self.formula

    def __repr__(self) -> str:
        return (
            f"<Molecule(formula={self.formula},"
            f" weighted_mass={self.averaged_mass})>"
        )

    def compare(self, experimental: tuple) -> dict:

        """
        Function to perform calculations for the theoretical and experimental spectrum
        Based on interpolation selected peaks are recalculated to the same mz_t value

        :param experimental: m/z and it of experimantal data

        :return: calculated metrics for the selected spectras
        """
        metrics: dict = {}
        mz_t, it_t = self.mz.copy(), self.it.copy()
        mz_e, it_e = experimental
        it_t = norm(it_t)
        it_e = norm(it_e)

        interpol_t = interp1d(
            mz_t, norm(it_t), bounds_error=False, fill_value=(0, 0)
        )
        interpol_e = interp1d(
            mz_e, norm(it_e), bounds_error=False, fill_value=(0, 0)
        )
        theory = interpol_t(mz_e) * 100
        exp = interpol_e(mz_e) * 100

        metrics["cosine"] = distance.cosine(theory, exp)
        # TODO: improve and add other statistics calculations
        return metrics

def plot_graph_for_comparing_molecules(
    mz: np.array,
    it: np.array,
    formula: str,
    *,
    save: bool = False,
    path: str = "./",
    step: float = 0.5,
    font: int = 18,
):
    """
    Provides plot with comparing stats with original spectrum
    :param mz:  original spectrum m/z values
    :param it: original spectrum intensity
    :param formula: formula to find in original spectrum
    :param save: save plot into provided path
    :param path: path to save plot
    :param step: parameter to tweak width of the spectrum
    """
    mol = Molecule(formula=formula)
    mol.calculate()
    mz, it = mz.copy(), it.copy()
    bar_mz: List = list()
    bar_it: List = list()
    visited: List = list()

    mz_f, it_f, _, _ = get_spectrum_by_close_values(
        mz, it, mol.mz[0], mol.mz[-1]
    )
    mpl.rcParams["xtick.labelsize"] = font
    mpl.rcParams["ytick.labelsize"] = font
    max_it_x_ind, max_it_t_ind = np.argmax(it_f), np.argmax(mol.it)
    m_x, it_max_x, m_t, _ = (
        mz_f[max_it_x_ind],
        it_f[max_it_x_ind],
        mol.mz[max_it_t_ind],
        mol.it[max_it_t_ind],
    )
    tree = ckdtree.cKDTree(np.array([mol.mz, mol.mz]).T)
    for _, val in enumerate(mol.mass_out):
        inds = tree.query_ball_point((val, val), step)
        S1, S2 = set(inds), set(visited)
        if S1.intersection(S2):
            continue
        visited.extend(inds)
        bar_mz.append(np.mean(mol.mz[inds]))
        bar_it.append(np.mean(mol.it[inds]))

    delta_b = m_x - m_t
    spectrum = (
        mz_f,
        it_f,
    )
    stats = {
        "delta": abs(delta_b),
        "metrics": mol.compare(spectrum),
        "relative": max(it_f) / max(it),
    }
    _, ax = plt.subplots(1, 1, figsize=(15, 5))
    ax.bar(
        bar_mz,
        (bar_it / max(bar_it)) * 100,
        width=step,
        align="center",
        alpha=1,
        color="r",
    )
    ax.plot(
        spectrum[0],
        (spectrum[1] / it_max_x) * 100,
        color="black",
    )
    plotted_formula = convert_into_formula_for_plot(formula)
    ax.set_xlabel("M/Z", fontsize=20)
    ax.set_ylabel("Intensity", fontsize=20)
    labels = [
        plotted_formula,
        f"Delta m/z: {stats['delta']:.3f}",
        f"Cosine: {stats['metrics']['cosine']:.3f}",
        f"Relative: {stats['relative']:.3f}",
    ]
    ax.text(
        0.8,
        0.8,
        "\n".join(labels),
        color="black",
        horizontalalignment="center",
        verticalalignment="center",
        fontsize=font,
        transform=ax.transAxes,
    )
    ax.set_title(f"{plotted_formula}", fontsize=20)
    if save and path:
        if not os.path.exists(path):
            os.makedirs(path)
        plt.savefig(f"{path}/{formula}.png", dpi=600)
    plt.show()
    plt.close()


"""
.. module:: adaptive_sampling
   :synopsis: Ways of finding the next point to evaluate in the adaptive phase

.. moduleauthor:: David Eriksson <dme65@cornell.edu>,
                David Bindel <bindel@cornell.edu>

:Module: adaptive_sampling
:Author: David Eriksson <dme65@cornell.edu>,
        David Bindel <bindel@cornell.edu>

"""

import math
import scipy.stats as stats
from merit_functions import *
import types

def __fix_docs(cls):
    """Help function for stealing docs from the parent"""
    for name, func in vars(cls).items():
        if isinstance(func, types.FunctionType) and not func.__doc__:
            for parent in cls.__bases__:
                parfunc = getattr(parent, name, None)
                if parfunc and getattr(parfunc, '__doc__', None):
                    func.__doc__ = parfunc.__doc__
                    break
    return cls

class CandidateSRBF(object):
    """An implementation of Stochastic RBF

    This is an implementation of the candidate points method that is
    proposed in the first SRBF paper. Candidate points are generated
    by making normally distributed perturbations with standard
    deviation sigma around the best solution. The candidate point that
    minimizes a specified merit function is selected as the next
    point to evaluate.

    :param data: Optimization problem object
    :type data: Object
    :param numcand: Number of candidate points to be used. Default is min([5000, 100*data.dim])
    :type numcand: int
    :param weights: Weights used for the merit function, to balance exploration vs exploitation
    :type weights: list of numpy.array

    :raise ValueError: If number of candidate points is
        incorrect or if the weights aren't a list in [0, 1]

    :ivar data: Optimization problem object
    :ivar fhat: Response surface object
    :ivar xrange: Variable ranges, xup - xlow
    :ivar dtol: Smallest allowed distance between evaluated points 1e-3 * sqrt(dim)
    :ivar weights: Weights used for the merit function
    :ivar proposed_points: List of points proposed to the optimization algorithm
    :ivar dmerit: Minimum distance between the points and the proposed points
    :ivar xcand: Candidate points
    :ivar fhvals: Predicted values by the surrogate model
    :ivar next_weight: Index of the next weight to be used
    :ivar numcand: Number of candidate points
    :ivar budget: Remaining evaluation budget

    .. note:: This object needs to be initialized with the init method. This is done when the
        initial phase has finished.

    .. todo:: Get rid of the proposed_points object and replace it by something that is
        controlled by the strategy.
    """

    def __init__(self, data, numcand=None, weights=None):
        self.data = data
        self.fhat = None
        self.xrange = self.data.xup - self.data.xlow
        self.dtol = 1e-3 * math.sqrt(data.dim)
        self.weights = weights
        if self.weights is None:
            self.weights = [0.3, 0.5, 0.8, 0.95]
        self.proposed_points = None
        self.dmerit = None
        self.xcand = None
        self.fhvals = None
        self.next_weight = 0
        self.numcand = numcand
        if self.numcand is None:
            self.numcand = min([5000, 100*data.dim])
        self.budget = None

        # Check that the inputs make sense
        if not(isinstance(self.numcand, int) and self.numcand > 0):
            raise ValueError("The number of candidate points has to be a positive integer")
        if not((isinstance(self.weights, np.ndarray) or isinstance(self.weights, list))
               and max(self.weights) <= 1 and min(self.weights) >= 0):
            raise ValueError("Incorrect weights")

    def init(self, start_sample, fhat, budget):
        """Initialize the sampling method after the initial phase

        This initializes the list of sampling methods after the initial phase
        has finished and the experimental design has been evaluated. The user
        provides the points in the experimental design, the surrogate model,
        and the remaining evaluation budget.

        :param start_sample: Points in the experimental design
        :type start_sample: numpy.array
        :param fhat: Surrogate model
        :type fhat: Object
        :param budget: Evaluation budget
        :type budget: int
        """

        self.proposed_points = start_sample
        self.budget = budget
        self.fhat = fhat

    def remove_point(self, x):
        """Remove x from proposed_points

        This removes x from the list of proposed points in the case where the optimization
        strategy decides to not evaluate x.

        :param x: Point to be removed
        :type x: numpy.array
        :return: True if points was removed, False otherwise
        :type: bool
        """

        idx = np.sum(np.abs(self.proposed_points - x), axis=1).argmin()
        if np.sum(np.abs(self.proposed_points[idx, :] - x)) < 1e-10:
            self.proposed_points = np.delete(self.proposed_points, idx, axis=0)
            return True
        return False

    def __generate_cand__(self, scalefactors, xbest, subset):
        self.xcand = np.ones((self.numcand,  self.data.dim)) * xbest
        for i in subset:
            lower, upper = self.data.xlow[i], self.data.xup[i]
            ssigma = scalefactors[i]
            self.xcand[:, i] = stats.truncnorm.rvs(
                (lower - xbest[i]) / ssigma, (upper - xbest[i]) / ssigma,
                loc=xbest[i], scale=ssigma, size=self.numcand)

    def make_points(self, npts, xbest, sigma, subset=None, proj_fun=None,
                    merit=candidate_merit_weighted_distance):
        """Proposes npts new points to evaluate

        :param npts: Number of points to select
        :type npts: int
        :param xbest: Best solution found so far
        :type xbest: numpy.array
        :param sigma: Current sampling radius w.r.t the unit box
        :type sigma: float
        :param subset: Coordinates to perturb, the others are fixed
        :type subset: numpy.array
        :param proj_fun: Routine for projecting infeasible points onto the feasible region
        :type proj_fun: Object
        :param merit: Merit function for selecting candidate points
        :type merit: Object

        :return: Points selected for evaluation, of size npts x dim
        :rtype: numpy.array

        .. todo:: Change the merit function from being hard-coded
        """

        if subset is None:
            subset = np.arange(0, self.data.dim)
        scalefactors = sigma * self.xrange

        # Make sure that the scale factors are correct for
        # the integer variables (at least 1)
        ind = np.intersect1d(self.data.integer, subset)
        if len(ind) > 0:
            scalefactors[ind] = np.maximum(scalefactors[ind], 1.0)

        # Generate candidate points
        self.__generate_cand__(scalefactors, xbest, subset)
        if proj_fun is not None:
            self.xcand = proj_fun(self.xcand)

        dists = scp.distance.cdist(self.xcand, self.proposed_points)
        fhvals = self.fhat.evals(self.xcand)

        self.dmerit = np.amin(np.asmatrix(dists), axis=1)
        self.fhvals = unit_rescale(fhvals)

        xnew = merit(self, npts)
        self.proposed_points = np.vstack((self.proposed_points,
                                          np.asmatrix(xnew)))
        return xnew

@__fix_docs
class CandidateDYCORS(CandidateSRBF):
    """An implementation of the DYCORS method

    The DYCORS method only perturbs a subset of the dimensions when
    perturbing the best solution. The probability for a dimension
    to be perturbed decreases after each evaluation and is capped
    in order to guarantee global convergence.

    :param data: Optimization problem object
    :type data: Object
    :param numcand: Number of candidate points to be used. Default is min([5000, 100*data.dim])
    :type numcand: int
    :param weights: Weights used for the merit function, to balance exploration vs exploitation
    :type weights: list of numpy.array

    :raise ValueError: If number of candidate points is
        incorrect or if the weights aren't a list in [0, 1]

    :ivar data: Optimization problem object
    :ivar fhat: Response surface object
    :ivar xrange: Variable ranges, xup - xlow
    :ivar dtol: Smallest allowed distance between evaluated points 1e-3 * sqrt(dim)
    :ivar weights: Weights used for the merit function
    :ivar proposed_points: List of points proposed to the optimization algorithm
    :ivar dmerit: Minimum distance between the points and the proposed points
    :ivar xcand: Candidate points
    :ivar fhvals: Predicted values by the surrogate model
    :ivar next_weight: Index of the next weight to be used
    :ivar numcand: Number of candidate points
    :ivar budget: Remaining evaluation budget
    :ivar minprob: Smallest allowed perturbation probability
    :ivar n0: Evaluations spent when the initial phase ended
    :ivar probfun: Function that computes the perturbation probability of a given iteration

    .. note:: This object needs to be initialized with the init method. This is done when the
        initial phase has finished.

    .. todo:: Get rid of the proposed_points object and replace it by something that is
        controlled by the strategy.
    """

    def __init__(self, data, numcand=None, weights=None):
        CandidateSRBF.__init__(self, data, numcand=numcand, weights=weights)
        self.minprob = np.min([1.0, 1.0/self.data.dim])
        self.n0 = None

        if data.dim <= 1:
            raise ValueError("You can't use DYCORS on a 1d problem")

        def probfun(numevals, budget):
            if budget < 2:
                return 0
            return min([20.0/data.dim, 1.0]) * (1.0 - (np.log(numevals + 1.0) / np.log(budget)))
        self.probfun = probfun

    def init(self, start_sample, fhat, budget):
        CandidateSRBF.init(self, start_sample, fhat, budget)
        self.n0 = start_sample.shape[0]

    def remove_point(self, x):
        return CandidateSRBF.remove_point(self, x)

    def make_points(self, npts, xbest, sigma, subset=None, proj_fun=None,
                    merit=candidate_merit_weighted_distance):
        return CandidateSRBF.make_points(self, npts, xbest, sigma, subset, proj_fun, merit)

    def __generate_cand__(self, scalefactors, xbest, subset):
        ddsprob = self.probfun(self.proposed_points.shape[0] - self.n0, self.budget - self.n0)
        ddsprob = np.max([self.minprob, ddsprob])

        nlen = len(subset)

        # Fix when nlen is 1
        # Todo: Use SRBF instead
        if nlen == 1:
            ar = np.ones((self.numcand, 1))
        else:
            ar = (np.random.rand(self.numcand, nlen) < ddsprob)
            ind = np.where(np.sum(ar, axis=1) == 0)[0]
            ar[ind, np.random.randint(0, nlen - 1, size=len(ind))] = 1

        self.xcand = np.ones((self.numcand, self.data.dim)) * xbest
        for i in range(nlen):
            lower, upper = self.data.xlow[i], self.data.xup[i]
            ssigma = scalefactors[subset[i]]
            ind = np.where(ar[:, i] == 1)[0]
            self.xcand[ind, subset[i]] = stats.truncnorm.rvs(
                (lower - xbest[subset[i]]) / ssigma, (upper - xbest[subset[i]]) / ssigma,
                loc=xbest[subset[i]], scale=ssigma, size=len(ind))


#  MINLP written by GAMS Convert at 08/13/20 17:38:08
#
#  Equation counts
#      Total        E        G        L        N        X        C        B
#         53       18        0       35        0        0        0        0
#
#  Variable counts
#                   x        b        i      s1s      s2s       sc       si
#      Total     cont   binary  integer     sos1     sos2    scont     sint
#         96       21       75        0        0        0        0        0
#  FX      0        0        0        0        0        0        0        0
#
#  Nonzero counts
#      Total    const       NL      DLL
#        306      261       45        0
#
#  Reformulation has removed 1 variable and 1 equation

from pyomo.environ import *

model = m = ConcreteModel()

m.b1 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b2 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b3 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b4 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b5 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b6 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b7 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b8 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b9 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b10 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b11 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b12 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b13 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b14 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b15 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b16 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b17 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b18 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b19 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b20 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b21 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b22 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b23 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b24 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b25 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b26 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b27 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b28 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b29 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b30 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b31 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b32 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b33 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b34 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b35 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b36 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b37 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b38 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b39 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b40 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b41 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b42 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b43 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b44 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b45 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b46 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b47 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b48 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b49 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b50 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b51 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b52 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b53 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b54 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b55 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b56 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b57 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b58 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b59 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b60 = Var(within=Binary,bounds=(0,1),initialize=0.2)
m.b61 = Var(within=Binary,bounds=(0,1),initialize=0.333333333333333)
m.b62 = Var(within=Binary,bounds=(0,1),initialize=0.333333333333333)
m.b63 = Var(within=Binary,bounds=(0,1),initialize=0.333333333333333)
m.b64 = Var(within=Binary,bounds=(0,1),initialize=0.333333333333333)
m.b65 = Var(within=Binary,bounds=(0,1),initialize=0.333333333333333)
m.b66 = Var(within=Binary,bounds=(0,1),initialize=0.333333333333333)
m.b67 = Var(within=Binary,bounds=(0,1),initialize=0.333333333333333)
m.b68 = Var(within=Binary,bounds=(0,1),initialize=0.333333333333333)
m.b69 = Var(within=Binary,bounds=(0,1),initialize=0.333333333333333)
m.b70 = Var(within=Binary,bounds=(0,1),initialize=0.333333333333333)
m.b71 = Var(within=Binary,bounds=(0,1),initialize=0.333333333333333)
m.b72 = Var(within=Binary,bounds=(0,1),initialize=0.333333333333333)
m.b73 = Var(within=Binary,bounds=(0,1),initialize=0.333333333333333)
m.b74 = Var(within=Binary,bounds=(0,1),initialize=0.333333333333333)
m.b75 = Var(within=Binary,bounds=(0,1),initialize=0.333333333333333)
m.x76 = Var(within=Reals,bounds=(0,None),initialize=1.18464727499703)
m.x77 = Var(within=Reals,bounds=(0,None),initialize=2.21055142184158)
m.x78 = Var(within=Reals,bounds=(0,None),initialize=1.19998063005095)
m.x79 = Var(within=Reals,bounds=(0,None),initialize=1.11549458684761)
m.x80 = Var(within=Reals,bounds=(0,None),initialize=1.13890807654545)
m.x81 = Var(within=Reals,bounds=(0,None),initialize=0.18075340102651)
m.x82 = Var(within=Reals,bounds=(0,None),initialize=0.18075340102651)
m.x83 = Var(within=Reals,bounds=(0,None),initialize=0.18075340102651)
m.x84 = Var(within=Reals,bounds=(0,None),initialize=0.229509008619004)
m.x85 = Var(within=Reals,bounds=(0,None),initialize=0.229509008619004)
m.x86 = Var(within=Reals,bounds=(0,None),initialize=0.229509008619004)
m.x87 = Var(within=Reals,bounds=(0,None),initialize=0.181816847787907)
m.x88 = Var(within=Reals,bounds=(0,None),initialize=0.181816847787907)
m.x89 = Var(within=Reals,bounds=(0,None),initialize=0.181816847787907)
m.x90 = Var(within=Reals,bounds=(0,None),initialize=0.175765767145396)
m.x91 = Var(within=Reals,bounds=(0,None),initialize=0.175765767145396)
m.x92 = Var(within=Reals,bounds=(0,None),initialize=0.175765767145396)
m.x93 = Var(within=Reals,bounds=(0,None),initialize=0.177490575531558)
m.x94 = Var(within=Reals,bounds=(0,None),initialize=0.177490575531558)
m.x95 = Var(within=Reals,bounds=(0,None),initialize=0.177490575531558)

m.obj = Objective(expr=   301.899928098152*m.b1 + 282.051473607022*m.b2 + 151.594044960674*m.b3 + 114.784185877557*m.b4
                        + 213.364530716922*m.b5 + 772.653148294131*m.b6 + 697.676211791334*m.b7 + 146.306371684975*m.b8
                        + 390.583393857486*m.b9 + 208.147527440482*m.b10 + 662.892902187869*m.b11
                        + 577.461337631217*m.b12 + 221.10047354739*m.b13 + 425.919826737657*m.b14
                        + 123.074770812851*m.b15 + 333.28129673946*m.b16 + 248.380746723092*m.b17
                        + 249.162942146638*m.b18 + 164.598799150643*m.b19 + 280.957171099846*m.b20
                        + 308.552481034871*m.b21 + 270.059605282374*m.b22 + 104.633483616243*m.b23
                        + 79.6631898566695*m.b24 + 170.696237801571*m.b25 + 237.754076296143*m.b26
                        + 189.862911729786*m.b27 + 107.217531395173*m.b28 + 131.358715293396*m.b29
                        + 103.406777059692*m.b30 + 626.417763832299*m.b31 + 487.184730842973*m.b32
                        + 502.300580630229*m.b33 + 506.426352475088*m.b34 + 463.185748318154*m.b35
                        + 358.178221555384*m.b36 + 281.629247221142*m.b37 + 230.4203839171*m.b38
                        + 251.915433121165*m.b39 + 209.261088879339*m.b40 + 303.899003044044*m.b41
                        + 243.197489456663*m.b42 + 237.390965850675*m.b43 + 57.1385835039462*m.b44
                        + 301.733744039334*m.b45 + 30.6123768510861*m.b46 + 21.3396948414106*m.b47
                        + 278.520865043453*m.b48 + 162.122145724483*m.b49 + 304.508803157003*m.b50
                        + 252.516206195527*m.b51 + 178.796029580139*m.b52 + 319.145634893211*m.b53
                        + 257.755103285795*m.b54 + 317.996864520235*m.b55 + 936.171150833806*m.b56
                        + 887.611963724196*m.b57 + 419.760722838682*m.b58 + 519.981401235063*m.b59
                        + 524.621957902125*m.b60 + 326.37044675*m.b61 + 119.610927362864*m.b62 + 76.800859418795*m.b63
                        + 338.15311375*m.b64 + 113.101546866718*m.b65 + 69.3762358590679*m.b66 + 313.6973235*m.b67
                        + 116.266585440261*m.b68 + 75.0744657614982*m.b69 + 401.4402965*m.b70 + 138.599587312691*m.b71
                        + 86.376825937843*m.b72 + 456.70672375*m.b73 + 150.554161322115*m.b74 + 91.6821859840903*m.b75
                        + 93617.1150833806*m.x76 + 93617.1150833806*m.x77 + 93617.1150833806*m.x78
                        + 93617.1150833806*m.x79 + 93617.1150833806*m.x80, sense=minimize)

m.c2 = Constraint(expr=   0.609376132*m.b1 + 1.180016336*m.b6 + 0.967493052*m.b11 + 1.004918785*m.b16
                        + 0.698898063*m.b21 + 0.540292599*m.b26 + 1.460452986*m.b31 + 0.811980791*m.b36
                        + 0.973180988*m.b41 + 0.544914116*m.b46 + 0.78515855*m.b51 + 1.312281472*m.b56
                        - 2.0080698912*m.x81 - 4.0161397824*m.x82 - 6.0242096736*m.x83 == 0)

m.c3 = Constraint(expr=   0.609376132*m.b2 + 1.180016336*m.b7 + 0.967493052*m.b12 + 1.004918785*m.b17
                        + 0.698898063*m.b22 + 0.540292599*m.b27 + 1.460452986*m.b32 + 0.811980791*m.b37
                        + 0.973180988*m.b42 + 0.544914116*m.b47 + 0.78515855*m.b52 + 1.312281472*m.b57
                        - 1.581486777*m.x84 - 3.162973554*m.x85 - 4.744460331*m.x86 == 0)

m.c4 = Constraint(expr=   0.609376132*m.b3 + 1.180016336*m.b8 + 0.967493052*m.b13 + 1.004918785*m.b18
                        + 0.698898063*m.b23 + 0.540292599*m.b28 + 1.460452986*m.b33 + 0.811980791*m.b38
                        + 0.973180988*m.b43 + 0.544914116*m.b48 + 0.78515855*m.b53 + 1.312281472*m.b58
                        - 1.9963246902*m.x87 - 3.9926493804*m.x88 - 5.9889740706*m.x89 == 0)

m.c5 = Constraint(expr=   0.609376132*m.b4 + 1.180016336*m.b9 + 0.967493052*m.b14 + 1.004918785*m.b19
                        + 0.698898063*m.b24 + 0.540292599*m.b29 + 1.460452986*m.b34 + 0.811980791*m.b39
                        + 0.973180988*m.b44 + 0.544914116*m.b49 + 0.78515855*m.b54 + 1.312281472*m.b59
                        - 2.065052076*m.x90 - 4.130104152*m.x91 - 6.195156228*m.x92 == 0)

m.c6 = Constraint(expr=   0.609376132*m.b5 + 1.180016336*m.b10 + 0.967493052*m.b15 + 1.004918785*m.b20
                        + 0.698898063*m.b25 + 0.540292599*m.b30 + 1.460452986*m.b35 + 0.811980791*m.b40
                        + 0.973180988*m.b45 + 0.544914116*m.b50 + 0.78515855*m.b55 + 1.312281472*m.b60
                        - 2.0449844238*m.x93 - 4.0899688476*m.x94 - 6.1349532714*m.x95 == 0)

m.c7 = Constraint(expr=   m.b1 + m.b2 + m.b3 + m.b4 + m.b5 == 1)

m.c8 = Constraint(expr=   m.b6 + m.b7 + m.b8 + m.b9 + m.b10 == 1)

m.c9 = Constraint(expr=   m.b11 + m.b12 + m.b13 + m.b14 + m.b15 == 1)

m.c10 = Constraint(expr=   m.b16 + m.b17 + m.b18 + m.b19 + m.b20 == 1)

m.c11 = Constraint(expr=   m.b21 + m.b22 + m.b23 + m.b24 + m.b25 == 1)

m.c12 = Constraint(expr=   m.b26 + m.b27 + m.b28 + m.b29 + m.b30 == 1)

m.c13 = Constraint(expr=   m.b31 + m.b32 + m.b33 + m.b34 + m.b35 == 1)

m.c14 = Constraint(expr=   m.b36 + m.b37 + m.b38 + m.b39 + m.b40 == 1)

m.c15 = Constraint(expr=   m.b41 + m.b42 + m.b43 + m.b44 + m.b45 == 1)

m.c16 = Constraint(expr=   m.b46 + m.b47 + m.b48 + m.b49 + m.b50 == 1)

m.c17 = Constraint(expr=   m.b51 + m.b52 + m.b53 + m.b54 + m.b55 == 1)

m.c18 = Constraint(expr=   m.b56 + m.b57 + m.b58 + m.b59 + m.b60 == 1)

m.c19 = Constraint(expr=   m.b61 + m.b62 + m.b63 <= 1)

m.c20 = Constraint(expr=   m.b64 + m.b65 + m.b66 <= 1)

m.c21 = Constraint(expr=   m.b67 + m.b68 + m.b69 <= 1)

m.c22 = Constraint(expr=   m.b70 + m.b71 + m.b72 <= 1)

m.c23 = Constraint(expr=   m.b73 + m.b74 + m.b75 <= 1)

m.c24 = Constraint(expr= - m.b61 + m.x81 <= 0)

m.c25 = Constraint(expr= - m.b62 + m.x82 <= 0)

m.c26 = Constraint(expr= - m.b63 + m.x83 <= 0)

m.c27 = Constraint(expr= - m.b64 + m.x84 <= 0)

m.c28 = Constraint(expr= - m.b65 + m.x85 <= 0)

m.c29 = Constraint(expr= - m.b66 + m.x86 <= 0)

m.c30 = Constraint(expr= - m.b67 + m.x87 <= 0)

m.c31 = Constraint(expr= - m.b68 + m.x88 <= 0)

m.c32 = Constraint(expr= - m.b69 + m.x89 <= 0)

m.c33 = Constraint(expr= - m.b70 + m.x90 <= 0)

m.c34 = Constraint(expr= - m.b71 + m.x91 <= 0)

m.c35 = Constraint(expr= - m.b72 + m.x92 <= 0)

m.c36 = Constraint(expr= - m.b73 + m.x93 <= 0)

m.c37 = Constraint(expr= - m.b74 + m.x94 <= 0)

m.c38 = Constraint(expr= - m.b75 + m.x95 <= 0)

m.c39 = Constraint(expr=m.x81*m.b61 + m.x81*m.x76 - m.x76*m.b61 <= 0)

m.c40 = Constraint(expr=m.x82*m.b62 + m.x82*m.x76 - m.x76*m.b62 <= 0)

m.c41 = Constraint(expr=m.x83*m.b63 + m.x83*m.x76 - m.x76*m.b63 <= 0)

m.c42 = Constraint(expr=m.x84*m.b64 + m.x84*m.x77 - m.x77*m.b64 <= 0)

m.c43 = Constraint(expr=m.x85*m.b65 + m.x85*m.x77 - m.x77*m.b65 <= 0)

m.c44 = Constraint(expr=m.x86*m.b66 + m.x86*m.x77 - m.x77*m.b66 <= 0)

m.c45 = Constraint(expr=m.x87*m.b67 + m.x87*m.x78 - m.x78*m.b67 <= 0)

m.c46 = Constraint(expr=m.x88*m.b68 + m.x88*m.x78 - m.x78*m.b68 <= 0)

m.c47 = Constraint(expr=m.x89*m.b69 + m.x89*m.x78 - m.x78*m.b69 <= 0)

m.c48 = Constraint(expr=m.x90*m.b70 + m.x90*m.x79 - m.x79*m.b70 <= 0)

m.c49 = Constraint(expr=m.x91*m.b71 + m.x91*m.x79 - m.x79*m.b71 <= 0)

m.c50 = Constraint(expr=m.x92*m.b72 + m.x92*m.x79 - m.x79*m.b72 <= 0)

m.c51 = Constraint(expr=m.x93*m.b73 + m.x93*m.x80 - m.x80*m.b73 <= 0)

m.c52 = Constraint(expr=m.x94*m.b74 + m.x94*m.x80 - m.x80*m.b74 <= 0)

m.c53 = Constraint(expr=m.x95*m.b75 + m.x95*m.x80 - m.x80*m.b75 <= 0)