"""
Classes for spectral analysis.
"""

import datetime
from copy import copy
from math import floor
from random import randint
from distutils.version import LooseVersion
from typing import Union, List

import numpy as np
from skimage import filters
from matplotlib import pyplot as plt
from matplotlib.colorbar import Colorbar
from matplotlib.figure import Figure
from matplotlib.ticker import FuncFormatter, IndexLocator, MaxNLocator
from numpy import ma
from scipy import ndimage
from sortedcontainers import SortedList
import ruptures as rpt

from sunpy import __version__
from sunpy.time import parse_time

from radiospectra.spectrum import Spectrum
from radiospectra.util import ConditionalDispatch, Parent, common_base, get_day, merge, to_signed

__all__ = ['Spectrogram', 'LinearTimeSpectrogram']

SUNPY_LT_1 = LooseVersion(__version__) < LooseVersion('1.0')

# 1080 because that usually is the maximum vertical pixel count on modern
# screens nowadays (2012).
DEFAULT_YRES = 1080

# This should not be necessary, as observations do not take more than a day
# but it is used for completeness' and extendibility's sake.
# XXX: Leap second?
SECONDS_PER_DAY = 86400

# Used for COPY_PROPERTIES
REFERENCE = 0
COPY = 1
DEEPCOPY = 2

def figure(*args, **kwargs):
    """
    Returns a new SpectroFigure, a figure extended with features useful for
    analysis of spectrograms.

    Compare pyplot.figure.
    """
    kw = {
        'FigureClass': SpectroFigure,
    }
    kw.update(kwargs)
    return plt.figure(*args, **kw)

def _min_delt(arr):
    deltas = (arr[:-1] - arr[1:])
    # Multiple values at the same frequency are just thrown away
    # in the process of linearizaion
    return deltas[deltas != 0].min()

def _list_formatter(lst, fun=None):
    """
    Returns a function that takes x, pos and returns fun(lst[x]) if fun is not
    None, else lst[x] or "" if x is out of range.
    """

    def _fun(x, pos):
        x = int(x)
        if x >= len(lst) or x < 0:
            return ""

        elem = lst[x]
        if fun is None:
            return elem
        return fun(elem)

    return _fun

def _union(sets):
    """
    Returns a union of sets.
    """
    union = set()
    for s in sets:
        union |= s
    return union

class _LinearView(object):
    """
    Helper class for frequency channel linearization.

    Attributes
    ----------
    arr : Spectrogram
        Spectrogram to linearize.
    delt : float
        Delta between frequency channels in linearized spectrogram. Defaults to
        (minimum delta / 2.) because of the Shannon sampling theorem.
    """

    def __init__(self, arr, delt=None):
        self.arr = arr
        if delt is None:
            # Nyquist–Shannon sampling theorem
            delt = _min_delt(arr.freq_axis) / 2.

        self.delt = delt

        midpoints = (self.arr.freq_axis[:-1] + self.arr.freq_axis[1:]) / 2
        self.midpoints = np.concatenate([midpoints, arr.freq_axis[-1:]])

        self.max_mp_delt = np.min(self.midpoints[1:] - self.midpoints[:-1])

        self.freq_axis = np.arange(
            self.arr.freq_axis[0], self.arr.freq_axis[-1], -self.delt
        )
        self.time_axis = self.arr.time_axis

        self.shape = (len(self), arr.data.shape[1])

    def __len__(self):
        return int(1 + (self.arr.freq_axis[0] - self.arr.freq_axis[-1]) /
                   self.delt)

    def _find(self, arr, item):
        if item < 0:
            item = item % len(self)
        if item >= len(self):
            raise IndexError

        freq_offset = item * self.delt
        freq = self.arr.freq_axis[0] - freq_offset
        # The idea is that when we take the biggest delta in the mid points,
        # we do not have to search anything that is between the beginning and
        # the first item that can possibly be that frequency.
        min_mid = int(max(0, (freq - self.midpoints[0]) // self.max_mp_delt))
        for n, mid in enumerate(self.midpoints[min_mid:]):
            if mid <= freq:
                return arr[min_mid + n]
        return arr[min_mid + n]

    def __getitem__(self, item):
        return self._find(self.arr, item)

    def get_freq(self, item):
        return self._find(self.arr.freq_axis, item)

    def make_mask(self, max_dist):
        mask = np.zeros(self.shape, dtype=np.bool)
        for n, item in enumerate(range(len(self))):
            freq = self.arr.freq_axis[0] - item * self.delt
            if abs(self.get_freq(item) - freq) > max_dist:
                mask[n, :] = True
        return mask

class SpectroFigure(Figure):
    def _init(self, data, freqs):
        self.data = data
        self.freqs = freqs

    def ginput_to_time(self, inp):
        return [
            self.data.start + datetime.timedelta(seconds=secs)
            for secs in self.ginput_to_time_secs(inp)
        ]

    def ginput_to_time_secs(self, inp):
        return np.array([float(self.data.time_axis[x]) for x, y in inp])

    def ginput_to_time_offset(self, inp):
        v = self.ginput_to_time_secs(inp)
        return v - v.min()

    def ginput_to_freq(self, inp):
        return np.array([self.freqs[y] for x, y in inp])

    def time_freq(self, points=0):
        inp = self.ginput(points)
        min_ = self.ginput_to_time_secs(inp).min()
        start = self.data.start + datetime.timedelta(seconds=min_)
        return TimeFreq(
            start, self.ginput_to_time_offset(inp), self.ginput_to_freq(inp)
        )

class TimeFreq(object):
    """
    Class to use for plotting frequency vs time.

    Attributes
    ----------
    start : `datetime.datetime`
        Start time of the plot.
    time : `~numpy.ndarray`
        Time of the data points as offset from start in seconds.
    freq : `~numpy.ndarray`
        Frequency of the data points in MHz.
    """

    def __init__(self, start, time, freq):
        self.start = start
        self.time = time
        self.freq = freq

    def plot(self, time_fmt="%H:%M:%S", **kwargs):
        """
        Plot the spectrum.

        Parameters
        ----------
        time_fmt : str
            The time format in a `~datetime.datetime` compatible format

        **kwargs : dict
            Any additional plot arguments that should be used
            when plotting.

        Returns
        -------
        fig : `~matplotlib.Figure`
            A plot figure.
        """
        figure = plt.gcf()
        axes = figure.add_subplot(111)
        axes.plot(self.time, self.freq, **kwargs)
        xa = axes.get_xaxis()
        xa.set_major_formatter(
            FuncFormatter(
                lambda x, pos: (
                    self.start + datetime.timedelta(seconds=x)
                ).strftime(time_fmt)
            )
        )

        axes.set_xlabel("Time [UT]")
        axes.set_ylabel("Frequency [MHz]")

        xa = axes.get_xaxis()
        for tl in xa.get_ticklabels():
            tl.set_fontsize(10)
            tl.set_rotation(30)
        figure.add_axes(axes)
        figure.subplots_adjust(bottom=0.2)
        figure.subplots_adjust(left=0.2)

        return figure

    def peek(self, *args, **kwargs):
        """
        Plot spectrum onto current axes.

        Parameters
        ----------
        *args : dict

        **kwargs : dict
            Any additional plot arguments that should be used
            when plotting.

        Returns
        -------
        fig : `~matplotlib.Figure`
            A plot figure.
        """
        plt.figure()
        ret = self.plot(*args, **kwargs)
        plt.show()
        return ret

class Spectrogram(Parent):
    """
    Spectrogram Class.

    .. warning:: This module is under development! Use at your own risk.

    Attributes
    ----------
    data : `~numpy.ndarray`
        two-dimensional array of the image data of the spectrogram.
    time_axis : `~numpy.ndarray`
        one-dimensional array containing the offset from the start
        for each column of data.
    freq_axis : `~numpy.ndarray`
        one-dimensional array containing information about the
        frequencies each row of the image corresponds to.
    start : `~datetime.datetime`
        starting time of the measurement
    end : `~datetime.datetime`
        end time of the measurement
    t_init : int
        offset from the start of the day the measurement began. If None
        gets automatically set from start.
    t_label : str
        label for the time axis
    f_label : str
        label for the frequency axis
    content : str
        header for the image
    instruments : str array
        instruments that recorded the data, may be more than one if
        it was constructed using combine_frequencies or join_many.
    """
    # Contrary to what pylint may think, this is not an old-style class.
    # pylint: disable=E1002,W0142,R0902

    # This needs to list all attributes that need to be
    # copied to maintain the object and how to handle them.
    COPY_PROPERTIES = [
        ('time_axis', COPY),
        ('freq_axis', COPY),
        ('instruments', COPY),
        ('start', REFERENCE),
        ('end', REFERENCE),
        ('t_label', REFERENCE),
        ('f_label', REFERENCE),
        ('content', REFERENCE),
        ('t_init', REFERENCE),
    ]
    _create = ConditionalDispatch.from_existing(Parent._create)

    @property
    def shape(self):
        return self.data.shape

    @property
    def dtype(self):
        return self.data.dtype

    def _get_params(self):
        """
        Implementation detail.
        """
        return {
            name: getattr(self, name) for name, _ in self.COPY_PROPERTIES
        }

    def _slice(self, y_range, x_range):
        """
        Return new spectrogram reduced to the values passed as slices.

        Implementation detail.
        """
        data = self.data[y_range, x_range]
        params = self._get_params()

        soffset = 0 if x_range.start is None else x_range.start
        soffset = int(soffset)
        eoffset = self.shape[1] if x_range.stop is None else x_range.stop  # pylint: disable=E1101
        eoffset -= 1
        eoffset = int(eoffset)

        params.update({
            'time_axis': self.time_axis[
                         x_range.start:x_range.stop:x_range.step
                         ] - self.time_axis[soffset],
            'freq_axis': self.freq_axis[
                         y_range.start:y_range.stop:y_range.step],
            'start': self.start + datetime.timedelta(
                seconds=self.time_axis[soffset]),
            'end': self.start + datetime.timedelta(
                seconds=self.time_axis[eoffset]),
            't_init': self.t_init + self.time_axis[soffset],
        })
        return self.__class__(data, **params)

    def _with_data(self, data):
        new = copy(self)
        new.data = data
        return new

    def __init__(self, data, time_axis, freq_axis, start, end, t_init=None,
                 t_label="Time", f_label="Frequency", content="",
                 instruments=None):
        # Because of how object creation works, there is no avoiding
        # unused arguments in this case.
        self.data = data

        if t_init is None:
            diff = start - get_day(start)
            t_init = diff.seconds
        if instruments is None:
            instruments = set()

        self.start = start
        self.end = end

        self.t_label = t_label
        self.f_label = f_label

        self.t_init = t_init

        self.time_axis = time_axis
        self.freq_axis = freq_axis
        self.rfi_freq_axis = np.array([])

        self.content = content
        self.instruments = instruments

    def time_formatter(self, x, pos):
        """
        This returns the label for the tick of value x at a specified pos on
        the time axis.
        """
        # Callback, cannot avoid unused arguments.
        # pylint: disable=W0613
        x = int(x)
        if x >= len(self.time_axis) or x < 0:
            return ""
        return self.format_time(
            self.start + datetime.timedelta(
                seconds=float(self.time_axis[x])
            )
        )

    @staticmethod
    def format_time(time):
        """
        Override to configure default plotting.
        """
        return time.strftime("%H:%M:%S")

    @staticmethod
    def format_freq(freq):
        """
        Override to configure default plotting.
        """
        return "{freq:0.1f}".format(freq=freq)

    def peek(self, *args, **kwargs):
        """
        Plot spectrum onto current axes.

        Parameters
        ----------
        *args : dict

        **kwargs : dict
            Any additional plot arguments that should be used
            when plotting.

        Returns
        -------
        fig : `~matplotlib.Figure`
            A plot figure.
        """
        figure()
        ret = self.plot(*args, **kwargs)
        plt.show()
        return ret

    def plot(self, figure=None, overlays=[], colorbar=True, vmin=None,
             vmax=None, linear=True, showz=True, yres=DEFAULT_YRES,
             max_dist=None, **matplotlib_args):
        """
        Plot spectrogram onto figure.

        Parameters
        ----------
        figure : `~matplotlib.Figure`
            Figure to plot the spectrogram on. If None, new Figure is created.
        overlays : list
            List of overlays (functions that receive figure and axes and return
            new ones) to be applied after drawing.
        colorbar : bool
            Flag that determines whether or not to draw a colorbar. If existing
            figure is passed, it is attempted to overdraw old colorbar.
        vmin : float
            Clip intensities lower than vmin before drawing.
        vmax : float
            Clip intensities higher than vmax before drawing.
        linear : bool
            If set to True, "stretch" image to make frequency axis linear.
        showz : bool
            If set to True, the value of the pixel that is hovered with the
            mouse is shown in the bottom right corner.
        yres : int or None
            To be used in combination with linear=True. If None, sample the
            image with half the minimum frequency delta. Else, sample the
            image to be at most yres pixels in vertical dimension. Defaults
            to 1080 because that's a common screen size.
        max_dist : float or None
            If not None, mask elements that are further than max_dist away
            from actual data points (ie, frequencies that actually have data
            from the receiver and are not just nearest-neighbour interpolated).
        """
        # [] as default argument is okay here because it is only read.
        # pylint: disable=W0102,R0914
        if linear:
            delt = yres
            if delt is not None:
                delt = max(
                    (self.freq_axis[0] - self.freq_axis[-1]) / (yres - 1),
                    _min_delt(self.freq_axis) / 2.
                )
                delt = float(delt)

            data = _LinearView(self.clip_values(vmin, vmax), delt)
            freqs = np.arange(
                self.freq_axis[0], self.freq_axis[-1], -data.delt
            )
        else:
            data = np.array(self.clip_values(vmin, vmax))
            freqs = self.freq_axis

        figure = plt.gcf()

        if figure.axes:
            axes = figure.axes[0]
        else:
            axes = figure.add_subplot(111)

        params = {
            'origin': 'lower',
            'aspect': 'auto',
        }
        params.update(matplotlib_args)
        if linear and max_dist is not None:
            toplot = ma.masked_array(data, mask=data.make_mask(max_dist))
        else:
            toplot = data
        im = axes.imshow(toplot, **params)

        xa = axes.get_xaxis()
        ya = axes.get_yaxis()

        xa.set_major_formatter(
            FuncFormatter(self.time_formatter)
        )

        if linear:
            # Start with a number that is divisible by 5.
            init = (self.freq_axis[0] % 5) / data.delt
            nticks = 15.
            # Calculate MHz difference between major ticks.
            dist = (self.freq_axis[0] - self.freq_axis[-1]) / nticks
            # Round to next multiple of 10, at least ten.
            dist = max(round(dist, -1), 10)
            # One pixel in image space is data.delt MHz, thus we can convert
            # our distance between the major ticks into image space by dividing
            # it by data.delt.

            ya.set_major_locator(
                IndexLocator(
                    dist / data.delt, init
                )
            )
            ya.set_minor_locator(
                IndexLocator(
                    dist / data.delt / 10, init
                )
            )

            def freq_fmt(x, pos):
                # This is necessary because matplotlib somehow tries to get
                # the mid-point of the row, which we do not need here.
                x = x + 0.5
                return self.format_freq(self.freq_axis[0] - x * data.delt)
        else:
            freq_fmt = _list_formatter(freqs, self.format_freq)
            ya.set_major_locator(MaxNLocator(integer=True, steps=[1, 5, 10]))

        ya.set_major_formatter(
            FuncFormatter(freq_fmt)
        )

        axes.set_xlabel(self.t_label)
        axes.set_ylabel(self.f_label)
        # figure.suptitle(self.content)

        figure.suptitle(
            ' '.join([
                get_day(self.start).strftime("%d %b %Y"),
                'Radio flux density',
                '(' + ', '.join(self.instruments) + ')',
            ])
        )

        for tl in xa.get_ticklabels():
            tl.set_fontsize(10)
            tl.set_rotation(30)
        figure.add_axes(axes)
        figure.subplots_adjust(bottom=0.2)
        figure.subplots_adjust(left=0.2)

        if showz:
            axes.format_coord = self._mk_format_coord(
                data, figure.gca().format_coord)

        if colorbar:
            if len(figure.axes) > 1:
                Colorbar(figure.axes[1], im).set_label("Intensity")
            else:
                figure.colorbar(im).set_label("Intensity")

        for overlay in overlays:
            figure, axes = overlay(figure, axes)

        for ax in figure.axes:
            ax.autoscale()
        if isinstance(figure, SpectroFigure):
            figure._init(self, freqs)
        return axes

    def __getitem__(self, key):
        only_y = not isinstance(key, tuple)

        if only_y:
            return self.data[int(key)]
        elif isinstance(key[0], slice) and isinstance(key[1], slice):
            return self._slice(key[0], key[1])
        elif isinstance(key[1], slice):
            # return Spectrum( # XXX: Right class
            #     super(Spectrogram, self).__getitem__(key),
            #     self.time_axis[key[1].start:key[1].stop:key[1].step]
            # )
            return np.array(self.data[key])
        elif isinstance(key[0], slice):
            return Spectrum(
                self.data[key],
                self.freq_axis[key[0].start:key[0].stop:key[0].step]
            )

        return self.data[int(key)]

    def clip_freq(self, vmin=None, vmax=None):
        """
        Return a new spectrogram only consisting of frequencies in the interval
        [vmin, vmax].

        Parameters
        ----------
        vmin : float
            All frequencies in the result are greater or equal to this.
        vmax : float
            All frequencies in the result are smaller or equal to this.
        """
        left = 0
        if vmax is not None:
            while self.freq_axis[left] > vmax:
                left += 1

        right = len(self.freq_axis) - 1

        if vmin is not None:
            while self.freq_axis[right] < vmin:
                right -= 1

        return self[left:right + 1, :]

    def auto_find_background(self, amount=0.05):
        """
        Automatically find the background. This is done by first subtracting
        the average value in each channel and then finding those times which
        have the lowest standard deviation.

        Parameters
        ----------
        amount : float
            The percent amount (out of 1) of lowest standard deviation to
            consider.
        """
        # pylint: disable=E1101,E1103
        data = self.data.astype(to_signed(self.dtype))
        # Subtract average value from every frequency channel.
        tmp = (data - np.average(self.data, 1).reshape(self.shape[0], 1))
        # Get standard deviation at every point of time.
        # Need to convert because otherwise this class's __getitem__
        # is used which assumes two-dimensionality.
        sdevs = np.asarray(np.std(tmp, 0))

        # Get indices of values with lowest standard deviation.
        cand = sorted(list(range(self.shape[1])), key=lambda y: sdevs[y])
        # Only consider the best 5 %.
        return cand[:max(1, int(amount * len(cand)))]

    def auto_const_bg(self):
        """
        Automatically determine background.
        """
        realcand = self.auto_find_background()
        bg = np.average(self.data[:, realcand], 1)
        return bg.reshape(self.shape[0], 1)

    def subtract_bg(self, *args, **kwargs):
        """
        Perform background subtraction, with the opportunity to choose between different procedures
        and the ability to remove the radio frequency interference (RFI).

        default:
        Default background subtraction of radiospectra by using the "auto_const_bg()" function

        constbacksub:
        Background subtraction method where the average and the standard deviation of each row
        will be calculated and subtracted from the image.

        subtract_bg_sliding_window:
        Performs background subtraction with the possibility of having a sliding window and changing points.

        glid_back_sub:
        A gliding background subtraction method, where the sum weighted from the
        coefficients of the evenly spaces values of each row will be subtracted from the spectrogram.

        elimwrongchannels:
        Removing the RFI (radio frequency interference) from the spectrogram.

        Parameters
        ----------
        *args:
        List of desired methods that should be called.

        **kwargs:
        List of arguments that should be passed to a subfunction

        """
        spec = copy(self)
        for arg in args:
            if arg == "default":
                # default background subtraction of radiospectra
                spec = spec._with_data(spec.data - spec.auto_const_bg())

            elif arg == "constbacksub":
                spec = spec.constbacksub(overwrite=False)

            elif arg == "subtract_bg_sliding_window":
                _sbg, _bg, _min_sdevs, _cps = spec.subtract_bg_sliding_window(**kwargs)
                spec = _sbg

            elif arg == "glid_back_sub":
                spec = spec.glid_back_sub(overwrite=False)

            elif arg == "elimwrongchannels":
                spec = spec.elimwrongchannels(overwrite=False)

        if len(args) == 0:
            # default background subtraction of radiospectra
            spec = spec._with_data(spec.data - spec.auto_const_bg())

        return spec

    def subtract_bg_sliding_window(self, amount: float = 0.05, window_width: int = 0, affected_width: int = 0,
                                   change_points: Union[bool, List[int]] = False):
        """
        Performs background subtraction with a sliding window. Change points where significant jumps in
        value are observed or expected can be specified or automatically estimated. If change points are present
        each one will split the spectrogram and each resulting part will have its background removed independently.

        Parameters
        ----------
        amount : float
        The percent amount (out of 1) of lowest standard deviation to consider.

        window_width : int
        The width of the sliding window that is used to calculate the background.

        affected_width : int
        The width of the section where the background calculated by the window_width gets subtracted.
        It is centered in the sliding window and is also the step size.

        change_points : list of int or bool
        If a list of ints is provided it will use these values as change points. If a bool is
        provided it will estimate the change points if true or won't include any change points if false.
        """

        _data = self.data.copy()

        _og_image_height = _data.shape[0]
        _og_image_width = _data.shape[1]

        _bg = np.zeros([_og_image_height, _og_image_width])
        _min_sdevs = np.zeros([_og_image_height, _og_image_width])
        _out = _data.copy()

        if isinstance(change_points, bool):
            if change_points:
                _cps = self.estimate_change_points()
            else:
                _cps = []
        else:
            _cps = change_points

        if len(_cps) == 0:
            _images = [(0, _og_image_width)]
        else:
            _images = []
            _temp = 0
            _cps = sorted(_cps)
            for _cp in _cps:
                _images.append((_temp, _cp))
                _temp = _cp
            _images.append((_temp, _og_image_width))

        for (_img_start, _img_end) in _images:

            _cwp = _img_start

            _img_width = _img_end - _img_start
            _img_data = _data[:, _img_start:_img_end]

            _window_height = _og_image_height
            _window_width = _img_width if (window_width == 0 or window_width > _img_width) else window_width
            _affected_height = _og_image_height
            _affected_width = _img_width if (affected_width == 0 or affected_width > _img_width) else (
                affected_width if affected_width <= _window_width else _window_width)

            _data_minus_avg = (_img_data - np.average(_img_data, 1).reshape(_img_data.shape[0], 1))
            _sdevs = [(index, std) for (index, std) in enumerate(np.std(_data_minus_avg, 0))]

            _half = max((_window_width - _affected_width) // 2, 0)
            _division_fix = _half + _half != max(_img_width - _affected_width, 0)
            _max_amount = max(1, int(amount * _img_width))

            # calc initial set of used columns
            _window_sdevs = [sdev for sdev in _sdevs[:_half]]
            _sorted_sdevs = sorted(_window_sdevs, key=lambda y: y[1])
            _bg_used_sdevs = SortedList(_sorted_sdevs, key=lambda y: y[1])

            while _cwp <= _img_end:

                _affected_left = _cwp
                _affected_right = min(_affected_left + _affected_width, _img_end)
                _window_left = max(_affected_left - _half if _division_fix else _affected_left - _half, _img_start)
                _window_right = min(_affected_right + _half, _img_end)

                for sdev in _sdevs[max(_window_left - _affected_width, 0) - _img_start:_window_left - _img_start]:
                    _bg_used_sdevs.discard(sdev)

                if _window_right <= _img_end:
                    _bg_used_sdevs.update(
                        _sdevs[_window_right - _affected_width - _img_start:_window_right - _img_start])

                # calc current background
                _current_background = np.average(_img_data[:, [sdev[0] for sdev in _bg_used_sdevs[:_max_amount]]], 1)
                for sdev in _bg_used_sdevs[:_max_amount]:
                    _min_sdevs[:, sdev[0] + _img_start] += 1
                _bg[:, _affected_left:_affected_right] = np.repeat(_current_background.reshape(_bg.shape[0], 1),
                                                                   (_affected_right - _affected_left), axis=1)

                _cwp += _affected_width

        _sbg = np.ma.subtract(_out, _bg)
        return self._with_data(_sbg), self._with_data(_bg), self._with_data(_min_sdevs), _cps

    def estimate_change_points(self, window_width=100, max_length_single_segment=20000, segment_width=10000):
        """
        Estimates the change points of the spectrogram and returns the indices.
        If the spectrogram is too big it will get segmented.
        These segments will overlap by 2*window_width to not miss change points where the spectrogram is segmented.

        Parameters
        ----------
        window_width : int
            width of the sliding window
        max_length_single_segment : int
            max width of data array. If bigger it will get segmented because of memory reasons
        segment_width : int
            width of the segments if the CallistoSpectrogram gets segmented
        """
        avgs = np.average(self.data, axis=0)
        penalty = np.log(len(avgs)) * 8 * np.std(avgs) ** 2
        changepoints = set()

        if len(avgs) > max_length_single_segment:
            num = len(avgs) // (segment_width - window_width * 2)
            segments = [(x, x + segment_width) for x in np.multiply(range(0, num), (segment_width - window_width * 2))]
            segments.append((len(avgs) - segment_width, len(avgs)))
        else:
            segments = [(0, len(avgs))]

        if (len(segments)) > 3:
            m = 'mahalanobis'
        else:
            m = 'rbf'

        for start, end in segments:
            mod = rpt.Window(model=m, width=window_width).fit(avgs[start:end])
            res = np.array(mod.predict(pen=penalty)[:-1]) + start
            changepoints.update(res)
        return sorted(changepoints)

    def constbacksub(self, overwrite=True):
        """
        Background subtraction method where the average and the standard deviation of each row
        will be calculated and subtracted from the image.

        Parameters
        ----------
        overwrite : bool
            If function constbacksub has been called directly, there will be a possibility to overwrite it's current
            spectrogram data
        """
        im = self.data.copy()

        nx = im.shape[0]
        ny = im.shape[1]

        average_arr = np.average(im, axis=1)
        for i in range(nx):
            im[i, :] = im[i, :] - average_arr[i]

        sdev_arr = np.zeros(ny)
        for i in range(ny):
            sdev_arr[i] = np.std(im[:, i])

        zist = np.argsort(sdev_arr)
        nPart = int(ny * 0.05)
        zist = zist[0: nPart + 1]
        bkgArr = im[:, zist]

        background = np.average(bkgArr, axis=1)

        for j in range(ny):
            im[:, j] = im[:, j] - background

        if overwrite:
            self.data = im

        return self._with_data(im)

    def elimwrongchannels(self, overwrite=True):
        """
        Removing the RFI (radio frequency interference) from the spectrogram

        Parameters
        ----------
        overwrite : bool
            If function elimwrongchannels has been called directly, there will be a possibility to overwrite it's current
            spectrogram data
        """
        im = self.data.copy()
        freq_axis = self.freq_axis.copy()
        rfi_freq_axis = self.rfi_freq_axis.copy()
        ny = len(im.data[:, 0])

        # optimize std part
        std = np.ndarray((ny,), dtype=np.double)
        for i in range(ny):
            # Calculate std for each row and ignore nan's
            std[i] = np.nanstd(im[i, :].data)

        # Byte scale
        std = ((std - np.nanmin(std)) * 255) / (np.nanmax(std) - np.nanmin(std))
        std[std > 255] = 255
        np.nan_to_num(std, copy=False)
        std = std.astype(int)

        mean_sigma = np.average(std)
        positions = std < 5 * mean_sigma
        zist = std[positions]
        eliminated_channels = len(std) - len(zist)
        print(str(eliminated_channels) + " channels eliminated")

        rfi_freq_axis = self.update_rfi_header(freq_axis, rfi_freq_axis, positions)
        if np.sum(zist) != -1 and eliminated_channels != 0:
            im = im[positions, :]
            freq_axis = freq_axis[positions]

        print("Eliminating sharp jumps between channels ...")
        y_profile = np.average((filters.roberts(im.astype(float)).astype(float)), 1)
        masked_data = np.ma.masked_array(y_profile, np.isnan(y_profile))
        y_profile = masked_data - np.ma.average(masked_data)
        mean_sigma = np.std(y_profile)

        positions = np.abs(y_profile) < 2 * mean_sigma
        zist = y_profile[positions]
        eliminated_channels = len(y_profile) - len(zist)
        print(str(eliminated_channels) + " channels eliminated")

        rfi_freq_axis = self.update_rfi_header(freq_axis, rfi_freq_axis, positions)
        if np.sum(zist) != -1 and eliminated_channels != 0:
            im = im[positions, :]
            freq_axis = freq_axis[positions]

        if np.sum(zist) == -1:
            print("Sorry, all channels are bad ...")
            im = 0

        if overwrite:
            self.data = im
            self.freq_axis = freq_axis
            self.rfi_freq_axis = rfi_freq_axis

        spec = self._with_data(im)
        spec.freq_axis = freq_axis
        spec.rfi_freq_axis = rfi_freq_axis
        return spec

    def update_rfi_header(self, freq_axis, rfi_freq_axis, positions):
        for curr_position, valid in enumerate(positions):
            if not valid:
                curr_freq = freq_axis[curr_position]
                idx = rfi_freq_axis.searchsorted(curr_freq)
                rfi_freq_axis = np.concatenate((rfi_freq_axis[:idx], [curr_freq], rfi_freq_axis[idx:]))
        return rfi_freq_axis

    def glid_back_sub(self, window_width=0, weighted=False, overwrite=True):
        """
        A gliding background subtraction method, where the sum weighted from the
        coefficients of the evenly spaces values of each row will be subtracted from the spectrogram.

        Parameters
        ----------
        window_width: int
            The width of the sliding window that is used to calculate the background.
        weighted : bool
            If true, the coefficients of each rows will be considered dependent on the image width.
        overwrite : bool
            If function glid_back_sub has been called directly, there will be a possibility to overwrite it's current
            spectrogram data
        """
        image = self.data.copy()

        nx = len(image[0, :])
        ny = len(image[:, 0])

        if window_width == 0:
            w_len_half = int(image.shape[1] / 2)
        else:
            w_len_half = int(window_width / 2)
        w_len = w_len_half * 2

        backgr = np.zeros((ny, nx), dtype=np.float)

        # i is of type float!
        i = 0

        if weighted:
            coeffs = w_len_half - np.arange(0, w_len_half, dtype=np.float)
            sum = np.sum(coeffs)
            while i < w_len_half:
                img_data = image[:, 0:w_len_half + i].flatten()
                min_array_size = min(coeffs.size, img_data.size)
                coeffs_sum = np.multiply(img_data[0:min_array_size], coeffs[0:min_array_size])
                backgr[:, i] = np.sum(coeffs_sum) / sum

                img_data = image[:, nx - i - 1:nx].flatten()
                flipped_coeffs = coeffs[::-1]
                min_array_size = min(flipped_coeffs.size, img_data.size)
                coeffs_sum = np.multiply(img_data[0:min_array_size], flipped_coeffs[0:min_array_size])

                backgr[:, nx - i - 1] = np.sum(coeffs_sum) / sum
                i += 1
                coeffs = np.insert(coeffs, 0, w_len_half - i)
                sum = sum + coeffs[i]

            while i < nx - w_len_half:
                img_data = image[:, i - w_len_half:i + w_len_half].flatten()
                min_array_size = min(coeffs.size, img_data.size)
                coeffs_sum = np.multiply(img_data[0:min_array_size], coeffs[0:min_array_size])
                backgr[:, i] = np.sum(coeffs_sum) / sum
                i += 1
        else:
            while i < w_len_half + 1:
                backgr[:, i] = np.average(image[:, 0:min(i + w_len_half, nx)] + 1, 1)
                i += 1

            i = w_len_half + 1
            while i < nx - w_len_half:
                backgr[:, i] = backgr[:, i - 1] + (image[:, i + w_len_half] - image[:, i - 1 - w_len_half]) / w_len
                i += 1

            while i <= nx - 1:
                backgr[:, i] = np.average(image[:, i:nx], 1)
                i += 1

        if overwrite:
            self.data = image

        return self._with_data(image - backgr)

    def randomized_auto_const_bg(self, amount):
        """
        Automatically determine background. Only consider a randomly chosen
        subset of the image.

        Parameters
        ----------
        amount : int
            Size of random sample that is considered for calculation of
            the background.
        """
        cols = [randint(0, self.shape[1] - 1) for _ in range(amount)]

        # pylint: disable=E1101,E1103
        data = self.data.astype(to_signed(self.dtype))
        # Subtract average value from every frequency channel.
        tmp = (data - np.average(self.data, 1).reshape(self.shape[0], 1))
        # Get standard deviation at every point of time.
        # Need to convert because otherwise this class's __getitem__
        # is used which assumes two-dimensionality.
        tmp = tmp[:, cols]
        sdevs = np.asarray(np.std(tmp, 0))

        # Get indices of values with lowest standard deviation.
        cand = sorted(list(range(amount)), key=lambda y: sdevs[y])
        # Only consider the best 5 %.
        realcand = cand[:max(1, int(0.05 * len(cand)))]

        # Average the best 5 %
        bg = np.average(self[:, [cols[r] for r in realcand]], 1)

        return bg.reshape(self.shape[0], 1)

    def randomized_subtract_bg(self, amount):
        """
        Perform randomized constant background subtraction. Does not produce
        the same result every time it is run.

        Parameters
        ----------
        amount : int
            Size of random sample that is considered for calculation of
            the background.
        """
        return self._with_data(self.data - self.randomized_auto_const_bg(amount))

    def clip_values(self, vmin=None, vmax=None, out=None):
        """
        Clip intensities to be in the interval [vmin, vmax].

        Any values greater than the maximum will be assigned the maximum,
        any values lower than the minimum will be assigned the minimum.
        If either is left out or None, do not clip at that side of the interval.

        Parameters
        ----------
        min : int or float
            New minimum value for intensities.
        max : int or float
            New maximum value for intensities
        """
        # pylint: disable=E1101
        if vmin is None:
            vmin = int(self.data.min())

        if vmax is None:
            vmax = int(self.data.max())

        return self._with_data(self.data.clip(vmin, vmax, out))

    def rescale(self, vmin=0, vmax=1, dtype=np.dtype('float32')):
        """
        Rescale intensities to [vmin, vmax]. Note that vmin ≠ vmax and
        spectrogram.min() ≠ spectrogram.max().

        Parameters
        ----------
        vmin : float or int
            New minimum value in the resulting spectrogram.
        vmax : float or int
            New maximum value in the resulting spectrogram.
        dtype : `numpy.dtype`
            Data-type of the resulting spectrogram.
        """
        if vmax == vmin:
            raise ValueError("Maximum and minimum must be different.")
        if self.data.max() == self.data.min():
            raise ValueError("Spectrogram needs to contain distinct values.")
        data = self.data.astype(dtype)  # pylint: disable=E1101
        return self._with_data(
            vmin + (vmax - vmin) * (data - self.data.min()) /  # pylint: disable=E1101
            (self.data.max() - self.data.min())  # pylint: disable=E1101
        )

    def interpolate(self, frequency):
        """
        Linearly interpolate intensity at unknown frequency using linear
        interpolation of its two neighbours.

        Parameters
        ----------
        frequency : float or int
            Unknown frequency for which to linearly interpolate the intensities.
            freq_axis[0] >= frequency >= self_freq_axis[-1]
        """
        lfreq, lvalue = None, None
        for freq, value in zip(self.freq_axis, self.data[:, :]):
            if freq < frequency:
                break
            lfreq, lvalue = freq, value
        else:
            raise ValueError("Frequency not in interpolation range")
        if lfreq is None:
            raise ValueError("Frequency not in interpolation range")
        diff = frequency - freq  # pylint: disable=W0631
        ldiff = lfreq - frequency
        return (ldiff * value + diff * lvalue) / (diff + ldiff)  # pylint: disable=W0631

    def linearize_freqs(self, delta_freq=None):
        """
        Rebin frequencies so that the frequency axis is linear.

        Parameters
        ----------
        delta_freq : float
            Difference between consecutive values on the new frequency axis.
            Defaults to half of smallest delta in current frequency axis.
            Compare Nyquist-Shannon sampling theorem.
        """
        if delta_freq is None:
            # Nyquist–Shannon sampling theorem
            delta_freq = _min_delt(self.freq_axis) / 2.
        nsize = int((self.freq_axis.max() - self.freq_axis.min()) /
                    delta_freq + 1)
        new = np.zeros((int(nsize), self.shape[1]), dtype=self.data.dtype)

        freqs = self.freq_axis - self.freq_axis.max()
        freqs = freqs / delta_freq

        midpoints = np.round((freqs[:-1] + freqs[1:]) / 2)
        fillto = np.concatenate(
            [midpoints - 1, np.round([freqs[-1]]) - 1]
        )
        fillfrom = np.concatenate(
            [np.round([freqs[0]]), midpoints - 1]
        )

        fillto = np.abs(fillto)
        fillfrom = np.abs(fillfrom)

        for row, from_, to_ in zip(self, fillfrom, fillto):
            new[int(from_): int(to_)] = row

        vrs = self._get_params()
        vrs.update({
            'freq_axis': np.linspace(
                self.freq_axis.max(), self.freq_axis.min(), nsize
            )
        })

        return self.__class__(new, **vrs)

    def freq_overlap(self, other):
        """
        Get frequency range present in both spectrograms. Returns (min, max)
        tuple.

        Parameters
        ----------
        other : Spectrogram
            other spectrogram with which to look for frequency overlap
        """
        lower = max(self.freq_axis[-1], other.freq_axis[-1])
        upper = min(self.freq_axis[0], other.freq_axis[0])
        if lower > upper:
            raise ValueError("No overlap.")
        return lower, upper

    def time_to_x(self, time):
        """
        Return x-coordinate in spectrogram that corresponds to the passed
        `~datetime.datetime` value.

        Parameters
        ----------
        time : `~sunpy.time.parse_time` compatible str
            `~datetime.datetime` to find the x coordinate for.
        """
        diff = time - self.start
        diff_s = SECONDS_PER_DAY * diff.days + diff.seconds
        if self.time_axis[-1] < diff_s < 0:
            raise ValueError("Out of bounds")
        for n, elem in enumerate(self.time_axis):
            if diff_s < elem:
                return n - 1
        # The last element is the searched one.
        return n

    def at_freq(self, freq):
        return self[np.nonzero(self.freq_axis == freq)[0], :]

    @staticmethod
    def _mk_format_coord(spec, fmt_coord):
        def format_coord(x, y):
            shape = list(map(int, spec.shape))

            xint, yint = int(x), int(y)
            if 0 <= xint < shape[1] and 0 <= yint < shape[0]:
                pixel = spec[yint][xint]
            else:
                pixel = ""

            return '{!s} z={!s}'.format(fmt_coord(x, y), pixel)

        return format_coord

class LinearTimeSpectrogram(Spectrogram):
    """
    Spectrogram evenly sampled in time.

    Attributes
    ----------
    t_delt : float
        difference between the items on the time axis
    """
    # pylint: disable=E1002
    COPY_PROPERTIES = Spectrogram.COPY_PROPERTIES + [
        ('t_delt', REFERENCE),
    ]

    def __init__(self, data, time_axis, freq_axis, start, end,
                 t_init=None, t_delt=None, t_label="Time", f_label="Frequency",
                 content="", instruments=None):
        if t_delt is None:
            t_delt = _min_delt(freq_axis)

        super(LinearTimeSpectrogram, self).__init__(
            data, time_axis, freq_axis, start, end, t_init, t_label, f_label,
            content, instruments
        )
        self.t_delt = t_delt

    @staticmethod
    def make_array(shape, dtype=np.dtype('float32')):
        """
        Function to create an array with shape and dtype.

        Parameters
        ----------
        shape : tuple
            shape of the array to create
        dtype : `numpy.dtype`
            data-type of the array to create
        """
        return np.zeros(shape, dtype=dtype)

    @staticmethod
    def memmap(filename):
        """
        Return function that takes shape and dtype and returns a memory mapped
        array.

        Parameters
        ----------
        filename : str
            File to store the memory mapped array in.
        """
        return (
            lambda shape, dtype=np.dtype('float32'): np.memmap(
                filename, mode="write", shape=shape, dtype=dtype
            )
        )

    def resample_time(self, new_delt):
        """
        Rescale image so that the difference in time between pixels is new_delt
        seconds.

        Parameters
        ----------
        new_delt : float
            New delta between consecutive values.
        """
        if self.t_delt == new_delt:
            return self
        factor = self.t_delt / float(new_delt)

        # The last data-point does not change!
        new_size = floor((self.shape[1] - 1) * factor + 1)  # pylint: disable=E1101
        data = ndimage.zoom(self.data, (1, new_size / self.shape[1]))  # pylint: disable=E1101

        params = self._get_params()
        params.update({
            'time_axis': np.linspace(
                self.time_axis[0],
                self.time_axis[int((new_size - 1) * new_delt / self.t_delt)],
                new_size
            ),
            't_delt': new_delt,
        })
        return self.__class__(data, **params)

    JOIN_REPEAT = object()

    @classmethod
    def join_many(cls, specs, mk_arr=None, nonlinear=False,
                  maxgap=0, fill=JOIN_REPEAT):
        """
        Produce new Spectrogram that contains spectrograms joined together in
        time.

        Parameters
        ----------
        specs : list
            List of spectrograms to join together in time.
        nonlinear : bool
            If True, leave out gaps between spectrograms. Else, fill them with
            the value specified in fill.
        maxgap : float, int or None
            Largest gap to allow in second. If None, allow gap of arbitrary
            size.
        fill : float or int
            Value to fill missing values (assuming nonlinear=False) with.
            Can be LinearTimeSpectrogram.JOIN_REPEAT to repeat the values for
            the time just before the gap.
        mk_array: function
            Function that is called to create the resulting array. Can be set
            to LinearTimeSpectrogram.memap(filename) to create a memory mapped
            result array.
        """
        # XXX: Only load header and load contents of files
        # on demand.
        mask = None

        if mk_arr is None:
            mk_arr = cls.make_array

        specs = sorted(specs, key=lambda x: x.start)

        freqs = specs[0].freq_axis
        if not all(np.array_equal(freqs, sp.freq_axis) for sp in specs):
            raise ValueError("Frequency channels do not match.")

        # Smallest time-delta becomes the common time-delta.
        min_delt = min(sp.t_delt for sp in specs)
        dtype_ = max(sp.dtype for sp in specs)

        specs = [sp.resample_time(min_delt) for sp in specs]
        size = sum(sp.shape[1] for sp in specs)

        data = specs[0]
        start_day = data.start

        xs = []
        last = data
        for elem in specs[1:]:
            e_init = (
                SECONDS_PER_DAY * (
                get_day(elem.start) - get_day(start_day)
            ).days + elem.t_init
            )
            x = int((e_init - last.t_init) / min_delt)
            xs.append(x)
            diff = last.shape[1] - x

            if maxgap is not None and -diff > maxgap / min_delt:
                raise ValueError("Too large gap.")

            # If we leave out undefined values, we do not want to
            # add values here if x > t_res.
            if nonlinear:
                size -= max(0, diff)
            else:
                size -= diff

            last = elem

        # The non existing element after the last one starts after
        # the last one. Needed to keep implementation below sane.
        xs.append(specs[-1].shape[1])

        # We do that here so the user can pass a memory mapped
        # array if they'd like to.
        arr = mk_arr((data.shape[0], size), dtype_)
        time_axis = np.zeros((size,))
        sx = 0
        # Amount of pixels left out due to non-linearity. Needs to be
        # considered for correct time axes.
        sd = 0
        for x, elem in zip(xs, specs):
            diff = x - elem.shape[1]
            e_time_axis = elem.time_axis

            elem = elem.data

            if x > elem.shape[1]:
                if nonlinear:
                    x = elem.shape[1]
                else:
                    # If we want to stay linear, fill up the missing
                    # pixels with placeholder zeros.
                    filler = np.zeros((data.shape[0], diff))
                    if fill is cls.JOIN_REPEAT:
                        filler[:, :] = elem[:, -1, np.newaxis]
                    else:
                        filler[:] = fill
                    minimum = e_time_axis[-1]
                    e_time_axis = np.concatenate([
                        e_time_axis,
                        np.linspace(
                            minimum + min_delt,
                            minimum + diff * min_delt,
                            diff
                        )
                    ])
                    elem = np.concatenate([elem, filler], 1)
            arr[:, sx:sx + x] = elem[:, :x]

            if diff > 0:
                if mask is None:
                    mask = np.zeros((data.shape[0], size), dtype=np.uint8)
                mask[:, sx + x - diff:sx + x] = 1
            time_axis[sx:sx + x] = e_time_axis[:x] + data.t_delt * (sx + sd)
            if nonlinear:
                sd += max(0, diff)
            sx += x
        params = {
            'time_axis': time_axis,
            'freq_axis': data.freq_axis,
            'start': data.start,
            'end': specs[-1].end,
            't_delt': data.t_delt,
            't_init': data.t_init,
            't_label': data.t_label,
            'f_label': data.f_label,
            'content': data.content,
            'instruments': _union(spec.instruments for spec in specs),
        }
        if mask is not None:
            arr = ma.array(arr, mask=mask)
        if nonlinear:
            del params['t_delt']
            return Spectrogram(arr, **params)
        return common_base(specs)(arr, **params)

    def time_to_x(self, time):
        """
        Return x-coordinate in spectrogram that corresponds to the passed
        datetime value.

        Parameters
        ----------
        time : `~sunpy.time.parse_time` compatible str
            `datetime.datetime` to find the x coordinate for.
        """
        # This is impossible for frequencies because that mapping
        # is not injective.
        if SUNPY_LT_1:
            time = parse_time(time)
        else:
            time = parse_time(time).datetime
        diff = time - self.start
        diff_s = SECONDS_PER_DAY * diff.days + diff.seconds
        result = diff_s // self.t_delt
        if 0 <= result <= self.shape[1]:  # pylint: disable=E1101
            return result
        raise ValueError("Out of range.")

    @staticmethod
    def intersect_time(specs):
        """
        Return slice of spectrograms that is present in all of the ones passed.

        Parameters
        ----------
        specs : list
            List of spectrograms of which to find the time intersections.
        """
        delt = min(sp.t_delt for sp in specs)
        start = max(sp.t_init for sp in specs)

        # XXX: Could do without resampling by using
        # sp.t_init below, not sure if good idea.
        specs = [sp.resample_time(delt) for sp in specs]
        cut = [sp[:, int((start - sp.t_init) / delt):] for sp in specs]

        length = min(sp.shape[1] for sp in cut)
        return [sp[:, :length] for sp in cut]

    @classmethod
    def combine_frequencies(cls, specs):
        """
        Return new spectrogram that contains frequencies from all the
        spectrograms in spec. Only returns time intersection of all of them.

        Parameters
        ----------
        spec : list
            List of spectrograms of which to combine the frequencies into one.
        """
        if not specs:
            raise ValueError("Need at least one spectrogram.")

        specs = cls.intersect_time(specs)

        one = specs[0]

        dtype_ = max(sp.dtype for sp in specs)
        fsize = sum(sp.shape[0] for sp in specs)

        new = np.zeros((fsize, one.shape[1]), dtype=dtype_)

        freq_axis = np.zeros((fsize,))

        for n, (data, row) in enumerate(merge(
            [
                [(sp, n) for n in range(sp.shape[0])] for sp in specs
            ],
            key=lambda x: x[0].freq_axis[x[1]]
        )):
            new[n, :] = data[row, :]
            freq_axis[n] = data.freq_axis[row]
        params = {
            'time_axis': one.time_axis,  # Should be equal
            'freq_axis': freq_axis,
            'start': one.start,
            'end': one.end,
            't_delt': one.t_delt,
            't_init': one.t_init,
            't_label': one.t_label,
            'f_label': one.f_label,
            'content': one.content,
            'instruments': _union(spec.instruments for spec in specs)
        }
        return common_base(specs)(new, **params)

    def check_linearity(self, err=None, err_factor=None):
        """
        Check linearity of time axis. If err is given, tolerate absolute
        derivation from average delta up to err. If err_factor is given,

        tolerate up to err_factor * average_delta. If both are given,
        TypeError is raised. Default to err=0.

        Parameters
        ----------
        err : float
            Absolute difference each delta is allowed to diverge from the
            average. Cannot be used in combination with err_factor.
        err_factor : float
            Relative difference each delta is allowed to diverge from the
            average, i.e. err_factor * average. Cannot be used in combination
            with err.
        """
        deltas = self.time_axis[:-1] - self.time_axis[1:]
        avg = np.average(deltas)
        if err is None and err_factor is None:
            err = 0
        elif err is None:
            err = abs(err_factor * avg)
        elif err_factor is not None:
            raise TypeError("Only supply err or err_factor, not both")
        return (abs(deltas - avg) <= err).all()

    def in_interval(self, start=None, end=None):
        """
        Return part of spectrogram that lies in [start, end).

        Parameters
        ----------
        start : None or `~datetime.datetime` or `~sunpy.time.parse_time` compatible string or time string
            Start time of the part of the spectrogram that is returned. If the
            measurement only spans over one day, a colon separated string
            representing the time can be passed.
        end : None or `~datetime.datetime` or `~sunpy.time.parse_time` compatible string or time string
            See start.
        """
        if start is not None:
            try:
                if SUNPY_LT_1:
                    start = parse_time(start)
                else:
                    start = parse_time(start).datetime
            except ValueError:
                # XXX: We could do better than that.
                if get_day(self.start) != get_day(self.end):
                    raise TypeError(
                        "Time ambiguous because data spans over more than one day"
                    )
                start = datetime.datetime(
                    self.start.year, self.start.month, self.start.day,
                    *list(map(int, start.split(":")))
                )
            start = self.time_to_x(start)
        if end is not None:
            try:
                if SUNPY_LT_1:
                    end = parse_time(end)
                else:
                    end = parse_time(end).datetime
            except ValueError:
                if get_day(self.start) != get_day(self.end):
                    raise TypeError(
                        "Time ambiguous because data spans over more than one day"
                    )
                end = datetime.datetime(
                    self.start.year, self.start.month, self.start.day,
                    *list(map(int, end.split(":")))
                )
            end = self.time_to_x(end)
        if start:
            start = int(start)
        if end:
            end = int(end)
        return self[:, start:end]


import os
from tensorboard_logger import configure, log_value
import torch
import torch.autograd as autograd
from torch.autograd import Variable
import torch.utils.data as torchdata
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
import tqdm
import utils
import torch.optim as optim
import pdb
from torch.distributions import Multinomial, Bernoulli
import torch.backends.cudnn as cudnn
cudnn.benchmark = True

import argparse
parser = argparse.ArgumentParser(description='Transfer Learning')
parser.add_argument('--lr', type=float, default=1e-4, help='learning rate')
parser.add_argument('--data_dir', default='data/', help='data directory')
parser.add_argument('--cv_dir', default='cv/tmp/', help='checkpoint directory (models and logs are saved here)')
parser.add_argument('--train_csv', default='cv/tmp/', help='train csv directory')
parser.add_argument('--val_csv', default='cv/tmp/', help='validation csv directory')
parser.add_argument('--batch_size', type=int, default=256, help='batch size')
parser.add_argument('--epoch_step', type=int, default=10000, help='epochs after which lr is decayed')
parser.add_argument('--max_epochs', type=int, default=10000, help='total epochs to run')
parser.add_argument('--load', help='Checkpoints for the pretrained model')
parser.add_argument('--parallel', action ='store_true', default=False, help='use multiple GPUs for training')

args = parser.parse_args()

if not os.path.exists(args.cv_dir):
    os.system('mkdir ' + args.cv_dir)
utils.save_args(__file__, args)

def train(epoch):

    net.train()
    matches, losses = [], []
    for batch_idx, (inputs, targets) in tqdm.tqdm(enumerate(trainloader), total=len(trainloader)):

        inputs, targets = Variable(inputs), Variable(targets).cuda(async=True)
        if not args.parallel:
    	    inputs = inputs.cuda()

        v_inputs = Variable(inputs.data, volatile=True)

        preds = net.forward(v_inputs)

        _, pred_idx = preds.max(1)
        match = (pred_idx==targets).data

        loss = F.cross_entropy(preds, targets)

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        matches.append(match.cpu())
        losses.append(loss.cpu())

    accuracy = torch.cat(matches, 0).float().mean()
    loss = torch.stack(losses).mean()
    log_str = 'E: %d | A: %.3f | L: %.3f'%(epoch, accuracy, loss)
    print log_str

    log_value('train_accuracy', accuracy, epoch)
    log_value('train_loss', loss, epoch)

def test(epoch):

    net.eval()
    matches = []
    for batch_idx, (inputs, targets) in tqdm.tqdm(enumerate(testloader), total=len(testloader)):

        inputs, targets = Variable(inputs, volatile=True), Variable(targets).cuda(async=True)
        if not args.parallel:
            inputs = inputs.cuda()

        v_inputs = Variable(inputs.data, volatile=True)

        preds = net.forward(v_inputs)

        _, pred_idx = preds.max(1)
        match = (pred_idx==targets).data

        matches.append(match.cpu())

    accuracy = torch.cat(matches, 0).float().mean()
    log_str = 'TS: %d | A: %.3f'%(epoch, accuracy)
    print log_str

    log_value('train_accuracy', accuracy, epoch)

    # save the model parameters
    net_state_dict = net.module.state_dict() if args.parallel else net.state_dict()

    state = {
      'state_dict': net_state_dict,
      'epoch': epoch,
      'acc': accuracy
    }
    torch.save(state, args.cv_dir+'/ckpt_E_%d_A_%.3f'%(epoch, accuracy))

#--------------------------------------------------------------------------------------------------------#
trainset, testset = utils.get_dataset(args.train_csv, args.val_csv)
trainloader = torchdata.DataLoader(trainset, batch_size=args.batch_size, shuffle=True, num_workers=16)
testloader = torchdata.DataLoader(testset, batch_size=args.batch_size, shuffle=False, num_workers=4)
net = utils.get_model()
if args.parallel:
    net = nn.DataParallel(net)
net.cuda()

if args.load:
    checkpoint = torch.load(args.load)
    torch.load_state_dict(checkpoint)

start_epoch = 0
optimizer = optim.Adam(net.parameters(), lr=args.lr)
configure(args.cv_dir+'/log', flush_secs=5)
for epoch in range(start_epoch, start_epoch+args.max_epochs+1):
    train(epoch)
    if epoch % 1 == 0:
        test(epoch)
    lr_scheduler.step()


from absl import app, flags, logging
from absl.flags import FLAGS
import tensorflow as tf
import numpy as np
import cv2
from tensorflow.keras.callbacks import (
    ReduceLROnPlateau,
    EarlyStopping,
    ModelCheckpoint,
    TensorBoard
)
from yolov3_tf2.models import (
    YoloV3, YoloV3Tiny, YoloLoss,
    yolo_anchors, yolo_anchor_masks,
    yolo_tiny_anchors, yolo_tiny_anchor_masks
)
from yolov3_tf2.utils import freeze_all
import yolov3_tf2.dataset as dataset

import horovod.tensorflow as hvd

flags.DEFINE_string('dataset', '', 'path to dataset')
flags.DEFINE_string('val_dataset', '', 'path to validation dataset')
flags.DEFINE_boolean('tiny', False, 'yolov3 or yolov3-tiny')
flags.DEFINE_string('weights', './checkpoints/yolov3.tf',
                    'path to weights file')
flags.DEFINE_string('classes', './data/coco.names', 'path to classes file')
flags.DEFINE_enum('mode', 'fit', ['fit', 'eager_fit', 'eager_tf'],
                  'fit: model.fit, '
                  'eager_fit: model.fit(run_eagerly=True), '
                  'eager_tf: custom GradientTape')
flags.DEFINE_enum('transfer', 'none',
                  ['none', 'darknet', 'no_output', 'frozen', 'fine_tune'],
                  'none: Training from scratch, '
                  'darknet: Transfer darknet, '
                  'no_output: Transfer all but output, '
                  'frozen: Transfer and freeze all, '
                  'fine_tune: Transfer all and freeze darknet only')
flags.DEFINE_integer('size', 416, 'image size')
flags.DEFINE_integer('epochs', 2, 'number of epochs')
flags.DEFINE_integer('batch_size', 8, 'batch size')
flags.DEFINE_float('learning_rate', 1e-3, 'learning rate')

def main(_argv):
    # Horovod: initialize Horovod.
    hvd.init()

    # Horovod: pin GPU to be used to process local rank (one GPU per process)
    gpus = tf.config.experimental.list_physical_devices('GPU')
    for gpu in gpus:
        tf.config.experimental.set_memory_growth(gpu, True)
    if gpus:
        tf.config.experimental.set_visible_devices(gpus[hvd.local_rank()], 'GPU')

    if FLAGS.tiny:
        model = YoloV3Tiny(FLAGS.size, training=True)
        anchors = yolo_tiny_anchors
        anchor_masks = yolo_tiny_anchor_masks
    else:
        model = YoloV3(FLAGS.size, training=True)
        anchors = yolo_anchors
        anchor_masks = yolo_anchor_masks

    train_dataset = dataset.load_fake_dataset()
    if FLAGS.dataset:
        train_dataset = dataset.load_tfrecord_dataset(
            FLAGS.dataset, FLAGS.classes)
    train_dataset = train_dataset.shuffle(buffer_size=1024)  # TODO: not 1024
    train_dataset = train_dataset.batch(FLAGS.batch_size)
    train_dataset = train_dataset.map(lambda x, y: (
        dataset.transform_images(x, FLAGS.size),
        dataset.transform_targets(y, anchors, anchor_masks, 80)))
    train_dataset = train_dataset.prefetch(
        buffer_size=tf.data.experimental.AUTOTUNE)

    val_dataset = dataset.load_fake_dataset()
    if FLAGS.val_dataset:
        val_dataset = dataset.load_tfrecord_dataset(
            FLAGS.val_dataset, FLAGS.classes)
    val_dataset = val_dataset.batch(FLAGS.batch_size)
    val_dataset = val_dataset.map(lambda x, y: (
        dataset.transform_images(x, FLAGS.size),
        dataset.transform_targets(y, anchors, anchor_masks, 80)))

    if FLAGS.transfer != 'none':
        model.load_weights(FLAGS.weights)
        if FLAGS.transfer == 'fine_tune':
            # freeze darknet
            darknet = model.get_layer('yolo_darknet')
            freeze_all(darknet)
        elif FLAGS.mode == 'frozen':
            # freeze everything
            freeze_all(model)
        else:
            # reset top layers
            if FLAGS.tiny:  # get initial weights
                init_model = YoloV3Tiny(FLAGS.size, training=True)
            else:
                init_model = YoloV3(FLAGS.size, training=True)

            if FLAGS.transfer == 'darknet':
                for l in model.layers:
                    if l.name != 'yolo_darknet' and l.name.startswith('yolo_'):
                        l.set_weights(init_model.get_layer(
                            l.name).get_weights())
                    else:
                        freeze_all(l)
            elif FLAGS.transfer == 'no_output':
                for l in model.layers:
                    if l.name.startswith('yolo_output'):
                        l.set_weights(init_model.get_layer(
                            l.name).get_weights())
                    else:
                        freeze_all(l)

    # Horovod: adjust learning rate based on number of GPUs.
    optimizer = tf.optimizers.Adam(FLAGS.learning_rate * hvd.size())
    # Horovod: add Horovod DistributedOptimizer.

    ###############################################
    loss = [YoloLoss(anchors[mask]) for mask in anchor_masks]

    if FLAGS.mode == 'eager_tf':
        # Eager mode is great for debugging
        # Non eager graph mode is recommended for real training
        avg_loss = tf.keras.metrics.Mean('loss', dtype=tf.float32)
        avg_val_loss = tf.keras.metrics.Mean('val_loss', dtype=tf.float32)

        for epoch in range(1, FLAGS.epochs + 1):
            for batch, (images, labels) in enumerate(train_dataset.take(5717//hvd.size())):
                with tf.GradientTape() as tape:
                    outputs = model(images, training=True)
                    regularization_loss = tf.reduce_sum(model.losses)
                    pred_loss = []
                    for output, label, loss_fn in zip(outputs, labels, loss):
                        pred_loss.append(loss_fn(label, output))
                    total_loss = tf.reduce_sum(pred_loss) + regularization_loss
                # Horovod: add Horovod Distributed GradientTape.
                tape = hvd.DistributedGradientTape(tape)

                grads = tape.gradient(total_loss, model.trainable_variables)
                optimizer.apply_gradients(
                    zip(grads, model.trainable_variables))
                # Horovod: broadcast initial variable states from rank 0 to all other processes.
                # This is necessary to ensure consistent initialization of all workers when
                # training is started with random weights or restored from a checkpoint.
                #
                # Note: broadcast should be done after the first gradient step to ensure optimizer
                # initialization.
                if batch==0:
                    hvd.broadcast_variables(model.variables, root_rank=0)
                    hvd.broadcast_variables(optimizer.variables(), root_rank=0)

                #############################
                if hvd.rank()==0:
                        logging.info("{}_train_{}, {}, {}".format(
                            epoch, batch, total_loss.numpy(),
                            list(map(lambda x: np.sum(x.numpy()), pred_loss))))
                ###########################
                avg_loss.update_state(total_loss)

            for batch, (images, labels) in enumerate(val_dataset):
                outputs = model(images)
                regularization_loss = tf.reduce_sum(model.losses)
                pred_loss = []
                for output, label, loss_fn in zip(outputs, labels, loss):
                    pred_loss.append(loss_fn(label, output))
                total_loss = tf.reduce_sum(pred_loss) + regularization_loss
                if hvd.rank()==0:
                    logging.info("{}_val_{}, {}, {}".format(
                        epoch, batch, total_loss.numpy(),
                        list(map(lambda x: np.sum(x.numpy()), pred_loss))))
                avg_val_loss.update_state(total_loss)
            if hvd.rank()==0:
                logging.info("{}, train: {}, val: {}".format(
                    epoch,
                    avg_loss.result().numpy(),
                    avg_val_loss.result().numpy()))

            avg_loss.reset_states()
            avg_val_loss.reset_states()
            if hvd.rank()==0:
                model.save_weights(
                    'checkpoints/horovod_yolov3_train_{}.tf'.format(epoch))
    else:
        model.compile(optimizer=optimizer, loss=loss,
                      run_eagerly=(FLAGS.mode == 'eager_fit'))

        callbacks = [
            ReduceLROnPlateau(verbose=1),
            EarlyStopping(patience=3, verbose=1),
            ModelCheckpoint('checkpoints/yolov3_train_{epoch}.tf',
                            verbose=1, save_weights_only=True),
            TensorBoard(log_dir='logs')
        ]

        history = model.fit(train_dataset,
                            epochs=FLAGS.epochs,
                            callbacks=callbacks,
                            validation_data=val_dataset)

if __name__ == '__main__':
    try:
        app.run(main)
    except SystemExit:
        pass
# from absl import app, flags, logging
# from absl.flags import FLAGS
# import tensorflow as tf
# import numpy as np
# import cv2
# from tensorflow.keras.callbacks import (
#     ReduceLROnPlateau,
#     EarlyStopping,
#     ModelCheckpoint,
#     TensorBoard
# )
# from yolov3_tf2.models import (
#     YoloV3, YoloV3Tiny, YoloLoss,
#     yolo_anchors, yolo_anchor_masks,
#     yolo_tiny_anchors, yolo_tiny_anchor_masks
# )
# from yolov3_tf2.utils import freeze_all
# import yolov3_tf2.dataset as dataset

# #for horovod training
# import horovod.tensorflow.keras as hvd

# flags.DEFINE_string('dataset', '', 'path to dataset')
# flags.DEFINE_string('val_dataset', '', 'path to validation dataset')
# flags.DEFINE_boolean('tiny', False, 'yolov3 or yolov3-tiny')
# flags.DEFINE_string('weights', './checkpoints/yolov3.tf',
#                     'path to weights file')
# flags.DEFINE_string('classes', './data/coco.names', 'path to classes file')
# flags.DEFINE_enum('mode', 'fit', ['fit', 'eager_fit', 'eager_tf'],
#                   'fit: model.fit, '
#                   'eager_fit: model.fit(run_eagerly=True), '
#                   'eager_tf: custom GradientTape')
# flags.DEFINE_enum('transfer', 'none',
#                   ['none', 'darknet', 'no_output', 'frozen', 'fine_tune'],
#                   'none: Training from scratch, '
#                   'darknet: Transfer darknet, '
#                   'no_output: Transfer all but output, '
#                   'frozen: Transfer and freeze all, '
#                   'fine_tune: Transfer all and freeze darknet only')
# flags.DEFINE_integer('size', 416, 'image size')
# flags.DEFINE_integer('epochs', 2, 'number of epochs')
# flags.DEFINE_integer('batch_size', 8, 'batch size')
# flags.DEFINE_float('learning_rate', 1e-3, 'learning rate')

# # def get_number(path):
# #     tf_records_filenames = path
# #     c = 0
# #     for record in tf.python_io.tf_record_iterator(fn):
# #         c += 1
# #     return c

# def main(_argv):
#     # Horovod: initialize Horovod.
#     hvd.init()
#     # Horovod: pin GPU to be used to process local rank (one GPU per process)
#     gpus = tf.config.experimental.list_physical_devices('GPU')
#     for gpu in gpus:
#       tf.config.experimental.set_memory_growth(gpu, True)
#     if gpus:
#       tf.config.experimental.set_visible_devices(gpus[hvd.local_rank()], 'GPU')
#      #####################
#     if FLAGS.tiny:
#         model = YoloV3Tiny(FLAGS.size, training=True)
#         anchors = yolo_tiny_anchors
#         anchor_masks = yolo_tiny_anchor_masks
#     else:
#         model = YoloV3(FLAGS.size, training=True)
#         anchors = yolo_anchors
#         anchor_masks = yolo_anchor_masks

#     train_dataset = dataset.load_fake_dataset()
#     if FLAGS.dataset:
#         train_dataset = dataset.load_tfrecord_dataset(
#             FLAGS.dataset, FLAGS.classes)
#     train_dataset = train_dataset.shuffle(buffer_size=1024)  # TODO: not 1024
#     train_dataset = train_dataset.batch(FLAGS.batch_size)
#     train_dataset = train_dataset.map(lambda x, y: (
#         dataset.transform_images(x, FLAGS.size),
#         dataset.transform_targets(y, anchors, anchor_masks, 80)))
#     train_dataset = train_dataset.prefetch(
#         buffer_size=tf.data.experimental.AUTOTUNE)

#     val_dataset = dataset.load_fake_dataset()
#     if FLAGS.val_dataset:
#         val_dataset = dataset.load_tfrecord_dataset(
#             FLAGS.val_dataset, FLAGS.classes)
#     val_dataset = val_dataset.batch(FLAGS.batch_size)
#     val_dataset = val_dataset.map(lambda x, y: (
#         dataset.transform_images(x, FLAGS.size),
#         dataset.transform_targets(y, anchors, anchor_masks, 80)))

#     if FLAGS.transfer != 'none':
#         model.load_weights(FLAGS.weights)
#         if FLAGS.transfer == 'fine_tune':
#             # freeze darknet
#             darknet = model.get_layer('yolo_darknet')
#             freeze_all(darknet)
#         elif FLAGS.mode == 'frozen':
#             # freeze everything
#             freeze_all(model)
#         else:
#             # reset top layers
#             if FLAGS.tiny:  # get initial weights
#                 init_model = YoloV3Tiny(FLAGS.size, training=True)
#             else:
#                 init_model = YoloV3(FLAGS.size, training=True)

#             if FLAGS.transfer == 'darknet':
#                 for l in model.layers:
#                     if l.name != 'yolo_darknet' and l.name.startswith('yolo_'):
#                         l.set_weights(init_model.get_layer(
#                             l.name).get_weights())
#                     else:
#                         freeze_all(l)
#             elif FLAGS.transfer == 'no_output':
#                 for l in model.layers:
#                     if l.name.startswith('yolo_output'):
#                         l.set_weights(init_model.get_layer(
#                             l.name).get_weights())
#                     else:
#                         freeze_all(l)

#     #optimizer = tf.keras.optimizers.Adam(lr=FLAGS.learning_rate)
#     # Horovod: adjust learning rate based on number of GPUs.
#     optimizer = tf.keras.optimizers.Adam(lr=FLAGS.learning_rate * hvd.size())
#     # Horovod: add Horovod DistributedOptimizer.
#     optimizer = hvd.DistributedOptimizer(optimizer)
#     ###############################################

#     loss = [YoloLoss(anchors[mask]) for mask in anchor_masks]

#     if FLAGS.mode == 'eager_tf':
#         # Eager mode is great for debugging
#         # Non eager graph mode is recommended for real training
#         avg_loss = tf.keras.metrics.Mean('loss', dtype=tf.float32)
#         avg_val_loss = tf.keras.metrics.Mean('val_loss', dtype=tf.float32)

#         for epoch in range(1, FLAGS.epochs + 1):
#             for batch, (images, labels) in enumerate(train_dataset):
#                 with tf.GradientTape() as tape:
#                     outputs = model(images, training=True)
#                     regularization_loss = tf.reduce_sum(model.losses)
#                     pred_loss = []
#                     for output, label, loss_fn in zip(outputs, labels, loss):
#                         pred_loss.append(loss_fn(label, output))
#                     total_loss = tf.reduce_sum(pred_loss) + regularization_loss

#                 grads = tape.gradient(total_loss, model.trainable_variables)
#                 optimizer.apply_gradients(
#                     zip(grads, model.trainable_variables))

#                 logging.info("{}_train_{}, {}, {}".format(
#                     epoch, batch, total_loss.numpy(),
#                     list(map(lambda x: np.sum(x.numpy()), pred_loss))))
#                 avg_loss.update_state(total_loss)

#             for batch, (images, labels) in enumerate(val_dataset):
#                 outputs = model(images)
#                 regularization_loss = tf.reduce_sum(model.losses)
#                 pred_loss = []
#                 for output, label, loss_fn in zip(outputs, labels, loss):
#                     pred_loss.append(loss_fn(label, output))
#                 total_loss = tf.reduce_sum(pred_loss) + regularization_loss

#                 logging.info("{}_val_{}, {}, {}".format(
#                     epoch, batch, total_loss.numpy(),
#                     list(map(lambda x: np.sum(x.numpy()), pred_loss))))
#                 avg_val_loss.update_state(total_loss)

#             logging.info("{}, train: {}, val: {}".format(
#                 epoch,
#                 avg_loss.result().numpy(),
#                 avg_val_loss.result().numpy()))

#             avg_loss.reset_states()
#             avg_val_loss.reset_states()
#             model.save_weights(
#                 'checkpoints/yolov3_train_{}.tf'.format(epoch))
#     else:
#         model.compile(optimizer=optimizer, loss=loss,
#                        run_eagerly=(FLAGS.mode == 'eager_fit'))

#         # callbacks = [
#         #     ReduceLROnPlateau(verbose=1),
#         #     EarlyStopping(patience=3, verbose=1),
#         #     ModelCheckpoint('checkpoints/yolov3_train_{epoch}.tf',
#         #                     verbose=1, save_weights_only=True),
#         #     TensorBoard(log_dir='logs')
#         # ]
#         callbacks = [

#                     # Horovod: broadcast initial variable states from rank 0 to all other processes.
#                     # This is necessary to ensure consistent initialization of all workers when
#                     # training is started with random weights or restored from a checkpoint.
#                     hvd.callbacks.BroadcastGlobalVariablesCallback(0),

#                     # Horovod: average metrics among workers at the end of every epoch.

#                     # Note: This callback must be in the list before the ReduceLROnPlateau,
#                     # TensorBoard or other metrics-based callbacks.
#                     hvd.callbacks.MetricAverageCallback(),

#                     # Horovod: using `lr = 1.0 * hvd.size()` from the very beginning leads to worse final
#                     # accuracy. Scale the learning rate `lr = 1.0` ---> `lr = 1.0 * hvd.size()` during
#                     # the first three epochs. See https://arxiv.org/abs/1706.02677 for details.
#                     hvd.callbacks.LearningRateWarmupCallback(warmup_epochs=3, verbose=1),
#         ]
#         # Horovod: save checkpoints only on worker 0 to prevent other workers from corrupting them.
#         if hvd.rank() == 0:
#             callbacks.append(tf.keras.callbacks.ModelCheckpoint('checkpoints/yolov3_train_{epoch}.tf',verbose=1, save_weights_only=True))

#         verbose = 1 if hvd.rank() == 0 else 0

#         history = model.fit(train_dataset,
#                             steps_per_epoch=5717//FLAGS.batch_size// hvd.size(),
#                             #steps_per_epoch=500,
#                             epochs=FLAGS.epochs,
#                             callbacks=callbacks,
#                             validation_data=val_dataset,
#                             verbose=verbose
#                             )

# if __name__ == '__main__':
#     try:
#         app.run(main)
#     except SystemExit:
#         pass


"""Spectral shaping."""

from scipy.interpolate import interp1d

import numpy as np

# most of the contents of this file is heritage or not used.  That reflects
# a change in CGI practice.  Formerly, we used CBE values for coatings, etc.
# that posture has changed: we now lump things into a scalar value, which
# removes the spectral shaping from anything other than the star, detector,
# or FPM.

CGI_PUPIL_AREA = 0.641*2.363*2.363  # 64.1% of the enclosing rectangle (close enough)

# standard LOWFS bandpass, 9 wavelengths of spectral sampling
LOWFS_LOWER = (575-64)/1e3
LOWFS_UPPER = (575+64)/1e3

def mk_bandpass(n):
    """Make a LOWFS bandpass with n wavelengths."""
    if n == 1:
        return np.array([.575])

    return np.linspace(LOWFS_LOWER, LOWFS_UPPER, n)

LOWFS_BANDPASS = mk_bandpass(9)

CGI_BASIC_COATINGS = {
    'HRC': 7,       # = OTA+TCA, per Bijan
    'FSS99': 7+1,   # = CGI front-end + rad shield mirror (LOCAM)
    'Al': 2,        # = DM1+DM2, CGI front-end
    'BBAR': 3,      # LOWFS lenses
}
# additions...
# HLC => +1 FSS99 (SPM)
# SPC => +1 Al (SPM)

class ThroughputDatabase:
    """Database of spectral throughputs."""

    def __init__(self, emccd):  # historical: first arg coatings
        """Create a new database of throughputs.

        Parameters
        ----------
        coatings : `dict`
            keys of strings, which are the names of coatings
            values of scipy.interpolation.interp1d returns
        emccd : `scipy.interpolation.interp1d`
            interp1d of emccd data

        """
        # self.coatings = coatings
        self.emccd_interpf = emccd

    def __call__(self, corongraph_mode, wavelengths):
        """Spectral shape andthroughput for mode & wavelengths."""
        # hrc = self.coatings['HRC']
        # fss99 = self.coatings['FSS99']
        # al = self.coatings['Al']
        # bbar = self.coatings['bbar']

        # n_hrc = CGI_BASIC_COATINGS['HRC']
        # n_fss99 = CGI_BASIC_COATINGS['FSS99']
        # n_al = CGI_BASIC_COATINGS['Al']
        # n_bbar = CGI_BASIC_COATINGS['BBAR']
        # if corongraph_mode == 'HLC':
        #     #  not += 1 to avoid any chance of modifying the dict
        #     n_fss99 = n_fss99 + 1
        # else:
        #     n_al = n_al + 1

        # hrc = hrc(wavelengths) ** n_hrc
        # fss99 = fss99(wavelengths) ** n_fss99
        # al = al(wavelengths) ** n_al
        # bbar = bbar(wavelengths) ** n_bbar
        emccd = self.emccd_interpf(wavelengths)
        # return hrc * fss99 * al * bbar * emccd
        corongraph_mode = corongraph_mode.upper()
        # from LOWFS_photometry 20200801, Throughput sheet.  BOL REQs
        if corongraph_mode == 'HLC':
            return 0.3443 * emccd
        elif corongraph_mode == 'SPEC':
            return 0.3092 * emccd
        elif corongraph_mode == 'WFOV':
            return 0.3092 * emccd
        else:
            raise ValueError('invalid coronagraph mode: must be HLC, SPEC, or WFOV')

    @classmethod
    def bijan_data(cls, data_fldr):
        """Load data from Bijan."""
        emccd_data = np.genfromtxt(data_fldr/'emccd-spectral-qe.csv', delimiter=',', skip_header=1)
        emccd_wvl = emccd_data[:, 0]/1e3
        emccd_dat = emccd_data[:, 1]
        emccd_interpf = interp1d(emccd_wvl, emccd_dat, bounds_error=False, fill_value=0)

        # coating_data = np.genfromtxt(data_fldr/'coatings.csv', delimiter=',', skip_header=1)
        # coating_wvl = coating_data[:, 0]
        # coatings = {}
        # for i, lbl in enumerate(['HRC', 'FSS99', 'Al', 'BBAR']):
        #     data = coating_data[:, i+1]
        #     interpf = interp1d(coating_wvl, data)
        #     coatings[lbl] = interpf

        # return ThroughputDatabase(coatings, emccd_interpf)
        return ThroughputDatabase(emccd_interpf)

class StellarDatabase:
    """Database of stellar spectra."""

    def __init__(self, stellar_table, stellar_labels):
        """Create a new database of stellar throughputs.

        Parameters
        ----------
        stellar_table : `numpy.ndarray`
            columnnar table of stellar spectral fluxes
        stellar_labels : `numpy.ndarray`
            labels associated with the columns, as [g0v, av5], and so forth.
            The index of the label identifies its column

        """
        self.stellar_interpfs = {}
        self.table = stellar_table
        self.stellar_labels = stellar_labels

    def __call__(self, star_type, wavelengths, lower_lim=-1, upper_lim=-1):
        """Spectral weights associated with a given star type at a given wavelength."""
        stellar_interpf = self.stellar_interpfs.get(star_type, None)
        if lower_lim == -1:
            lower_lim = LOWFS_LOWER
        if upper_lim == -1:
            upper_lim = LOWFS_UPPER

        if stellar_interpf is None:
            # this line throws value error if star_type not in list
            i = self.stellar_labels.index(star_type)

            # table has units of W/(m^2 m) == J/(s m^2 m)
            # => divide by J/ph, multiply by density, multiply by area
            # == ph/s.
            # this could be cleaner with astropy units, but c'est la vie
            wvl = self.table[:, 0]*1e6  # 1e6; m => um
            photon_energy = self.table[:, 1]
            dat = self.table[:, i]
            dat = dat / photon_energy * 1e-9 * CGI_PUPIL_AREA  # 1e-9 = sampling density, 1 nm
            stellar_interpf = interp1d(wvl, dat)
            self.stellar_interpfs[star_type] = stellar_interpf

        ret = stellar_interpf(wavelengths)
        ret[wavelengths < lower_lim] = 0
        ret[wavelengths > upper_lim] = 0
        return ret

    def sparsity_fudge_factor(self, star_type, wavelengths, lower_lim=-1, upper_lim=-1):
        """Fudge factor to deal with spectral sparsity for a given set of wavelengths."""
        true = self(star_type, wavelengths, lower_lim=lower_lim, upper_lim=upper_lim).sum()
        wvl_dense = self.table[:, 0] * 1e6
        dense = self(star_type, wvl_dense, lower_lim=lower_lim, upper_lim=upper_lim).sum()
        return dense/true

    @classmethod
    def bijan_data(cls, data_fldr):
        """Load data from Bijan."""
        stellar_data = np.genfromtxt(data_fldr/'stellar-spectra.csv', delimiter=',', skip_header=1)
        labels = [
            'wvl',
            'photon energy',
            'b3v',
            'a0v',
            'a5v',
            'f5v',
            'g0v',
            'f5v',
            'k0v',
            'k5v',
            'm0v',
            'm5v'
        ]

        return StellarDatabase(stellar_data, labels)


import sys
import os
import schnarc
import logging
import shutil
import argparse
import numpy as np
import datetime
import copy
import time
(tc, tt) = (time.process_time(), time.time())
#import schnet
import schnetpack as spk
import torch
import torch.nn as nn
from torch.optim import Adam
from torch.utils.data.sampler import RandomSampler
from sharc.pysharc.interface import SHARC_INTERFACE
import run_schnarc as rs
#import sharc and schnarc
from functools import partial
from schnarc.calculators import SchNarculator, EnsembleSchNarculator

class SHARC_NN(SHARC_INTERFACE):
    """
    Class for SHARC NN
    """
    # Name of the interface
    interface = 'NN'
    # store atom ids
    save_atids = True
    # store atom names
    save_atnames = True
    # accepted units:  0 : Bohr, 1 : Angstrom
    iunit = 0
    # not supported keys
    not_supported = ['nacdt', 'dmdr' ]

    def initial_setup(self, **kwargs):
        return

    def do_qm_job(self, tasks, Crd):
        """

        Here you should perform all your qm calculations

        depending on the tasks, that were asked
        Hamiltonian matrix and dipole moment matrix are complex
        Gradient and NACs are real
        """
        QMout = self.schnarc_init.calculate(Crd)
        return QMout

    def final_print(self):
        self.sharc_writeQMin()

    def parseTasks(self, tasks):
        """
        these things should be interface dependent

        so write what you love, it covers basically everything
        after savedir information in QMin

        """

        # find init, samestep, restart
        QMin = { key : value for key, value in self.QMin.items() }
        QMin['natom'] = self.NAtoms
        QMin['atname'] = self.AtNames
        self.n_atoms=self.NAtoms
        key_tasks = tasks['tasks'].lower().split()

        if any( [self.not_supported in key_tasks ] ):
            print( "not supported keys: ", self.not_supported )
            sys.exit(16)

        for key in key_tasks:
            QMin[key] = []

        for key in self.states:
            QMin[key] = self.states[key]

        if 'init' in QMin:
            checkscratch(QMin['savedir'])
        if not 'init' in QMin and not 'samestep' in QMin and not 'restart' in QMin:
            fromfile=os.path.join(QMin['savedir'],'U.out')
            if not os.path.isfile(fromfile):
                print( 'ERROR: savedir does not contain U.out! Maybe you need to add "init" to QM.in.' )
                sys.exit(1)
            tofile=os.path.join(QMin['savedir'],'Uold.out')
            shutil.copy(fromfile,tofile)

        for key in ['grad', 'nacdr']:
            if tasks[key].strip() != "":
                QMin[key] = []

        QMin['pwd'] = os.getcwd()

        return QMin

    def readParameter(self, param,  *args, **kwargs):

        # Get device
        self.device = torch.device("cuda" if param.cuda else "cpu")
        self.NAtoms = vars(self)['NAtoms']
        self.AtNames = vars(self)['AtNames']
        self.dummy_crd = np.zeros((self.NAtoms,3))
        self.hessian = True if param.hessian else False
        self.adaptive = param.adaptive
        #self.nac_approx=param.nac_approx
        if param.thresholds is not None:
            self.thresholds = {}
            self.thresholds['energy'] = param.thresholds[0]
            self.thresholds['forces'] = param.thresholds[1]
            self.thresholds['dipoles'] = param.thresholds[2]
            self.thresholds['nacs'] = param.thresholds[3]
            self.thresholds['socs'] = param.thresholds[4]
            self.thresholds['diab'] = np.inf
            self.thresholds['diab2'] = np.inf
        if param.nac_approx is not None:
            self.nac_approx=param.nac_approx
        else:
            self.nac_approx = None
        if self.adaptive is not None:
            self.NNnumber = int(2)
            self.modelpaths=[]
            self.modelpaths.append(param.modelpath)
            self.modelpaths.append(param.modelpaths)
            self.schnarc_init = EnsembleSchNarculator(self.dummy_crd,self.AtNames,self.modelpaths,self.device,hessian=self.hessian,nac_approx=self.nac_approx,adaptive=self.adaptive,thresholds=self.thresholds,print_uncertainty=param.print_uncertainty)
        else:
            self.NNnumber = int(1) #self.options["NNnumber"]
            self.schnarc_init = SchNarculator(self.dummy_crd,self.AtNames,param.modelpath,self.device,hessian=self.hessian,nac_approx=self.nac_approx)
        return

def main():
    """
        Main Function if program is called as standalone
    """
    parser = rs.get_parser()
    args = parser.parse_args()
    print( "Initialize: CPU time:% .3f s, wall time: %.3f s"%(time.process_time() - tc, time.time() - tt))
    if args.adaptive is not None:
        adaptive = args.adaptive
    else:
        adaptive = float(1)
    param = "schnarc_options"
    # init SHARC_NN class
    nn = SHARC_NN()
    print( "init SHARC_NN:  CPU time: % .3f s, wall time: %.3f s"%(time.process_time() - tc, time.time() - tt))
    # run sharc dynamics
    nn.run_sharc("input",args,adaptive, initial_param=param)
    print( "run dynamics:  CPU time: % .3f s, wall time: %.3f s"%(time.process_time() - tc, time.time() - tt))
if __name__ == "__main__":
    main()


import keras
import numpy as np
import librosa

class livePredictions:

    def __init__(self, path, file):

        self.path = path
        self.file = file

    def load_model(self):
        '''
        I am here to load you model.

        :param path: path to your h5 model.
        :return: summary of the model with the .summary() function.

        '''
        self.loaded_model = keras.models.load_model(self.path)
        return self.loaded_model.summary()

    def makepredictions(self, file):
        '''
        I am here to process the files and create your features.
        '''
        data, sampling_rate = librosa.load(file)
        mfccs = np.mean(librosa.feature.mfcc(y=data, sr=sampling_rate, n_mfcc=40).T, axis=0)
        x = np.expand_dims(mfccs, axis=2)
        x = np.expand_dims(x, axis=0)
        predictions = self.loaded_model.predict_classes(x)
        print( "Prediction is", " ", self.convertclasstoemotion(predictions))

    def convertclasstoemotion(self, pred):
        '''
        I am here to convert the predictions (int) into human readable strings.
        '''
        self.pred  = pred

        if pred == 0:
            pred = "neutral"
            return pred
        elif pred == 1:
            pred = "calm"
            return pred
        elif pred == 2:
            pred = "happy"
            return pred
        elif pred == 3:
            pred = "sad"
            return pred
        elif pred == 4:
            pred = "angry"
            return pred
        elif pred == 5:
            pred = "fearful"
            return pred
        elif pred == 6:
            pred = "disgust"
            return pred
        elif pred == 7:
            pred = "surprised"
            return pred

# Here you can replace path and file with the path of your model and of the file from the RAVDESS dataset you want to use for the prediction,
# Below, I have used a neutral file: the prediction made is neutral.

pred = livePredictions(path='./Emotion_Voice_Detection_Model.h5',
                       file='/Users/marcogdepinto/Desktop/Ravdess_V2/01-01-01-01-01-01-01.wav')

pred.load_model()

from BaseHTTPServer import BaseHTTPRequestHandler,HTTPServer
import simplejson

PORT_NUMBER = 8080

#This class will handles any incoming request from
#the browser
class myHandler(BaseHTTPRequestHandler):

    #Handler for the GET requests
    def do_GET(self):
        self.send_response(200)
        self.send_header('Content-type','text/html')
        self.end_headers()
        # Send the html message
        self.wfile.write("Hello World !")
        return

    def do_POST(self):
        self._set_headers()
        print "in post method"
        self.data_string = self.rfile.read(int(self.headers['Content-Length']))

        self.send_response(200)
        self.end_headers()

        data = simplejson.loads(self.data_string)
        self.wfile.write(pred.makepredictions(data["file"]))
        return

try:
    #Create a web server and define the handler to manage the
    #incoming request
    server = HTTPServer(('', PORT_NUMBER), myHandler)
    print 'Started httpserver on port ' , PORT_NUMBER

    #Wait forever for incoming htto requests
    server.serve_forever()

except KeyboardInterrupt:
    print '^C received, shutting down the web server'
    server.socket.close()