"""
This module contains basic measurement classes for data acquired with the ROACH.
"""
from __future__ import division
import time
from collections import OrderedDict
import logging

import numpy as np
import pandas as pd
from matplotlib.pyplot import mlab
from scipy import signal
from memoized_property import memoized_property

from kid_readout.measurement import core
from kid_readout.analysis.resonator import lmfit_resonator
from kid_readout.analysis.timeseries import binning, despike, iqnoise, periodic
from kid_readout.roach import calculate

logger = logging.getLogger(__name__)

class RoachMeasurement(core.Measurement):
    """
    An abstract base class for measurements taken with the ROACH.
    """

    _version = 0

    def start_epoch(self):
        """
        Return self.epoch, if it exists, and if not return the earliest epoch of any RoachMeasurement that this
        measurement contains. Measurements that are not RoachMeasurements are ignored.

        Returns
        -------
        float
            The epoch of this Measurement or the earliest epoch of its contents; np.nan if neither is found.
        """
        if hasattr(self, 'epoch'):
            return self.epoch
        else:
            possible_epochs = []
            public_nodes = [(k, v) for k, v in self.__dict__.items()
                            if not k.startswith('_') and isinstance(v, core.Node)]
            for name, node in public_nodes:
                if isinstance(node, RoachMeasurement):
                    possible_epochs.append(node.start_epoch())
                elif isinstance(node, core.MeasurementList):
                    possible_epochs.append(np.min([m.start_epoch() for m in node if isinstance(m, RoachMeasurement)]))
            if possible_epochs:
                return np.min(possible_epochs)
            else:
                return np.nan

    def _delete_memoized_property_caches(self):
        for attr in dir(self):
            if hasattr(self, '_' + attr) and attr not in core.PRIVATE:
                delattr(self, '_' + attr)
        if self._parent is not None:
            self._parent._delete_memoized_property_caches()

    @property
    def cryostat(self):
        try:
            return self._io.metadata.cryostat
        except (AttributeError, KeyError) as e:
            return None

class RoachStream(RoachMeasurement):

    _version = 1

    def __init__(self, tone_bin, tone_amplitude, tone_phase, tone_index, filterbank_bin, epoch, sequence_start_number,
                 s21_raw, data_demodulated, roach_state, state=None, description='', validate=True):
        """
        Return a new RoachStream instance. This class has no dimensions and is intended to be subclassed.

        Parameters
        ----------
        tone_bin : numpy.ndarray(int)
            An array of integers representing the frequencies of the tones played during the measurement.
        tone_amplitude : numpy.ndarray(float)
            An array of floats representing the amplitudes of the tones played during the measurement.
        tone_phase : numpy.ndarray(float)
            An array of floats representing the radian phases of the tones played during the measurement.
        tone_index : int or numpy.ndarray(int)
            tone_bin[tone_index] corresponds to the frequency used to produce s21_raw.
        filterbank_bin : int or numpy.ndarray(int)
            The filter bank bin(s) containing the tone(s).
        epoch : float
            The unix timestamp of the first sample of the time-ordered data.
        sequence_start_number : int
            The ROACH sequence number for the first sample of the time-ordered data.
        s21_raw : numpy.ndarray(complex)
            The data, demodulated or not.
        data_demodulated : bool
            True if the data is demodulated.
        roach_state : dict
            State information for the roach; the result of roach.state.
        state : dict
            All non-roach state information.
        description : str
            A human-readable description of this measurement.
        validate : bool
            If True, check that the array shapes match the dimensions OrderedDict.
        """
        self.tone_bin = tone_bin
        self.tone_amplitude = tone_amplitude
        self.tone_phase = tone_phase
        self.tone_index = tone_index
        self.filterbank_bin = filterbank_bin
        self.epoch = epoch
        self.sequence_start_number = sequence_start_number
        self.s21_raw = s21_raw
        self.data_demodulated = data_demodulated
        self.roach_state = core.StateDict(roach_state)
        super(RoachStream, self).__init__(state=state, description=description, validate=validate)

    @memoized_property
    def sample_time(self):
        """numpy.ndarray(float): The time of each sample relative to the first sample in the stream."""
        return (np.arange(self.s21_raw.shape[-1], dtype='float') /
                self.stream_sample_rate)

    @memoized_property
    def frequency(self):
        return calculate.frequency(self.roach_state, self.tone_bin[self.tone_index])

    @property
    def frequency_MHz(self):
        return 1e-6 * self.frequency

    @memoized_property
    def baseband_frequency(self):
        return calculate.baseband_frequency(self.roach_state, self.tone_bin[self.tone_index])

    @property
    def baseband_frequency_MHz(self):
        return 1e-6 * self.baseband_frequency

    @property
    def stream_sample_rate(self):
        return calculate.stream_sample_rate(self.roach_state)

    @memoized_property
    def s21_raw_mean(self):
        """
        Return the mean of s21_raw for each channel. NaN samples are excluded from the calculation.

        Raises
        ------
        RuntimeWarning
            If all of the samples for a channel are NaN; in this case the return value for that channel will be NaN.

        Returns
        -------
        numpy.ndarray(complex)
           The mean of s21_raw for each channel.
        """
        return np.nanmean(self.s21_raw, axis=-1)

    @memoized_property
    def s21_raw_mean_error(self):
        """
        Estimate the error in s21_raw_mean for each channel. NaN samples are excluded from the calculation.

        The method assumes that the samples are independent, which is probably not true.

        Raises
        ------
        RuntimeWarning
            If all of the samples for a channel are NaN; in this case the return value for that channel will be NaN.

        Returns
        -------
        numpy.ndarray(complex)
            An estimate of the complex standard error of the mean of s21_raw.
        """
        # The float cast allows conversion to NaN.
        num_good_samples = np.sum(~np.isnan(self.s21_raw), axis=-1).astype(np.float)
        if isinstance(num_good_samples, np.ndarray):
            # Avoid a ZeroDivisionError if some of the channels have no good samples.
            num_good_samples[num_good_samples == 0] = np.nan
        elif num_good_samples == 0:  # num_good_samples is a scalar; avoid a ZeroDivisionError.
            num_good_samples = np.nan
        return ((np.nanstd(self.s21_raw.real, axis=-1) + 1j * np.nanstd(self.s21_raw.imag, axis=-1)) /
                np.sqrt(num_good_samples))

    @property
    def s21_point(self):
        """
        Return one s21 point per stream, calculated using the best available method.

        The method used to calculate this point may change.
        """
        return self.s21_raw_mean

    @property
    def s21_point_error(self):
        """
        Return an estimate of the standard error of the mean for s21_point, calculated using the best available method.

        The method used to calculate this error may change.
        """
        return self.s21_raw_mean_error

    def fold(self, array, period_samples=None, reduce=np.mean):
        if period_samples is None:
            period_samples = calculate.modulation_period_samples(self.roach_state)
        return periodic.fold(array, period_samples, reduce=reduce)

    def epochs(self, start=-np.inf, stop=np.inf):
        """
        Return a StreamArray containing only the data between the given start and stop epochs.

        The returned StreamArray has the same state.

        The indexing follows the Python convention that the first value is inclusive and the second is exclusive:
        start <= epoch < stop
        Thus, the two slices stream_array.epochs(t0, t1) and stream_array.epochs(t1, t2) will contain all the data
        occurring at or after t0 and before t2, with no duplication.
        """
        start_index = np.searchsorted(self.epoch + self.sample_time, (start,), side='left')[0] # Need to be ints
        stop_index = np.searchsorted(self.epoch + self.sample_time, (stop,), side='right')[0]  # This index not included
        return self.__class__(tone_bin=self.tone_bin, tone_amplitude=self.tone_amplitude,
                              tone_phase=self.tone_phase, tone_index=self.tone_index,
                              filterbank_bin=self.filterbank_bin, epoch=self.epoch + self.sample_time[start_index],
                              sequence_start_number=np.nan,  # This may no longer be valid for the sliced data.
                              s21_raw=self.s21_raw[..., start_index:stop_index],
                              data_demodulated=self.data_demodulated, roach_state=self.roach_state,
                              state=self.state, description=self.description)

class RoachStream0(RoachMeasurement):
    """
    This class is a factory for producing RoachStream version 1 instances from version 0 data.

    Version 0 did not save the sequence_start_number integer.
    """
    def __new__(cls, *args, **kwargs):
        kwargs['sequence_start_number'] = None
        return RoachStream(*args, **kwargs)

class StreamArray(RoachStream):
    """
    This class represents simultaneously-sampled data from multiple channels.
    """

    _version = 1

    dimensions = OrderedDict([('tone_bin', ('tone_bin',)),
                              ('tone_amplitude', ('tone_bin',)),
                              ('tone_phase', ('tone_bin',)),
                              ('tone_index', ('tone_index',)),
                              ('filterbank_bin', ('tone_index',)),
                              ('s21_raw', ('tone_index', 'sample_time'))])

    def __init__(self, tone_bin, tone_amplitude, tone_phase, tone_index, filterbank_bin, epoch, sequence_start_number,
                 s21_raw, data_demodulated, roach_state, state=None, description='', validate=True):
        """
        Return a new StreamArray instance. The integer array tone_index contains the indices of tone_bin,
        tone_amplitude, and tone_phase for the tones demodulated to produce the time-ordered s21_raw data.
        The tone_bin, tone_amplitude, tone_phase, tone_index, and filterbank_bin arrays are 1-D, while s21_raw is 2-D:
        s21_raw.shape == (tone_index.size, sample_time.size)

        Parameters
        ----------
        tone_bin : numpy.ndarray(int)
            An array of integers representing the frequencies of the tones played during the measurement.
        tone_amplitude : numpy.ndarray(float)
            An array of floats representing the amplitudes of the tones played during the measurement.
        tone_phase : numpy.ndarray(float)
            An array of floats representing the radian phases of the tones played during the measurement.
        tone_index : numpy.ndarray(int)
            tone_bin[tone_index] corresponds to the frequency used to produce s21_raw.
        filterbank_bin : numpy.ndarray(int)
            The filter bank bins in which the tones lie.
        epoch : float
            The unix timestamp of the first sample of the time-ordered data.
        sequence_start_number : int
            The ROACH sequence number for the first sample of the time-ordered data.
        s21_raw : numpy.ndarray(complex)
            The data, demodulated or not.
        data_demodulated : bool
            True if the data is demodulated.
        roach_state : dict
            State information for the roach; the result of roach.state.
        state : dict
            All non-roach state information.
        description : str
            A human-readable description of this measurement.
        validate : bool
            If True, check that the array shapes match the dimensions OrderedDict.
        """
        super(StreamArray, self).__init__(tone_bin=tone_bin, tone_amplitude=tone_amplitude, tone_phase=tone_phase,
                                          tone_index=tone_index, filterbank_bin=filterbank_bin, epoch=epoch,
                                          sequence_start_number=sequence_start_number, s21_raw=s21_raw,
                                          data_demodulated=data_demodulated, roach_state=roach_state, state=state,
                                          description=description, validate=validate)

    def __getitem__(self, number):
        """
        See stream().
        """
        number = int(number)  # Avoid weird indexing bugs
        ss = SingleStream(tone_bin=self.tone_bin, tone_amplitude=self.tone_amplitude, tone_phase=self.tone_phase,
                          tone_index=self.tone_index[number], filterbank_bin=self.filterbank_bin[number],
                          epoch=self.epoch, sequence_start_number=self.sequence_start_number,
                          s21_raw=self.s21_raw[number, :], data_demodulated=self.data_demodulated,
                          roach_state=self.roach_state, number=number, state=self.state,
                          description=self.description)
        ss._io = self._io
        ss._io_node_path = self._io_node_path
        return ss

    def stream(self, number):
        """
        Return a SingleStream object containing the data from the channel corresponding to the given integer.

        Parameters
        ----------
        number : int
            The index of the stream to use to create the new single-channel object.

        Returns
        -------
        SingleStream
        """
        return self[number]

    def tone_offset_frequency(self, normalized_frequency=True):
        offset = calculate.tone_offset_frequency(self.tone_bin,self.roach_state.num_tone_samples,self.filterbank_bin,
                                     self.roach_state.num_filterbank_channels)
        if not normalized_frequency:
            offset = offset * self.stream_sample_rate
        return offset

    def to_dataframe(self, add_origin = False):
        dataframes = []
        for number in range(self.tone_bin.shape[0]):
            dataframes.append(self.stream(number).to_dataframe(add_origin=add_origin))
        return pd.concat(dataframes, ignore_index=True)

class StreamArray0(RoachStream0):
    """
    This class is a factory for producing StreamArray version 1 instances from version 0 data.

    Version 0 did not save the sequence_start_number integer.
    """
    def __new__(cls, *args, **kwargs):
        kwargs['sequence_start_number'] = None
        return StreamArray(*args, **kwargs)

class SingleStream(RoachStream):
    """
    This class contains time-ordered data from a single channel.
    """

    _version = 1

    dimensions = OrderedDict([('tone_bin', ('tone_bin',)),
                              ('tone_amplitude', ('tone_bin',)),
                              ('tone_phase', ('tone_bin',)),
                              ('s21_raw', ('sample_time',))])

    def __init__(self, tone_bin, tone_amplitude, tone_phase, tone_index, filterbank_bin, epoch, sequence_start_number,
                 s21_raw, data_demodulated, roach_state, number=None, state=None, description='', validate=True):
        """
        Return a new SingleStream instance. The single integer tone_index is the common index of tone_bin,
        tone_amplitude, and tone_phase for the tone used to produce the time-ordered s21_raw data.

        The tone_bin, tone_amplitude, tone_phase, tone_index, filterbank_bin, and s21_raw arrays are all 1-D.

        Parameters
        ----------
        tone_bin : numpy.ndarray(int)
            An array of integers representing the frequencies of the tones played during the measurement.
        tone_amplitude : numpy.ndarray(float)
            An array of floats representing the amplitudes of the tones played during the measurement.
        tone_phase : numpy.ndarray(float)
            An array of floats representing the radian phases of the tones played during the measurement.
        tone_index : int
            tone_bin[tone_index] corresponds to the frequency used to produce s21_raw.
        filterbank_bin : int
            An int that is the filter bank bin in which the tone lies.
        epoch : float
            The unix timestamp of the first sample of the time-ordered data.
        sequence_start_number : int
            The ROACH sequence number for the first sample of the time-ordered data.
        s21_raw : numpy.ndarray(complex)
            The data, demodulated or not.
        data_demodulated : bool
            True if the data is demodulated.
        roach_state : dict
            State information for the roach; the result of roach.state.
        number : int or None
            The number of this instance in some larger structure, such as a StreamArray.
        state : dict
            All non-roach state information.
        description : str
            A human-readable description of this measurement.
        validate : bool
            If True, check that the array shapes match the dimensions OrderedDict.
        """
        self.number = number
        super(SingleStream, self).__init__(tone_bin=tone_bin, tone_amplitude=tone_amplitude, tone_phase=tone_phase,
                                           tone_index=tone_index, filterbank_bin=filterbank_bin, epoch=epoch,
                                           sequence_start_number=sequence_start_number, s21_raw=s21_raw,
                                           data_demodulated=data_demodulated, roach_state=roach_state, state=state,
                                           description=description, validate=validate)

    # ToDo: use tone_index?
    def tone_offset_frequency(self, normalized_frequency=True):
        offset = calculate.tone_offset_frequency(self.tone_bin,self.roach_state.num_tone_samples,self.filterbank_bin,
                                                 self.roach_state.num_filterbank_channels)
        if not normalized_frequency:
            offset = offset * self.stream_sample_rate
        return offset

    def to_dataframe(self, add_origin=True):
        data = {'number': self.number, 'analysis_epoch': time.time(), 'start_epoch': self.start_epoch()}
        try:
            for key, value in self.roach_state.items():
                data['roach_{}'.format(key)] = value
        except KeyError:
            pass

        data.update(self.state.flatten(wrap_lists=True))

        data['s21_point'] = [self.s21_point]
        data['s21_point_error'] = [self.s21_point_error]
        data['frequency'] = [self.frequency]
        data['frequency_MHz'] = [self.frequency_MHz]

        dataframe = pd.DataFrame(data, index=[0])
        if add_origin:
            self.add_origin(dataframe)
        return dataframe

class SingleStream0(RoachStream0):
    """
    This class is a factory for producing SingleStream version 1 instances from version 0 data.

    Version 0 did not save the sequence_start_number integer.
    """
    def __new__(cls, *args, **kwargs):
        kwargs['sequence_start_number'] = None
        return SingleStream(*args, **kwargs)

class SweepArray(RoachMeasurement):
    """
    This class contains a list of StreamArrays.

    The analysis methods and properties are designed for two modes of operation.

    For simultaneous sweeps across individual resonators, use the sweep() method (equivalent to __getitem__ access)
    to return SingleSweep instances containing the data from individual resonators.

    For scans over many unknown resonators, use the properties to return values in ascending frequency order, and use
    find_resonators() and resonator() to locate resonance frequencies and extract resonator objects.
    """

    _version = 0

    def __init__(self, stream_arrays, state=None, description=''):
        """
        Parameters
        ----------
        stream_arrays : iterable(StreamArray)
            The streams that make up the sweep.
        state : dict
            All non-roach state information.
        description : str
            A human-readable description of this measurement.
        """
        if not isinstance(stream_arrays, core.MeasurementList):
            stream_arrays = core.MeasurementList(stream_arrays)
        self.stream_arrays = stream_arrays
        super(SweepArray, self).__init__(state=state, description=description)

    def __getitem__(self, number):
        """
        See sweep().
        """
        number = int(number)  # Avoid weird indexing bugs
        ss = SingleSweep(streams=core.MeasurementList(sa.stream(number) for sa in self.stream_arrays),
                           number=number, state=self.state, description=self.description)
        ss._io = self._io
        ss._io_node_path = self._io_node_path
        return ss

    def sweep(self, number):
        """
        Return a SingleSweep object containing the data from the channel corresponding to the given integer.

        Parameters
        ----------
        number : int
            The index of the sweep to use to create the new single-channel object.

        Returns
        -------
        SingleSweep
        """
        return self[number]

    @property
    def num_channels(self):
        try:
            if np.any(np.diff([sa.tone_index.size for sa in self.stream_arrays])):
                raise ValueError("Channel numbers differ between stream arrays.")
            else:
                return self.stream_arrays[0].tone_index.size
        except IndexError:
            return 0

    @memoized_property
    def ascending_order(self):
        """numpy.ndarray[int]: Re-arranges values for this SweepArray in ascending frequency order."""
        return np.concatenate([sa.frequency for sa in self.stream_arrays]).argsort()

    @memoized_property
    def frequency(self):
        """numpy.ndarray[float]: The frequencies of all data points in ascending order."""
        return np.concatenate([sa.frequency for sa in self.stream_arrays])[self.ascending_order]

    @property
    def frequency_MHz(self):
        """numpy.ndarray[float]: The frequencies in MHz of all data points, in ascending order."""
        return 1e-6 * self.frequency

    @memoized_property
    def s21_point(self):
        """numpy.ndarray[complex]: The s21_point values of all data points, in ascending frequency order."""
        return np.concatenate([sa.s21_point for sa in self.stream_arrays])[self.ascending_order]

    @memoized_property
    def s21_point_error(self):
        """numpy.ndarray[complex]: The s21_point_error values of all data points, in ascending frequency order."""
        return np.concatenate([sa.s21_point_error for sa in self.stream_arrays])[self.ascending_order]

    @memoized_property
    def s21_raw(self):
        """numpy.ndarray[complex]: The raw s21 streams of all data points, in ascending frequency order."""
        return np.vstack([stream_array.s21_raw for stream_array in self.stream_arrays])[self.ascending_order, :]

    @property
    def s21_point_foreground(self):
        return self.s21_point / self.background

    @property
    def s21_point_error_foreground(self):
        return self.s21_point_error / self.background

    @property
    def background(self):
        return self.fit_background(frequency=self.frequency, s21=self.s21_point)

    # ToDo: swap this out for a lmfit model
    def fit_background(self, frequency=None, s21=None, amp_degree=3, phi_degree=3, weights=None, mask=None):
        if frequency is None:
            frequency = self.frequency
        if s21 is None:
            s21 = self.s21_point
        if weights is None:  # Attempt to down-weight resonances
            weights = np.abs(s21) ** 2
        if mask is None:
            mask = np.ones(frequency.size, dtype=np.bool)
        weights *= mask
        amp_poly = np.polyfit(frequency, np.abs(s21), deg=amp_degree, w=weights)
        phi_poly = np.polyfit(frequency, np.unwrap(np.angle(s21)), deg=phi_degree, w=weights)
        return np.polyval(amp_poly, frequency) * np.exp(1j * np.polyval(phi_poly, frequency))

    # ToDo: this could be much smarter about identifying large peaks
    def find_peaks(self, expected_Q=30000, num_widths=100, threshold=1, minimum_linewidth_separation=1,
                   **find_peaks_cwt_kwargs):
        linewidth = self.frequency.mean() / expected_Q
        width = linewidth / (self.frequency[1] - self.frequency[0])  # in samples
        data = 1 / np.abs(self.s21_point_foreground) - 1
        widths = np.linspace(width / 10, 10 * width, num_widths)
        peaks = np.array(signal.find_peaks_cwt(vector=data, widths=widths, **find_peaks_cwt_kwargs))
        mask = (width < peaks) & (peaks < data.size - width) & (data[peaks] > threshold * np.std(data))
        return peaks[mask]

    def resonator(self, frequency, width, model=lmfit_resonator.LinearLossResonatorWithCable):
        mask = (frequency - width / 2 <= self.frequency) & (self.frequency <= frequency + width / 2)
        return model(frequency=self.frequency[mask], s21=self.s21_point_foreground[mask],
                     errors=self.s21_point_error_foreground[mask])

    def to_dataframe(self, add_origin=True, one_sweep_per_row=True):
        """

        Parameters
        ----------
        add_origin
        one_sweep_per_row: bool, default True
            If True, return a dataframe with one row per sweep, usually what you want if each sweep corresponds to a
            resonator.
            If False, return a single row with frequency and s21_point arrays for VNA style sweep data.

        Returns
        -------
        pandas.DataFrame
        """
        dataframes = []
        if one_sweep_per_row:
            for number in range(self.num_channels):
                dataframes.append(self.sweep(number).to_dataframe(add_origin=add_origin))
        else:
            data = {'analysis_epoch': time.time(), 'start_epoch': self.start_epoch()}
            try:
                for thermometer, temperature in self.state['temperature'].items():
                    data['temperature_{}'.format(thermometer)] = temperature
            except KeyError:
                pass
            try:
                for key, value in self.stream_arrays[0].stream(0).roach_state.items():
                    data['roach_{}'.format(key)] = value
            except KeyError:
                pass

            flat_state = self.state.flatten(wrap_lists=True)
            data.update(flat_state)
            data['frequency'] = [self.frequency]
            data['s21_point'] = [self.s21_point]
            data['s21_point_error'] = [self.s21_point_error]
            dataframe = pd.DataFrame(data, index=[0])
            self.add_origin(dataframe)
            dataframes = [dataframe]
        return pd.concat(dataframes, ignore_index=True)

class SingleSweep(RoachMeasurement):
    """
    This class contains a list of SingleStreams with different frequencies.

    The properties return values in ascending frequency order.
    """

    _version = 0

    def __init__(self, streams, number=None, state=None, description=''):
        """
        Parameters
        ----------
        streams: iterable(SingleStream)
            The streams that make up the sweep.
        number : int or None
            The number of this instance in the SweepArray from which it was created.
        state : dict
            All non-roach state information.
        description : str
            A human-readable description of this measurement.
        """
        if not isinstance(streams, core.MeasurementList):
            streams = core.MeasurementList(streams)
        self.streams = streams
        self.number = number
        super(SingleSweep, self).__init__(state=state, description=description)

    @memoized_property
    def ascending_order(self):
        """numpy.ndarray[int]: Re-arranges values for this SingleSweep in ascending frequency order."""
        return np.array([s.frequency for s in self.streams]).argsort()

    @memoized_property
    def frequency(self):
        """numpy.ndarray[float]: The frequencies of all data points in ascending order."""
        return np.array([s.frequency for s in self.streams])[self.ascending_order]

    @property
    def frequency_MHz(self):
        """numpy.ndarray[float]: The frequencies in MHz of all data points, in ascending order."""
        return 1e-6 * self.frequency

    @memoized_property
    def s21_point(self):
        """numpy.ndarray[complex]: The s21_point values of all data points, in ascending frequency order."""
        return np.array([s.s21_point for s in self.streams])[self.ascending_order]

    @memoized_property
    def s21_point_error(self):
        """numpy.ndarray[complex]: The s21_point_error values of all data points, in ascending frequency order."""
        return np.array([s.s21_point_error for s in self.streams])[self.ascending_order]

    @memoized_property
    def s21_raw(self):
        """numpy.ndarray[complex]: The raw s21 streams of all data points, in ascending frequency order."""
        return np.vstack([s.s21_raw for s in self.streams])[self.ascending_order, :]

    @memoized_property
    def s21_normalized(self):
        return np.array([self.resonator.remove_background(f, s21) for f, s21 in zip(self.frequency, self.s21_point)])

    @memoized_property
    def s21_normalized_error(self):
        return np.array([self.resonator.remove_background(f, s21_error)
                         for f, s21_error in zip(self.frequency, self.s21_point_error)])

    @memoized_property
    def resonator(self):
        """BaseResonator: the result of the last call to fit_resonator()."""
        return self.fit_resonator()

    def fit_resonator(self, model=lmfit_resonator.LinearResonatorWithCable, params=None):
        """
        Fit the s21 data with the given resonator model and, if given, the initial Parameters.

        Parameters
        ----------
        model : BaseResonator
            The resonator model to use for the fit.
        params : lmfit.Parameters
            A parameters object to use for initial values and limits in the fit.
        """
        self._delete_memoized_property_caches()
        self._resonator = model(frequency=self.frequency, s21=self.s21_point, errors=self.s21_point_error)
        self._resonator.fit(params)
        return self._resonator

    def to_dataframe(self, add_origin=True):
        data = {'number': self.number, 'analysis_epoch': time.time(), 'start_epoch': self.start_epoch()}
        try:
            for key, value in self.streams[0].roach_state.items():
                data['roach_{}'.format(key)] = value
        except KeyError:
            pass

        data.update(self.state.flatten(wrap_lists=True))

        for param in self.resonator.current_result.params.values():
            data['res_{}'.format(param.name)] = param.value
            data['res_{}_error'.format(param.name)] = param.stderr
        data['res_redchi'] = self.resonator.current_result.redchi
        data['res_Q_i'] = self.resonator.Q_i
        data['res_Q_e'] = self.resonator.Q_e

        data['res_s21_data'] = [self.resonator.data]
        data['res_frequency_data'] = [self.resonator.frequency]
        data['res_s21_errors'] = [self.resonator.errors]
        modelf = np.linspace(self.resonator.frequency.min(), self.resonator.frequency.max(), 1000)
        models21 = self.resonator.model.eval(params=self.resonator.current_params, f=modelf)
        data['res_model_frequency'] = [modelf]
        data['res_model_s21'] = [models21]

        dataframe = pd.DataFrame(data, index=[0])
        if add_origin:
            self.add_origin(dataframe)
        return dataframe

class SweepStreamArray(RoachMeasurement):

    _version = 0

    def __init__(self, sweep_array, stream_array, state=None, description=''):
        """
        Parameters
        ----------
        sweep_array : SweepArray
        stream_array : StreamArray
        state : dict
        description : str
        """
        if sweep_array.num_channels != stream_array.tone_index.size:
            raise core.MeasurementError("The number of SweepArray channels does not match the StreamArray number.")
        self.sweep_array = sweep_array
        self.stream_array = stream_array
        super(SweepStreamArray, self).__init__(state=state, description=description)

    def epochs(self, start=-np.inf, stop=np.inf):
        """
        Return a new SweepStreamArray containing the same sweep and only the stream data between the given epochs.

        Parameters
        ----------
        start : float
            The start time of the slice.
        stop : float
            The stop time of the slice.

        Returns
        -------
        SweepStreamArray
        """
        return SweepStreamArray(self.sweep_array, self.stream_array.epochs(start, stop), state=self.state,
                                description=self.description)

    @property
    def num_channels(self):
        return self.sweep_array.num_channels

    def __getitem__(self, number):
        """
        See sweep_stream()
        """
        number = int(number)  # Avoid weird indexing bugs
        sss = SingleSweepStream(sweep=self.sweep_array.sweep(number), stream=self.stream_array.stream(number),
                                number=number, state=self.state, description=self.description)
        sss._io = self._io
        sss._io_node_path = self._io_node_path
        return sss

    def sweep_stream(self, number):
        """
        Return a SingleSweepStream object containing the data from the channel corresponding to the given integer.

        Parameters
        ----------
        number : int
            The index of the sweep and stream to use to create the new single-channel object.

        Returns
        -------
        SingleSweepStream
        """
        return self[number]

    def to_dataframe(self):
        dataframes = []
        for number in range(self.num_channels):
            dataframes.append(self.sweep_stream(number).to_dataframe())
        return pd.concat(dataframes, ignore_index=True)

class SingleSweepStream(RoachMeasurement):

    _version = 0

    def __init__(self, sweep, stream, number=None, state=None, description=''):
        """
        Parameters
        ----------
        sweep : SingleSweep
        stream : SingleStream
        number : int or None
        state : dict
        description : str
        """
        self.sweep = sweep
        self.stream = stream
        self.number = number
        super(SingleSweepStream, self).__init__(state=state, description=description)

    def epochs(self, start=-np.inf, stop=np.inf):
        """
        Return a new SingleSweepStream containing the same sweep and only the stream data between the given epochs.

        Parameters
        ----------
        start : float
            The start time of the slice.
        stop : float
            The stop time of the slice.

        Returns
        -------
        SingleSweepStream
        """
        return SingleSweepStream(self.sweep, self.stream.epochs(start, stop), state=self.state,
                                 number=self.number, description=self.description)

    @property
    def resonator(self):
        return self.sweep.resonator

    @property
    def glitch_mask(self):
        if not hasattr(self,'_glitch_mask'):
            logger.debug("glitch_mask running deglitch")
            self.deglitch()
        return self._glitch_mask

    @property
    def number_of_masked_samples(self):
        if not hasattr(self,'_number_of_masked_samples'):
            logger.debug("number_of_masked_samples running deglitch")
            self.deglitch()
        return self._number_of_masked_samples

    def deglitch(self, threshold=8, window_in_seconds=1, mask_extend_samples=50):
        window_samples = int(2 ** np.ceil(np.log2(window_in_seconds * self.stream.stream_sample_rate)))
        logger.debug("deglitching with threshold %f, window %.f seconds, %d samples, extending mask by %d samples"
                     % (threshold, window_in_seconds,window_samples, mask_extend_samples))
        try:
            self._glitch_mask = despike.deglitch_mask_mad(self.x_raw, thresh=threshold, window_length=window_samples,
                                                          mask_extend=mask_extend_samples)
            self._number_of_masked_samples = self._glitch_mask.sum()
            self._x, self._q, self._stream_s21_normalized_deglitched = despike.mask_glitches([self.x_raw, self.q_raw,
                                                                                self.stream_s21_normalized],
                                                                                mask=self._glitch_mask,
                                                                                window_length=window_samples)
        except:
            self._glitch_mask = np.zeros(self.x_raw.shape, dtype='bool')
            self._number_of_masked_samples = self._glitch_mask.sum()
            self._x = self.x_raw
            self._q = self.q_raw
            self._stream_s21_normalized_deglitched = self.stream_s21_normalized

        logger.debug("masked %d samples out of %d total, fraction: %f" % (self._number_of_masked_samples,
                                                                          self._x_raw.size,
                                                                          (self._number_of_masked_samples /
                                                                           self._x_raw.size)))

    @memoized_property
    def stream_s21_normalized(self):
        return self.sweep.resonator.remove_background(self.stream.frequency, self.stream.s21_raw)

    @property
    def stream_s21_normalized_deglitched(self):
        if not hasattr(self,'_stream_s21_normalized_deglitched'):
            logger.debug("stream_s21_normalized_deglitch running deglitch")
            self.deglitch()
        return self._stream_s21_normalized_deglitched

    @property
    def q_raw(self):
        """
        Return the inverse internal quality factor q = 1 / Q_i calculated by inverting the resonator model.

        Returns
        -------
        numpy.ndarray (float)
            Values of q from self.stream corresponding to self.stream.sample_time
        """
        if not hasattr(self, '_q_raw'):
            self.set_raw_q_and_x()
        return self._q_raw

    @property
    def y_raw(self):
        """
        Return half the inverse internal quality factor y = q / 2 calculated by inverting the resonator model. The
        purpose of this is that S_yy = S_xx when amplifier-noise dominated.

        Returns
        -------
        numpy.ndarray (float)
            Values of y from self.stream corresponding to self.stream.sample_time.
        """
        return self.q_raw / 2

    @property
    def x_raw(self):
        """
        Return the fractional frequency shift x = f / f_r - 1 calculated by inverting the resonator model.

        Returns
        -------
        numpy.ndarray (float)
            Values of x from self.stream corresponding to self.stream.sample_time.
        """
        if not hasattr(self, '_x_raw'):
            self.set_raw_q_and_x()
        return self._x_raw

    @property
    def q(self):
        """
        Return the inverse internal quality factor q = 1 / Q_i calculated by inverting the resonator model.

        Returns
        -------
        numpy.ndarray (float)
            Values of q from self.stream corresponding to self.stream.sample_time
        """
        if not hasattr(self, '_q'):
            self.deglitch()
        return self._q

    @property
    def y(self):
        """
        Return half the inverse internal quality factor y = q / 2 calculated by inverting the resonator model. The
        purpose of this is that S_yy = S_xx when amplifier-noise dominated.

        Returns
        -------
        numpy.ndarray (float)
            Values of y from self.stream corresponding to self.stream.sample_time.
        """
        return self.q / 2

    @property
    def x(self):
        """
        Return the fractional frequency shift x = f / f_r - 1 calculated by inverting the resonator model.

        Returns
        -------
        numpy.ndarray (float)
            Values of x from self.stream corresponding to self.stream.sample_time.
        """
        if not hasattr(self, '_x'):
            self.deglitch()
        return self._x

    def set_raw_q_and_x(self):
        """
        Use the resonator model to calculate time-ordered resonator parameters from the time-ordered s21 data.

        The parameters are q and x: q is the inverse internal quality factor and x is the fractional frequency shift.

        Returns
        -------
        None
        """
        self._x_raw, self._q_raw = self.resonator.invert(self.stream_s21_normalized)

    @property
    def S_frequency(self):
        """
        Return the frequencies used in calculating the single-sided spectral densities.

        Returns
        -------
        numpy.ndarray (float)
             Positive frequencies through the Nyquist frequency.
        """
        if not hasattr(self, '_S_frequency'):
            self.set_S()
        return self._S_frequency

    @property
    def S_qq(self):
        """
        The single-sided spectral density of q(t), S_qq(f), where f is self.S_frequency.

        Returns
        -------
        numpy.ndarray (complex)
            :return: an array of complex values representing the spectral density of q(t)
        """
        if not hasattr(self, '_S_qq'):
            self.set_S()
        return self._S_qq

    @property
    def S_yy(self):
        """
        The single-sided spectral density of y(t), S_yy(f), where f is self.S_frequency.
        :return: an array of complex values representing the spectral density of y(t)
        """
        return self.S_qq / 4

    @property
    def S_xx(self):
        """
        The single-sided spectral density of x(t), S_xx(f), where f is self.S_frequency.
        :return: an array of complex values representing the spectral density of x(t)
        """
        if not hasattr(self, '_S_xx'):
            self.set_S()
        return self._S_xx

    @property
    def S_xq(self):
        """
        The single-sided cross-spectral density of q(t) and x(t), S_xq(f), where f is self.S_frequency.

        Returns
        -------
        numpy.ndarray (complex)
        """
        if not hasattr(self, '_S_xq'):
            self.set_S()
        return self._S_xq

    @property
    def S_xy(self):
        """
        The single-sided cross-spectral density of y(t) and x(t), S_yx(f), where f is self.S_frequency.

        Returns
        -------
        numpy.ndarray (complex)
        """
        return self.S_xq / 2

    @property
    def S_edges(self):
        """
        The edges of the bins used for log-binning of the spectral densities, or None if the spectra were not binned.

        Returns
        -------
        numpy.ndarray(float)
        """
        if not hasattr(self, '_S_edges'):
            self.set_S()
        return self._S_edges

    @property
    def S_counts(self):
        """
        The number of spectral samples that contributed to each frequency bin
        Returns
        -------

        """
        if not hasattr(self,'_S_counts'):
            self.set_S()
        return self._S_counts

    @property
    def S_xx_variance(self):
        """
        The variance on S_xx
        Returns
        -------

        """
        if not hasattr(self,'_S_xx_variance'):
            self.set_S()
        return self._S_xx_variance

    @property
    def S_qq_variance(self):
        """
        The variance on S_qq
        Returns
        -------

        """
        if not hasattr(self,'_S_qq_variance'):
            self.set_S()
        return self._S_qq_variance

    @property
    def S_yy_variance(self):
        """
        The variance on S_yy
        Returns
        -------

        """
        return self.S_qq_variance / 16

    def set_S(self, NFFT=None, window=mlab.window_none, detrend=mlab.detrend_none, noverlap=None, binned=True,
              bins_per_decade=30, masking_function=None, **psd_kwds):
        """
        Calculate the spectral density of self.x and self.q and set the related properties.

        Parameters
        ----------
        NFFT : int or None
            The number of samples to use for each FFT chunk; should be a power of two for speed; if None, a reasonable
            default is calculated that averages about eight spectra.
        detrend : callable
            A function that takes a complex time series as argument and returns a detrended time series.
        window  : callable
            A function that takes a complex time series as argument and returns a windowed time series.
        noverlap : int or None
            The number of samples to overlap in each chunk; if None, a value equal to half the NFFT value is used.
        binned : bool
            If True, the result is binned using bin sizes that increase with frequency.
        bins_per_decade : int
            If binned is True, this is the number of frequency bins per decade that will be used.
        masking_function : callable
            A function that takes the frequency and all spectral densities as inputs and produces a boolean mask used
            to remove points from them.
        psd_kwds : dict
            Additional keywords to pass to mlab.psd and mlab.csd.

        Returns
        -------
        None
        """
        if NFFT is None:
            NFFT = int(2**(np.floor(np.log2(self.stream.s21_raw.size)) - 3))
        if noverlap is None:
            noverlap = NFFT // 2
        S_qq, f = mlab.psd(self.q, Fs=self.stream.stream_sample_rate, NFFT=NFFT, window=window, detrend=detrend,
                           noverlap=noverlap, **psd_kwds)
        S_xx, f = mlab.psd(self.x, Fs=self.stream.stream_sample_rate, NFFT=NFFT, window=window, detrend=detrend,
                           noverlap=noverlap, **psd_kwds)
        S_xq, f = mlab.csd(self.x, self.q, Fs=self.stream.stream_sample_rate, NFFT=NFFT, window=window, detrend=detrend,
                           noverlap=noverlap, **psd_kwds)
        if masking_function is not None:
            mask = masking_function(f, S_xx, S_qq, S_xq)
            f = f[mask]
            S_qq = S_qq[mask]
            S_xx = S_xx[mask]
            S_xq = S_xq[mask]
            self._S_mask = mask
            logger.debug("Masked %d frequencies from raw power spectra" % (~mask).sum())
        # Drop the DC and Nyquist bins since they're not helpful and make plots look messy.
        f = f[1:-1]
        S_xx = S_xx[1:-1]
        S_qq = S_qq[1:-1]
        S_xq = S_xq[1:-1]
        # The value in each bin is chi-squared distributed with degrees of freedom equal to two times the number of
        # spectra that are averaged. Assume that the variance of the value in a bin is equal to the square of the value
        # in that bin divided by the number of degrees of freedom. Using nonzero overlap will complicate this, but let's
        # ignore that. It's also not clear that this is correct for the cross-spectrum.
        ndof = 2 * self.x.size // NFFT
        if binned:
            edges, counts, f_mean, d_and_v = binning.log_bin_with_variance(f, bins_per_decade,
                                                                           (S_xx, S_xx**2 / ndof),
                                                                           (S_qq, S_qq**2 / ndof),
                                                                           (S_xq, S_xq**2 / ndof))
            (S_xx, V_xx), (S_qq, V_qq), (S_xq, V_xq) = d_and_v
        else:
            edges = None
            counts = np.ones(f.size, dtype=int)
            f_mean = f
            V_xx = S_xx**2 / ndof
            V_qq = S_qq**2 / ndof
            V_xq = S_xq**2 / ndof
        self._S_edges = edges
        self._S_counts = counts
        self._S_frequency = f_mean
        self._S_qq = S_qq
        self._S_xx = S_xx
        self._S_xq = S_xq
        self._S_xx_variance = V_xx
        self._S_qq_variance = V_qq
        self._S_xq_variance = V_xq

    @property
    def pca_S_frequency(self):
        if not hasattr(self, '_pca_frequency'):
            self.set_pca()
        return self._pca_S_frequency

    @property
    def pca_S_00(self):
        if not hasattr(self, '_pca_S_00'):
            self.set_pca()
        return self._pca_S_00

    @property
    def pca_S_11(self):
        if not hasattr(self, '_pca_S_11'):
            self.set_pca()
        return self._pca_S_11

    @property
    def pca_angles(self):
        if not hasattr(self, '_pca_angles'):
            self.set_pca()
        return self._pca_angles

    def set_pca(self, NFFT=None, window=mlab.window_none, detrend=mlab.detrend_none, binned=True):
        """
        Calculate the spectral densities of the complex time series self.x + 1j * self.y in the two directions that,
        at each point, correspond to minimal and maximal fluctuation. We use self.y = self.q / 2 here because
        isotropic fluctuations in S_{21} have the same amplitude in these units.

        Note that if `binned` is True then binning is performed before PCA, and this can produce quite different results
        from binning after PCA. At frequencies where amplifier noise dominates, PCA without binning will select the
        direction corresponding to the largest fluctuations, so pca_S_00 will be less than pca_S_11 even though the
        noise is isotropic.

        Parameters
        ----------
        NFFT : int
            The number of samples to use for each FFT chunk; should be a power of two for speed.
        window  : callable
            A function that takes a complex time series as argument and returns a windowed time series.
        detrend : callable
            A function that takes a complex time series as argument and returns a detrended time series.
        binned : bool
            If True, the PSDs are binned using bin sizes that increase with frequency before PCA is performed.

        Returns
        -------
        None
        """
        if NFFT is None:
            NFFT = int(2**(np.floor(np.log2(self.stream.s21_raw.size)) - 3))
        fr, S, evals, evects, angles, piq = iqnoise.pca_noise(self.x + 1j * self.y, NFFT=NFFT,
                                                              Fs=self.stream.stream_sample_rate, window=window,
                                                              detrend=detrend, use_log_bins=binned,
                                                              use_full_spectral_helper=True)
        self._pca_S_frequency = fr
        self._pca_S_00 = evals[0]
        self._pca_S_11 = evals[1]
        self._pca_angles = angles

    def to_dataframe(self, add_origin=True, num_model_points=1000):
        data = {'number': self.number, 'analysis_epoch': time.time(), 'start_epoch': self.start_epoch()}

        data.update(self.state.flatten(wrap_lists=True))
        data.update(self.stream.state.flatten(prefix='stream', wrap_lists=True))
        data.update(self.sweep.state.flatten(prefix='sweep', wrap_lists=True))

        try:
            for key, value in self.stream.roach_state.items():
                data['roach_{}'.format(key)] = value
        except KeyError:
            pass

        data['stream_frequency'] = [self.stream.frequency]

        for param in self.resonator.current_result.params.values():
            data['res_{}'.format(param.name)] = param.value
            data['res_{}_error'.format(param.name)] = param.stderr
        data['res_redchi'] = self.resonator.current_result.redchi
        data['res_Q_i'] = self.resonator.Q_i
        data['res_Q_e'] = self.resonator.Q_e

        data['res_frequency_data'] = [self.resonator.frequency]
        data['res_s21_data'] = [self.resonator.data]
        data['res_s21_errors'] = [self.resonator.errors]
        model_f = np.linspace(self.resonator.frequency.min(), self.resonator.frequency.max(), num_model_points)
        data['res_model_frequency'] = [model_f]
        model_s21 = self.resonator.model.eval(params=self.resonator.current_params, f=model_f)
        data['res_model_s21'] = [model_s21]
        data['res_s21_data_normalized'] = [self.resonator.remove_background(self.resonator.frequency,
                                                                            self.resonator.data)]
        data['res_model_s21_normalized'] = [self.resonator.remove_background(model_f, model_s21)]
        s21_at_f_0 = self.resonator.model.eval(params=self.resonator.current_params, f=self.resonator.f_0)
        data['res_model_s21_at_f_0'] = s21_at_f_0
        data['res_model_s21_normalized_at_f_0'] = self.resonator.remove_background(self.resonator.f_0, s21_at_f_0)

        try:
            data['folded_x'] = [self.stream.fold(self.x_raw)]
            data['folded_q'] = [self.stream.fold(self.q_raw)]
            data['folded_normalized_s21'] = [self.stream.fold(self.stream_s21_normalized)]
        except ValueError:
            pass

        data['S_xx'] = [self.S_xx]
        data['S_yy'] = [self.S_yy]
        data['S_xy'] = [self.S_xy]
        data['S_xx_variance'] = [self.S_xx_variance]
        data['S_yy_variance'] = [self.S_yy_variance]
        data['S_counts'] = [self.S_counts]
        #data['S_xy_variance'] = [self.S_xy_variance]
        data['S_frequency'] = [self.S_frequency]

        dataframe = pd.DataFrame(data, index=[0])
        if add_origin:
            self.add_origin(dataframe)
        return dataframe

    def fold(self, array, period_samples=None, reduce=np.mean):
        if period_samples is None:
            period_samples = calculate.modulation_period_samples(self.stream.roach_state)
        return periodic.fold(array, period_samples, reduce=reduce)

class SweepStreamList(RoachMeasurement):

    _version = 0

    def __init__(self, sweep, stream_list, state=None, description=''):
        """
        Parameters
        ----------
        sweep : SweepArray
        stream_list : iterable(StreamArray)
        state : dict
        description : str
        """
        self.sweep = sweep
        if not isinstance(stream_list, core.MeasurementList):
            stream_list = core.MeasurementList(stream_list)
        self.stream_list = stream_list
        super(SweepStreamList, self).__init__(state=state, description=description)

    def __getitem__(self, number):
        """
        See single_sweep_stream_list().
        """
        number = int(number)  # Avoid weird indexing bugs
        sssl = SingleSweepStreamList(self.sweep[number],
                                     core.MeasurementList(sa[number] for sa in self.stream_list),
                                     number=number, state=self.state, description=self.description)
        sssl._io = self._io
        sssl._io_node_path = self._io_node_path
        return sssl

    def single_sweep_stream_list(self, number):
        """
        Return a SingleSweepStreamList object containing the data from the channel corresponding to the given integer.

        Parameters
        ----------
        number : int
            The index of the sweep and streams to use to create the new single-channel object.

        Returns
        -------
        SingleSweepStreamList
        """
        return self[number]

class SingleSweepStreamList(RoachMeasurement):

    _version = 0

    def __init__(self, single_sweep, stream_list, number=None, state=None, description=''):
        """
        Parameters
        ----------
        single_sweep : SingleSweep
        stream_list : iterable(SingleStream)
        number : int or None
        state : dict
        description : str
        """
        self.single_sweep = single_sweep
        if not isinstance(stream_list, core.MeasurementList):
            stream_list = core.MeasurementList(stream_list)
        self.stream_list = stream_list
        self.number = number
        super(SingleSweepStreamList, self).__init__(state=state, description=description)

    @property
    def resonator(self):
        return self.single_sweep.resonator

    def x_and_q(self, index):
        stream = self.stream_list[index]
        return self.resonator.invert_raw(frequency=stream.frequency, s21_raw=stream.s21_raw)

    def state_vector(self, *keys):
        vector = []
        for stream in self.stream_list:
            state = stream.state
            for key in keys:
                if state is np.nan:  # Not clear that this actually works
                    break
                state = state.get(key, np.nan)
            vector.append(state)
        return np.array(vector)

class Scan(RoachMeasurement):
    """
    This class contains a MeasurementList of SweepArrays, and allows them to be addressed as a single scan across the
    entire range of frequencies. It can handle overlap by averaging.
    """
    _version = 0

    def __init__(self, sweep_arrays, state=None, description=''):
        """
        Parameters
        ----------
        sweep_arrays : list (SweepArray)
        state : dict
        description : str
        """
        if not isinstance(sweep_arrays, core.MeasurementList):
            sweep_arrays = core.MeasurementList(sweep_arrays)
        self.sweep_arrays = sweep_arrays
        super(Scan, self).__init__(state=state, description=description)

    @property
    def frequency(self):
        return np.concatenate([sa.frequency for sa in self.sweep_arrays])

    @property
    def s21_point(self):
        return np.concatenate([sa.s21_point for sa in self.sweep_arrays])

    @property
    def s21_point_foreground(self):
        return np.concatenate([sa.s21_point_foreground for sa in self.sweep_arrays])

    # ToDo: test and finish
    def stitch(self, kernel=None):
        n = self.sweep_arrays[0].frequency.size
        duplicates = np.where(self.sweep_arrays[0].frequency == self.sweep_arrays[1].frequency[0])[0]
        if duplicates.size == 0:
            overlap = 0
        else:
            overlap = self.sweep_arrays[0].frequency.size - int(duplicates)  # will fail if more than 1 duplicate
        left_weight = np.linspace(1, 0, overlap)
        right_weight = np.linspace(0, 1, overlap)
        frequencies = [self.sweep_arrays[0].frequency[:overlap]]
        amplitudes = [np.abs(self.sweep_arrays[0].s21_point[:overlap])]
        for left, right in zip(self.sweep_arrays[:-1], self.sweep_arrays[1:]):
            frequencies.append(left.frequency[overlap:])
            amplitudes.append(left_weight * np.abs(left.s21_point[overlap:]) +
                              right_weight * np.abs(right.s21_point[:overlap]))
        frequencies.append(self.sweep_arrays[-1].frequency[overlap:])
        amplitudes.append(np.abs(self.sweep_arrays[-1].s21_point[overlap:]))
        frequency = np.concatenate(frequencies)
        amplitude = np.concatenate(amplitudes)
        if kernel is None:
            width = int(n / 10)
            kernel = np.exp(-np.linspace(-4, 4, width) ** 2)
            kernel /= kernel.sum()
        smoothed = np.convolve(kernel, amplitude, mode='same')
        smoothed[:width] = smoothed[width + 1]
        smoothed[-width:] = smoothed[-(width + 1)]
        return frequency, amplitude, smoothed


# !/usr/bin/python
# -*- coding: utf-8 -*-

import tensorflow as tf
import numpy as np
import os
import sys
import argparse

import network
import provider
import tf_utils
import math

BASE_DIR = os.path.dirname(__file__)
DATA_ROOT = os.path.join(BASE_DIR, '../data/Class1/')

parser = argparse.ArgumentParser()
parser.add_argument('--gpu', type=int, default=0)
parser.add_argument('--batch_size', type=int, default=32)
parser.add_argument('--base_learning_rate', type=float, default=0.001)
parser.add_argument('--max_epoch', type=int, default=10)
parser.add_argument('--max_epoch_seg', type=int, default=25)
parser.add_argument('--max_epoch_cls', type=int, default=10)
parser.add_argument('--optimizer', default='adam')
parser.add_argument('--image_size', type=int, default=512)
parser.add_argument('--decay_step', type=int, default=1000)
parser.add_argument('--decay_rate', type=float, default=0.7)
FLAGS = parser.parse_args()

GPU_IDX = FLAGS.gpu
BATCH_SIZE = FLAGS.batch_size
BASE_LEARNING_RATE = FLAGS.base_learning_rate
MAX_EPOCH = FLAGS.max_epoch
MAX_EPOCH_seg = FLAGS.max_epoch_seg
MAX_EPOCH_cls = FLAGS.max_epoch_cls
OPTIMIZER = FLAGS.optimizer
IMAGE_SIZE = FLAGS.image_size
DECAY_STEP = FLAGS.decay_step
DECAY_RATE = FLAGS.decay_rate

LOG_DIR = os.path.join(BASE_DIR, '../log/')
if not os.path.exists(LOG_DIR): os.mkdir(LOG_DIR)
LOG_FOUT = open(os.path.join(LOG_DIR, 'log_train.txt'), 'w')
LOG_FOUT.write(str(FLAGS)+'\n')

BN_INIT_DECAY = 0.5
BN_DECAY_DECAY_RATE = 0.5
BN_DECAY_DECAY_STEP = float(DECAY_STEP)
BN_DECAY_CLIP = 0.99

def log_string(out_str): # 把每次print的东西都写入log
    LOG_FOUT.write(out_str+'\n')
    LOG_FOUT.flush()
    print(out_str)

def get_learning_rate(global_step):
    learning_rate = tf.train.exponential_decay(
        BASE_LEARNING_RATE, global_step*BATCH_SIZE, DECAY_STEP,
        DECAY_RATE, staircase=True)
    learning_rate = tf.maximum(learning_rate, 0.00001)
    return learning_rate

def get_bn_decay(global_step):
    bn_momentum = tf.train.exponential_decay(
        BN_INIT_DECAY, global_step*BATCH_SIZE, BN_DECAY_DECAY_STEP,
        BN_DECAY_DECAY_STEP, staircase=True)
    bn_decay = tf.minimum(BN_DECAY_CLIP, 1-bn_momentum)
    return bn_decay

def train():
    with tf.device('/gpu:'+str(GPU_IDX)):
        # placeholder的作用是什么，我们现在需要用到一些参数构建graph，但这些参数要到后面才能给
        # 而且不同的阶段这些参数不同，因此不能直接将它们给定，先用placeholder占个位
        # 在后面向graph喂数据的时候再把这些数据具体输进去
        img_pl, label_seg_pl, label_cls_pl = \
            network.placeholder_inputs(BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE)
        is_training_pl = tf.placeholder(tf.bool, shape=())
        print(is_training_pl)

        global_step = tf.Variable(0)
        bn_decay = get_bn_decay(global_step)

        seg_out, seg_out_former = network.seg_model(img_pl, is_training_pl, bn_decay)
        seg_pred, loss_seg = network.get_loss(label_seg_pl, seg_out, 'seg')

        cls_out = network.cls_model(seg_out, seg_out_former, is_training_pl, bn_decay)
        cls_pred, loss_cls = network.get_loss(label_cls_pl, cls_out, 'cls')

        learning_rate = get_learning_rate(global_step)
        tf.summary.scalar('learning_rate', learning_rate)

        if OPTIMIZER == 'momentum':
            optimizer = tf.train.MomentumOptimizer(learning_rate, momentum='MOMENTUM')
        elif OPTIMIZER == 'adam':
            optimizer = tf.train.AdamOptimizer(learning_rate)

        train_op_seg = optimizer.minimize(loss_seg, global_step=global_step)
        train_op_cls = optimizer.minimize(loss_cls, global_step=global_step)
        # global_step: Optional 'Variable' to increment by one after the
        # variables have been updated every time
        # 所有数据集训练完一次，称为一个epoch。
        # 在一个epoch内，每训练一个batchsize，参数更新一次，称为一个iteration/step，
        # 累计在所有epoch的训练过程中，iteration的数目成为global_step

        saver = tf.train.Saver()

    # Create a session
    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    config.allow_soft_placement = True
    config.log_device_placement = True
    sess = tf.Session(config=config)

    init = tf.global_variables_initializer()

    sess.run(init)

    ops_seg = {'stage': 'seg',
               'img_pl': img_pl,
               'label_pl': label_seg_pl,
               'is_training_pl': is_training_pl,
               'pred': seg_pred,
               'loss': loss_seg,
               'train_op': train_op_seg,
               'step': global_step}

    ops_cls = {'stage': 'cls',
               'img_pl': img_pl,
               'label_pl': label_cls_pl,
               'is_training_pl': is_training_pl,
               'pred': cls_pred,
               'loss': loss_cls,
               'train_op': train_op_cls,
               'step': global_step}

    for e in range(MAX_EPOCH):
        log_string('----- EPOCH %03d -----'% e)
        for epoch in range(MAX_EPOCH_seg):
            log_string('----- SEG EPOCH %03d -----'% epoch)
            train_one_epoch(sess, ops_seg)
            eval_one_epoch(sess, ops_seg)

        for epoch in range(MAX_EPOCH_cls):
            log_string('----- CLS EPOCH %03d -----' % epoch)
            train_one_epoch(sess, ops_cls)
            eval_one_epoch(sess, ops_cls)

        if e % 1 == 0:
            save_path = saver.save(sess, os.path.join(LOG_DIR, "model.ckpt"))
            log_string("Model saved in file: %s" % save_path)

def train_one_epoch(sess, ops):
    is_training = True
    img, seg_label, cls_label = provider.LOAD_DATA(DATA_ROOT + 'Train/')
    img, seg_label, cls_label = provider.shuffle_data(img, seg_label, cls_label)
    label = {'seg': seg_label, 'cls': cls_label}
    img_num = img.shape[0]
    batch_num = img_num / BATCH_SIZE

    loss_sum = 0

    for batch_idx in range(batch_num):
        start_idx = batch_idx * BATCH_SIZE
        end_idx = (batch_idx+1) * BATCH_SIZE

        current_imgs = img[start_idx:end_idx]
        current_label = label[ops['stage']][start_idx:end_idx]

        feed_dict = {ops['img_pl']: current_imgs,
                     ops['label_pl']: current_label,
                     ops['is_training_pl']: is_training}

        global_step, pred, loss, _ = sess.run([ops['step'], ops['pred'], ops['loss'], ops['train_op']],
                                               feed_dict=feed_dict)

        log_string('global_step: %d; iter: %d; loss: %f' % (global_step, batch_idx+1, loss))
        loss_sum += loss

    log_string('train mean loss: %f' % (loss_sum / float(batch_num)))
    # print('accuracy: %f' % (total_correct / float(total_seen)))

def eval_one_epoch(sess, ops):
    is_training = False
    img, seg_label, cls_label = provider.LOAD_DATA(DATA_ROOT + 'Test/')
    label = {'seg': seg_label, 'cls': cls_label}
    img_num = len(img)
    batch_num = img_num / BATCH_SIZE

    loss_sum = 0

    for batch_idx in range(batch_num):
        start_idx = batch_idx * BATCH_SIZE
        end_idx = (batch_idx + 1) * BATCH_SIZE

        current_imgs = img[start_idx:end_idx]
        current_label = label[ops['stage']][start_idx:end_idx]

        feed_dict = {ops['img_pl']: current_imgs,
                     ops['label_pl']: current_label,
                     ops['is_training_pl']: is_training}

        global_step, pred, loss = sess.run([ops['step'], ops['pred'], ops['loss']],
                                            feed_dict=feed_dict)

        loss_sum += loss

    log_string('eval mean loss: %f' % (loss_sum / float(batch_num)))
    # print('accuracy: %f' % (total_correct / float(total_seen)))

if __name__ == "__main__":
    train()
    LOG_FOUT.close()


# Header variables and parameters.
import time
import os
import numpy as np
from IPython.core.debugger import set_trace
import sys
import importlib
from masif_modules.train_masif_site import run_masif_site
from default_config.masif_opts import masif_opts

"""
masif_site_predict.py: Evaluate one or multiple proteins on MaSIF-site.
Pablo Gainza - LPDI STI EPFL 2019
This file is part of MaSIF.
Released under an Apache License 2.0
"""

# Apply mask to input_feat
def mask_input_feat(input_feat, mask):
    mymask = np.where(np.array(mask) == 0.0)[0]
    return np.delete(input_feat, mymask, axis=2)

params = masif_opts["site"]
custom_params_file = sys.argv[1]
custom_params = importlib.import_module(custom_params_file, package=None)
custom_params = custom_params.custom_params

for key in custom_params:
    print("Setting {} to {} ".format(key, custom_params[key]))
    params[key] = custom_params[key]

# Shape precomputation dir.
parent_in_dir = params["masif_precomputation_dir"]
eval_list = []

if len(sys.argv) == 3:
    ppi_pair_ids = [sys.argv[2]]
# Read a list of pdb_chain entries to evaluate.
elif len(sys.argv) == 4 and sys.argv[2] == "-l":
    listfile = open(sys.argv[3])
    ppi_pair_ids = []
    for line in listfile:
        eval_list.append(line.rstrip())
    for mydir in os.listdir(parent_in_dir):
        ppi_pair_ids.append(mydir)
else:
    sys.exit(1)

# Build the neural network model
from masif_modules.MaSIF_site import MaSIF_site

learning_obj = MaSIF_site(
    params["max_distance"],
    n_thetas=4,
    n_rhos=3,
    n_rotations=4,
    idx_gpu="/gpu:0",
    feat_mask=params["feat_mask"],
    n_conv_layers=params["n_conv_layers"],
)
print("Restoring model from: " + params["model_dir"] + "model")
learning_obj.saver.restore(learning_obj.session, params["model_dir"] + "model")

if not os.path.exists(params["out_pred_dir"]):
    os.makedirs(params["out_pred_dir"])

idx_count = 0
for ppi_pair_id in ppi_pair_ids:
    print(ppi_pair_id)
    in_dir = parent_in_dir + ppi_pair_id + "/"

    pdbid = ppi_pair_id.split("_")[0]
    chain1 = ppi_pair_id.split("_")[1]
    chain2 = ppi_pair_id.split("_")[2]
    pids = ["p1"]
    chains = [chain1]
    if chain2 != "":
        pids = ["p1", "p2"]
        chains = [chain1, chain2]

    for ix, pid in enumerate(pids):
        pdb_chain_id = pdbid + "_" + chains[ix]
        if (
            len(eval_list) > 0
            and pdb_chain_id not in eval_list
            and pdb_chain_id + "_" not in eval_list
        ):
            continue

        print("Evaluating {}".format(pdb_chain_id))

        try:
            rho_wrt_center = np.load(in_dir + pid + "_rho_wrt_center.npy")
        except:
            print("File not found: {}".format(in_dir + pid + "_rho_wrt_center.npy"))
            continue
        theta_wrt_center = np.load(in_dir + pid + "_theta_wrt_center.npy")
        input_feat = np.load(in_dir + pid + "_input_feat.npy")
        input_feat = mask_input_feat(input_feat, params["feat_mask"])
        mask = np.load(in_dir + pid + "_mask.npy")
        indices = np.load(in_dir + pid + "_list_indices.npy", encoding="latin1")
        labels = np.zeros((len(mask)))

        print("Total number of patches:{} \n".format(len(mask)))

        tic = time.time()
        scores = run_masif_site(
            params,
            learning_obj,
            rho_wrt_center,
            theta_wrt_center,
            input_feat,
            mask,
            indices,
        )
        toc = time.time()
        print(
            "Total number of patches for which scores were computed: {}\n".format(
                len(scores[0])
            )
        )
        print("GPU time (real time, not actual GPU time): {:.3f}s".format(toc-tic))
        np.save(
            params["out_pred_dir"] + "/pred_" + pdbid + "_" + chains[ix] + ".npy",
            scores,
        )


#!/usr/bin/env python
#
# Licensed under the BSD license.  See full license in LICENSE file.
# http://www.lightshowpi.com/
#
# Author: Todd Giles (todd@lightshowpi.com)
# Author: Chris Usey (chris.usey@gmail.com)
# Author: Ryan Jennings
# Author: Paul Dunn (dunnsept@gmail.com)
"""Play any audio file and synchronize lights to the music

When executed, this script will play an audio file, as well as turn on and off N channels
of lights to the music (by default the first 8 GPIO channels on the Rasberry Pi), based upon
music it is playing. Many types of audio files are supported (see decoder.py below), but
it has only been tested with wav and mp3 at the time of this writing.

The timing of the lights turning on and off is based upon the frequency response of the music
being played.  A short segment of the music is analyzed via FFT to get the frequency response
across each defined channel in the audio range.  Each light channel is then faded in and out based
upon the amplitude of the frequency response in the corresponding audio channel.  Fading is
accomplished with a software PWM output.  Each channel can also be configured to simply turn on
and off as the frequency response in the corresponding channel crosses a threshold.

FFT calculation can be CPU intensive and in some cases can adversely affect playback of songs
(especially if attempting to decode the song as well, as is the case for an mp3).  For this reason,
the FFT cacluations are cached after the first time a new song is played.  The values are cached
in a gzip'd text file in the same location as the song itself.  Subsequent requests to play the
same song will use the cached information and not recompute the FFT, thus reducing CPU utilization
dramatically and allowing for clear music playback of all audio file types.

Recent optimizations have improved this dramatically and most users are no longer reporting
adverse playback of songs even on the first playback.

Sample usage:
To play an entire list -
sudo python synchronized_lights.py --playlist=/home/pi/music/.playlist

To play a specific song -
sudo python synchronized_lights.py --file=/home/pi/music/jingle_bells.mp3

Third party dependencies:

alsaaudio: for audio input/output - http://pyalsaaudio.sourceforge.net/
decoder.py: decoding mp3, ogg, wma, ... - https://pypi.python.org/pypi/decoder.py/1.5XB
numpy: for FFT calculation - http://www.numpy.org/
"""

import argparse
import csv
import fcntl
import json
import logging
import os
import random
import subprocess
import sys
from threading import Timer
import wave

import alsaaudio as aa
import fft
import configuration_manager as cm
import decoder
import hardware_controller as hc
import numpy as np

from preshow import Preshow

# Configurations - TODO(todd): Move more of this into configuration manager
_CONFIG = cm.CONFIG
_MODE = cm.lightshow()['mode']
_MIN_FREQUENCY = _CONFIG.getfloat('audio_processing', 'min_frequency')
_MAX_FREQUENCY = _CONFIG.getfloat('audio_processing', 'max_frequency')
_RANDOMIZE_PLAYLIST = _CONFIG.getboolean('lightshow', 'randomize_playlist')
try:
    _CUSTOM_CHANNEL_MAPPING = [int(channel) for channel in
                               _CONFIG.get('audio_processing', 'custom_channel_mapping').split(',')]
except:
    _CUSTOM_CHANNEL_MAPPING = 0
try:
    _CUSTOM_CHANNEL_FREQUENCIES = [int(channel) for channel in
                                   _CONFIG.get('audio_processing',
                                               'custom_channel_frequencies').split(',')]
except:
    _CUSTOM_CHANNEL_FREQUENCIES = 0
try:
    _PLAYLIST_PATH = cm.lightshow()['playlist_path'].replace('$SYNCHRONIZED_LIGHTS_HOME', cm.HOME_DIR)
except:
    _PLAYLIST_PATH = "/home/pi/music/.playlist"
try:
    _usefm=_CONFIG.get('audio_processing','fm');
    frequency =_CONFIG.get('audio_processing','frequency');
    play_stereo = True
    music_pipe_r,music_pipe_w = os.pipe()
except:
    _usefm='false'
CHUNK_SIZE = 2048  # Use a multiple of 8 (move this to config)

def calculate_channel_frequency(min_frequency, max_frequency, custom_channel_mapping,
                                custom_channel_frequencies):
    '''Calculate frequency values for each channel, taking into account custom settings.'''

    # How many channels do we need to calculate the frequency for
    if custom_channel_mapping != 0 and len(custom_channel_mapping) == hc.GPIOLEN:
        logging.debug("Custom Channel Mapping is being used: %s", str(custom_channel_mapping))
        channel_length = max(custom_channel_mapping)
    else:
        logging.debug("Normal Channel Mapping is being used.")
        channel_length = hc.GPIOLEN

    logging.debug("Calculating frequencies for %d channels.", channel_length)
    octaves = (np.log(max_frequency / min_frequency)) / np.log(2)
    logging.debug("octaves in selected frequency range ... %s", octaves)
    octaves_per_channel = octaves / channel_length
    frequency_limits = []
    frequency_store = []

    frequency_limits.append(min_frequency)
    if custom_channel_frequencies != 0 and (len(custom_channel_frequencies) >= channel_length + 1):
        logging.debug("Custom channel frequencies are being used")
        frequency_limits = custom_channel_frequencies
    else:
        logging.debug("Custom channel frequencies are not being used")
        for i in range(1, hc.GPIOLEN + 1):
            frequency_limits.append(frequency_limits[-1]
                                    * 10 ** (3 / (10 * (1 / octaves_per_channel))))
    for i in range(0, channel_length):
        frequency_store.append((frequency_limits[i], frequency_limits[i + 1]))
        logging.debug("channel %d is %6.2f to %6.2f ", i, frequency_limits[i],
                      frequency_limits[i + 1])

    # we have the frequencies now lets map them if custom mapping is defined
    if custom_channel_mapping != 0 and len(custom_channel_mapping) == hc.GPIOLEN:
        frequency_map = []
        for i in range(0, hc.GPIOLEN):
            mapped_channel = custom_channel_mapping[i] - 1
            mapped_frequency_set = frequency_store[mapped_channel]
            mapped_frequency_set_low = mapped_frequency_set[0]
            mapped_frequency_set_high = mapped_frequency_set[1]
            logging.debug("mapped channel: " + str(mapped_channel) + " will hold LOW: "
                          + str(mapped_frequency_set_low) + " HIGH: "
                          + str(mapped_frequency_set_high))
            frequency_map.append(mapped_frequency_set)
        return frequency_map
    else:
        return frequency_store

def update_lights(matrix, mean, std):
    '''Update the state of all the lights based upon the current frequency response matrix'''
    for i in range(0, hc.GPIOLEN):
        # Calculate output pwm, where off is at some portion of the std below
        # the mean and full on is at some portion of the std above the mean.
        brightness = matrix[i] - mean[i] + 0.5 * std[i]
        brightness = brightness / (1.25 * std[i])
        if brightness > 1.0:
            brightness = 1.0
        if brightness < 0:
            brightness = 0
        if not hc.is_pin_pwm(i):
            # If pin is on / off mode we'll turn on at 1/2 brightness
            if (brightness > 0.5):
                hc.turn_on_light(i, True)
            else:
                hc.turn_off_light(i, True)
        else:
            hc.turn_on_light(i, True, brightness)

def audio_in():
    '''Control the lightshow from audio coming in from a USB audio card'''
    sample_rate = cm.lightshow()['audio_in_sample_rate']
    input_channels = cm.lightshow()['audio_in_channels']

    # Open the input stream from default input device
    stream = aa.PCM(aa.PCM_CAPTURE, aa.PCM_NORMAL, cm.lightshow()['audio_in_card'])
    stream.setchannels(input_channels)
    stream.setformat(aa.PCM_FORMAT_S16_LE) # Expose in config if needed
    stream.setrate(sample_rate)
    stream.setperiodsize(CHUNK_SIZE)

    logging.debug("Running in audio-in mode - will run until Ctrl+C is pressed")
    print "Running in audio-in mode, use Ctrl+C to stop"
    try:
        hc.initialize()
        frequency_limits = calculate_channel_frequency(_MIN_FREQUENCY,
                                                       _MAX_FREQUENCY,
                                                       _CUSTOM_CHANNEL_MAPPING,
                                                       _CUSTOM_CHANNEL_FREQUENCIES)

        # Start with these as our initial guesses - will calculate a rolling mean / std
        # as we get input data.
        mean = [12.0 for _ in range(hc.GPIOLEN)]
        std = [0.5 for _ in range(hc.GPIOLEN)]
        recent_samples = np.empty((250, hc.GPIOLEN))
        num_samples = 0

        # Listen on the audio input device until CTRL-C is pressed
        while True:
            l, data = stream.read()

            if l:
                try:
                    matrix = fft.calculate_levels(data, CHUNK_SIZE, sample_rate, frequency_limits, input_channels)
                    if not np.isfinite(np.sum(matrix)):
                        # Bad data --- skip it
                        continue
                except ValueError as e:
                    # TODO(todd): This is most likely occuring due to extra time in calculating
                    # mean/std every 250 samples which causes more to be read than expected the
                    # next time around.  Would be good to update mean/std in separate thread to
                    # avoid this --- but for now, skip it when we run into this error is good
                    # enough ;)
                    logging.debug("skipping update: " + str(e))
                    continue

                update_lights(matrix, mean, std)

                # Keep track of the last N samples to compute a running std / mean
                #
                # TODO(todd): Look into using this algorithm to compute this on a per sample basis:
                # http://www.johndcook.com/blog/standard_deviation/
                if num_samples >= 250:
                    no_connection_ct = 0
                    for i in range(0, hc.GPIOLEN):
                        mean[i] = np.mean([item for item in recent_samples[:, i] if item > 0])
                        std[i] = np.std([item for item in recent_samples[:, i] if item > 0])

                        # Count how many channels are below 10, if more than 1/2, assume noise (no connection)
                        if mean[i] < 10.0:
                            no_connection_ct += 1

                    # If more than 1/2 of the channels appear to be not connected, turn all off
                    if no_connection_ct > hc.GPIOLEN / 2:
                        logging.debug("no input detected, turning all lights off")
                        mean = [20 for _ in range(hc.GPIOLEN)]
                    else:
                        logging.debug("std: " + str(std) + ", mean: " + str(mean))
                    num_samples = 0
                else:
                    for i in range(0, hc.GPIOLEN):
                        recent_samples[num_samples][i] = matrix[i]
                    num_samples += 1

    except KeyboardInterrupt:
        pass
    finally:
        print "\nStopping"
        hc.clean_up()

# TODO(todd): Refactor more of this to make it more readable / modular.
def play_song():
    '''Play the next song from the play list (or --file argument).'''
    song_to_play = int(cm.get_state('song_to_play', 0))
    play_now = int(cm.get_state('play_now', 0))

    # Arguments
    parser = argparse.ArgumentParser()
    filegroup = parser.add_mutually_exclusive_group()
    filegroup.add_argument('--playlist', default=_PLAYLIST_PATH,
                           help='Playlist to choose song from.')
    filegroup.add_argument('--file', help='path to the song to play (required if no'
                           'playlist is designated)')
    parser.add_argument('--readcache', type=int, default=1,
                        help='read light timing from cache if available. Default: true')
    args = parser.parse_args()

    # Make sure one of --playlist or --file was specified
    if args.file == None and args.playlist == None:
        print "One of --playlist or --file must be specified"
        sys.exit()

    # Initialize Lights
    hc.initialize()

    # Handle the pre-show
    if not play_now:
        result = Preshow().execute()
        if result == Preshow.PlayNowInterrupt:
            play_now = True

    # Determine the next file to play
    song_filename = args.file
    if args.playlist != None and args.file == None:
        most_votes = [None, None, []]
        current_song = None
        with open(args.playlist, 'rb') as playlist_fp:
            fcntl.lockf(playlist_fp, fcntl.LOCK_SH)
            playlist = csv.reader(playlist_fp, delimiter='\t')
            songs = []
            for song in playlist:
                if len(song) < 2 or len(song) > 4:
                    logging.error('Invalid playlist.  Each line should be in the form: '
                                 '<song name><tab><path to song>')
                    sys.exit()
                elif len(song) == 2:
                    song.append(set())
                else:
                    song[2] = set(song[2].split(','))
                    if len(song) == 3 and len(song[2]) >= len(most_votes[2]):
                        most_votes = song
                songs.append(song)
            fcntl.lockf(playlist_fp, fcntl.LOCK_UN)

        if most_votes[0] != None:
            logging.info("Most Votes: " + str(most_votes))
            current_song = most_votes

            # Update playlist with latest votes
            with open(args.playlist, 'wb') as playlist_fp:
                fcntl.lockf(playlist_fp, fcntl.LOCK_EX)
                writer = csv.writer(playlist_fp, delimiter='\t')
                for song in songs:
                    if current_song == song and len(song) == 3:
                        song.append("playing!")
                    if len(song[2]) > 0:
                        song[2] = ",".join(song[2])
                    else:
                        del song[2]
                writer.writerows(songs)
                fcntl.lockf(playlist_fp, fcntl.LOCK_UN)

        else:
            # Get a "play now" requested song
            if play_now > 0 and play_now <= len(songs):
                current_song = songs[play_now - 1]
            # Get random song
            elif _RANDOMIZE_PLAYLIST:
                # Use python's random.randrange() to get a random song
                current_song = songs[random.randrange(0, len(songs))]

            # Play next song in the lineup
            else:
                song_to_play = song_to_play if (song_to_play <= len(songs) - 1) else 0
                current_song = songs[song_to_play]
                next_song = (song_to_play + 1) if ((song_to_play + 1) <= len(songs) - 1) else 0
                cm.update_state('song_to_play', next_song)

        # Get filename to play and store the current song playing in state cfg
        song_filename = current_song[1]
        cm.update_state('current_song', songs.index(current_song))

    song_filename = song_filename.replace("$SYNCHRONIZED_LIGHTS_HOME", cm.HOME_DIR)

    # Ensure play_now is reset before beginning playback
    if play_now:
        cm.update_state('play_now', 0)
        play_now = 0

    # Initialize FFT stats
    matrix = [0 for _ in range(hc.GPIOLEN)]

    # Set up audio
    if song_filename.endswith('.wav'):
        musicfile = wave.open(song_filename, 'r')
    else:
        musicfile = decoder.open(song_filename)

    sample_rate = musicfile.getframerate()
    num_channels = musicfile.getnchannels()

    if _usefm=='true':
        logging.info("Sending output as fm transmission")
        with open(os.devnull, "w") as dev_null:
            fm_process = subprocess.Popen(["sudo",cm.HOME_DIR + "/bin/pifm","-",str(frequency),"44100", "stereo" if play_stereo else "mono"], stdin=music_pipe_r, stdout=dev_null)
    else:
        output = aa.PCM(aa.PCM_PLAYBACK, aa.PCM_NORMAL)
        output.setchannels(num_channels)
        output.setrate(sample_rate)
        output.setformat(aa.PCM_FORMAT_S16_LE)
        output.setperiodsize(CHUNK_SIZE)

    logging.info("Playing: " + song_filename + " (" + str(musicfile.getnframes() / sample_rate)
                 + " sec)")
    # Output a bit about what we're about to play to the logs
    song_filename = os.path.abspath(song_filename)

    # create empty array for the cache_matrix
    cache_matrix = np.empty(shape=[0, hc.GPIOLEN])
    cache_found = False
    cache_filename = os.path.dirname(song_filename) + "/." + os.path.basename(song_filename) \
        + ".sync"
    # The values 12 and 1.5 are good estimates for first time playing back (i.e. before we have
    # the actual mean and standard deviations calculated for each channel).
    mean = [12.0 for _ in range(hc.GPIOLEN)]
    std = [1.5 for _ in range(hc.GPIOLEN)]

    if args.readcache:
        # Read in cached fft
        try:
            # load cache from file using numpy loadtxt
            cache_matrix = np.loadtxt(cache_filename)
            cache_found = True

            # get std from matrix / located at index 0
            std = np.array(cache_matrix[0])

            # get mean from matrix / located at index 1
            mean = np.array(cache_matrix[1])

            # delete mean and std from the array
            cache_matrix = np.delete(cache_matrix, (0), axis = 0)
            cache_matrix = np.delete(cache_matrix, (0), axis = 0)

            logging.debug("std: " + str(std) + ", mean: " + str(mean))
        except IOError:
            logging.warn("Cached sync data file not found: '" + cache_filename
                         + ".  One will be generated.")

    # Determine amount to delay showing the lights
    delay_lights_s = -cm.lightshow()['delay_audio_s']
    skip_rows = 0
    if delay_lights_s < 0:
        # We need to start lights before music, so we need to skip ahead on the lights
        chunk_period = CHUNK_SIZE / sample_rate
        skip_rows = int(-delay_lights_s / chunk_period) + 1
        delay_lights_s = chunk_period * skip_rows + delay_lights_s

    # Process audio file
    row = skip_rows
    data = musicfile.readframes(CHUNK_SIZE)
    frequency_limits = calculate_channel_frequency(_MIN_FREQUENCY,
                                                   _MAX_FREQUENCY,
                                                   _CUSTOM_CHANNEL_MAPPING,
                                                   _CUSTOM_CHANNEL_FREQUENCIES)

    while data != '' and not play_now:
        if _usefm=='true':
            os.write(music_pipe_w, data)
        else:
            output.write(data)

        # Control lights with cached timing values if they exist
        matrix = None
        if cache_found and args.readcache:
            if row < len(cache_matrix):
                matrix = cache_matrix[row]
            elif row < len(cache_matrix) + skip_rows:
                # No more lights to play due to audio delay
                continue
            else:
                logging.warning("Ran out of cached FFT values, will update the cache.")
                cache_found = False

        if matrix == None:
            # No cache - Compute FFT in this chunk, and cache results
            matrix = fft.calculate_levels(data, CHUNK_SIZE, sample_rate, frequency_limits)

            # Add the matrix to the end of the cache
            cache_matrix = np.vstack([cache_matrix, matrix])

        # Update the lights (delayed if necessary)
        if delay_lights_s > 0:
            Timer(delay_lights_s, update_lights(matrix, mean, std)).start()
        else:
            update_lights(matrix, mean, std)

        # Read next chunk of data from music file
        data = musicfile.readframes(CHUNK_SIZE)
        row = row + 1

        # Load new application state in case we've been interrupted
        cm.load_state()
        play_now = int(cm.get_state('play_now', 0))

    if not cache_found:
        # Compute the standard deviation and mean values for the cache
        for i in range(0, hc.GPIOLEN):
            std[i] = np.std([item for item in cache_matrix[:, i] if item > 0])
            mean[i] = np.mean([item for item in cache_matrix[:, i] if item > 0])

        # Add mean and std to the top of the cache
        cache_matrix = np.vstack([mean, cache_matrix])
        cache_matrix = np.vstack([std, cache_matrix])

        # Save the cache using numpy savetxt
        np.savetxt(cache_filename, cache_matrix)

        logging.info("Cached sync data written to '." + cache_filename
                        + "' [" + str(len(cache_matrix)) + " rows]")

    # Cleanup the pifm process
    if _usefm=='true':
        fm_process.kill()

    # We're done, turn it all off and clean up things ;)
    hc.clean_up()

if __name__ == "__main__":
    # Log everything to our log file
    # TODO(todd): Add logging configuration options.
    logging.basicConfig(filename=cm.LOG_DIR + '/music_and_lights.play.dbg',
                        format='[%(asctime)s] %(levelname)s {%(pathname)s:%(lineno)d}'
                        ' - %(message)s',
                        level=logging.DEBUG)

    if cm.lightshow()['mode'] == 'audio-in':
        audio_in()
    else:
        play_song()