#!/usr/bin/env python
"""
# =================================================================================================================== #
# - Written by Enrico Ciraci - 04/24/2019
# =================================================================================================================== #
# - Create Basins Binary Mask given:
# -
# - input basin shapefile;
# - reference lat/lon grid;
# -
# =================================================================================================================== #
# -  INPUT PARAMETERS:
# - "-S", "--basins" - list of basins to consider passed as csv
# -                    (def. 'indus_river', 'lower_indus_river', 'upper_indus_river')
# - "-B", "--buffer" - boundary shapefile buffer used in the intersection operation. (def. 0)
# - "-N", "--nproc" - number of maximum simultaneous processes (def. 32)
# =================================================================================================================== #
#  OUTPUTS - (see introduction)
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #
#     To Run this procedure digit:
#
#     python create_basins_mask.py
# =================================================================================================================== #
# PYTHON DEPENDENCIES:
#	numpy: Scientific Computing Tools For Python (http://www.numpy.org)
#   netCDF4: python/numpy interface to netCDF library (https://pypi.python.org/pypi/netCDF4)
#   getopt: C-style parser for command line options (https://docs.python.org/2/library/getopt.html)
#   xarray: xarray: N-D labeled arrays and datasets in Python (http://xarray.pydata.org)
#   pyshp: This library reads and writes ESRI Shapefiles in pure Python. (https://github.com/GeospatialPython/pyshp)
#   shapely: Manipulation and analysis of geometric objects in the Cartesian plane. (https://shapely.readthedocs.
#            io/en/stable/manual.html)
#   concurrent.futures: https://docs.python.org/3/library/concurrent.futures.html
# PROGRAM DEPENDENCIES:
#   enrico_library: https://github.com/uci-gravity/Enrico
# =================================================================================================================== #
# - UPDATE -
# =================================================================================================================== #
# IMPORTANT:
# =================================================================================================================== #
"""
# - python dependencies
from __future__ import print_function
import os
import sys
import numpy as np
import netCDF4 as nC4
import getopt
import shapefile
from shapely.geometry import shape, Point, polygon
from concurrent.futures import ThreadPoolExecutor, as_completed
from tqdm import tqdm
from time import time

def write_point_shapefile(output_path_shp, lat_vect, lon_vect, attribute_vect, attribute_name=''):
    """
    Save a Point Type esri shapefile with a single attribute per point
    :param output_path_shp: absolute path to the output file
    :param lat_vect: vector containing the latitude points coordinates
    :param lon_vect: vector containing the longitude points coordinates
    :param attribute_vect: vector containing the discharge magnitude
    :param attribute_name: name of the attribute
    :return:
    """
    # - python
    w = shapefile.Writer(output_path_shp[:-4], shapeType=1)
    w.autoBalance = 1
    w.field(attribute_name, 'F', 10, 8)
    # -
    for ind in range(0, len(attribute_vect)):
        w.point(np.float(lon_vect[ind]), np.float(lat_vect[ind]))
        w.record(float(attribute_vect[ind]))
    w.close()
    # - create the PRJ file
    with open(output_path_shp[:-3] + 'prj', "w") as prj:
        epsg = 'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563]],                ' \
               'PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]]'
        prj.write(epsg)

def verify_and_create_dir(abs_path, dir_name):
    """
    Create directory
    :param abs_path:absolute path to the output directory
    :param dir_name: new directory name
    :return: path to the new directory
    """
    directory = os.path.join(abs_path, dir_name)

    if not os.path.exists(directory):
        os.mkdir(directory)
    # -
    return directory

def calculate_area_mask_vect(lat_vect, lon_vect):
    """
    Calculate Latitude/Longitude Area mask at a selected degree resolution grid
    defined employing the input latitude and longitude vector.
    :param lat_vect: latitude vector
    :param lon_vect: latitude vector
    :return: Area Mask - Expressed in cm2
    """
    # - General Parameters
    dtr = np.pi / 180.    # - Coefficient for the deg to radiant conversion
    rad_e = 6.371e8       # -  Earth Radius in cm
    out_lat = lat_vect
    out_lon = lon_vect
    resolution_lat = np.abs(lat_vect[1] - lat_vect[0])
    resolution_lon = np.abs(lon_vect[1] - lon_vect[0])
    area0 = resolution_lat * resolution_lon * dtr * dtr  # - Area of a single point mass
    out_lat_grid, out_lon_grid = np.meshgrid(out_lat, out_lon)
    # - output area mask
    radius = np.sqrt(area0 * np.cos(out_lat_grid * dtr) / np.pi) * rad_e
    out_mask = np.pi * (radius ** 2)  # - Circular Area in cm^2
    # -
    return out_mask

def from_shp_to_polygon(path_to_shape, buffer_p=0.5):
    """
    Read the input shapefile and return a list of polygon objects
    :param path_to_shape: absolute path to the input shapefile
    :param buffer_p: boundary buffer in degree
    :return:
    """
    # - open the shapefile using the python fiona package
    region_list = list()
    # - read the input regional shapefile
    pol = shapefile.Reader(path_to_shape)
    # - extract polygon-shapes
    sub = pol.shapes()
    # - Build the regional ice-covered region domain
    for ss in range(0, len(sub)):
        if len(sub[ss].parts) > 1:
            # - the shapefile is composed by multiple parts defining an external ring and one ore more
            # - interior holes
            limits = sub[ss].parts
            holes = []  # - holes boundaries
            for x in range(2, len(limits)):
                holes.append(sub[ss].points[limits[x - 1]:limits[x]])
            # - define polygon with holes
            shp_tmp = polygon.Polygon(sub[ss].points[limits[0]:limits[1]], holes)
        else:
            # - polygon composed only by an external ring
            shp_tmp = shape(sub[ss]).buffer(buffer_p)

        region_list.append(shp_tmp)
    # -
    return region_list

def write_netcdf_mask(binary_mask, area_mask, lat, lon, file_to_save):
    """
    Write spatial field in a netcdf archive
    :param binary_mask: input binary mask
    :param area_mask: regional area mask in cm2
    :param lat: latitude axis
    :param lon: longitude axis
    :param file_to_save: absolute path to the output fi;e
    :return:
    """
    # - Writing the Output file
    rootgrp = nC4.Dataset(file_to_save, mode='w', format='NETCDF4')
    # - Create Variable dimensions
    rootgrp.createDimension('lat', len(lat))
    rootgrp.createDimension('lon', len(lon))
    # - Create output variables:
    var_lat = rootgrp.createVariable('lat', 'f4', 'lat')
    var_lon = rootgrp.createVariable('lon', 'f4', 'lon')
    var_mask = rootgrp.createVariable('area', 'f8', ('lat', 'lon'))
    var_mask_binary = rootgrp.createVariable('binary', 'f8', ('lat', 'lon'))

    # - Longitude attributes
    var_lon.units = 'degree east'
    var_lon.long_name = 'Longitude'
    var_lon.actual_range = [np.min(lon), np.max(lon)]
    var_lon.standard_name = 'longitude'
    var_lon.axis = 'X'
    var_lon.coordinate_defines = 'point'
    # - Latitude Attributes
    var_lat.units = 'degree north'
    var_lat.long_name = 'Latitude'
    var_lat.actual_range = [np.min(lat), np.max(lat)]
    var_lat.standard_name = 'latitude'
    var_lat.axis = 'Y'
    var_lat.coordinate_defines = 'point'
    # - Mask Attributes
    var_mask.units = 'cm2'
    var_mask.var_desc = 'Basin Area Mask'
    var_mask.actual_range = [np.min(area_mask), np.max(area_mask)]
    # - Binary Mask Attributes
    var_mask_binary.units = ''
    var_mask_binary.var_desc = 'basin binary mask'
    var_mask_binary.actual_range = [np.min(binary_mask), np.max(binary_mask)]

    var_lon[:] = lon
    var_lat[:] = lat
    var_mask[:, :] = area_mask
    var_mask_binary[:, :] = binary_mask
    # -  close the netcdf file created
    rootgrp.close()

def parallel_code(data_dict):
    """
    Contains the portion of the code that is executed in parallel
    :param data_dict: python dictionary containing the input parameters
    :return:
    """
    lon_s = data_dict['lon_s']
    lat_s = data_dict['lat_s']
    xy_point = Point(lon_s, lat_s)
    region_list = data_dict['rl']
    out_index = -9999.
    for rl in region_list:
        if xy_point.within(rl):
            out_index = data_dict['index']
            break
    return out_index

def main():
    """
    Main Section of the script.
    :return:
    """
    # -- Read the system arguments listed after the program and run the program
    long_options = ['basins=', 'buffer=', 'nproc=']
    try:
        optlist = getopt.getopt(sys.argv[1:], 'S:B:N:', long_options)
    except ValueError:
        optlist = list()

    # - list of the river basin to consider
    basin_list = ['indus_river']
    # - uses by default a number of simultaneous processes equal to the number of CPUs
    max_processes = None
    # - basin boundaries buffer
    buffer_p = 0.
    try:
        for opt, arg in optlist[0]:
            if opt in ("-S", "--basins"):
                # - list of the basins to process
                # - passed as csv
                basin_list = arg.split(',')
            elif opt in ("-B", "--buffer"):
                # - Basin boundary buffer
                buffer_p = float(arg)
            elif opt in ("-N", "--nproc"):
                # - number of simultaneous processes
                max_processes = int(arg)
    except ValueError:
        pass
    start = time()

    # - input/output data directory
    input_dir = os.path.join('.', 'input')
    output_dir = verify_and_create_dir('.', 'output')

    # - lat/lon arrays
    lat_vect = np.arange(-90,  90+1, 1)
    lon_vect = np.arange(-180,  180+1, 1)
    # - create lat lon domain grid
    l_lon, l_lat = np.meshgrid(lon_vect, lat_vect)
    l_lon_vect = l_lon.flatten()
    l_lat_vect = l_lat.flatten()

    # - calculate a global area mask in cm2 ad the model resolution
    area_mask = np.transpose(calculate_area_mask_vect(lat_vect, lon_vect))

    # - crop the model mask using the selected river mask
    if basin_list:
        print('# - Create Basins Mask.')
        for basin in basin_list:
            print('# - ' + basin)
            # - load basin mask
            b_info = dict()
            b_info['basin_boundary'] = os.path.join(input_dir, basin, basin+'.shp')
            out_dir_reg = verify_and_create_dir(output_dir, basin)
            # - read regional shapefile and convert it to a shapely polygon object
            region_list = from_shp_to_polygon(b_info['basin_boundary'], buffer_p=buffer_p)
            # - list that will contain the indexes of the points within the region of interest
            tot_ind = list()
            # - parallel portion of the code
            processes = []
            with ThreadPoolExecutor(max_workers=max_processes) as executor:
                for ll in tqdm(range(0, len(l_lon_vect)), ncols=50):
                    lon_s = l_lon_vect[ll]
                    lat_s = l_lat_vect[ll]
                    # -
                    data_dict = dict()
                    data_dict['lon_s'] = lon_s
                    data_dict['lat_s'] = lat_s
                    data_dict['index'] = ll
                    data_dict['rl'] = region_list
                    processes.append(executor.submit(parallel_code, data_dict))

            for res in as_completed(processes):
                res_out = res.result()
                if res_out != -9999.:
                    tot_ind.append(res_out)
            # - Save the obtained mask
            out_lat = np.array(l_lat_vect)[tot_ind]
            out_lon = np.array(l_lon_vect)[tot_ind]
            id_y = np.arange(len(out_lon)) + 1
            output_path_shp = os.path.join(out_dir_reg, basin + '.shp')
            write_point_shapefile(output_path_shp, out_lat, out_lon, id_y, attribute_name='id')

            # - Save the mask also in netcdf format
            binary_mask = np.zeros(np.shape(l_lon))
            # - fill the binary mask
            for k in range(0, len(out_lat)):
                binary_mask[np.where(lat_vect == out_lat[k]), np.where(lon_vect == out_lon[k])] = 1.
            output_path_shp = os.path.join(out_dir_reg, basin + '.nc')
            write_netcdf_mask(binary_mask, area_mask*binary_mask, lat_vect, lon_vect, output_path_shp)

    end = time()
    hours, rem = divmod(end - start, 3600)
    minutes, seconds = divmod(rem, 60)
    print(f'Execution Time: ' + "{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), seconds))

# -- run main program
if __name__ == '__main__':
    main()


# Copyright (C) 2013 Lindley Graham

"""
This module contains a set of methods and a class for interacting with NCSU
Subdomain Modeling Python code and associated files. The focus of this module
is the :class:`subdomain`.
"""

import glob, os, sys, subprocess, re, math
import numpy as np
import scipy.io as sio
import py.gensub as gensub
import polyadcirc.run_framework.domain as dom
import polyadcirc.pyADCIRC.fort15_management as f15
import polyadcirc.pyADCIRC.fort13_management as f13
import polyadcirc.pyADCIRC.output as output
import polyadcirc.run_framework.random_manningsn as rmn
import polyadcirc.pyADCIRC.post_management as post
import polyadcirc.pyGriddata.file_management as fm
from polyadcirc.pyADCIRC.basic import comm

def loadmat(save_file, base_dir, grid_dir, save_dir, basis_dir):
    """
    Loads data from ``save_file`` into a
    :class:`~polyadcirc.run_framwork.random_manningsn.runSet` object.
    Reconstructs :class:`~polyadcirc.run_framwork.random_manningsn.subdomain`.

    :param string save_file: local file name
    :param string grid_dir: directory containing ``fort.14``, ``fort.15``, and
        ``fort.22``
    :param string save_dir: directory where ``RF_directory_*`` are
        saved, and where fort.13 is located
    :param string basis_dir: directory where ``landuse_*`` folders are located
    :param string base_dir: directory that contains ADCIRC executables, and
        machine specific ``in.prep#`` files
    :rtype: tuple of :class:`~polyadcirc.run_framwork.random_manningsn.runSet`
        and :class:`~polyadcirc.run_framwork.random_manningsn.domain` objects
    :returns: (main_run, domain)

    """

    # the lines below are only necessary if you need to update what the
    # directories are when swithcing from euclid to your desktop/laptop
    # assumes that the landuse directory and ADCIRC_landuse directory are in
    # the same directory
    domain = subdomain(grid_dir)
    domain.update()
    domain.get_Triangulation()
    #domain.set_station_bathymetry()

    main_run = rmn.runSet(grid_dir, save_dir, basis_dir, base_dir=base_dir)
    main_run.ts_error = {}
    main_run.nts_error = {}
    main_run.time_obs = {}

    # load the data from at *.mat file
    mdat = sio.loadmat(os.path.join(save_dir, save_file))

    for k, v in mdat.iteritems():
        skey = k.split('_')
        if skey[-1] == 'time':
            # check to see if the key is "*_time"
            main_run.time_obs[skey[0]] = v
        elif f15.filetype.has_key(skey[0]):
            if not re.match('fort', skey[0]):
                # check to see if key is nts_data
                main_run.nts_error[skey[0]] = v
            else:
                # check to see if key is ts_data
                main_run.ts_error[skey[0]] = v
        #print k, v

    return (main_run, domain)

class subdomain(dom.domain):
    """
    Objects of this class contain all the data needed by :mod:`py.genbcs`,
    :mod:`py.genfull`, and :mod:`py.gensub` for a particular subdomain
    mesh/grid. References to :class:`polyadcirc.run_framework.subdomain` objects
    are also contained in an instantiation of this class.
    """
    def __init__(self, path, node_num=0, element_num=0,
                 node=None, element=None):
        """
        Initialization
        """
        super(subdomain, self).__init__(path, node_num, element_num, node,
                                        element)

        # figure out where the script dir for the ncsu subdomain code is
        for sys_path in sys.path:
            potential_file_list = glob.glob(os.path.join(sys_path, 'py'))
            if potential_file_list:
                self.script_dir = potential_file_list[0]
                break

        fm.mkdir(path)

        #: flag for shape of subdomain (0 ellipse, 1 circle)
        self.flag = None

    def set_fulldomain(self, fulldomain):
        """
        Sets the fulldomain of this subdomain to fulldomain and adds this
        subdomain to that fulldomain.

        :type fulldomain: :class:`~polyadcirc.run_framework.fulldomain`
        :param fulldomain: the fulldomain for this subdomain

        """
        self.fulldomain = fulldomain
        self.fulldomain.subdomains.append(self)

    def gensub(self, bound_ele=1, bound_vel=1, bound_wd=1, winddir=None):
        """
        Generate the subdomain input files (``fort.13``, ``fort.14``,
        ``fort.015``, ``py.141``, ``py.140``) and shape file. Creates
        ``fort.15`` based on the ``fort.15`` in
        :class:`polyadcirc.run_framework.fulldomain` to ``self.path``, and
        creates symbolic links to meterological forcing files (``fort.22*``).

        :param int bound_ele: a flag determining whether surface elevations of
            the boundary nodes of a subdomain are enforced using a boundary
            condition file.
        :param int bound_vel: a flag determining whether velocities of
            the boundary nodes of a subdomain are enforced using a boundary
            condition file.
        :param int bound_wd: a flag determining whether wet/dry status of
            the boundary nodes of a subdomain are enforced using a boundary
            condition file.
        :rtype: string
        :returns: command line for invoking gensub.py

        """
        with open(os.path.join(self.path, 'gensub.in'), 'w') as fid:
            fid.write(str(bound_ele)+'\n')
            fid.write(str(bound_vel)+'\n')
            fid.write(str(bound_wd)+'\n')
        command = 'python '+os.path.join(self.script_dir, 'gensub.py')+' '
        command += os.path.join(self.fulldomain.path, 'fort.14')
        command += ' '+str(self.flag)+' fort.14 '

        if os.path.exists(os.path.join(self.fulldomain.path, 'fort.13')):
            command += os.path.join(self.fulldomain.path, 'fort.13')+' '
            command += 'fort.13 '

        command += '< gensub.in'
        subprocess.call(command, shell=True, cwd=self.path)

        #self.update_sub2full_map()
        self.create_fort15()
        self.link_fort22(winddir)
        return command

    def link_fort22(self, fdir=None):
        """
        Create symboolic links to ``fort.22*`` meterological files in this
        subdomain folder from the fulldomain folder.
        """
        if fdir is None:
            fdir = self.fulldomain.path
        fort22_files = glob.glob(os.path.join(fdir, 'fort.22*'))
        for fid in fort22_files:
            fm.symlink(fid, os.path.join(self.path, fid.rpartition('/')[-1]))

    def genfull(self, noutgs=1, nspoolgs=1):
        """
        Generate the full domain control file, ``fort.015``, and save it to
        ``self.fulldomain.path``.

        :param int noutgs: flag controlling whether or not ``fort.06*`` will be
            written out
        :param int nspoolgs: the number of timesteps at which information is
            written to the new output files ``fort.06*``
        :rtype: string
        :returns: command line for invoking genfull.py

        """
        return self.fulldomain.genfull(noutgs, nspoolgs, [self])

    def genbcs(self, forcing_freq=1, dt=None, nspoolgs=1, h0=None, L=False):
        """
        Generate the ``fort.019`` which is the boundary conditions file needed
        for a subdomain run of :program:`ADCIRC`. This requires the presence of
        the output files from a fulldomain run, ``fort.06*``.

        :param int forcing_freq: number of timesteps at which infomration
            is written to a boudnary conditions file (``fort.019``) THIS MUST
            BE A MULTIPLE OF NSPOOLGS
        :param float dt: One timestep in seconds
        :param int nspoolgs: the number of timesteps at which information is
            written to the new output files ``fort.06*``
        :param float h0: minimum water depth for a node to be wet
        :param bool L: flag whether or not :program:`PADCIRC` was run with
            ``-L`` flag and if local files need to be post-processed into
            global files
        :rtype: string
        :returns: command line for invoking genbcs.py

        """
        if L:
            # create post-processing input file
            post.write_sub(self.fulldomain.path)
            # run ADCPOST
            subprocess.call('./adcpost < in.postsub > post_o.txt', shell=True,
                            cwd=self.fulldomain.path)

        self.create_fort15()
        self.link_fort22()
        self.read_recording_data()

        if self.check_fulldomain():
            if h0 is None:
                h0 = self.h0
            if dt is None:
                dt = self.fulldomain.time.dt
            command = "python "+self.script_dir+"/genbcs.py -p "
            command += self.fulldomain.path+'/ '+self.path+'/ '
            command += str(forcing_freq)+' '+str(dt)+' '+str(nspoolgs)
            command += ' '+str(h0)
            print command
            subprocess.call(command, shell=True, cwd=self.path)
            return command
        else:
            print "Output files from the fulldomain run do not exist"
            return "Output files from the fulldomain run do not exist"

    def circle(self, x, y, r):
        """
        Generate a subdomain shape file for a circular subdomain

        :param float x: x coordinate of circle center
        :param float y: y coordinate of circle center
        :param float r: radius of circle
        :rtype: int
        :returns: flag for :meth:`py.gensub`

        """
        with open(os.path.join(self.path, 'shape.c14'), 'w') as fid:
            fid.write('{:17.15f} {:17.15f}\n'.format(x, y))
            fid.write(str(r))
        self.flag = 1
        return self.flag

    def ellipse(self, x, y, w):
        """
        Generate a subdomain shape file for an elliptical subdomain

        :param list x: x coordinates of the first and second focal points
        :param list y: y coordinates of the first and second focal points
        :param float w: width of ellipse
        :rtype: int
        :returns: flag for :meth:`py.gensub`

        """
        with open(os.path.join(self.path, 'shape.e14'), 'w') as fid:
            fid.write('{:17.15f} {:17.15f}\n'.format(x[0], y[0]))
            fid.write('{:17.15f} {:17.15f}\n'.format(x[1], y[1]))
            fid.write(str(w))
        self.flag = 0
        return self.flag

    def read_circle(self):
        """
        Read in the parameters used to define the circular subdomain cut out and
        store them as ``self.x``, ``self.y``, and ``self.r``

        :rtype: tuple
        :returns: (x, y, r)

        """
        with open(os.path.join(self.path, "shape.c14"), "r") as fid:
            t = fid.readline().split()
            x = float(t[0])
            y = float(t[1])
            r = float(fid.readline())

        self.x = x
        self.y = y
        self.r = r
        return (x, y, r)

    def read_ellipse(self):
        """
        Read in the parameters used to define the circular subdomain cut out and
        store them as ``self.x``, ``self.y``, and ``self.r``

        :rtype: tuple
        :returns: (x, y, w)

        """

        with open(os.path.join(self.path, "shape.e14"), "r") as fid:
            point1 = fid.readline().split()
            point2 = fid.readline().split()
            x = [float(point1[0]), float(point2[0])]
            y = [float(point1[1]), float(point2[1])]
            w = float(fid.readline())
            self.x = x
            self.y = y
            self.w = w
        return (x, y, w)

    def ellipse_properties(self, x, y, w):
        """
        Given a the (x,y) locations of the foci of the ellipse and the width
        return the center of the ellipse, width, height, and angle relative to
        the x-axis.

        :param double x: x-coordinates of the foci
        :param double y: y-coordinates of the foci
        :param double w: width of the ellipse
        :rtype: tuple of doubles
        :returns: (center_coordinates, width, height, angle_in_rads)

        """
        return ellipse_properties(x, y, w)

    def setup(self, flag=None, bound_ele=1, bound_vel=1, bound_wd=1,
              winddir=None):
        """
        Generate the subdomain input files (``fort.13``, ``fort.14``,
        ``fort.015``, ``py.141``, ``py.140``) and shape file. Creates
        ``fort.15`` based on the ``fort.15`` in
        :class:`polyadcirc.run_framework.fulldomain` to ``self.path``, and
        creates symbolic links to meterological forcing files (``fort.22*``).

        :param int flag: flag determining whether or not the subdomain is an
            ellipse (0) or a circle (1)
        :param int bound_ele: a flag determining whether surface elevations of
            the boundary nodes of a subdomain are enforced using a boundary
            condition file.
        :param int bound_vel: a flag determining whether velocities of
            the boundary nodes of a subdomain are enforced using a boundary
            condition file.
        :param int bound_wd: a flag determining whether wet/dry status of
            the boundary nodes of a subdomain are enforced using a boundary
            condition file.
        :rtype: string
        :returns: command line for invoking gensub.py

        """
        # Appropriately flag subdomain
        if flag is None and self.flag is None:
            circle = glob.glob(os.path.join(self.path, 'shape.c14'))
            if len(circle) > 0:
                self.flag = 1
            ellipse = glob.glob(os.path.join(self.path, 'shape.e14'))
            if len(ellipse) > 0:
                self.flag = 0
        elif flag != None:
            self.flag = flag #: flag for :meth:`py.gensub`
        # Get rid of old files
        f_list = ['fort.015', 'fort.13', 'fort.14', 'bv.nodes', 'py.140',
                  'py.141']
        for fid in f_list:
            if os.path.exists(os.path.join(self.path, fid)):
                os.remove(os.path.join(self.path, fid))
        return self.gensub(bound_ele, bound_vel, bound_wd, winddir)

    def check_fulldomain(self):
        """
        Check to see if the ``fort.06*`` and ``PE*/fort.065`` files exist

        :rtype: bool
        :returns: False if the ``fort.06*`` files don't exist

        """
        return self.fulldomain.check_fulldomain()

    def check(self):
        """
        Check to make sure the ``fort.019`` file exists

        :rtype: bool
        :returns: False the ``fort.019`` doesn't exist

        """
        fort019 = glob.glob(os.path.join(self.path, 'fort.019'))
        return len(fort019) > 0

    def compare_runSet(self, ts_data, nts_data, ts_names=None,
                       nts_names=None, save_file=None):
        """
        Reads in :class:`polyadcirc.random_manningsn.runSet` output from this
        subdomain and from it's fulldomain and compares them.

        NOTE THIS DOES NOT CURRENTLY WORK FOR STATION DATA! ONLY USE FOR GLOBAL
        DATA i.e files that are fort.*3 or fort.*4

        comparision_data = fulldomain_data - subdomain_data

        :param list ts_data: (ts_data_subdomain, ts_data_fulldomain)
        :param list nts_data: (nts_data_subdomain, nts_data_fulldomain)
        :param list ts_names: names of ADCIRC timeseries
            output files to be recorded from each run
        :param list nts_names: names of ADCIRC non timeseries
            output files to be recorded from each run
        :param string save_file: name of file to save comparision matricies to
        :rtype: tuple
        :returns: (ts_error, nts_error)

        """

        if save_file is None:
            save_file = os.path.join(self.path, 'compare_s2f_runSet.mat')

        nts_keys = []
        if nts_names is None:
            nts_keys = nts_data[0].keys()
        else:
            for fid in nts_names:
                nts_keys.append(fid.replace('.', ''))

        ts_keys = []
        if ts_names is None:
            ts_keys = ts_data[0].keys()
        else:
            for fid in ts_names:
                ts_keys.append(fid.replace('.', ''))

        # Save matricies to *.mat file for use by MATLAB or Python
        mdict = dict()

        # Pre-allocate arrays for non-timeseries data
        nts_error = {}
        ts_error = {}

        fulldom_nodes = [v-1 for v in self.sub2full_node.values()]

        # Get nts_error
        for key in nts_keys:
            full_data = nts_data[1][key][fulldom_nodes]
            sub_data = nts_data[0][key]
            nts_error[key] = (full_data - sub_data)#/full_data

        # fix dry nodes
        if 'fort63' in ts_keys:
            ts_data[0] = rmn.fix_dry_nodes(ts_data[0], self)
            ts_data[1] = rmn.fix_dry_nodes(ts_data[1], self)

        # fix dry data
        if 'fort61' in ts_keys:
            self.set_station_bathymetry()
            ts_data[0] = rmn.fix_dry_data(ts_data[0], self)
            ts_data[1] = rmn.fix_dry_data(ts_data[1], self)

        # Get ts_data
        for key in ts_keys:
            # Theres a bug wrt stations here either we need a mapping between
            # fulldomain stations and subdomain stations or these stations MUST
            # be the same.
            if key == 'fort61' or key == 'fort62':
                continue
            sub_data = ts_data[0][key]
            total_obs = sub_data.shape[1]
            if self.recording[key][2] == 1:
                full_data = ts_data[1][key][fulldom_nodes, 0:total_obs]
            else:
                full_data = ts_data[1][key][fulldom_nodes, 0:total_obs, :]
            ts_error[key] = (full_data - sub_data)#/full_data

        # Update and save
        # export nontimeseries data
        for k, v in nts_error.iteritems():
            mdict[k] = v
            print k
            b = np.ma.fix_invalid(v, fill_value=0)
            print np.max(abs(b)), np.argmax(abs(b))
        # export timeseries data
        for k, v in ts_error.iteritems():
            mdict[k] = v
            print k
            b = np.ma.fix_invalid(v, fill_value=0)
            print np.max(abs(b)), np.argmax(abs(b))

        sio.savemat(save_file, mdict, do_compression=True)
        return (ts_error, nts_error)

    def compare_to_fulldomain(self, ts_names, nts_names, save_file=None,
                              timesteps=None, savefull=True, readmatfull=False,
                              savesub=True, readmatsub=False, fulldict=None):
        """
        Reads in output files from this subdomain and from it's fulldomain and
        compares them.

        NOTE THIS DOES NOT CURRENTLY WORK FOR STATION DATA! ONLY USE FOR GLOBAL
        DATA i.e files that are fort.*3 or fort.*4

        NOTE THIS DOES NOT CURRENTLY WORK FOR ANY NTS DATA EXCEPT FOR MAXELE

        comparision_data = fulldomain_data - subdomain_data

        :param list ts_names: names of ADCIRC timeseries
            output files to be recorded from each run
        :param list nts_names: names of ADCIRC non timeseries
            output files to be recorded from each run
        :param string save_file: name of file to save comparision matricies to
        :param int timesteps: number of timesteps to read from file
        :rtype: tuple
        :returns: (ts_error, nts_error, time_obs, ts_data, nts_data)

        """

        if save_file is None:
            save_file = os.path.join(self.path, 'compare_s2f.mat')

        full_file = os.path.join(self.fulldomain.path, 'full.mat')
        sub_file = os.path.join(self.path, 'sub.mat')

        # Save matricies to *.mat file for use by MATLAB or Python
        mdict = dict()
        if readmatsub:
            subdict = sio.loadmat(sub_file)
            savesub = False
        else:
            subdict = dict()

        if fulldict is None:
            if readmatfull:
                fulldict = sio.loadmat(full_file)
                savefull = False
            else:
                fulldict = dict()
        else:
            readmatfull = True
            savefull = False

        # Pre-allocate arrays for non-timeseries data
        nts_error = {}
        ts_error = {}
        time_obs = {}

        fulldom_nodes = [v-1 for v in self.sub2full_node.values()]

        # Get nts_data
        for fid in nts_names:
            key = fid.replace('.', '')
            if not readmatfull:
                fulldict[key] = output.get_nts_sr(self.fulldomain.path,
                                                  self.fulldomain, fid)
            if not readmatsub:
                subdict[key] = output.get_nts_sr(self.path, self, fid)

        # Get ts_data
        for fid in ts_names:
            key = fid.replace('.', '')
            if not readmatsub:
                subdict[key], time_obs[key] = output.get_ts_sr(self.path, fid,
                                                               True,
                                                               ihot=self.ihot)
                subdict[key+'_time'] = time_obs[key]
            if not readmatfull:
                fulldict[key] = output.get_ts_sr(self.fulldomain.path,
                                                 fid, timesteps=timesteps,
                                                 ihot=self.fulldomain.ihot)[0]

        if not readmatsub:
            # fix dry nodes
            if subdict.has_key('fort63'):
                subdict['fort63'] = np.expand_dims(subdict['fort63'], axis=2)
                subdict = rmn.fix_dry_nodes(subdict, self)
                subdict['fort63'] = np.squeeze(subdict['fort63'])
            # fix dry data
            if subdict.has_key('fort61'):
                subdict['fort61'] = np.expand_dims(subdict['fort61'], axis=1)
                subdict = rmn.fix_dry_data(subdict, self)
                subdict['fort61'] = np.squeeze(subdict['fort61'])
            # fix dry nodes nts
            if subdict.has_key('maxele63'):
                subdict['maxele63'] = np.expand_dims(subdict['maxele63'],
                                                     axis=1)
                subdict = rmn.fix_dry_nodes_nts(subdict, self)
                subdict['maxele63'] = np.squeeze(subdict['maxele63'])

        if not readmatfull:
            # fix dry nodes
            if fulldict.has_key('fort63'):
                fulldict['fort63'] = np.expand_dims(fulldict['fort63'], axis=2)
                fulldict = rmn.fix_dry_nodes(fulldict, self)
                fulldict['fort63'] = np.squeeze(fulldict['fort63'])
            # fix dry data
            if fulldict.has_key('fort61'):
                fulldict['fort61'] = np.expand_dims(fulldict['fort61'], axis=1)
                fulldict = rmn.fix_dry_data(fulldict, self)
                fulldict['fort61'] = np.squeeze(fulldict['fort61'])
            # fix dry nodes nts
            if fulldict.has_key('maxele63'):
                fulldict['maxele63'] = np.expand_dims(fulldict['maxele63'],
                                                      axis=1)
                fulldict = rmn.fix_dry_nodes_nts(fulldict, self)
                fulldict['maxele63'] = np.squeeze(fulldict['maxele63'])

        # Get ts_error
        for fid in ts_names:
            key = fid.replace('.', '')
            sub_data, time_obs[key] = subdict[key], subdict[key+'_time']
            total_obs = sub_data.shape[1]
            if timesteps and timesteps < total_obs:
                total_obs = timesteps
            full_data = fulldict[key]
            if self.recording[key][2] == 1:
                full_data = full_data[fulldom_nodes, 0:total_obs]
            else:
                full_data = full_data[fulldom_nodes, 0:total_obs, :]
            ts_error[key] = (full_data - sub_data)
            if key == 'fort63' and (subdict.has_key('maxele63') or\
                    subdict.has_key('maxele.63')):
                nts_error['maxele63'] = np.max(full_data,
                                               axis=1) - subdict['maxele63']
            if key == 'fort64' and (subdict.has_key('maxvel63') or\
                    subdict.has_key('maxvel.63')):
                nts_error['maxvel63'] = np.max(np.sqrt(full_data[:, :, 0]**2 +
                                                       full_data[:, :, 1]**2),
                                               axis=1) - subdict['maxvel63']

        # Get nts_error
        for fid in nts_names:
            key = fid.replace('.', '')
            if key != 'maxele63' and key != 'maxevel63':
                nts_error[key] = fulldict[key][fulldom_nodes] - subdict[key]

        # Update and save
        # export nontimeseries data
        for k, v in nts_error.iteritems():
            mdict[k] = v
            print k
            #b = np.ma.fix_invalid(v, fill_value=0)
            b = v
            print np.max(abs(b)), np.argmax(abs(b))
            print "Nodes above threshold", sum(np.abs(b) > 1e-2)
        # export timeseries data
        for k, v in ts_error.iteritems():
            mdict[k] = v
            print k
            #b = np.ma.fix_invalid(v, fill_value=0)
            b = v
            print np.max(abs(b)), np.argmax(abs(b))
            print "Nodes above threshold", sum(np.abs(b) > 1e-2)

        # export time_obs data
        for k, v in time_obs.iteritems():
            mdict[k+'_time'] = v

        sio.savemat(save_file, mdict, do_compression=True)
        if savesub:
            sio.savemat(sub_file, subdict, do_compression=True)
        if savefull:
            sio.savemat(full_file, fulldict, do_compression=True)

        return (ts_error, nts_error, time_obs, None, None)

    def create_fort15(self):
        """
        Copy the ``fort.15`` from ``fulldomain.path`` to ``self.path`` and
        modify for a subdomain run.

        .. seealso:: :meth:`polyadcirc.pyADCIRC.fort15_management.subdomain`
        """

        f15.subdomain(self.fulldomain.path, self.path)

    def read_py_node(self):
        """
        Read in the subdomain to fulldomain node map

        Store as ::

            self.sub2full_node = dict()

        where key = subdomain node #, value = fulldomain node #
        """
        self.sub2full_node = {}
        with open(os.path.join(self.path, 'py.140'), 'r') as fid:
            fid.readline() # skip header
            for line in fid:
                k, v = np.fromstring(line, dtype=int, sep=' ')
                self.sub2full_node[k] = v

    def read_py_ele(self):
        """
        Read in the subdomain to fulldomain element map

        Store as ::

            self.sub2full_element = dict()

        where key = subdomain element #, value = fulldomain element #
        """
        self.sub2full_element = {}
        with open(os.path.join(self.path, 'py.141'), 'r') as fid:
            fid.readline() # skip header
            for line in fid:
                k, v = np.fromstring(line, dtype=int, sep=' ')
                self.sub2full_element[k] = v

    def read_bv_nodes(self):
        """
        Read in the nodes on the boundary and store in ``self.bv_nodes`` as a
        list.

        :rtype: list
        :returns: list of boundary nodes
        """
        self.bv_nodes = []
        with open(os.path.join(self.path, 'bv.nodes'), 'r') as fid:
            for line in fid:
                self.bv_nodes.append(int(np.fromstring(line, sep=' ')[0]))
        return self.bv_nodes

    def update_sub2full_map(self):
        """
        Read in the subdomain to fulldomain element and node maps
        """
        #: dict where key = subdomain node #, value = fulldomain node #
        self.read_py_node()
        #: dict where key = subdomain element #, value = fulldomain element #
        self.read_py_ele()
        #: list of boundary nodes
        self.read_bv_fort13()

    def trim_fort13(self, old_fort13, new_fort13):
        """
        Trim ``old_fort13`` to match the nodes in this subdomain and save as
        ``new_fort13``. This only assumes that a ``py.140`` file exists.

        :param string old_fort13: path to the old ``fort.13`` file to be
            trimmed
        :param string new_fort13: path to save the new ``fort.13`` file
        """
        trim_fort13(old_fort13, new_fort13, os.path.join(self.path, 'py.140'))

    def trim_multiple_fort13(self, old_fort13, new_fort13):
        """
        Trim ``old_fort13`` to match the nodes in this subdomain and save as
        ``new_fort13``. This only assumes that a ``py.140`` file exists.

        :param string old_fort13: path to the old ``fort.13`` file to be
            trimmed
        :param string new_fort13: path to save the new ``fort.13`` file
        """
        trim_multiple_fort13(old_fort13, new_fort13, os.path.join(self.path,
                                                                  'py.140'))

    def read_bv_fort13(self):
        """
        Read the boundary value nodal values for Manning's n from
        ``self.fulldomain`` and save as a ``dict`` as ``self.bv_fort13``.
        """
        self.read_bv_nodes()
        self.bv_fort13 = f13.read_nodal_attr(self, self.path,
                                             nums=self.bv_nodes)

    def set_bv_fort13(self, mann_data):
        """
        Replace the boundary nodal values with the boundary nodal values in the
        fulldomain.

        :type mann_data: :class:`numpy.ndarray` or :class:`dict`
        :param mann_data: containing the nodal attribute information

        :rtype: :class:`numpy.ndarray` or :class:`dict`
        :returns: dictionary or array of nodal values

        """
        if isinstance(mann_data, dict):
            for k, v in self.bv_fort13.iteritems():
                mann_data[k] = v
        else:
            for k, v in self.bv_fort13.iteritems():
                mann_data[k-1] = v
        return mann_data

    def update_mann(self, data, path=None, default=None, file_name='fort.13'):
        """
        Write out fort.13 to path with the attributes contained in Data.

        :type data: :class:`numpy.ndarray` or :class:`dict`
        :param data: containing the nodal attribute information
        :type path: string or None
        :param path: the directory to which the fort.13 file will be written
        :type default: None or float
        :param default: default value
        :type file_name: string
        :param file_name: the name of the ``fort.13`` formatted file

        """
        data = self.set_bv_fort13(data)
        f13.update_mann(data, path, default, file_name)

def trim_fort13(old_fort13, new_fort13, pynode_map):
    """
    Trim ``old_fort13`` to match the nodes in this subdomain and save as
    ``new_fort13``. This only assumes that a ``py.140`` file exists.

    :param string old_fort13: path to the old ``fort.13`` file to be
        trimmed
    :param string new_fort13: path to save the new ``fort.13`` file
    """
    print pynode_map
    gensub.main13(None, old_fort13, new_fort13, pynode_map)

def trim_multiple_fort13(old_fort13, new_fort13, pynode_map):
    """
    Trim ``old_fort13`` to match the nodes in this subdomain and save as
    ``new_fort13``. This only assumes that a ``py.140`` file exists.

    :param string old_fort13: path to the old ``fort.13`` file to be
        trimmed
    :param string new_fort13: path to save the new ``fort.13`` file
    """
    size = comm.Get_size()
    rank = comm.Get_rank()

    print "len old, len new", len(old_fort13), len(new_fort13)

    for i in range(0+rank, len(old_fort13), size):
        if not os.path.exists(os.path.dirname(new_fort13[i])):
            os.makedirs(os.path.dirname(new_fort13[i]))
        trim_fort13(old_fort13[i], new_fort13[i], pynode_map)

def ellipse_properties(x, y, w):
    """
    Given a the (x,y) locations of the foci of the ellipse and the width return
    the center of the ellipse, width, height, and angle relative to the x-axis.

    :param double x: x-coordinates of the foci
    :param double y: y-coordinates of the foci
    :param double w: width of the ellipse
    :rtype: tuple of doubles
    :returns: (center_coordinates, width, height, angle_in_rads)

    """
    p1 = [x[0], y[0]]
    p2 = [x[1], y[1]]

    #center point
    xy = [(p1[0] + p2[0])/2, (p1[1] + p2[1])/2]
    #distance between points
    d = ((p1[0] - p2[0])**2 + (p1[1] - p2[1])**2)**(0.5)
    #theta to positive Xaxis
    angle = math.atan((p1[1] - p2[1])/(p1[0] - p2[0]))
    #sin = math.sin(-angle)
    #cos = math.cos(-angle)
    #width will be the axis the points lie on
    width = 2*((0.5*d)**2 + (0.5*w)**2)**(0.5)
    height = w

    return (xy, width, height, angle*180/math.pi)


# The MIT License (MIT)
# Copyright (c) 2016, 2017 by the ESA CCI Toolbox development team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

"""
Description
===========

This module provides aggregation operations

Components
==========
"""
from datetime import timezone

import numpy as np
import pandas as pd
import xarray as xr
from xarray.core.resample import DatasetResample as resampler

from cate.core.op import op, op_input, op_return
from cate.core.types import VarNamesLike, DatasetLike, ValidationError, DimNamesLike
from cate.ops.normalize import adjust_temporal_attrs
from cate.ops.select import select_var
from cate.util.monitor import Monitor

@op(tags=['aggregate', 'temporal'], version='1.5')
@op_input('ds', data_type=DatasetLike)
@op_input('var', value_set_source='ds', data_type=VarNamesLike)
@op_return(add_history=True)
def long_term_average(ds: DatasetLike.TYPE,
                      var: VarNamesLike.TYPE = None,
                      monitor: Monitor = Monitor.NONE) -> xr.Dataset:
    """
    Create a 'mean over years' dataset by averaging the values of the given input
    dataset over all years. The output is a climatological dataset with the same
    resolution as the input dataset. E.g. a daily input dataset will create a daily
    climatology consisting of 365 days, a monthly input dataset will create a monthly
    climatology, etc.

    Seasonal input datasets must have matching seasons over all years denoted by the
    same date each year. E.g., first date of each quarter. The output dataset will
    then be a seasonal climatology where each season is denoted with the same date
    as in the input dataset.

    For further information on climatological datasets, see
    http://cfconventions.org/cf-conventions/v1.6.0/cf-conventions.html#climatological-statistics

    :param ds: A dataset to average
    :param var: If given, only these variables will be preserved in the resulting dataset
    :param monitor: A progress monitor
    :return: A climatological long term average dataset
    """
    ds = DatasetLike.convert(ds)
    # Check if time dtype is what we want
    if 'datetime64[ns]' != ds.time.dtype:
        raise ValidationError('Long term average operation expects a dataset with the'
                              ' time coordinate of type datetime64[ns], but received'
                              ' {}. Running the normalize operation on this'
                              ' dataset may help'.format(ds.time.dtype))

    try:
        t_resolution = ds.attrs['time_coverage_resolution']
    except KeyError:
        raise ValidationError('Could not determine temporal resolution. Running'
                              ' the adjust_temporal_attrs operation beforehand may'
                              ' help.')

    var = VarNamesLike.convert(var)
    # Shallow

    if var:
        ds = select_var(ds, var)

    if t_resolution == 'P1D':
        return _lta_daily(ds)
    elif t_resolution == 'P1M':
        return _lta_monthly(ds, monitor)
    else:
        return _lta_general(ds, monitor)

def _lta_monthly(ds: xr.Dataset, monitor: Monitor):
    """
    Carry out a long term average on a monthly dataset

    :param ds: Dataset to aggregate
    :param monitor: Progress monitor
    :return: Aggregated dataset
    """
    time_min = pd.Timestamp(ds.time.values[0], tzinfo=timezone.utc)
    time_max = pd.Timestamp(ds.time.values[-1], tzinfo=timezone.utc)
    total_work = 100
    retset = ds

    with monitor.starting('LTA', total_work=total_work):
        monitor.progress(work=0)
        step = total_work / 12
        kwargs = {'monitor': monitor, 'step': step}
        retset = retset.groupby('time.month', squeeze=False).apply(_mean, **kwargs)

    # Make the return dataset CF compliant
    retset = retset.rename({'month': 'time'})
    retset['time'] = pd.date_range('{}-01-01'.format(time_min.year),
                                   freq='MS',
                                   periods=12)

    climatology_bounds = xr.DataArray(data=np.tile([time_min, time_max],
                                                   (12, 1)),
                                      dims=['time', 'nv'],
                                      name='climatology_bounds')
    retset['climatology_bounds'] = climatology_bounds
    retset.time.attrs = ds.time.attrs
    retset.time.attrs['climatology'] = 'climatology_bounds'

    for var in retset.data_vars:
        try:
            retset[var].attrs['cell_methods'] = \
                retset[var].attrs['cell_methods'] + ' time: mean over years'
        except KeyError:
            retset[var].attrs['cell_methods'] = 'time: mean over years'

    return retset

def _groupby_day(ds: xr.Dataset, monitor: Monitor, step: float):
    """
    Groupby the given dataset by day of month and apply mean to it

    :param ds: Dataset to aggregate
    :param monitor: Progress monitor
    :param step: Progress step
    """
    kwargs = {'monitor': monitor, 'step': step}
    return ds.groupby('time.day', squeeze=False).apply(_mean, **kwargs)

def _lta_daily(ds: xr.Dataset):
    """
    Carry out a long term average of a daily dataset

    :param ds: Dataset to aggregate
    :return: Aggregated dataset
    """

    retset = ds.groupby('time.dayofyear', squeeze=False).mean('time')

    for var in retset.data_vars:
        try:
            retset[var].attrs['cell_methods'] = \
                retset[var].attrs['cell_methods'] + ' time: mean over years'
        except KeyError:
            retset[var].attrs['cell_methods'] = 'time: mean over years'

    return retset

def _lta_general(ds: xr.Dataset, monitor: Monitor):
    """
    Try to carry out a long term average in a general case, notably
    in the case of having seasonal datasets

    :param ds: Dataset to aggregate
    :param monitor: Progress monitor
    :return: Aggregated dataset
    """
    time_min = pd.Timestamp(ds.time.values[0], tzinfo=timezone.utc)
    time_max = pd.Timestamp(ds.time.values[-1], tzinfo=timezone.utc)
    total_work = 100
    retset = ds

    # The dataset should feature time periods consistent over years
    # and denoted with the same dates each year
    if not _is_seasonal(ds.time):
        raise ValidationError("A long term average dataset can not be created for"
                              " a dataset with inconsistent seasons.")

    # Get 'representative year'
    c = 0
    for group in ds.time.groupby('time.year'):
        c = c + 1
        if c == 1:
            rep_year = group[1].time
            continue
        if c == 2 and len(group[1].time) > len(rep_year):
            rep_year = group[1].time
            break

    with monitor.starting('LTA', total_work=total_work):
        monitor.progress(work=0)
        step = total_work / len(rep_year.time)
        kwargs = {'monitor': monitor, 'step': step}
        retset = retset.groupby('time.month', squeeze=False).apply(_groupby_day, **kwargs)

    # Make the return dataset CF compliant
    retset = retset.stack(time=('month', 'day'))

    # Turn month, day coordinates to time
    retset = retset.reset_index('time')
    retset = retset.drop(['month', 'day'])
    retset['time'] = rep_year.time

    climatology_bounds = xr.DataArray(data=np.tile([time_min, time_max],
                                                   (len(rep_year), 1)),
                                      dims=['time', 'nv'],
                                      name='climatology_bounds')
    retset['climatology_bounds'] = climatology_bounds
    retset.time.attrs = ds.time.attrs
    retset.time.attrs['climatology'] = 'climatology_bounds'

    for var in retset.data_vars:
        try:
            retset[var].attrs['cell_methods'] = \
                retset[var].attrs['cell_methods'] + ' time: mean over years'
        except KeyError:
            retset[var].attrs['cell_methods'] = 'time: mean over years'

    return retset

def _is_seasonal(time: xr.DataArray):
    """
    Check if the given timestamp dataarray features consistent
    seasons. E.g. Each year has the same date-month values in it.
    """
    c = 0
    test = None
    for group in time.groupby('time.year'):
        # Test (month, day) dates of all years against
        # (month, day) dates of the first year, or second
        # year in case the first year is not full
        c = c + 1
        np_time = group[1].time.values
        months = pd.DatetimeIndex(np_time).month
        days = pd.DatetimeIndex(np_time).day
        if c == 1:
            first_months = months
            first_days = days
            continue
        elif c == 2:
            second_months = months
            second_days = days
            if len(second_months) > len(first_months):
                test = list(zip(second_months, second_days))
                for date in zip(first_months, first_days):
                    if date not in test:
                        return False
            else:
                test = list(zip(first_months, first_days))
                for date in zip(second_months, second_days):
                    if date not in test:
                        return False
            continue

        for date in zip(months, days):
            if date not in test:
                return False

    return True

def _mean(ds: xr.Dataset, monitor: Monitor, step: float):
    """
    Calculate mean of the given dataset and update the given monitor.

    :param ds: Dataset to take the mean of
    :param monitor: Monitor to update
    :param step: Work step
    """
    retset = ds.mean(dim='time', keep_attrs=True)
    monitor.progress(work=step)
    return retset

@op(tags=['aggregate', 'temporal'], version='1.5')
@op_input('ds', data_type=DatasetLike)
@op_input('method', value_set=['mean', 'max', 'median', 'prod', 'sum', 'std',
                               'var', 'argmax', 'argmin', 'first', 'last'])
@op_input('output_resolution', value_set=['month', 'season'])
@op_return(add_history=True)
def temporal_aggregation(ds: DatasetLike.TYPE,
                         method: str = 'mean',
                         output_resolution: str = 'month',
                         custom_resolution: str = None,
                         monitor: Monitor = Monitor.NONE) -> xr.Dataset:
    """
    Perform aggregation of dataset according to the given
    method and output resolution.

    Note that the operation does not perform weighting. Depending on the
    combination of input and output resolutions, as well as aggregation
    method, the resulting dataset might yield unexpected results.

    Resolution 'month' will result in a monthly dataset with each month
    denoted by its first date. Resolution 'season' will result in a dataset
    aggregated to DJF, MAM, JJA, SON seasons, each denoted by the first
    date of the season.

    The operation also works with custom resolution strings, see:
    http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
    If ``custom_resolution`` is provided, it will override ``output_resolution``.

    Some examples:
      'QS-JUN' produces an output dataset on a quarterly resolution where the
      year ends in 1st of June and each quarter is denoted by its first date
      '8MS' produces an output dataset on an eight-month resolution where each
      period is denoted by the first date. Note that such periods will not be
      consistent over years.
      '8D' produces a dataset on an eight day resolution

    :param ds: Dataset to aggregate
    :param method: Aggregation method
    :param output_resolution: Desired temporal resolution of the output dataset
    :param custom_resolution: Custom temporal resolution, overrides output_resolution
    :return: Aggregated dataset
    """
    ds = DatasetLike.convert(ds)
    # Check if time dtype is what we want
    if 'datetime64[ns]' != ds.time.dtype:
        raise ValidationError('Temporal aggregation operation expects a dataset with the'
                              ' time coordinate of type datetime64[ns], but received'
                              ' {}. Running the normalize operation on this'
                              ' dataset may help'.format(ds.time.dtype))

    # Try to figure out the input frequency
    try:
        in_freq = ds.attrs['time_coverage_resolution']
    except KeyError:
        raise ValidationError('Could not determine temporal resolution of input dataset.'
                              ' Running the adjust_temporal_attrs operation beforehand may'
                              ' help.')

    if custom_resolution:
        freq = custom_resolution
    else:
        frequencies = {'month': 'MS', 'season': 'QS-DEC'}
        freq = frequencies[output_resolution]

    _validate_freq(in_freq, freq)

    with monitor.observing("resample dataset"):
        try:
            retset = getattr(resampler, method)(ds.resample(time=freq, keep_attrs=True))
        except AttributeError:
            raise ValidationError(f'Provided aggregation method {method} is not valid.')

    for var in retset.data_vars:
        try:
            retset[var].attrs['cell_methods'] = \
                retset[var].attrs['cell_methods'] + \
                ' time: {} within years'.format(method)
        except KeyError:
            retset[var].attrs['cell_methods'] = 'time: {} within years'.format(method)

    return adjust_temporal_attrs(retset)

def _validate_freq(in_res: str, out_res: str) -> None:
    """
    Validate the aggregation step

    See also: `ISO 8601 Durations <https://en.wikipedia.org/wiki/ISO_8601#Durations>`_
    """
    # Validate output frequency as a valid offset string
    try:
        dates = pd.date_range('2000-01-01', periods=5, freq=out_res)
    except ValueError:
        raise ValidationError('Invalid custom resolution: {}.'
                              ' Please check operation documentation.'.format(out_res))

    # Assuming simple ISO_8601 periods: PXXD/M
    try:
        count = int(in_res[1:-1])
    except ValueError:
        raise ValidationError('Could not interpret time coverage resolution of'
                              ' the given dataset: {}'.format(in_res))

    if in_res == 'P1M' and out_res == 'MS':
        raise ValidationError('Input dataset is already at the requested output resolution.'
                              ' Execution stopped.')

    in_delta = pd.Timedelta(count, unit=in_res[-1])
    out_delta = dates[1] - dates[0]

    if out_delta < in_delta:
        raise ValidationError('Requested output resolution is smaller than dataset resolution.'
                              ' This operation only performs aggregation to larger resolutions.')
    elif out_delta == in_delta:
        raise ValidationError('Input dataset is already at the requested output resolution.'
                              'Execution stopped.')

    return

@op(tags=['aggregate'], version='1.0')
@op_input('ds', data_type=DatasetLike)
@op_input('var', value_set_source='ds', data_type=VarNamesLike)
@op_input('dim', value_set_source='ds', data_type=DimNamesLike)
@op_input('method', value_set=['mean', 'min', 'max', 'sum', 'median'])
@op_return(add_history=True)
def reduce(ds: DatasetLike.TYPE,
           var: VarNamesLike.TYPE = None,
           dim: DimNamesLike.TYPE = None,
           method: str = 'mean',
           monitor: Monitor = Monitor.NONE) -> xr.Dataset:
    """
    Reduce the given variables of the given dataset along the given dimensions.
    If no variables are given, all variables of the dataset will be reduced. If
    no dimensions are given, all dimensions will be reduced. If no variables
    have been given explicitly, it can be set that only variables featuring numeric
    values should be reduced.

    :param ds: Dataset to reduce
    :param var: Variables in the dataset to reduce
    :param dim: Dataset dimensions along which to reduce
    :param method: reduction method
    :param monitor: A progress monitor
    """
    ufuncs = {'min': np.nanmin, 'max': np.nanmax, 'mean': np.nanmean,
              'median': np.nanmedian, 'sum': np.nansum}

    ds = DatasetLike.convert(ds)

    if not var:
        var = list(ds.data_vars.keys())
    var_names = VarNamesLike.convert(var)

    if not dim:
        dim = list(ds.coords.keys())
    else:
        dim = DimNamesLike.convert(dim)

    retset = ds.copy()

    for var_name in var_names:
        intersection = [value for value in dim if value in retset[var_name].dims]
        with monitor.starting("Reduce dataset", total_work=100):
            monitor.progress(5)
            with monitor.child(95).observing("Reduce"):
                retset[var_name] = retset[var_name].reduce(ufuncs[method],
                                                           dim=intersection,
                                                           keep_attrs=True)

    return retset