"""
Sea Ice Diagnostics.
====================

Diagnostic to produce a series of images which are useful for evaluating
the behaviour of the a sea ice model.

There are three kinds of plots shown here.
1. Sea ice Extent maps plots with a stereoscoic projection.
2. Maps plots of individual models ice fracrtion.
3. Time series plots for the total ice extent.

All three kinds of plots are made for both Summer and Winter in both the
North and Southern hemisphere.

Note that this diagnostic assumes that the preprocessors do the bulk of the
hard work, and that the cube received by this diagnostic (via the settings.yml
and metadata.yml files) has no time component, a small number of depth layers,
and a latitude and longitude coordinates.

This diagnostic takes data from either North or South hemisphere, and
from either December-January-February or June-July-August. This diagnostic
requires the data to be 2D+time, and typically expects the data field to be
the sea ice cover.
An approproate preprocessor would be::

  preprocessors:
    timeseries_NHW_ice_extent: # North Hemisphere Winter ice_extent
      custom_order: true
      extract_time:
          start_year: 1960
          start_month: 12
          start_day: 1
          end_year: 2005
          end_month: 9
          end_day: 31
      extract_season:
        season: DJF
      extract_region:
        start_longitude: -180.
        end_longitude: 180.
        start_latitude: 0.
        end_latitude: 90.

Note that this recipe may not function on machines with no access to the
internet, as cartopy may try to download the shapefiles. The solution to
this issue is the put the relevant cartopy shapefiles on a disk visible to your
machine, then link that path to ESMValTool via the `auxiliary_data_dir`
variable. The cartopy masking files can be downloaded from::

    https://www.naturalearthdata.com/downloads/

Here, cartopy uses the 1:10, physical coastlines and land files::

        110m_coastline.dbf  110m_coastline.shp  110m_coastline.shx
        110m_land.dbf  110m_land.shp  110m_land.shx

This tool is part of the ocean diagnostic tools package in the ESMValTool.

Author: Lee de Mora (PML)
        ledm@pml.ac.uk
"""
import logging
import os
import sys
from itertools import product

import cartopy
import iris
import iris.coord_categorisation
import iris.quickplot as qplt
import matplotlib
import matplotlib.pyplot as plt
import numpy as np

from esmvaltool.diag_scripts.ocean import diagnostic_tools as diagtools
from esmvaltool.diag_scripts.shared import run_diagnostic

# This part sends debug statements to stdout
logger = logging.getLogger(os.path.basename(__file__))
logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))

# Note that this recipe may not function on machines with no access to
# the internet, as cartopy may try to download geographic files.

def create_ice_cmap(threshold=0.15):
    """
    Create colour map with ocean blue below a threshold and white above.

    Parameters
    ----------
    threshold: float
        The threshold for the line between blue and white.

    Returns
    -------
    matplotlib.colors.LinearSegmentedColormap:
        The resulting colour map.

    """
    threshold = threshold / 100.
    ice_cmap_dict = {
        'red': ((0., 0.0313, 0.0313), (threshold, 0.0313, 1.), (1., 1., 1.)),
        'green': ((0., 0.237, 0.237), (threshold, 0.237, 1.), (1., 1., 1.)),
        'blue': ((0., 0.456, 0.456), (threshold, 0.456, 1.), (1., 1., 1.))
    }

    return matplotlib.colors.LinearSegmentedColormap('ice_cmap', ice_cmap_dict)

def calculate_area_time_series(cube, plot_type, threshold):
    """
    Calculate the area of unmasked cube cells.

    Requires a cube with two spacial dimensions. (no depth coordinate).

    Parameters
    ----------
    cube: iris.cube.Cube
        Data Cube
    plot_type: str
        The type of plot: ice extent or ice area
    threshold: float
        The threshold for ice fraction (typically 15%)

    Returns
    -------
    numpy array:
        An numpy array containing the time points.
    numpy.array:
        An numpy array containing the total ice extent or total ice area.

    """
    data = []
    times = diagtools.cube_time_to_float(cube)
    for time_itr, time in enumerate(times):
        icedata = cube[time_itr].data

        area = iris.analysis.cartography.area_weights(cube[time_itr])
        if plot_type.lower() == 'ice extent':
            # Ice extend is the area with more than 15% ice cover.
            icedata = np.ma.masked_where(icedata < threshold, icedata)
            total_area = np.ma.masked_where(icedata.mask, area.data).sum()
        if plot_type.lower() == 'ice area':
            # Ice area is cover * cell area
            total_area = np.sum(icedata * area)

        logger.debug('Calculating time series area: %s, %s, %s,', time_itr,
                     time, total_area)
        data.append(total_area)

    ######
    # Create a small dummy output array
    data = np.array(data)
    return times, data

def make_ts_plots(
        cfg,
        metadata,
        filename,
):
    """
    Make a ice extent and ice area time series plot for an individual model.

    Parameters
    ----------
    cfg: dict
        the opened global config dictionairy, passed by ESMValTool.
    metadata: dict
        The metadata dictionairy for a specific model.
    filename: str
        The preprocessed model file.

    """
    # Load cube and set up units
    cube = iris.load_cube(filename)
    iris.coord_categorisation.add_year(cube, 'time')
    cube = diagtools.bgc_units(cube, metadata['short_name'])
    cube = agregate_by_season(cube)

    # Is this data is a multi-model dataset?
    multi_model = metadata['dataset'].find('MultiModel') > -1

    # Make a dict of cubes for each layer.
    cubes = diagtools.make_cube_layer_dict(cube)

    # Load image format extention
    image_extention = diagtools.get_image_format(cfg)

    # # Load threshold, pole, season.
    threshold = float(cfg['threshold'])
    pole = get_pole(cube)
    season = get_season(cube)

    # Making plots for each layer
    for plot_type in ['Ice Extent', 'Ice Area']:
        for layer_index, (layer, cube_layer) in enumerate(cubes.items()):
            layer = str(layer)

            times, data = calculate_area_time_series(cube_layer, plot_type,
                                                     threshold)

            plt.plot(times, data)

            # Add title to plot
            title = ' '.join(
                [metadata['dataset'], pole, 'hemisphere', season, plot_type])
            if layer:
                title = ' '.join([
                    title, '(', layer,
                    str(cube_layer.coords('depth')[0].units), ')'
                ])
            plt.title(title)

            # y axis label:
            plt.ylabel(' '.join([plot_type, 'm^2']))

            # Determine image filename:
            suffix = '_'.join(['ts', metadata['preprocessor'], season, pole,
                               plot_type, str(layer_index)])\
                     + image_extention
            suffix = suffix.replace(' ', '')
            if multi_model:
                path = diagtools.folder(
                    cfg['plot_dir']) + os.path.basename(filename)
                path = path.replace('.nc', suffix)
            else:
                path = diagtools.get_image_path(
                    cfg,
                    metadata,
                    suffix=suffix,
                )

            # Saving files:
            if cfg['write_plots']:
                logger.info('Saving plots to %s', path)
                plt.savefig(path)

            plt.close()

def make_polar_map(
        cube,
        pole='North',
        cmap='Blues_r',
):
    """
    Make a polar stereoscopic map plot.

    The cube is the opened cube (two dimensional),
    pole is the polar region (North/South)
    cmap is the colourmap,

    Parameters
    ----------
    cube: iris.cube.Cube
        Data Cube
    pole: str
        The hemisphere
    cmap: str
        The string describing the matplotlib colourmap.

    Returns
    ----------
    matplotlib.pyplot.figure:
        The matplotlib figure where the map was drawn.
    matplotlib.pyplot.axes:
        The matplotlib axes where the map was drawn.

    """
    fig = plt.figure()
    fig.set_size_inches(7, 7)

    # ####
    # Set  limits, based on https://nedbatchelder.com/blog/200806/pylint.html

    if pole not in ['North', 'South']:
        logger.fatal('make_polar_map: hemisphere not provided.')

    if pole == 'North':  # North Hemisphere
        ax1 = plt.subplot(111, projection=cartopy.crs.NorthPolarStereo())
        ax1.set_extent([-180, 180, 50, 90], cartopy.crs.PlateCarree())

    if pole == 'South':  # South Hemisphere
        ax1 = plt.subplot(111, projection=cartopy.crs.SouthPolarStereo())
        ax1.set_extent([-180, 180, -90, -50], cartopy.crs.PlateCarree())

    linrange = np.linspace(0., 100., 21.)
    qplt.contourf(cube, linrange, cmap=cmap, linewidth=0, rasterized=True)
    plt.tight_layout()

    try:
        ax1.add_feature(
            cartopy.feature.LAND,
            zorder=10,
            facecolor=[0.8, 0.8, 0.8],
        )
    except ConnectionRefusedError:
        logger.error('Cartopy was unable add coastlines due to  a '
                     'connection error.')
    ax1.gridlines(
        linewidth=0.5, color='black', zorder=20, alpha=0.5, linestyle='--')
    try:
        plt.gca().coastlines()
    except AttributeError:
        logger.warning('make_polar_map: Not able to add coastlines')
    return fig

def get_pole(cube):
    """
    Figure out the hemisphere and returns it as a string (North or South).

    Parameters
    ----------
    cube: iris.cube.Cube
        Data Cube

    Returns
    ----------
    str:
        The hemisphere (North or South)

    """
    margin = 5.
    if np.max(cube.coord('latitude').points) < 0. + margin:
        return 'South'
    if np.min(cube.coord('latitude').points) > 0. - margin:
        return 'North'
    logger.fatal('get_pole: Not able to determine hemisphere.')
    return False

def get_time_string(cube):
    """
    Return a climatological season string in the format: "year season".

    Parameters
    ----------
    cube: iris.cube.Cube
        Data Cube

    Returns
    ----------
    str:
        The climatological season as a string

    """
    season = cube.coord('clim_season').points
    year = cube.coord('year').points
    return str(int(year[0])) + ' ' + season[0].upper()

def get_year(cube):
    """
    Return the cube year as a string.

    Parameters
    ----------
    cube: iris.cube.Cube
        Data Cube

    Returns
    ----------
    str:
        The year as a string

    """
    year = cube.coord('year').points
    return str(int(year))

def get_season(cube):
    """
    Return a climatological season time string.

    Parameters
    ----------
    cube: iris.cube.Cube
        Data Cube

    Returns
    ----------
    str:
        The climatological season as a string

    """
    season = cube.coord('clim_season').points
    return season[0].upper()

def make_map_plots(
        cfg,
        metadata,
        filename,
):
    """
    Make a simple map plot for an individual model.

    Parameters
    ----------
    cfg: dict
        the opened global config dictionairy, passed by ESMValTool.
    metadata: dict
        The metadata dictionairy for a specific model.
    filename: str
        The preprocessed model file.

    """
    # Load cube and set up units
    cube = iris.load_cube(filename)
    iris.coord_categorisation.add_year(cube, 'time')
    cube = diagtools.bgc_units(cube, metadata['short_name'])
    cube = agregate_by_season(cube)

    # Is this data is a multi-model dataset?
    multi_model = metadata['dataset'].find('MultiModel') > -1

    # Make a dict of cubes for each layer.
    cubes = diagtools.make_cube_layer_dict(cube)

    # Load image format extention and threshold.
    image_extention = diagtools.get_image_format(cfg)
    threshold = float(cfg['threshold'])

    # Making plots for each layer
    plot_types = ['Fractional cover', 'Ice Extent']
    plot_times = [0, -1]
    for plot_type, plot_time in product(plot_types, plot_times):
        for layer_index, (layer, cube_layer) in enumerate(cubes.items()):
            layer = str(layer)

            if plot_type == 'Fractional cover':
                cmap = 'Blues_r'
            if plot_type == 'Ice Extent':
                cmap = create_ice_cmap(threshold)

            cube = cube_layer[plot_time]

            # use cube to determine which hemisphere, season and year.
            pole = get_pole(cube)
            time_str = get_time_string(cube)

            # Make the polar map.
            make_polar_map(cube, pole=pole, cmap=cmap)

            # Add title to plot
            title = ' '.join([metadata['dataset'], plot_type, time_str])
            if layer:
                title = ' '.join([
                    title, '(', layer,
                    str(cube_layer.coords('depth')[0].units), ')'
                ])
            plt.title(title)

            # Determine image filename:
            suffix = '_'.join(
                ['ortho_map', plot_type, time_str,
                 str(layer_index)])
            suffix = suffix.replace(' ', '') + image_extention
            if multi_model:
                path = diagtools.folder(cfg['plot_dir'])
                path = path + os.path.basename(filename)
                path = path.replace('.nc', suffix)
            else:
                path = diagtools.get_image_path(
                    cfg,
                    metadata,
                    suffix=suffix,
                )

            # Saving files:
            if cfg['write_plots']:
                logger.info('Saving plots to %s', path)
                plt.savefig(path)

            plt.close()

def agregate_by_season(cube):
    """
    Aggregate the cube into seasonal means.

    Note that it is not currently possible to do this in the preprocessor,
    as the seasonal mean changes the cube units.

    Parameters
    ----------
    cube: iris.cube.Cube
        Data Cube

    Returns
    ----------
    iris.cube.Cube:
        Data Cube with the seasonal means

    """
    if not cube.coords('clim_season'):
        iris.coord_categorisation.add_season(cube, 'time', name='clim_season')
    if not cube.coords('season_year'):
        iris.coord_categorisation.add_season_year(
            cube, 'time', name='season_year')
    return cube.aggregated_by(['clim_season', 'season_year'],
                              iris.analysis.MEAN)

def make_map_extent_plots(
        cfg,
        metadata,
        filename,
):
    """
    Make an extent map plot showing several times for an individual model.

    Parameters
    ----------
    cfg: dict
        the opened global config dictionairy, passed by ESMValTool.
    metadata: dict
        The metadata dictionairy for a specific model.
    filename: str
        The preprocessed model file.

    """
    # Load cube and set up units
    cube = iris.load_cube(filename)
    iris.coord_categorisation.add_year(cube, 'time')
    cube = diagtools.bgc_units(cube, metadata['short_name'])
    cube = agregate_by_season(cube)

    # Is this data is a multi-model dataset?
    multi_model = metadata['dataset'].find('MultiModel') > -1

    # Make a dict of cubes for each layer.
    cubes = diagtools.make_cube_layer_dict(cube)

    # Load image format extention
    image_extention = diagtools.get_image_format(cfg)

    # Load threshold, pole and season
    threshold = float(cfg['threshold'])
    pole = get_pole(cube)
    season = get_season(cube)

    # Start making figure
    for layer_index, (layer, cube_layer) in enumerate(cubes.items()):

        fig = plt.figure()
        fig.set_size_inches(7, 7)

        if pole == 'North':  # North Hemisphere
            projection = cartopy.crs.NorthPolarStereo()
            ax1 = plt.subplot(111, projection=projection)
            ax1.set_extent([-180, 180, 50, 90], cartopy.crs.PlateCarree())

        if pole == 'South':  # South Hemisphere
            projection = cartopy.crs.SouthPolarStereo()
            ax1 = plt.subplot(111, projection=projection)
            ax1.set_extent([-180, 180, -90, -50], cartopy.crs.PlateCarree())
        try:
            ax1.add_feature(
                cartopy.feature.LAND, zorder=10, facecolor=[0.8, 0.8, 0.8])
        except ConnectionRefusedError:
            logger.error('Cartopy was unable add coastlines due to  a '
                         'connection error.')

        ax1.gridlines(
            linewidth=0.5, color='black', zorder=20, alpha=0.5, linestyle='--')

        try:
            plt.gca().coastlines()
        except AttributeError:
            logger.warning('make_polar_map: Not able to add coastlines')

        times = np.array(cube.coord('time').points.astype(float))
        plot_desc = {}
        for time_itr, time in enumerate(times):
            cube = cube_layer[time_itr]
            line_width = 1
            color = plt.cm.jet(float(time_itr) / float(len(times)))
            label = get_year(cube)
            plot_desc[time] = {'label': label,
                               'c': [color, ],
                               'lw': [line_width, ],
                               'ls': ['-', ]}

            layer = str(layer)
            qplt.contour(cube,
                         [threshold, ],
                         colors=plot_desc[time]['c'],
                         linewidths=plot_desc[time]['lw'],
                         linestyles=plot_desc[time]['ls'],
                         rasterized=True)

        # Add legend
        legend_size = len(plot_desc) + 1
        ncols = int(legend_size / 25) + 1
        ax1.set_position([
            ax1.get_position().x0,
            ax1.get_position().y0,
            ax1.get_position().width * (1. - 0.1 * ncols),
            ax1.get_position().height
        ])

        fig.set_size_inches(7 + ncols * 1.2, 7)

        # Construct dummy plots.
        for i in sorted(plot_desc):
            plt.plot(
                [],
                [],
                c=plot_desc[i]['c'][0],
                lw=plot_desc[i]['lw'][0],
                ls=plot_desc[i]['ls'][0],
                label=plot_desc[i]['label'],
            )

        legd = ax1.legend(
            loc='center left',
            ncol=ncols,
            prop={'size': 10},
            bbox_to_anchor=(1., 0.5))
        legd.draw_frame(False)
        legd.get_frame().set_alpha(0.)

        # Add title to plot
        title = ' '.join([
            metadata['dataset'],
        ])
        if layer:
            title = ' '.join([
                title, '(', layer,
                str(cube_layer.coords('depth')[0].units), ')'
            ])
        plt.title(title)

        # Determine image filename:
        suffix = '_'.join(['ortho_map', pole, season, str(layer_index)])
        suffix = suffix.replace(' ', '') + image_extention
        if multi_model:
            path = diagtools.folder(cfg['plot_dir'])
            path = path + os.path.basename(filename)
            path = path.replace('.nc', suffix)
        else:
            path = diagtools.get_image_path(
                cfg,
                metadata,
                suffix=suffix,
            )

        # Saving files:
        if cfg['write_plots']:
            logger.info('Saving plots to %s', path)
            plt.savefig(path)
        plt.close()

def main(cfg):
    """
    Load the config file and metadata, then pass them the plot making tools.

    Parameters
    ----------
    cfg: dict
        the opened global config dictionairy, passed by ESMValTool.

    """
    cartopy.config['data_dir'] = cfg['auxiliary_data_dir']

    for index, metadata_filename in enumerate(cfg['input_files']):
        logger.info(
            'metadata filename:\t%s',
            metadata_filename,
        )

        metadatas = diagtools.get_input_files(cfg, index=index)
        for filename in sorted(metadatas):

            logger.info('-----------------')
            logger.info(
                'model filenames:\t%s',
                filename,
            )
            ######
            # extent maps plots of individual models
            make_map_extent_plots(cfg, metadatas[filename], filename)

            ######
            # maps plots of individual models
            make_map_plots(cfg, metadatas[filename], filename)

            ######
            # time series plots o
            make_ts_plots(cfg, metadatas[filename], filename)

    logger.info('Success')

if __name__ == '__main__':
    with run_diagnostic() as config:
        main(config)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""

@author: Frank Nussbaum (frank.nussbaum@uni-jena.de), 2019

"""
import abc
import numpy as np

#import time

# pylint: disable=R0914

class BaseAdmm(abc.ABC):
    """A base class for ADMM solvers.

    Attributes:
        admm_param (float): initial admm parameter
        opts (dict): dictionary of solver options
    """

    def __init__(self):
        super().__init__()
        self.admm_param = 1  # TODO(franknu): external access
        self.opts = {}
        self.set_defaults_admm()

    @abc.abstractmethod
    def _do_iter_admm(self, current_vars: tuple):
        """perform computations of one ADMM iteration"""
        raise NotImplementedError  # override in derived classes

    @abc.abstractmethod
    def _initialize_admm(self):
        """initialize ADMM variables"""
        raise NotImplementedError  # override in derived classes

#    def _collect(self, current_vars: tuple):
#        """store information of ADMM variables in dictionairy"""
#        raise NotImplementedError  # override in derived classes

    def _report(self, out):
        """print stats after solving"""
        print('>', out['message'])
        print('> (P)ADMM: obj: %e, iter: %d, resid:%e' %
              (out['obj'], out['iter'], out['resid']))
        try:
            print('> regparams used:', self.get_regparams())
        except AttributeError:
            # Method get_regparams does not exist
            pass

        for i, var in enumerate(out['solution']):
            print('norm optvar%d=%f' % (i, np.linalg.norm(var)))

    def set_defaults_admm(self):
        """Set default solver options in class attribute opt.

        Adjustable solver options contain abstol, reltol,
        maxiter, stoptol, verb, etc."""
        self.opts = {}
        self.opts.setdefault('abstol', 1e-5)
        self.opts.setdefault('reltol', 1e-5)

        self.opts.setdefault('continuation', 1)
        self.opts.setdefault('num_continuation', 10)
        #        self.opts.setdefault('cont_adaptive', 10)
        self.opts.setdefault('continuation_fac', 2)
        #        self.opts.setdefault('eta', .25)
        #        self.opts.setdefault('muf', 1e-6)
        self.opts.setdefault('maxiter', 500)
        self.opts.setdefault('stoptol', 1e-5)

        self.opts.setdefault('verb', 0)

    def _solve(self, warminit=None):
        """
        solve a generic ADMM problem
        """
        ## initialization
        current_vars = self._initialize_admm()
        if not warminit is None:
            current_vars = warminit

        hist = np.empty((5, self.opts['maxiter'] + 1))
        history = {
            'objval': hist[0, :],
            'r_norm': hist[1, :],
            's_norm': hist[2, :],
            'eps_pri': hist[3, :],
            'eps_dual': hist[4, :]
        }

        for i in range(1, self.opts['maxiter'] + 1):
            ## update Theta
            current_vars, residuals, stats = self._do_iter_admm(current_vars)

            ## diagnostics, reporting, termination checks
            rnorm, snorm, eps_pri, eps_dual = residuals

            history['objval'][i] = stats['admm_obj']

            history['r_norm'][i] = rnorm
            history['s_norm'][i] = snorm

            history['eps_pri'][i] = eps_pri
            history['eps_dual'][i] = eps_dual

#            print(i % self.opts['verb'])
            if self.opts['verb'] and i % self.opts['verb'] == 0:
                print('%3d\t%10.4f %10.4f %10.4f %10.4f %10.2f' %
                      (i, history['r_norm'][i], history['eps_pri'][i],
                       history['s_norm'][i], history['eps_dual'][i],
                       history['objval'][i]))  # resid %10.2f

            ## check stop
            pridualresids_below_tolerance = (rnorm < eps_pri and
                                             snorm < eps_dual)

            if pridualresids_below_tolerance:
                break

            if self.opts['continuation'] and \
                i % self.opts['num_continuation'] == 0:
                # self-adaptive update
                # generally: larger admm_param leads to lower penalty on
                # primal residual (resid)
                admm_param_old = self.admm_param

                admm_state = eps_pri, eps_dual, rnorm, snorm

                if (i / self.opts['num_continuation']) % 5 == 1:
                    self._cont_update_2019(admm_state)
                else:
                    self._cont_update_s2a(admm_state)

                if self.opts['verb'] and self.admm_param != admm_param_old:
                    print('>> New ADMM parameter', self.admm_param,
                          '(old was %f)' % (admm_param_old))

        out = {}
        if pridualresids_below_tolerance:
            out['message'] = b'CONVERGENCE: primal and dual residual below tolerance'
        else:
            out['message'] = b'STOP: TOTAL NO. of ITERATIONS EXCEEDS LIMIT'

        out['history'] = history
        out['iter'] = i
        # out['admm_obj'] = history['objval'][i] # already in stats, see below

        out.update(stats)

        return out

    def _cont_update_s2a(self, admm_state, criticalratio: int = 5):
        """ update S2a from my master thesis
        uses more robust constant size updates of admm_param"""

        eps_pri, eps_dual, rnorm, snorm = admm_state

        snorm_rel = snorm / eps_dual
        rnorm_rel = rnorm / eps_pri
        if snorm_rel < 1 and rnorm_rel < 1:
            return  # residuals are already smaller than tolerancies

#        print(snorm_rel, rnorm_rel)

        ## do scaling of admm_param if necessary
        if rnorm_rel > snorm_rel * criticalratio:
            # decrease admm_param
            self.admm_param /= self.opts['continuation_fac']
        elif snorm_rel > rnorm_rel * criticalratio:
            # increase admm_param
            self.admm_param *= self.opts['continuation_fac']

    def _cont_update_2019(self,
                         admm_state,
                         criticalratio: int = 5,
                         cutoff: tuple = (1e-2, 1e2)):
        """ update that respects current ratio of residuals with tolerancies"""
        # TODO(franknu): consider speed of improvement instead of
        # just current state? - hist as class variable??
        # ML to learn update?
        eps_pri, eps_dual, rnorm, snorm = admm_state
        snorm_rel = snorm / eps_dual
        rnorm_rel = rnorm / eps_pri

        if snorm_rel < 1 and rnorm_rel < 1:
            return  # residuals are already smaller than tolerancies

        # do cutoff
        lower, upper = cutoff
        snorm_rel = min(upper, max((snorm_rel, lower)))
        rnorm_rel = min(upper, max((rnorm_rel, lower)))

        if snorm_rel < 1:
            # decrease ADMM param
            # increases penalty for violations of primal feasibiity
            scaling_factor = 1 / rnorm_rel
        elif rnorm_rel < 1:
            # increase ADMM param
            # decreases penalty for violations of primal feasibiity
            scaling_factor = snorm_rel
        else:
            scaling_factor = snorm_rel / rnorm_rel

#        if self.opts['verb']:
#            print(snorm_rel, rnorm_rel)

        ## do scaling of admm_param if necessary
        if max((scaling_factor, 1 / scaling_factor)) > criticalratio:

            if rnorm_rel >= 1 and snorm_rel >= 1:
                scaling_factor = (scaling_factor)**.8
                # inhibition, draw towards 1

            if self.opts['verb']:
                print('scaling_factor', scaling_factor)
            self.admm_param *= scaling_factor

    def solve(self, report=False, warminit=None, **kwargs):
        """solve the problem

        Args:
            report (bool): whether to print additional summary.
            warminit (solver state): starting point for the solver.
            kwargs (optional): additional solver options to replace defaults.

        Returns:
            dict: dictionary with info about solution/solving process
        """

        self.opts.update(**kwargs) # update solver options

        if report:
            print('>solver options', self.opts)

        ## select appropriate sparsity-inducing norms and corresponding proximity operators

        out = self._solve(warminit=warminit)

        if report:
            self._report(out)

        if not out['message'].startswith(b'CONV'):
            print(out['message'])

        return out


"""
Copyright 2020 The OneFlow Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import os
import unittest
from collections import OrderedDict

import numpy as np
import oneflow as flow

import oneflow.unittest
from test_util import GenArgList

def _test_rand(test_case, device, shape, low, high):
    y1 = flow.randint(low, high, shape, device=flow.device(device))
    y2 = flow.randint(low, high, shape, device=flow.device(device))
    test_case.assertFalse(np.all(y1.numpy() == y2.numpy()))
    test_case.assertTrue(shape == y1.shape)

def _test_0d_rand(test_case, device, shape, low, high):
    y1 = flow.randint(low, high, shape, device=flow.device(device))
    y2 = flow.randint(low, high, shape, device=flow.device(device))
    test_case.assertTrue(
        np.allclose(y1.numpy(), y2.numpy(), atol=1e-4, rtol=1e-4)
    )  # 0d is [] and []
    test_case.assertTrue(shape == y1.shape)

def _test_different_dtype(test_case, device, shape, low, high):
    y1 = flow.randint(low, high, shape, dtype=flow.float32, device=flow.device(device))
    y2 = flow.randint(low, high, shape, dtype=flow.float64, device=flow.device(device))
    test_case.assertTrue(not np.allclose(y1.numpy(), y2.numpy(), atol=1e-4, rtol=1e-4))
    test_case.assertTrue(shape == y1.shape)

def _test_backward(test_case, device, shape, low, high):
    x = flow.randint(low, high, shape, device=flow.device(device), requires_grad=True)
    y = x.sum()
    y.backward()
    test_case.assertTrue(
        np.allclose(np.ones(shape), x.grad.numpy(), atol=1e-4, rtol=1e-4)
    )

def _test_with_generator(test_case, device, shape, low, high):
    gen = flow.Generator()
    gen.manual_seed(0)
    y1 = flow.randint(
        low, high, shape, dtype=flow.float32, device=flow.device(device), generator=gen
    )
    y1_np = y1.numpy()
    gen.manual_seed(0)
    y2 = flow.randint(
        low, high, shape, dtype=flow.float32, device=flow.device(device), generator=gen
    )
    test_case.assertTrue(np.allclose(y1_np, y2.numpy(), atol=1e-4, rtol=1e-4))

def _test_high(test_case, device, shape, low, high):
    y1 = flow.randint(low, high, shape, device=flow.device(device))
    y2 = flow.randint(low, high, shape, device=flow.device(device))
    test_case.assertFalse(np.all(y1.numpy() == y2.numpy()))
    test_case.assertTrue(shape == y1.shape)

def _test_0rank(test_case, device, shape, low, high):
    y1 = flow.randint(low, high, shape, device=flow.device(device))
    test_case.assertTrue(y1.shape == shape)

@flow.unittest.skip_unless_1n1d()
@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
class TestRandint(flow.unittest.TestCase):
    def test_consistent_naive(test_case):
        placement = flow.placement("cpu", {0: [0]})
        sbp = (flow.sbp.broadcast,)
        x = flow.randint(0, 16, (10, 1), placement=placement, sbp=sbp)
        test_case.assertEqual(x.sbp, sbp)
        test_case.assertEqual(x.placement, placement)

    def test_randint(test_case):
        arg_dict = OrderedDict()
        arg_dict["test_fun"] = [
            _test_rand,
            _test_different_dtype,
            _test_backward,
            _test_with_generator,
        ]
        arg_dict["device"] = ["cpu", "cuda"]
        arg_dict["shape"] = [(2, 3), (2, 3, 4), (2, 3, 4, 5)]
        arg_dict["low"] = [i for i in range(10)]
        arg_dict["high"] = [10 + np.random.randint(10, 20) for i in range(10)]
        for arg in GenArgList(arg_dict):
            arg[0](test_case, *arg[1:])

    def test_0d_randint(test_case):
        arg_dict = OrderedDict()
        arg_dict["test_fun"] = [_test_0d_rand]
        arg_dict["device"] = ["cpu", "cuda"]
        arg_dict["shape"] = [(2, 0, 4), (2, 0, 2)]
        arg_dict["low"] = [i for i in range(10)]
        arg_dict["high"] = [10 + np.random.randint(1, 20) for i in range(10)]
        for arg in GenArgList(arg_dict):
            arg[0](test_case, *arg[1:])

    def test_high_randint(test_case):
        arg_dict = OrderedDict()
        arg_dict["test_fun"] = [_test_high]
        arg_dict["device"] = ["cpu", "cuda"]
        arg_dict["shape"] = [(2, 3, 4), (2, 5, 2)]
        arg_dict["low"] = [i for i in range(10)]
        arg_dict["high"] = [10 + np.random.randint(10, 20) for i in range(10)]
        for arg in GenArgList(arg_dict):
            arg[0](test_case, *arg[1:])

    def test_0rank_randint(test_case):
        arg_dict = OrderedDict()
        arg_dict["test_fun"] = [_test_0rank]
        arg_dict["device"] = ["cpu", "cuda"]
        arg_dict["shape"] = [()]
        arg_dict["low"] = [i for i in range(10)]
        arg_dict["high"] = [1000 + np.random.randint(1, 10) for i in range(10)]
        for arg in GenArgList(arg_dict):
            arg[0](test_case, *arg[1:])

if __name__ == "__main__":
    unittest.main()


# Programming for Data Science with Python Nanodegree Program (module 2 Introduction to Python)
# Python code to import US bike share data and answer interesting questions about it
# Author: Sel Pasos
# 8th January 2021
# 13th Jan 2021 Updated to correct initial errors

import time
import pandas as pd
import numpy as np

CITY_DATA = { 'chicago': 'chicago.csv',
              'new york city': 'new_york_city.csv',
              'washington': 'washington.csv' }
cities = ['chicago', 'new york city', 'washington']
months = ['all','january', 'february', 'march', 'april', 'may', 'june']
days = ['all', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']

def get_filters():
    """
    Asks user to specify a city, month, and day to analyze.

    Returns:
        (str) city - name of the city to analyze
        (str) month - name of the month to filter by, or "all" to apply no month filter
        (str) day - name of the day of week to filter by, or "all" to apply no day filter
    """
    print('Hello! Let\'s explore some US bikeshare data!')
    # TO DO: get user input for city (chicago, new york city, washington). HINT: Use a while loop to handle invalid inputs
    city_choice = ''
    while city_choice.lower() not in cities:
        city_choice = input("Which city do you want to explore data from: Chicago, New York, Washington?")
        if city_choice in cities:
            city = city_choice.lower()
        else:
            print('Invalid city choice, please select from: Chicago, New York, Washington')

    # TO DO: get user input for month (all, january, february, ... , june)
    month_choice = ''
    while month_choice.lower() not in months:
        month_choice = input("Which month would you like to analyse data for: January, February, March, May, June or all?")
        if month_choice.lower() in months:
            month = month_choice.lower()
        else:
            print('Invalid month choice, please choose from: January, February, March, May, June or all')

    # TO DO: get user input for day of week (all, monday, tuesday, ... sunday)
    day_choice = ''
    while day_choice.lower() not in days:
        day_choice = input("Which day would you like to anayse data for: Monday, Tuesday, Wednesday, Thursday, Friday, Saturday, Sunday or all")
        if day_choice.lower() in days:
            day = day_choice.lower()
        else:
            print('Invaid day choise, please enter any weekday or all to analyse data for this day')

    print('-'*40)
    return city, month, day

def load_data(city, month, day):
    """
    Loads data for the specified city and filters by month and day if applicable.

    Args:
        (str) city - name of the city to analyze
        (str) month - name of the month to filter by, or "all" to apply no month filter
        (str) day - name of the day of week to filter by, or "all" to apply no day filter
    Returns:
        df - Pandas DataFrame containing city data filtered by month and day
    """

    #load data
    filename = CITY_DATA.get(city)
    df = pd.read_csv(filename)

    # convert the Start Time column to datetime
    df['Start Time'] = pd.to_datetime(df['Start Time'])

    # extract month and day of week from Start Time to create new columns
    df['month'] = df['Start Time'].dt.month
    df['day_of_week'] = df['Start Time'].dt.weekday_name
    df['hour'] = df['Start Time'].dt.hour

    #filter data by month if applicable

    if month != 'all':
        # use the index of the months list to get the corresponding int
        month = months.index(month)

        # filter by month to create the new dataframe
        df = df[df['month'] == month]

    # filter data by day of week if applicable
    if day != 'all':
        # filter by day of week to create the new dataframe
        df = df[df['day_of_week'] == day.title()]

    return df

def time_stats(df):
    """Displays statistics on the most frequent times of travel."""

    print('\nCalculating The Most Frequent Times of Travel...\n')
    start_time = time.time()

    # TO DO: display the most common month

    pop_month = df['month'].mode()[0]

    print('The most frequent month of travel is: ', pop_month.title())

    # TO DO: display the most common day of week
    pop_day = df['day_of_week'].mode()[0]

    print('The most frequent day of travel is: ', pop_day.title())

    # TO DO: display the most common start hour
    pop_hour = df['hour'].mode()[0]

    print('The most frequent start hour of travel is: ', pop_hour)

    print("\nThis took %s seconds." % (time.time() - start_time))
    print('-'*40)

def station_stats(df):
    """Displays statistics on the most popular stations and trip."""

    print('\nCalculating The Most Popular Stations and Trip...\n')
    start_time = time.time()

    # TO DO: display most commonly used start station
    pop_start_st = df['Start Station'].mode()[0]
    print('The most commonly used start station is: ', pop_start_st)

    # TO DO: display most commonly used end station
    pop_end_st = df['End Station'].mode()[0]
    print('The most commonly used end station is: ', pop_end_st)

    # TO DO: display most frequent combination of start station and end station trip
    pop_combination = (df['Start Station'] + "||" + df['End Station']).mode()[0]
    print('The most frequent combination of start station and end station trip is : ' + str(pop_combination.split("||")))

    print("\nThis took %s seconds." % (time.time() - start_time))
    print('-'*40)

def trip_duration_stats(df):
    """Displays statistics on the total and average trip duration."""

    print('\nCalculating Trip Duration...\n')
    start_time = time.time()

    # TO DO: display total travel time
    total_travel_time = df['Trip Duration'].sum()
    print('The total travel time the selected data is: ', total_travel_time)

    # TO DO: display mean travel time
    mean_travel_time = df['Trip Duration'].mean()
    print('The average travel time the selected data is: ', mean_travel_time)

    print("\nThis took %s seconds." % (time.time() - start_time))
    print('-'*40)

def user_stats(df):
    """Displays statistics on bikeshare users."""

    print('\nCalculating User Stats...\n')
    start_time = time.time()

    # TO DO: Display counts of user types
    user_types = df['User Type'].value_counts()
    print('The count of user types for the selected data is: \n', user_types)

    # TO DO: Display counts of gender
    if city == 'washington':
        print('No gender or birth date data has been collected for Washington, select another city to analyse this data')
    else:
        gender = df['Gender'].value_counts()
        print("The count of user gender for the selected data is: \n", gender)

        # TO DO: Display earliest, most recent, and most common year of birth

        earliest_year = df['Birth Year'].min()
        recent_year = df['Birth Year'].max()
        common_year = df['Birth Year'].mode()[0]
        print('Earliest birth for the selected data is: \n', earliest_year)
        print('Most recent birth for the selected data is: \n', recent_year)
        print('Most common birth for the selected data is: \n', common_year)

    print("\nThis took %s seconds." % (time.time() - start_time))
    print('-'*40)

def main():
    while True:
        city, month, day = get_filters()
        df = load_data(city, month, day)

        time_stats(df)
        station_stats(df)
        trip_duration_stats(df)
        user_stats(df)

        restart = input('\nWould you like to restart? Enter yes or no.\n')
        if restart.lower() != 'yes':
            break

if __name__ == "__main__":
	main()


# -*- coding: utf-8 -*-
# BioSTEAM: The Biorefinery Simulation and Techno-Economic Analysis Modules
# Copyright (C) 2020, Yoel Cortes-Pena <yoelcortes@gmail.com>
#
# This module extends the elements module from the chemicals library:
# https://github.com/CalebBell/chemicals
# Copyright (C) 2020 Caleb Bell <Caleb.Andrew.Bell@gmail.com>
#
# This module is under a dual license:
# 1. The UIUC open-source license. See
# github.com/BioSTEAMDevelopmentGroup/biosteam/blob/master/LICENSE.txt
# for license details.
#
# 2. The MIT open-source license. See
# https://github.com/CalebBell/chemicals/blob/master/LICENSE.txt for details.
from chemicals import elements, periodic_table
import numpy as np

elements.__all__.extend([
    'atoms_to_array', 'array_to_atoms',
])

#: Dict[str, dict[str, int]] Cache of atomic counts.
formula_to_atoms = {}

#: Dict[str, int] Symbol - index pairs for atomic arrays.
symbol_to_index = {e.symbol: e.number - 1 for e in periodic_table}

#: tuple[str] Symbols for atomic arrays.
symbols = tuple(symbol_to_index)

def atoms_to_array(atoms: dict) -> np.ndarray:
    index = symbol_to_index
    array = np.zeros(118)
    for symbol, value in atoms.items():
        array[index[symbol]] = value
    return array

def array_to_atoms(array: np.ndarray) -> dict:
    index, = np.where(array != 0.)
    return dict(zip([symbols[i] for i in index], array[index]))

def get_atoms(formula):
    if formula in formula_to_atoms:
        return formula_to_atoms[formula]
    else:
        formula_to_atoms[formula] = atoms = elements.simple_formula_parser(formula)
        if len(formula_to_atoms) > 50: del formula_to_atoms[next(iter(formula_to_atoms))]
    return atoms.copy() # Prevent cached atoms from being altered

elements.get_atoms = get_atoms
elements.atoms_to_array = atoms_to_array
elements.array_to_atoms = array_to_atoms
elements.symbol_to_index = symbol_to_index
elements.symbols = symbols


"""
GMSH CAD support module
=======================
:synopsis: Manipulates ``.geo`` input files for ``gmsh``.

.. moduleauthor:: Pavel Ferkl <pavel.ferkl@gmail.com>
"""
from __future__ import print_function, division
import os
import re
import shutil
import subprocess as sp
import numpy as np
NAMES = {
    'point': 'Point',
    'line': 'Line',
    'line_loop': 'Line Loop',
    'surface': 'Plane Surface',
    'surface_loop': 'Surface Loop',
    'volume': 'Volume',
    'periodic_surface_X': 'Periodic Surface',
    'periodic_surface_Y': 'Periodic Surface',
    'physical_surface': 'Physical Surface',
    'physical_volume': 'Physical Volume'
}
NAME_LIST = [
    'point',
    'line',
    'line_loop',
    'surface',
    'surface_loop',
    'volume',
    'periodic_surface_X',
    'periodic_surface_Y',
    'physical_surface',
    'physical_volume'
]

def findall_top(regex, text):
    """Like ``re.findall``, but returns only top level group in list.

    Args:
        regex (str): regex patern
        text (str): text to search for patern

    Returns:
        list: list of all matches
    """
    matches = re.finditer(regex, text)
    lst = []
    for match in matches:
        lst.append(match.group(0))
    return lst

def read_geo(geo_file, plane_surface=True):
    """Read ``gmsh`` input file and extract geometry information.

    Uses regular expressions. Some geo files use Surface, some Plane Surface.
    You should specify what you want to read.

    Args:
        geo_file (str): input filename
        plane_surface (bool, optional): input file contains "Plane Surface"
            keyword

    Returns:
        dict: dictionary with read lines separated into points, lines, etc.
    """
    with open(geo_file, "r") as text_file:
        text = text_file.read()
        sdat = {}
        rexp = {}
        rexp['point'] = r'Point\s?[(][0-9]+[)]\s[=]\s[{](.*?)[}][;]'
        rexp['line'] = r'Line\s?[(][0-9]+[)]\s[=]\s[{][0-9]+[,]\s?[0-9]+[}][;]'
        rexp['line_loop'] = (
            r'Line\sLoop\s?[(][0-9]+[)]\s[=]\s[{]([+-]?[0-9]+[,]?\s?)+[}][;]'
        )
        if plane_surface:
            rexp['surface'] = (
                r'Plane\sSurface\s?[(][0-9]+[)]\s[=]\s[{]([0-9]+[,]?\s?)+[}][;]'
            )
        else:
            rexp['surface'] = (
                r'(Surface\s[(][0-9]+[)]\s[=]\s[{]([0-9]+[,]?)+[}][;])'
                + r'(?!.*Physical.*)',
            )
        rexp['physical_surface'] = (
            r'Physical\sSurface\s?[(][0-9]+[)]\s[=]\s[{]([0-9]+[,]?\s?)+[}][;]'
        )
        rexp['surface_loop'] = (
            r'Surface\sLoop\s?[(][0-9]+[)]\s[=]\s[{]([+-]?[0-9]+[,]?\s?)+[}][;]'
        )
        rexp['volume'] = (
            r'Volume\s?[(][0-9]+[)]\s[=]\s[{]([0-9]+[,]?\s?)+[}][;]'
        )
        rexp['physical_volume'] = (
            r'Physical\sVolume\s?[(]["][a-z]+["][)]\s[=]\s'
            + r'[{]([0-9]+[,]?\s?)+[}][;]'
        )
        for key in rexp:
            sdat[key] = findall_top(rexp[key], text)
        return sdat

def fix_strings(strings):
    """Remove negative signs (orientation) from loops.

    Used for OpenCASCADE kernel compatibility.

    Args:
        strings (list): list of line or surface loops in string format
    """
    for i, line in enumerate(strings):
        strings[i] = re.sub('[-]', '', line)

def save_geo(geo_file, sdat, opencascade=True):
    """Save ``gmsh`` CAD geometry input to file.

    Input is a dictionary with prepared string lines.

    Args:
        geo_file (str): filename
        sdat (dict): characterized geometry in string format
        opencascade (bool, optional): prepend OpenCASCADE keyword if True
    """
    with open(geo_file, "w") as fhl:
        if opencascade:
            fhl.write('SetFactory("OpenCASCADE");\n')
        for key in NAME_LIST:
            if key in sdat:
                for line in sdat[key]:
                    fhl.write("{}\n".format(line))

def geo2brep(geo_file, brep_file):
    """Convert ``gmsh`` CAD geometry to BREP format.

    ``gmsh`` is used for the conversion. A temporary file ``geo2brep.geo`` is
    created (overwrites existing file if it exists).

    Args:
        geo_file (str): input filename
        brep_file (str): output filename
    """
    wfile = 'geo2brep.geo'
    with open(wfile, "w") as fhl:
        fhl.write('SetFactory("OpenCASCADE");\n')
        fhl.write('Merge "{}";\n'.format(geo_file))
        fhl.write('Save "{}";\n'.format(brep_file))
    sp.Popen(['gmsh', wfile, '-parse_and_exit']).wait()
    os.remove(wfile)

def brep2geo(brep_file, geo_file):
    """Convert BREP CAD geometry to ``gmsh`` native format.

    ``gmsh`` is used for the conversion. A temporary file ``brep2geo.geo`` is
    created (overwrites existing file if it exists).

    Args:
        brep_file (str): input filename
        geo_file (str): output filename
    """
    wfile = 'brep2geo.geo'
    with open(wfile, "w") as fhl:
        fhl.write('SetFactory("OpenCASCADE");\n')
        fhl.write('Merge "{}";\n'.format(brep_file))
    sp.Popen(['gmsh', wfile, '-0']).wait()
    shutil.move(wfile + '_unrolled', geo_file)
    os.remove(wfile)

def extract_data(sdat):
    """Extract ``gmsh`` geometry data read by :func:`read_geo`.

    Only coordinates are taken from points. Point sizing if any is discarded.

    Opposite of :func:`collect_strings`.

    Args:
        sdat(dict): geometry data in string format

    Returns:
        dict: extracted geometry data
    """
    edat = {}
    for key in sdat:
        lines = dict()
        for line in sdat[key]:
            part = line.split("(")
            if key == "physical_volume":
                ind = part[1].split(")")[0]  # ID of the element
                if ind == '"cells"':
                    ind = 1
                elif ind == '"walls"':
                    ind = 2
            else:
                ind = int(part[1].split(")")[0])  # ID of the element
            fraction = line.split("{")
            fraction = fraction[1].split("}")
            fraction = fraction[0].split(",")
            if key == "point":  # point data consists of floats
                # ignore the optional fourth argument (defines mesh coarseness)
                fraction = np.array(fraction[0:3])
                fraction = fraction.astype(np.float)
                for j, number in enumerate(fraction):
                    if abs(number) < 1e-8:
                        fraction[j] = 0
            else:  # other data consists of integers
                fraction = np.array(fraction)
                fraction = np.absolute(fraction.astype(np.int)).tolist()
            lines[ind] = fraction
        edat[key] = lines
    return edat

def collect_strings(edat):
    """Convert extracted data to string format.

    Opposite of :func:`extract_data`.

    Args:
        edat(dict): extracted geometry data

    Returns:
        dict: geometry data in string format
    """
    sdat = {}
    for key in edat:
        sdat[key] = []
        if key == 'periodic_surface_X':
            for j in edat[key]:
                sdat[key].append(
                    '{0} {{{1}}} = {{{2}}} Translate{{-1,0,0}};'.format(
                        NAMES[key], j[0], j[1]
                    )
                )
        elif key == 'periodic_surface_Y':
            for j in edat[key]:
                sdat[key].append(
                    '{0} {{{1}}} = {{{2}}} Translate{{0,-1,0}};'.format(
                        NAMES[key], j[0], j[1]
                    )
                )
        else:
            for i, j in edat[key].items():
                j = ','.join(str(e) for e in j)
                sdat[key].append('{0} ({1}) = {{{2}}};'.format(
                    NAMES[key], i, j
                ))
    return sdat

def surfaces_in_plane(edat, coord, direction):
    """Finds surfaces that lie completely in specified plane.

    Plane must be normal to one of cartesian axes.

    Args:
        edat (dict): extracted geometry data
        coord (float): point on the chosen axis
        direction (int): order of coordinate axis

    Returns:
        list: line loops in specified plane
    """
    points_in_plane = []
    for i, point in edat['point'].items():
        if point[direction] == coord:
            points_in_plane.append(i)
    lines_in_plane = []
    for i, line in edat['line'].items():
        if line[0] in points_in_plane and line[1] in points_in_plane:
            lines_in_plane.append(i)
    line_loops_in_plane = []
    for i, line_loop in edat['line_loop'].items():
        log = True
        for line in line_loop:
            if line not in lines_in_plane:
                log = False
        if log:
            line_loops_in_plane.append(i)
    return line_loops_in_plane

def other_surfaces(edat, surfs):
    """Find boundary surfaces, which are not in ``surfs``.

    Assumes that inner surfaces are shared by two volumes. Remove duplicates
    before calling this function.

    Args:
        edat (dict): extracted geometry data
        surfs (list): list of surfaces, which should not be returned

    Returns:
        list: boundary surfaces, which are not in ``surfs``
    """
    all_surfaces = []
    for surface_loops in edat['volume'].values():
        for surface_loop in surface_loops:
            for surfaces in edat['surface_loop'][surface_loop]:
                all_surfaces += edat['surface'][surfaces]
    count = dict()
    for surface in all_surfaces:
        if surface in count:
            count[surface] += 1
        else:
            count[surface] = 1
    surf = [
        i for i, j in count.items() if j == 1
        and i not in surfs
    ]
    return surf

def periodic_surfaces(edat, surfaces, vec, eps=1e-8):
    """Find periodic surface pairs in specified direction.

    Only linear periodicity is supported. Checks for surfaces with points
    offset by specified vector within a tolerance.

    Args:
        edat (dict): extracted geometry data
        surfaces (list): boundary surfaces
        vec (ndarray): offset vector specification
        eps (float, optional): tolerance

    Returns:
        list: periodic surface pairs
    """
    surface_points = dict()  # point IDs for each boundary surface
    boundary_points = dict()  # dictionary with only boundary points
    for surface in surfaces:
        surface_points[surface] = []
        for line in edat['line_loop'][surface]:
            for point in edat['line'][line]:
                if point not in surface_points[surface]:
                    surface_points[surface] += [point]
                if point not in boundary_points:
                    boundary_points[point] = edat['point'][point]
    # sort point IDs so that you can compare later
    for point in surface_points.values():
        point.sort()
    # dictionary with ID of periodic point for each point that has one
    periodic_points = dict()
    for i, point in boundary_points.items():
        for j, secondpoint in boundary_points.items():
            if np.sum(np.abs(point + vec - secondpoint)) < eps:
                periodic_points[i] = j
    psurfs = []  # list of periodic surface pairs (IDs)
    for i, surface in surface_points.items():
        # Try to create surface using IDs of periodic points. Use None if there
        # is no periodic point in specified direction.
        per_surf = [
            periodic_points[point] if point in periodic_points else None
            for point in surface
        ]
        if None not in per_surf:
            per_surf.sort()  # sort so you can find it
            # use ID of current surface and find ID of periodic surface
            psurfs.append(
                [
                    i,
                    list(surface_points.keys())[
                        list(surface_points.values()).index(per_surf)
                    ]
                ]
            )
    return psurfs

def identify_duplicity(edat, key, number, eps):
    """Core algorithm for removing duplicities.

    User should call :func:`remove_duplicity` instead.

    Args:
        edat (dict): extracted geometry data
        key (str): type of geometry
        number (str): number type (float or integer)
        eps (float): tolerance

    Returns:
        dict: duplicit objects
    """
    dupl = dict()
    if number == 'float':
        for i, item1 in edat[key].items():
            for j, item2 in edat[key].items():
                if i != j and i > j and np.sum(np.abs(item1 - item2)) < eps:
                    if i not in dupl:
                        dupl[i] = []
                    dupl[i].append(j)
    elif number == 'integer':
        for i, item1 in edat[key].items():
            for j, item2 in edat[key].items():
                if i != j and i > j and sorted(item1) == sorted(item2):
                    if i not in dupl:
                        dupl[i] = []
                    dupl[i].append(j)
    else:
        raise Exception('number argument must be float or integer')
    return dupl

def remove_duplicit_ids_from_keys(edat, dupl, key):
    """Removes duplicit IDs from IDs of entities.

    Args:
        edat (dict): extracted geometry data
        dupl (dict): duplicit objects
        key (str): type of geometry
    """
    for i in dupl:
        del edat[key][i]

def remove_duplicit_ids_from_values(edat, dupl, key):
    """Removes duplicit IDs from values of entities.

    Args:
        edat (dict): extracted geometry data
        dupl (dict): duplicit objects
        key (str): type of geometry
    """
    for values in edat[key].values():
        for j, value in enumerate(values):
            if value in dupl:
                values[j] = min(dupl[value])

def remove_duplicity(edat, eps=1e-10):
    """Removes duplicit points, lines, etc.

    Args:
        edat (dict): extracted geometry data
        eps (float): tolerance
    """
    # points
    dupl = identify_duplicity(edat, 'point', 'float', eps)
    remove_duplicit_ids_from_keys(edat, dupl, 'point')
    remove_duplicit_ids_from_values(edat, dupl, 'line')
    # lines
    dupl = identify_duplicity(edat, 'line', 'integer', eps)
    remove_duplicit_ids_from_keys(edat, dupl, 'line')
    remove_duplicit_ids_from_values(edat, dupl, 'line_loop')
    # line loops
    dupl = identify_duplicity(edat, 'line_loop', 'integer', eps)
    remove_duplicit_ids_from_keys(edat, dupl, 'line_loop')
    remove_duplicit_ids_from_keys(edat, dupl, 'surface')
    remove_duplicit_ids_from_values(edat, dupl, 'surface_loop')
    # there are no duplicit volumes

def split_loops(edat, key):
    """Makes sure that line and surface loops contain only one loop.

    Surfaces and volumes with holes are instead defined in Surface and Volume
    entries, respectively. Needed because gmsh unrolls geometry in a way, which
    is unusable with OpenCASCADE kernel.

    This function is slow. It does not catch all loops.

    Args:
        edat (dict): extracted geometry data
        key (str): type of geometry
    """
    if key == 'line_loop':
        key2 = 'surface'
    elif key == 'surface_loop':
        key2 = 'volume'
    else:
        raise Exception('can be called only for line_loop or surface_loop')
    for i, item1 in edat[key].items():
        for j, item2 in edat[key].items():
            if i != j and set(item2).issubset((set(item1))):
                for value in item2:
                    item1.remove(value)
                edat[key][i] = item1
                edat[key2][i] = [i, j]
                break

def move_to_box(infile, wfile, outfile, mvol):
    """Moves periodic closed foam to periodic box.

    Uses gmsh, specifically boolean operations and transformations from
    OpenCASCADE. The result is unrolled to another geo file so that it can be
    quickly read and worked with in the follow-up work. Operations are
    performed two times. First for walls (first half of volumes) and then for
    cells.

    Save output to ``outfile``.

    Args:
        infile (str): input filename
        wfile (str): working filename
        outfile (str): output filename
        mvol (int): number of volumes
    """
    with open(wfile, 'w') as wfl:
        hvol = int(mvol / 2)
        wfl.write('SetFactory("OpenCASCADE");\n\n')
        wfl.write('Include "{0}";\n\n'.format(infile))
        wfl.write('Block({0}) = {{-1,-1,-1,3,3,1}};\n'.format(mvol + 1))
        wfl.write('Block({0}) = {{-1,-1, 1,3,3,1}};\n'.format(mvol + 2))
        wfl.write('Block({0}) = {{-1,-1, 0,3,3,1}};\n'.format(mvol + 3))
        wfl.write('Block({0}) = {{-1,-1,-1,3,1,3}};\n'.format(mvol + 4))
        wfl.write('Block({0}) = {{-1, 1,-1,3,1,3}};\n'.format(mvol + 5))
        wfl.write('Block({0}) = {{-1, 0,-1,3,1,3}};\n'.format(mvol + 6))
        wfl.write('Block({0}) = {{-1,-1,-1,1,3,3}};\n'.format(mvol + 7))
        wfl.write('Block({0}) = {{ 1,-1,-1,1,3,3}};\n'.format(mvol + 8))
        wfl.write('Block({0}) = {{ 0,-1,-1,1,3,3}};\n'.format(mvol + 9))
        wfl.write('\n')
        wfl.write(
            'zol() = BooleanIntersection'
            + '{{Volume{{1:{0}}};}}'.format(hvol)
            + '{{Volume{{{0}}};}};\n'.format(mvol + 1)
        )
        wfl.write(
            'zoh() = BooleanIntersection'
            + '{{Volume{{1:{0}}};}}'.format(hvol)
            + '{{Volume{{{0}}};}};\n'.format(mvol + 2)
        )
        wfl.write(
            'zin() = BooleanIntersection'
            + '{{Volume{{1:{0}}}; Delete;}}'.format(hvol)
            + '{{Volume{{{0}}};}};\n'.format(mvol + 3)
        )
        wfl.write('Translate{0,0, 1}{Volume{zol()};}\n')
        wfl.write('Translate{0,0,-1}{Volume{zoh()};}\n\n')
        wfl.write(
            'yol() = BooleanIntersection'
            + '{Volume{zol(),zoh(),zin()};}'
            + '{{Volume{{{0}}};}};\n'.format(mvol + 4)
        )
        wfl.write(
            'yoh() = BooleanIntersection'
            + '{Volume{zol(),zoh(),zin()};}'
            + '{{Volume{{{0}}};}};\n'.format(mvol + 5)
        )
        wfl.write(
            'yin() = BooleanIntersection'
            + '{Volume{zol(),zoh(),zin()}; Delete;}'
            + '{{Volume{{{0}}};}};\n'.format(mvol + 6)
        )
        wfl.write('Translate{0, 1,0}{Volume{yol()};}\n')
        wfl.write('Translate{0,-1,0}{Volume{yoh()};}\n\n')
        wfl.write(
            'xol() = BooleanIntersection'
            + '{Volume{yol(),yoh(),yin()};}'
            + '{{Volume{{{0}}};}};\n'.format(mvol + 7)
        )
        wfl.write(
            'xoh() = BooleanIntersection'
            + '{Volume{yol(),yoh(),yin()};}'
            + '{{Volume{{{0}}};}};\n'.format(mvol + 8)
        )
        wfl.write(
            'xin() = BooleanIntersection'
            + '{Volume{yol(),yoh(),yin()}; Delete;}'
            + '{{Volume{{{0}}};}};\n'.format(mvol + 9)
        )
        wfl.write('Translate{ 1,0,0}{Volume{xol()};}\n')
        wfl.write('Translate{-1,0,0}{Volume{xoh()};}\n\n')
        wfl.write(
            'zol2() = BooleanIntersection'
            + '{{Volume{{{0}:{1}}};}}'.format(hvol + 1, mvol)
            + '{{Volume{{{0}}}; Delete;}};\n'.format(mvol + 1)
        )
        wfl.write(
            'zoh2() = BooleanIntersection'
            + '{{Volume{{{0}:{1}}};}}'.format(hvol + 1, mvol)
            + '{{Volume{{{0}}}; Delete;}};\n'.format(mvol + 2)
        )
        wfl.write(
            'zin2() = BooleanIntersection'
            + '{{Volume{{{0}:{1}}}; Delete;}}'.format(hvol + 1, mvol)
            + '{{Volume{{{0}}}; Delete;}};\n'.format(mvol + 3)
        )
        wfl.write('Translate{0,0, 1}{Volume{zol2()};}\n')
        wfl.write('Translate{0,0,-1}{Volume{zoh2()};}\n\n')
        wfl.write(
            'yol2() = BooleanIntersection'
            + '{Volume{zol2(),zoh2(),zin2()};}'
            + '{{Volume{{{0}}}; Delete;}};\n'.format(mvol + 4)
        )
        wfl.write(
            'yoh2() = BooleanIntersection'
            + '{Volume{zol2(),zoh2(),zin2()};}'
            + '{{Volume{{{0}}}; Delete;}};\n'.format(mvol + 5)
        )
        wfl.write(
            'yin2() = BooleanIntersection'
            + '{Volume{zol2(),zoh2(),zin2()}; Delete;}'
            + '{{Volume{{{0}}}; Delete;}};\n'.format(mvol + 6)
        )
        wfl.write('Translate{0, 1,0}{Volume{yol2()};}\n')
        wfl.write('Translate{0,-1,0}{Volume{yoh2()};}\n\n')
        wfl.write(
            'xol2() = BooleanIntersection'
            + '{Volume{yol2(),yoh2(),yin2()};}'
            + '{{Volume{{{0}}}; Delete;}};\n'.format(mvol + 7)
        )
        wfl.write(
            'xoh2() = BooleanIntersection'
            + '{Volume{yol2(),yoh2(),yin2()};}'
            + '{{Volume{{{0}}}; Delete;}};\n'.format(mvol + 8)
        )
        wfl.write(
            'xin2() = BooleanIntersection'
            + '{Volume{yol2(),yoh2(),yin2()}; Delete;}'
            + '{{Volume{{{0}}}; Delete;}};\n'.format(mvol + 9)
        )
        wfl.write('Translate{ 1,0,0}{Volume{xol2()};}\n')
        wfl.write('Translate{-1,0,0}{Volume{xoh2()};}\n\n')
        wfl.write('Physical Volume ("walls") = {xol(),xoh(),xin()};\n')
        wfl.write('Physical Volume ("cells") = {xol2(),xoh2(),xin2()};\n\n')
    sp.Popen(['gmsh', wfile, '-0']).wait()
    shutil.move(wfile + '_unrolled', outfile)

def create_walls(edat, wall_thickness=0.01):
    """Creates walls by shring each cell.

    Each vertex is moved by toward the cell centroid as:

    .. math::

        v_n = v_o + w (c - v_o)

    where :math:`v_n` is new vertex position, :math:`v_o` is old vertex
    position, :math:`w` is the ``wall_thickness``, and :math:`c` is the
    centroid position.

    Args:
        edat (dict): extracted geometry data
        wall_thickness (float, optional): shrinking parameter

    Returns:
        list: [cell data, wall data]
    """
    xdat = dict()  # new cell data
    xdat['point'] = dict()
    xdat['line'] = dict()
    xdat['line_loop'] = dict()
    xdat['surface'] = dict()
    xdat['surface_loop'] = dict()
    xdat['volume'] = dict()
    volume_points = dict()  # point IDs for each volume
    for volume in edat['surface_loop']:
        volume_points[volume] = []
        for surface in edat['surface_loop'][volume]:
            for line in edat['line_loop'][surface]:
                for point in edat['line'][line]:
                    if point not in volume_points[volume]:
                        volume_points[volume] += [point]
        volume_points[volume].sort()
    centroids = dict()  # centroid for each volume
    for volume in edat['surface_loop']:
        total = 0
        for point in volume_points[volume]:
            total += edat['point'][point]
        total /= len(volume_points[volume])
        centroids[volume] = total
    npoints = len(edat['point'])
    nlines = len(edat['line'])
    nsurfaces = len(edat['line_loop'])
    nvolumes = len(edat['surface_loop'])
    for volume in list(edat['surface_loop']):
        point_map = dict()  # mapping of old points to new points
        nvolumes += 1
        edat['surface_loop'][nvolumes] = []
        xdat['surface_loop'][nvolumes] = []
        for point in volume_points[volume]:
            npoints += 1
            edat['point'][npoints] = edat['point'][point] + wall_thickness * (
                centroids[volume] - edat['point'][point])
            xdat['point'][npoints] = edat['point'][point] + wall_thickness * (
                centroids[volume] - edat['point'][point])
            point_map[point] = npoints
        for surface in edat['surface_loop'][volume]:
            nsurfaces += 1
            edat['line_loop'][nsurfaces] = []
            xdat['line_loop'][nsurfaces] = []
            for line in edat['line_loop'][surface]:
                nlines += 1
                edat['line'][nlines] = [
                    point_map[edat['line'][line][0]],
                    point_map[edat['line'][line][1]],
                ]
                xdat['line'][nlines] = [
                    point_map[edat['line'][line][0]],
                    point_map[edat['line'][line][1]],
                ]
                edat['line_loop'][nsurfaces] += [nlines]
                xdat['line_loop'][nsurfaces] += [nlines]
            edat['surface'][nsurfaces] = [nsurfaces]
            edat['surface_loop'][nvolumes] += [nsurfaces]
            xdat['surface'][nsurfaces] = [nsurfaces]
            xdat['surface_loop'][nvolumes] += [nsurfaces]
        # edat['volume'][nvolumes] = [nvolumes]
        xdat['volume'][nvolumes] = [nvolumes]
        edat['volume'][volume] += [nvolumes]
    remove_duplicity(edat)
    remove_duplicity(xdat)
    return xdat, edat

def restore_sizing(edat):
    """Add sizing info to all points.

    Adds fourth argument called "psize" to each point.

    Args:
        edat (dict): extracted geometry data
    """
    for ind in edat['point'].keys():
        edat['point'][ind] = list(edat['point'][ind]) + ['psize']

def prep_mesh_config(iname, oname, sizing, char_length=0.1):
    """Create file specifying meshing parameters.

    Sizing specified at points, edges and cells and implemented through
    thresholds.

    Additional info about gmsh mesh sizing `here
    <http://gmsh.info/doc/texinfo/gmsh.html#Specifying-mesh-element-sizes>`_.

    Args:
        iname (str): input filename
        oname (str): output filename
        sizing (list): mesh size near points, edges and in cells
        char_length (float, optional): gmsh Mesh.CharacteristicLengthMax
    """
    eps = 1e-6
    xmin = ymin = zmin = 0
    xmax = ymax = zmax = 1
    base = '{{{0}, {1}, {2}, {3}, {4}, {5}}}'
    with open(oname, "w") as fhl:
        fhl.write('Merge "{}";\n'.format(iname))
        fhl.write('e1() = Line In BoundingBox ' + base.format(
            xmin - eps, ymin - eps, zmin - eps,
            xmax + eps, ymax + eps, zmax + eps) + ';\n')
        fhl.write('Mesh.CharacteristicLengthMax = {0};\n'.format(char_length))
        fhl.write('psize = {0};\n'.format(sizing[0]))
        fhl.write('esize = {0};\n'.format(sizing[1]))
        fhl.write('csize = {0};\n'.format(sizing[2]))
        fhl.write('p1() = Point In BoundingBox ' + base.format(
            xmin - eps, ymin - eps, zmin - eps,
            xmax + eps, ymax + eps, zmax + eps) + ';\n')
        fhl.write('Field[1] = Distance;\n')
        fhl.write('Field[1].NodesList = {p1()};' + '\n')
        fhl.write('Field[2] = Threshold;\n')
        fhl.write('Field[2].IField = 1;\n')
        fhl.write('Field[2].LcMin = psize;\n')
        fhl.write('Field[2].LcMax = csize;\n')
        fhl.write('Field[2].DistMin = 0;\n')
        fhl.write('Field[2].DistMax = 3*csize;\n')
        fhl.write('Field[3] = Distance;\n')
        fhl.write('Field[3].NNodesByEdge = 10;\n')
        fhl.write('Field[3].EdgesList = {e1()};\n')
        fhl.write('Field[4] = Threshold;\n')
        fhl.write('Field[4].IField = 2;\n')
        fhl.write('Field[4].LcMin = esize;\n')
        fhl.write('Field[4].LcMax = csize;\n')
        fhl.write('Field[4].DistMin = 0;\n')
        fhl.write('Field[4].DistMax = 3*csize;\n')
        fhl.write('Field[5] = Min;\n')
        fhl.write('Field[5].FieldsList = {2, 4};' + '\n')
        fhl.write('Background Field = 5;\n')
        fhl.write('Mesh.CharacteristicLengthExtendFromBoundary = 0;\n')

def merge_and_label_geo(inames, oname):
    """Merge geometry files. Define periodic surfaces and physical volume.

    Assumes bounding box [0, 1] in all directions.

    Args:
        inames (list): input filenames
        oname (str): output filename
    """
    eps = 1e-6
    xmin = ymin = zmin = 0
    xmax = ymax = zmax = 1
    base = '{{{0}, {1}, {2}, {3}, {4}, {5}}}'
    base2 = '{{{0}, {1}, {2}}}'
    with open(oname, 'w') as fhl:
        for i, iname in enumerate(inames):
            fhl.write('Merge "{}";\n'.format(iname))
            fhl.write('v{}() = Volume In BoundingBox '.format(i + 1)
                      + base.format(xmin - eps, ymin - eps, zmin - eps,
                                    xmax + eps, ymax + eps, zmax + eps)
                      + ';\n')
            for j in range(i):
                fhl.write('v{0}() -= v{1}();\n'.format(i + 1, j + 1))
            fhl.write('Physical Volume({0}) = {{v{0}()}};\n'.format(i + 1))
        fhl.write('s1() = Surface In BoundingBox ' + base.format(
            xmin - eps, ymin - eps, zmin - eps,
            xmin + eps, ymax + eps, zmax + eps) + ';\n')
        fhl.write('s2() = Surface In BoundingBox ' + base.format(
            xmax - eps, ymin - eps, zmin - eps,
            xmax + eps, ymax + eps, zmax + eps) + ';\n')
        fhl.write('s3() = Surface In BoundingBox ' + base.format(
            xmin - eps, ymin - eps, zmin - eps,
            xmax + eps, ymin + eps, zmax + eps) + ';\n')
        fhl.write('s4() = Surface In BoundingBox ' + base.format(
            xmin - eps, ymax - eps, zmin - eps,
            xmax + eps, ymax + eps, zmax + eps) + ';\n')
        fhl.write('Periodic Surface {s2()} = {s1()} Translate'
                  + base2.format(xmax - xmin, 0, 0) + ';\n')
        fhl.write('Periodic Surface {s4()} = {s3()} Translate'
                  + base2.format(0, ymax - ymin, 0) + ';\n')


# -*- coding: utf-8 -*-
"""
script name
===========

Script :   ......py

Author :   Dan_Patterson@carleton.ca

Modified : 2018-06-04

Purpose:  tools for working with numpy arrays

Useage :

References
----------
`<http://pro.arcgis.com/en/pro-app/arcpy/data-access/numpyarraytotable.htm>`_.
`<http://pro.arcgis.com/en/pro-app/arcpy/data-access/tabletonumpyarray.htm>`_.
---------------------------------------------------------------------
"""

import sys
import numpy as np
import xlrd
import arcpy.da
from arcpy import env

env.overwriteOutput = True
#from arcpytools import fc_info, tweet  #, frmt_rec, _col_format
#import arcpy

ft = {'bool': lambda x: repr(x.astype(np.int32)),
      'float_kind': '{: 0.3f}'.format}
np.set_printoptions(edgeitems=10, linewidth=80, precision=2, suppress=True,
                    threshold=100, formatter=ft)
np.ma.masked_print_option.set_display('-')  # change to a single -

script = sys.argv[0]  # print this should you need to locate the script

# ----------------------------------------------------------------------
# (10) excel_np
def excel_np(path, sheet_num=0, int_null=-999):
    """Read excel files to numpy structured/record arrays.  Your spreadsheet
    must adhere to simple rules::
      - first row must contain the field names for the output array
      - no blank rows or columns, basically, no fluff or formatting
      - if you have nodata values, put them in, since blank cells will be
        'corrected' as best as possible.
      - text and numbers in a column, results in a text column

    See arraytools.a_io for excel_np for complete description
    """
    def isfloat(a):
        """float check"""
        try:
            i = float(a)
            return i
        except ValueError:
            return np.nan

    def punc_space(name):
        """delete punctuation and spaces and replace with '_'"""
        punc = list('!"#$%&\'()*+,-./:;<=>?@[\\]^`{|}~ ')
        return "".join([[i, '_'][i in punc] for i in name])

    # import xlrd
    w = xlrd.open_workbook(path)        # xlrd.book.Book class
    sheets = len(w.sheets())
    if sheet_num > sheets:
        return None
    sheet = w.sheet_by_index(sheet_num) # sheet by number
    # sheet = w.sheet_by_name('test')   # case sensitive, not implemented
    names = sheet.row_values(0)         # clean these up later
    cols = sheet.ncols
    rows = sheet.nrows
    col_data = [sheet.col_values(i, 1, rows) for i in range(cols)]
    row_guess = sheet.row_values(1)
    row_dts = [np.asarray(i).dtype.kind for i in row_guess]
    col_dts = [np.asarray(col_data[i]).dtype.kind
               for i in range(cols)]
    clean = []
    for i in range(len(row_dts)):
        c = col_data[i]
        if row_dts[i] == col_dts[i]:    # same dtype... send to array
            ar = np.asarray(c)
        if row_dts[i] == 'f':           # float? if so, substitute np.nan
            ar = np.array([isfloat(i) for i in c])
            is_nan = np.isnan(ar)       # find the nan values, then check
            not_nan = ar[~is_nan]       # are the floats == ints?
            if np.all(np.equal(not_nan, not_nan.astype('int'))):  # integer?
                ar[is_nan] = int_null   # assign the integer null
                ar = ar.astype('int')
        elif row_dts[i] in ('U', 'S'):  # unicode/string... send to array
            ar = np.char.strip(ar)
            ar = np.where(np.char.str_len(ar) == 0, 'None', ar)
        else:
            ar = np.asarray(c)
        clean.append(ar)
    # ---- assemble the columns for the array ----
    dt_str = [i.dtype.str for i in clean]
    names = [i.strip() for i in names]      # clean up leading/trailing spaces
    names = [punc_space(i) for i in names]  # replace punctuation and spaces
    dts_name = list(zip(names, dt_str))
    arr = np.empty((rows-1,), dtype= dts_name)
    cnt = 0
    for i in names:
        arr[i] = clean[cnt]
        cnt +=1
    return arr

# ----------------------------------------------------------------------
# .... final code section producing the featureclass and extendtable
if len(sys.argv) == 1:
    testing = True
    in_excel = script.rpartition("/")[0] + "/Data/test.xlsx"
    sheet_num = 0
    int_null = -999
    arr = excel_np(in_excel, sheet_num=sheet_num , int_null=int_null)
    print("Array returned...\n{}".format(arr))
    # parameters here
else:
    testing = False
    in_excel = sys.argv[1]
    sheet_num = int(sys.argv[2])
    int_null = sys.argv[3]
    if int_null in ('-2147483648', '-32768', '-128', '-9', '-1'):
        int_null == int(int_null)
    else:
        int_null = '-2147483648'
    out_tbl = sys.argv[4]
    arr = excel_np(in_excel, sheet_num=sheet_num , int_null=int_null)
    if arr is None:
        print("not a sheet number")
    else:
        arcpy.da.NumPyArrayToTable(arr, out_tbl)

    # parameters here
#
if testing:
    print('\nScript source... {}'.format(script))
# ----------------------------------------------------------------------
# __main__ .... code section
if __name__ == "__main__":
    """Optionally...
    : - print the script source name.
    : - run the _demo
    """


#!/usr/bin/env python
# coding: utf-8

import datetime
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import pytz
import random

# Date and Time
# =============

print(datetime.datetime(2000, 1, 1))
print(datetime.datetime.strptime("2000/1/1", "%Y/%m/%d"))
print(datetime.datetime(2000, 1, 1, 0, 0).strftime("%Y%m%d"))

# to_datetime
# ===========

print(pd.to_datetime("4th of July"))
print(pd.to_datetime("13.01.2000"))
print(pd.to_datetime("7/8/2000"))
print(pd.to_datetime("7/8/2000", dayfirst=True))
print(issubclass(pd.Timestamp, datetime.datetime))

ts = pd.to_datetime(946684800000000000)

print(ts.year, ts.month, ts.day, ts.weekday())

index = [pd.Timestamp("2000-01-01"),
         pd.Timestamp("2000-01-02"),
         pd.Timestamp("2000-01-03")]

ts = pd.Series(np.random.randn(len(index)), index=index)
print(ts)
print(ts.index)

ts = pd.Series(np.random.randn(len(index)),
               index=["2000-01-01", "2000-01-02", "2000-01-03"])
print(ts.index)

index = pd.to_datetime(["2000-01-01", "2000-01-02", "2000-01-03"])
ts = pd.Series(np.random.randn(len(index)), index=index)
print(ts.index)

print(pd.date_range(start="2000-01-01", periods=3, freq='H'))
print(pd.date_range(start="2000-01-01", periods=3, freq='T'))
print(pd.date_range(start="2000-01-01", periods=3, freq='S'))
print(pd.date_range(start="2000-01-01", periods=3, freq='B'))
print(pd.date_range(start="2000-01-01", periods=5, freq='1D1h1min10s'))
print(pd.date_range(start="2000-01-01", periods=5, freq='12BH'))

bh = pd.tseries.offsets.BusinessHour(start='07:00', end='22:00')
print(bh)

print(pd.date_range(start="2000-01-01", periods=5, freq=12 * bh))
print(pd.date_range(start="2000-01-01", periods=5, freq='W-FRI'))
print(pd.date_range(start="2000-01-01", periods=5, freq='WOM-2TUE'))

s = pd.date_range(start="2000-01-01", periods=10, freq='BAS-JAN')
t = pd.date_range(start="2000-01-01", periods=10, freq='A-FEB')
s.union(t)
index = pd.date_range(start='2000-01-01', periods=200, freq='B')
print(index)

ts = pd.Series(np.random.randn(len(index)), index=index)
walk = ts.cumsum()
walk.plot()
plt.savefig('random_walk.png')

print(ts.head())
print(ts[0])
print(ts[1:3])
print(ts['2000-01-03'])
print(ts[datetime.datetime(2000, 1, 3)])
print(ts['2000-01-03':'2000-01-05'])
print(ts['2000-01-03':datetime.datetime(2000, 1, 5)])
print(ts['2000-01-03':datetime.date(2000, 1, 5)])
print(ts['2000-02'])
print(ts['2000-03':'2000-05'])

small_ts = ts['2000-02-01':'2000-02-05']

print(small_ts)
print(small_ts.shift(2))
print(small_ts.shift(-2))

# Downsampling
# ============

rng = pd.date_range('4/29/2015 8:00', periods=600, freq='T')
ts = pd.Series(np.random.randint(0, 100, len(rng)), index=rng)

print(ts.head())
print(ts.resample('10min').head())
print(ts.resample('10min', how='sum').head())
print(ts.resample('1h', how='sum').head())
print(ts.resample('1h', how='max').head())

print(ts.resample('1h', how=lambda m: random.choice(m)).head())
print(ts.resample('1h', how='ohlc').head())

# Upsampling
# ==========

rng = pd.date_range('4/29/2015 8:00', periods=10, freq='H')
ts = pd.Series(np.random.randint(0, 100, len(rng)), index=rng)

print(ts.head())
print(ts.resample('15min'))
print(ts.head())
print(ts.resample('15min', fill_method='ffill').head())
print(ts.resample('15min', fill_method='bfill').head())
print(ts.resample('15min', fill_method='ffill', limit=2).head())
print(ts.resample('15min', fill_method='ffill', limit=2, loffset='5min').head())

tsx = ts.resample('15min')
print(tsx.interpolate().head())

# Time zone handling
# ==================

t = pd.Timestamp('2000-01-01')
print(t.tz is None)

t = pd.Timestamp('2000-01-01', tz='Europe/Berlin')
print(t.tz)

rng = pd.date_range('1/1/2000 00:00', periods=10, freq='D', tz='Europe/London')
print(rng)

tz = pytz.timezone('Europe/London')
rng = pd.date_range('1/1/2000 00:00', periods=10, freq='D', tz=tz)
print(rng)

rng = pd.date_range('1/1/2000 00:00', periods=10, freq='D')
ts = pd.Series(np.random.randn(len(rng)), rng)
print(ts.index.tz is None)

ts_utc = ts.tz_localize('UTC')

print(ts_utc.index.tz)
print(ts_utc.tz_convert('Europe/Berlin').index.tz)
print(ts_utc.tz_convert(None).index.tz is None)
print(ts_utc.tz_localize(None).index.tz is None)

# Time deltas
# ===========

print(pd.Timedelta('1 days'))
print(pd.Timedelta('-1 days 2 min 10s 3us'))
print(pd.Timedelta(days=1,seconds=1))
print(pd.Timedelta(days=1) + pd.Timedelta(seconds=1))
print(pd.to_timedelta('20.1s'))
print(pd.to_timedelta(np.arange(7), unit='D'))

# Time series plotting
# ====================

rng = pd.date_range(start='2000', periods=120, freq='MS')
ts = pd.Series(np.random.randint(-10, 10, size=len(rng)), rng).cumsum()

print(ts.head())

plt.clf()
ts.plot(c='k', title='Example time series')
plt.savefig('time_series_1.png')

ts.resample('2A').plot(c='0.75', ls='--')
ts.resample('5A').plot(c='0.25', ls='-.')

plt.clf()

tsx = ts.resample('1A')
ax = tsx.plot(kind='bar', color='k')
plt.savefig('time_series_2.png')

ax.set_xticklabels(tsx.index.year)
plt.clf()
ts = pd.Series(np.random.randn(1000), index=pd.date_range('1/1/2000', periods=1000))
df = pd.DataFrame(np.random.randn(1000, 4), index=ts.index, columns=['A', 'B', 'C', 'D'])
df = df.cumsum()
df.plot(color=['k', '0.75', '0.5', '0.25'], ls='--')
plt.savefig('time_series_3.png')


# $Id: text_tools.py,v 1.8 2011-09-13 21:50:16 wirawan Exp $
#
# wpylib.text_tools
# Created: 20091204
# Wirawan Purwanto
#
# Simple and dirty text tools
#

"""
wpylib.text_tools

Frequently used text tools.
"""

import numpy
from wpylib.sugar import ifelse

def read_text_table(F, maps={}, sep=None, comment_char="#"):
  """Reads in a 2-D table from a text stream.
  Returns a list of lists containing the table content, in each cell by
  default as a string, unless a mapping function is provided (for simple
  data conversion only)."""
  rows = []
  for L in F:
    if comment_char != None:
      L = L.split(comment_char,1)[0]
    flds = L.split(sep)
    if len(flds) == 0:
      continue
    if maps:
      for i in xrange(len(flds)):
        if i in maps:
          flds[i] = maps[i](flds[i])
    rows.append(flds)
  return rows

def make_matrix(Str, debug=None):
  """Simple tool to convert a string like
    '''1 2 3
    4 5 6
    7 8 9'''
  into a numpy matrix (or, actually, an array object).
  This is for convenience in programming quick scripts, much like octave matrix
  format (but without the evaluation of math expressions that octave has,
  of course)."""
  if isinstance(Str, numpy.matrix):
    return numpy.array(Str)
  elif isinstance(Str, numpy.ndarray):
    if len(Str.shape) == 2:
      return Str.copy()
    else:
      raise ValueError, "Cannot make matrix out of non-2D array"
  Str2 = ";".join([ row.split("#",1)[0].rstrip().rstrip(";")
                      for row in Str.split("\n")
                        if row.split("#",1)[0].strip() != ""
                  ])
  rslt = numpy.matrix(Str2)
  if debug: print rslt
  return numpy.array(rslt)

def vector_str(M, fmt="%22.15g", V=False, prefix="", suffix=""):
  if len(M.shape) != 1:
    raise ValueError, "Wrong shape: expecting a one-dimensional array."
  if V:
    return prefix + (suffix + "\n" + prefix).join([ fmt % m for m in M ]) + suffix
  else:
    return prefix + " ".join([ fmt % m for m in M ]) + suffix

def matrix_str(M, fmt=None, prefix="", suffix=""):
  """Prints a matrix in a textual format.
  Applicable for integer, float, and complex 2-D arrays.

  COMPLEX NUMBER SUPPORT

  By default, we print in full precision and in python-friendly format, like:

      (+3.200000000000000e+00+4.700000000000000e+00j)

  To print in C++ and Fortran friendly format, use:

      >>> A_str = text_tools.matrix_str(A, '(%+#22.15e,%+#22.15e)')

  The resulting output will be:

      (+3.200000000000000e+00,+4.700000000000000e+00)
  """
  linesep = suffix + "\n" + prefix
  if isinstance(M, numpy.matrix):
    M = numpy.asarray(M)
  elif not isinstance(M, numpy.ndarray):
    M = numpy.asarray(M)
  if len(M.shape) != 2:
    raise ValueError, "Wrong shape: expecting a two-dimensional array."
  if numpy.iscomplex(M[0,0]):
    if fmt is None:
      fmt = "(%+22.15e%+22.15ej)"
    mkfmt = lambda z: fmt % (z.real, z.imag)
    return prefix + linesep.join([ " ".join([ mkfmt(c) for c in R ]) for R in M ]) + suffix
  else:
    if fmt is None:
      fmt = "%22.15g"
    return prefix + linesep.join([ " ".join([ fmt % c for c in R ]) for R in M ]) + suffix

def str_indent(text, indent=" "*4):
  """Indents a text block by a given prefix.
  If the indent is a number 'N', a string of N white spaces are taken as
  the prefix.

  In python 3, textwrap.indent can accomplish the same thing."""
  if not isinstance(indent, basestring):
    # Assume this is a numeric:
    indent = " " * indent
  return indent + ('\n'+indent).join(x for x in str(text).splitlines())

def str_unindent(S, amount=None):
  """Automatically unidents a string based on the first indentation found
  on a nonempty string line. Assuming UNIX LF end-of-line.

  Note: textwrap.dedent accomplishes a similar function (but the amount of
  white spaces to remove is automatically detected)."""

  if amount == None:
    nindent = -1  # autodetect, default
  else:
    nindent = amount
    indent_whsp = " " * nindent

  strs = S.splitlines()
  rslt = []
  for s in strs:
    if s.strip() != "":
      if nindent == -1:
        nindent = len(s) - len(s.lstrip())
        indent_whsp = " " * nindent
      if s[:nindent] == indent_whsp:
        s = s[nindent:]
      # else, quietly accept all strings that are not properly indented
      # at their beginning
    rslt.append(s)

  return "\n".join(rslt)

def str_snippet(S):
  """Standard processing for input snippet:
  Unindent a string and strip the trailing whitespaces (mainly for input
  file segments."""
  return str_unindent(S).rstrip()

def str_trunc_begin(S, L):
  """Returns a possibly truncated S (ellipsis added at the string's
  beginning) if the length of S is greater than L.
  L should be equal to or greater than 3 to be making sense."""
  if len(S) > L:
    return "..." + S[min(-L+3,0):]
  else:
    return S

def str_trunc_end(S, L):
  """Returns a possibly truncated S (ellipsis added at the string's
  ending) if the length of S is greater than L.
  L should be equal to or greater than 3 to be making sense."""
  if len(S) > L:
    return S[:max(L-3,0)] + "..."
  else:
    return S

def str_lstrip(S, substr):
  """Strips a prefix from S if it matches the string in `substr'.
  """
  # This is akin to ${VAR#$substr} in Bourne-shell dialect.
  if len(substr) > 0 and S.startswith(substr):
    return S[len(substr):]
  else:
    return S

def str_rstrip(S, substr):
  """Strips a suffix from S if it matches the string in `substr'.
  """
  # This is akin to ${VAR%$substr} in Bourne-shell dialect.
  if len(substr) > 0 and S.endswith(substr):
    return S[:-len(substr)]
  else:
    return S

def str_save_to_file(filename, s1, *more_str, **opts):
  """Save one or more string (or iterables) to a file with a given file.

  Additional options (with their defaults shown below):
  * append=False: if True, then the string(s) are appended to the file.
  * eol=False: if True, then an EOLN is added between strings.
  """
  add_eol = opts.get("eol", False)
  append = opts.get("append", False)
  if append:
    F = open(filename, "a")
  else:
    F = open(filename, "w")

  for S in (s1,) + more_str:
    if getattr(S, "__iter__", False):
      for S2 in S:
        F.write(S2)
        if add_eol: F.write("\n")
    else:
      F.write(S)
      if add_eol: F.write("\n")

  F.close()

def str_expand(template, params, maxiter=100):
  """Doing iterative python-style %(KWD)* substitution until no more
  substitution takes place.
  This is used to constructively build input string, etc. that contain
  parameter within parameter."""
  str1 = None
  str2 = template
  i = 0
  while str1 != str2 and (maxiter > 0 and i < maxiter):
    str1 = str2
    str2 = str1 % params
    i += 1

  if str1 != str2: raise RuntimeError, "Iteration limit exceeded"
  return str1

# Internal variable: don't mess!
_str_fmt_heading_rx = None
def str_fmt_heading(fmt):
  """Replaces a printf-style formatting with one suitable for table heading:
  all non-string conversions are replaced with string conversions,
  preserving the minimum widths."""
  # Originally from: $PWQMC77/scripts/cost.py and later Cr2_analysis_cbs.py .
  #
  #_str_fmt_heading_rx = None # only for development purposes
  import re
  global _str_fmt_heading_rx
  if _str_fmt_heading_rx == None:
    # Because of complicated regex, I verbosely write it out here:
    _str_fmt_heading_rx = re.compile(r"""
      (
        %                 # % sign
        (?:\([^)]+\))?    # optional '(keyname)' mapping key
        [-+#0 hlL]*       # optional conversion flag
        [0-9*]*           # optional minimum field width
      )
      ((?:\.[0-9]*)?)     # optional precision
      [^-+#*0 hlL0-9.%s]  # not conv flag, dimensions, nor literal '%',
                          # nor 's' conversion specifiers
    """, re.VERBOSE)
  return _str_fmt_heading_rx.sub(r'\1s', fmt)

def str_grep(S, strs):
  """Returns a list of strings wherein the substring S is found."""
  return [s for s in strs if s.find(S) >= 0]

def str_igrep(S, strs):
  """Returns a list of the indices of the strings wherein the substring S
  is found."""
  return [i for (i,s) in enumerate(strs) if s.find(S) >= 0]
  #return [i for (s,i) in zip(strs,xrange(len(strs))) if s.find(S) >= 0]

def slice_str(s):
  return "%s:%s:%s" % (
    ifelse(s.start == None, "", str(s.start)),
    ifelse(s.stop == None, "", str(s.stop)),
    ifelse(s.step == None, "", str(s.step)),
  )


# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for ragged.to_tensor."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from absl.testing import parameterized
import numpy as np

from tensorflow.python.framework import ops
from tensorflow.python.framework import test_util
from tensorflow.python.ops.ragged import ragged_factory_ops
from tensorflow.python.platform import googletest

@test_util.run_all_in_graph_and_eager_modes
class RaggedTensorToTensorOpTest(test_util.TensorFlowTestCase,
                                 parameterized.TestCase):

  def testDocStringExamples(self):
    """Example from ragged_to_tensor.__doc__."""
    rt = ragged_factory_ops.constant([[9, 8, 7], [], [6, 5], [4]])
    dt = rt.to_tensor()
    self.assertAllEqual(dt, [[9, 8, 7], [0, 0, 0], [6, 5, 0], [4, 0, 0]])

  @parameterized.parameters(
      {
          'rt_input': [],
          'ragged_rank': 1,
          'expected': [],
          'expected_shape': [0, 0],
      },
      {
          'rt_input': [[1, 2, 3], [], [4], [5, 6]],
          'expected': [[1, 2, 3], [0, 0, 0], [4, 0, 0], [5, 6, 0]]
      },
      {
          'rt_input': [[1, 2, 3], [], [4], [5, 6]],
          'default': 9,
          'expected': [[1, 2, 3], [9, 9, 9], [4, 9, 9], [5, 6, 9]]
      },
      {
          'rt_input': [[[1], [2], [3]], [], [[4]], [[5], [6]]],
          'ragged_rank':
              1,
          'default': [9],
          'expected': [[[1], [2], [3]], [[9], [9], [9]], [[4], [9], [9]],
                       [[5], [6], [9]]]
      },
      {
          'rt_input': [[[1, 2], [], [3, 4]], [], [[5]], [[6, 7], [8]]],
          'expected': [
              [[1, 2], [0, 0], [3, 4]],  #
              [[0, 0], [0, 0], [0, 0]],  #
              [[5, 0], [0, 0], [0, 0]],  #
              [[6, 7], [8, 0], [0, 0]],  #
          ]
      },
      {
          'rt_input': [[[1, 2], [], [3, 4]], [], [[5]], [[6, 7], [8]]],
          'default':
              9,
          'expected': [
              [[1, 2], [9, 9], [3, 4]],  #
              [[9, 9], [9, 9], [9, 9]],  #
              [[5, 9], [9, 9], [9, 9]],  #
              [[6, 7], [8, 9], [9, 9]],  #
          ]
      },
      {
          'rt_input': [[[1], [2], [3]]],
          'ragged_rank': 1,
          'default': 0,
          'expected': [[[1], [2], [3]]],
      },
      {
          'rt_input': [[[[1], [2]], [], [[3]]]],
          'default': 9,
          'expected': [[[[1], [2]], [[9], [9]], [[3], [9]]]],
      },
  )
  def testRaggedTensorToTensor(self,
                               rt_input,
                               expected,
                               ragged_rank=None,
                               default=None,
                               expected_shape=None):
    rt = ragged_factory_ops.constant(rt_input, ragged_rank=ragged_rank)
    dt = rt.to_tensor(default)
    self.assertIsInstance(dt, ops.Tensor)
    self.assertEqual(rt.dtype, dt.dtype)
    self.assertTrue(dt.shape.is_compatible_with(rt.shape))
    if expected_shape is not None:
      expected = np.ndarray(expected_shape, buffer=np.array(expected))
    self.assertAllEqual(dt, expected)

  @parameterized.parameters(
      {
          'rt_input': [[1, 2, 3]],
          'default': [0],
          'error': (ValueError, r'Shape \(1,\) must have rank at most 0'),
      },
      {
          'rt_input': [[[1, 2], [3, 4]], [[5, 6]]],
          'ragged_rank': 1,
          'default': [7, 8, 9],
          'error': (ValueError, r'Shapes \(3,\) and \(2,\) are incompatible'),
      },
      {
          'rt_input': [[1, 2, 3]],
          'default': 'a',
          'error': (TypeError, '.*'),
      },
  )
  def testError(self, rt_input, default, error, ragged_rank=None):
    rt = ragged_factory_ops.constant(rt_input, ragged_rank=ragged_rank)
    with self.assertRaisesRegexp(error[0], error[1]):
      rt.to_tensor(default)

if __name__ == '__main__':
  googletest.main()


import numpy as np
from ai.parameter import Parameter
from ai.graph import ComputationalGraph, G
from ai.module import Module

# dropout layer - non-parametrized layer
class Dropout(Module):
    def __init__(self, p=0.5, graph=G):
        super(Dropout, self).__init__()
        self.p = p
        self.graph = graph

    def __str__(self):
        return('Dropout(p={})'.format(self.p))

    def __call__(self, x):  # easy callable
        return self.forward(x)

    def forward(self, x):

        if not isinstance(x, Parameter):
            x = Parameter(data=x, eval_grad=False, graph=self.graph)

        out = self.graph.dropout(x, p=self.p)

        return out


# -*- coding: utf-8 -*-
#
from helpers import assert_equality

def plot():
    import numpy as np
    import matplotlib.pyplot as plt

    fig, ax = plt.subplots(figsize=(17, 6))
    ax.plot(np.array([1, 5]), label="Test 1")
    ax.plot(np.array([5, 1]), label="Test 2")
    ax.legend(ncol=2, loc="upper center")
    return fig

def test():
    assert_equality(plot, "test_legend_columns_reference.tex")
    return


import functools
import operator
import os
import random
import re
import sys
import sqlite3

import linearmodels
import numpy as np
import pandas as pd
import scipy as sp
import statsmodels.api as sm
import statsmodels.formula.api as smf

import matplotlib.pyplot as plt
import seaborn as sns
import plotly.express as px
import altair as alt
from vega_datasets import data

# pandas settings
pd.set_option('display.max_rows', 120)
pd.set_option('display.max_columns', 120)
pd.set_option('max_colwidth', None)
pd.set_option('precision', 4)

# seaborn settings
sns.set_context("notebook")
# sns.set(rc={'figure.figsize': (16, 9.)})
sns.set_style("whitegrid")

# global vars
TEMPDIR = '/Users/fgu/tmp/'
SAMPLEDATA = '/Users/fgu/tmp/mdb/data_777.parquet'

- [Fluent Python](https://www.oreilly.com/library/view/fluent-python/9781491946237/)
- [Python Cookbook](https://www.oreilly.com/library/view/python-cookbook-3rd/9781449357337/)
- [Learning Python](https://www.oreilly.com/library/view/learning-python-5th/9781449355722/)
- [The Hitchhiker's Guide to Python](https://docs.python-guide.org/writing/structure/)
- [Effective Python](https://effectivepython.com)
- [Python for Data Analysis](https://www.oreilly.com/library/view/python-for-data/9781491957653/)
- [Python Data Science Handbook](https://www.oreilly.com/library/view/python-data-science/9781491912126/)
- [Pandas cookbook](https://pandas.pydata.org/pandas-docs/stable/user_guide/cookbook.html)
- [Numpy docs](https://numpy.org/doc/stable/)

- [An introduction to statistical learning](https://www.statlearning.com)
- [Applied predictive modeling](http://appliedpredictivemodeling.com)
- [Hands on machine learning with scikit-learn, keras, and tenserflow](https://www.oreilly.com/library/view/hands-on-machine-learning/9781492032632/)
- [The hundred-page machine learning book](http://themlbook.com)

import numpy,import scipy,import statsmodels