from __future__ import division, absolute_import, print_function

import numpy as np
from itertools import product
from numpy.compat import asbytes
from numpy.testing import *
import sys, warnings

class TestIndexing(TestCase):
    def test_none_index(self):
        # `None` index adds newaxis
        a = np.array([1, 2, 3])
        assert_equal(a[None], a[np.newaxis])
        assert_equal(a[None].ndim, a.ndim + 1)

    def test_empty_tuple_index(self):
        # Empty tuple index creates a view
        a = np.array([1, 2, 3])
        assert_equal(a[()], a)
        assert_(a[()].base is a)
        a = np.array(0)
        assert_(isinstance(a[()], np.int_))

    def test_empty_fancy_index(self):
        # Empty list index creates an empty array
        # with the same dtype (but with weird shape)
        a = np.array([1, 2, 3])
        assert_equal(a[[]], [])
        assert_equal(a[[]].dtype, a.dtype)

        b = np.array([], dtype=np.intp)
        assert_equal(a[[]], [])
        assert_equal(a[[]].dtype, a.dtype)

        b = np.array([])
        assert_raises(IndexError, a.__getitem__, b)

    def test_ellipsis_index(self):
        # Ellipsis index does not create a view
        a = np.array([[1, 2, 3],
                      [4 ,5, 6],
                      [7, 8, 9]])
        assert_equal(a[...], a)
        assert_(a[...] is a)

        # Slicing with ellipsis can skip an
        # arbitrary number of dimensions
        assert_equal(a[0, ...], a[0])
        assert_equal(a[0, ...], a[0, :])
        assert_equal(a[..., 0], a[:, 0])

        # Slicing with ellipsis always results
        # in an array, not a scalar
        assert_equal(a[0, ..., 1], np.array(2))

    def test_single_int_index(self):
        # Single integer index selects one row
        a = np.array([[1, 2, 3],
                      [4 ,5, 6],
                      [7, 8, 9]])

        assert_equal(a[0], [1, 2, 3])
        assert_equal(a[-1], [7, 8, 9])

        # Index out of bounds produces IndexError
        assert_raises(IndexError, a.__getitem__, 1<<30)
        # Index overflow produces IndexError
        assert_raises(IndexError, a.__getitem__, 1<<64)

    def test_single_bool_index(self):
        # Single boolean index
        a = np.array([[1, 2, 3],
                      [4 ,5, 6],
                      [7, 8, 9]])

        # Python boolean converts to integer
        # These are being deprecated (and test in test_deprecations)
        #assert_equal(a[True], a[1])
        #assert_equal(a[False], a[0])

        # Same with NumPy boolean scalar
        assert_equal(a[np.array(True)], a[1])
        assert_equal(a[np.array(False)], a[0])

    def test_boolean_indexing_onedim(self):
        # Indexing a 2-dimensional array with
        # boolean array of length one
        a = np.array([[ 0.,  0.,  0.]])
        b = np.array([ True], dtype=bool)
        assert_equal(a[b], a)
        # boolean assignment
        a[b] = 1.
        assert_equal(a, [[1., 1., 1.]])

    def test_boolean_indexing_twodim(self):
        # Indexing a 2-dimensional array with
        # 2-dimensional boolean array
        a = np.array([[1, 2, 3],
                      [4 ,5, 6],
                      [7, 8, 9]])
        b = np.array([[ True, False,  True],
                      [False,  True, False],
                      [ True, False,  True]])
        assert_equal(a[b], [1, 3, 5, 7, 9])
        assert_equal(a[b[1]], [[4, 5, 6]])
        assert_equal(a[b[0]], a[b[2]])

        # boolean assignment
        a[b] = 0
        assert_equal(a, [[0, 2, 0],
                         [4, 0, 6],
                         [0, 8, 0]])

class TestMultiIndexingAutomated(TestCase):
    """
     These test use code to mimic the C-Code indexing for selection.

     NOTE: * This still lacks tests for complex item setting.
           * If you change behavoir of indexing, you might want to modify
             these tests to try more combinations.
           * Behavior was written to match numpy version 1.8. (though a
             first version matched 1.7.)
           * Only tuple indicies are supported by the mimicing code.
             (and tested as of writing this)
           * Error types should match most of the time as long as there
             is only one error. For multiple errors, what gets raised
             will usually not be the same one. They are *not* tested.
    """
    def setUp(self):
        self.a = np.arange(np.prod([3,1,5,6])).reshape(3,1,5,6)
        self.b = np.empty((3,0,5,6))
        self.complex_indices = ['skip', Ellipsis,
            0,
            # Boolean indices, up to 3-d for some special cases of eating up
            # dimensions, also need to test all False
            np.array(False),
            np.array([True, False, False]),
            np.array([[True, False], [False, True]]),
            np.array([[[False, False], [False, False]]]),
            # Some slices:
            slice(-5, 5, 2),
            slice(1, 1, 100),
            slice(4, -1, -2),
            slice(None,None,-3),
            # Some Fancy indexes:
            np.empty((0,1,1), dtype=np.intp), # empty broadcastable
            np.array([0,1,-2]),
            np.array([[2],[0],[1]]),
            np.array([[0,-1], [0,1]]),
            np.array([2,-1]),
            np.zeros([1]*31, dtype=int), # trigger too large array.
            np.array([0., 1.])] # invalid datatype
        # Some simpler indices that still cover a bit more
        self.simple_indices = [Ellipsis, None, -1, [1], np.array([True]), 'skip']
        # Very simple ones to fill the rest:
        self.fill_indices = [slice(None,None), 0]

    def _get_multi_index(self, arr, indices):
        """Mimic multi dimensional indexing.

        Parameters
        ----------
        arr : ndarray
            Array to be indexed.
        indices : tuple of index objects

        Returns
        -------
        out : ndarray
            An array equivalent to the indexing operation (but always a copy).
            `arr[indices]` should be identical.
        no_copy : bool
            Whether the indexing operation requires a copy. If this is `True`,
            `np.may_share_memory(arr, arr[indicies])` should be `True` (with
            some exceptions for scalars and possibly 0-d arrays).

        Notes
        -----
        While the function may mostly match the errors of normal indexing this
        is generally not the case.
        """
        in_indices = list(indices)
        indices = []
        # if False, this is a fancy or boolean index
        no_copy = True
        # number of fancy/scalar indexes that are not consecutive
        num_fancy = 0
        # number of dimensions indexed by a "fancy" index
        fancy_dim = 0
        # NOTE: This is a funny twist (and probably OK to change).
        # The boolean array has illegal indexes, but this is
        # allowed if the broadcasted fancy-indices are 0-sized.
        # This variable is to catch that case.
        error_unless_broadcast_to_empty = False

        # We need to handle Ellipsis and make arrays from indices, also
        # check if this is fancy indexing (set no_copy).
        ndim = 0
        ellipsis_pos = None # define here mostly to replace all but first.
        for i, indx in enumerate(in_indices):
            if indx is None:
                continue
            if isinstance(indx, np.ndarray) and indx.dtype == bool:
                no_copy = False
                if indx.ndim == 0:
                    raise IndexError
                # boolean indices can have higher dimensions
                ndim += indx.ndim
                fancy_dim += indx.ndim
                continue
            if indx is Ellipsis:
                if ellipsis_pos is None:
                    ellipsis_pos = i
                    continue # do not increment ndim counter
                in_indices[i] = slice(None,None)
                ndim += 1
                continue
            if isinstance(indx, slice):
                ndim += 1
                continue
            if not isinstance(indx, np.ndarray):
                # This could be open for changes in numpy.
                # numpy should maybe raise an error if casting to intp
                # is not safe. It rejects np.array([1., 2.]) but not
                # [1., 2.] as index (same for ie. np.take).
                # (Note the importance of empty lists if changing this here)
                indx = np.array(indx, dtype=np.intp)
                in_indices[i] = indx
            elif indx.dtype.kind != 'b' and indx.dtype.kind != 'i':
                raise IndexError('arrays used as indices must be of integer (or boolean) type')
            if indx.ndim != 0:
                no_copy = False
            ndim += 1
            fancy_dim += 1

        if arr.ndim - ndim < 0:
            # we can't take more dimensions then we have, not even for 0-d arrays.
            # since a[()] makes sense, but not a[(),]. We will raise an error
            # lateron, unless a broadcasting error occurs first.
            raise IndexError

        if ndim == 0 and not None in in_indices:
            # Well we have no indexes or one Ellipsis. This is legal.
            return arr.copy(), no_copy

        if ellipsis_pos is not None:
            in_indices[ellipsis_pos:ellipsis_pos+1] = [slice(None,None)] * (arr.ndim - ndim)

        for ax, indx in enumerate(in_indices):
            if isinstance(indx, slice):
                # convert to an index array anways:
                indx = np.arange(*indx.indices(arr.shape[ax]))
                indices.append(['s', indx])
                continue
            elif indx is None:
                # this is like taking a slice with one element from a new axis:
                indices.append(['n', np.array([0], dtype=np.intp)])
                arr = arr.reshape((arr.shape[:ax] + (1,) + arr.shape[ax:]))
                continue
            if isinstance(indx, np.ndarray) and indx.dtype == bool:
                # This may be open for improvement in numpy.
                # numpy should probably cast boolean lists to boolean indices
                # instead of intp!

                # Numpy supports for a boolean index with
                # non-matching shape as long as the True values are not
                # out of bounds. Numpy maybe should maybe not allow this,
                # (at least not array that are larger then the original one).
                try:
                    flat_indx = np.ravel_multi_index(np.nonzero(indx),
                                    arr.shape[ax:ax+indx.ndim], mode='raise')
                except:
                    error_unless_broadcast_to_empty = True
                    # fill with 0s instead, and raise error later
                    flat_indx = np.array([0]*indx.sum(), dtype=np.intp)
                # concatenate axis into a single one:
                if indx.ndim != 0:
                    arr = arr.reshape((arr.shape[:ax]
                                  + (np.prod(arr.shape[ax:ax+indx.ndim]),)
                                  + arr.shape[ax+indx.ndim:]))
                    indx = flat_indx
                else:
                    # This could be changed, a 0-d boolean index can
                    # make sense (even outide the 0-d indexed array case)
                    # Note that originally this is could be interpreted as
                    # integer in the full integer special case.
                    raise IndexError
            if len(indices) > 0 and indices[-1][0] == 'f' and ax != ellipsis_pos:
                # NOTE: There could still have been a 0-sized Ellipsis
                # between them. Checked that with ellipsis_pos.
                indices[-1].append(indx)
            else:
                # We have a fancy index that is not after an existing one.
                # NOTE: A 0-d array triggers this as well, while
                # one may expect it to not trigger it, since a scalar
                # would not be considered fancy indexing.
                num_fancy += 1
                indices.append(['f', indx])

        if num_fancy > 1 and not no_copy:
            # We have to flush the fancy indexes left
            new_indices = indices[:]
            axes = list(range(arr.ndim))
            fancy_axes = []
            new_indices.insert(0, ['f'])
            ni = 0
            ai = 0
            for indx in indices:
                ni += 1
                if indx[0] == 'f':
                    new_indices[0].extend(indx[1:])
                    del new_indices[ni]
                    ni -= 1
                    for ax in range(ai, ai + len(indx[1:])):
                        fancy_axes.append(ax)
                        axes.remove(ax)
                ai += len(indx) - 1 # axis we are at
            indices = new_indices
            # and now we need to transpose arr:
            arr = arr.transpose(*(fancy_axes + axes))

        # We only have one 'f' index now and arr is transposed accordingly.
        # Now handle newaxes by reshaping...
        ax = 0
        for indx in indices:
            if indx[0] == 'f':
                if len(indx) == 1:
                    continue
                # First of all, reshape arr to combine fancy axes into one:
                orig_shape = arr.shape
                orig_slice = orig_shape[ax:ax + len(indx[1:])]
                arr = arr.reshape((arr.shape[:ax]
                                    + (np.prod(orig_slice).astype(int),)
                                    + arr.shape[ax + len(indx[1:]):]))

                # Check if broadcasting works
                if len(indx[1:]) != 1:
                    res = np.broadcast(*indx[1:]) # raises ValueError...
                else:
                    res = indx[1]
                # unfortunatly the indices might be out of bounds. So check
                # that first, and use mode='wrap' then. However only if
                # there are any indices...
                if res.size != 0:
                    if error_unless_broadcast_to_empty:
                        raise IndexError
                    for _indx, _size in zip(indx[1:], orig_slice):
                        if _indx.size == 0:
                            continue
                        if np.any(_indx >= _size) or np.any(_indx < -_size):
                                raise IndexError
                if len(indx[1:]) == len(orig_slice):
                    if np.product(orig_slice) == 0:
                        # Work around for a crash or IndexError with 'wrap'
                        # in some 0-sized cases.
                        try:
                            mi = np.ravel_multi_index(indx[1:], orig_slice, mode='raise')
                        except:
                            # This happens with 0-sized orig_slice (sometimes?)
                            # here it is a ValueError, but indexing gives a:
                            raise IndexError('invalid index into 0-sized')
                    else:
                        mi = np.ravel_multi_index(indx[1:], orig_slice, mode='wrap')
                else:
                    # Maybe never happens...
                    raise ValueError
                arr = arr.take(mi.ravel(), axis=ax)
                arr = arr.reshape((arr.shape[:ax]
                                    + mi.shape
                                    + arr.shape[ax+1:]))
                ax += mi.ndim
                continue

            # If we are here, we have a 1D array for take:
            arr = arr.take(indx[1], axis=ax)
            ax += 1

        return arr, no_copy

    def _check_multi_index(self, arr, index):
        """Check a multi index item getting and simple setting.

        Parameters
        ----------
        arr : ndarray
            Array to be indexed, must be a reshaped arange.
        index : tuple of indexing objects
            Index being tested.
        """
        # Test item getting
        try:
            mimic_get, no_copy = self._get_multi_index(arr, index)
        except Exception as e:
            assert_raises(Exception, arr.__getitem__, index)
            assert_raises(Exception, arr.__setitem__, index, 0)
            return

        self._compare_index_result(arr, index, mimic_get, no_copy)

    def _check_single_index(self, arr, index):
        """Check a single index item getting and simple setting.

        Parameters
        ----------
        arr : ndarray
            Array to be indexed, must be an arange.
        index : indexing object
            Index being tested. Must be a single index and not a tuple
            of indexing objects (see also `_check_multi_index`).
        """
        try:
            mimic_get, no_copy = self._get_multi_index(arr, (index,))
        except Exception as e:
            assert_raises(Exception, arr.__getitem__, index)
            assert_raises(Exception, arr.__setitem__, index, 0)
            return

        self._compare_index_result(arr, index, mimic_get, no_copy)

    def _compare_index_result(self, arr, index, mimic_get, no_copy):
        """Compare mimicked result to indexing result.
        """
        arr = arr.copy()
        indexed_arr = arr[index]
        assert_array_equal(indexed_arr, mimic_get)
        # Check if we got a view, unless its a 0-sized or 0-d array.
        # (then its not a view, and that does not matter)
        if indexed_arr.size != 0 and indexed_arr.ndim != 0:
            assert_(np.may_share_memory(indexed_arr, arr) == no_copy)
            # Check reference count of the original array
            if no_copy:
                # refcount increases by one:
                assert_equal(sys.getrefcount(arr), 3)
            else:
                assert_equal(sys.getrefcount(arr), 2)

        # Test non-broadcast setitem:
        b = arr.copy()
        b[index] = mimic_get + 1000
        if b.size == 0:
            return # nothing to compare here...
        if no_copy and indexed_arr.ndim != 0:
            # change indexed_arr in-place to manipulate original:
            indexed_arr += 1000
            assert_array_equal(arr, b)
            return
        # Use the fact that the array is originally an arange:
        arr.flat[indexed_arr.ravel()] += 1000
        assert_array_equal(arr, b)

    def test_boolean(self):
        a = np.array(5)
        assert_equal(a[np.array(True)], 5)
        a[np.array(True)] = 1
        assert_equal(a, 1)
        # NOTE: This is different from normal broadcasting, as
        # arr[boolean_array] works like in a multi index. Which means
        # it is aligned to the left. This is probably correct for
        # consistency with arr[boolean_array,] also no broadcasting
        # is done at all
        self._check_multi_index(self.a, (np.zeros_like(self.a, dtype=bool),))
        self._check_multi_index(self.a, (np.zeros_like(self.a, dtype=bool)[...,0],))
        self._check_multi_index(self.a, (np.zeros_like(self.a, dtype=bool)[None,...],))

    def test_multidim(self):
        # Automatically test combinations with complex indexes on 2nd (or 1st)
        # spot and the simple ones in one other spot.
        with warnings.catch_warnings():
            # This is so that np.array(True) is not accepted in a full integer
            # index, when running the file seperatly.
            warnings.filterwarnings('error', '', DeprecationWarning)
            for simple_pos in [0,2,3]:
                tocheck = [self.fill_indices, self.complex_indices,
                           self.fill_indices, self.fill_indices]
                tocheck[simple_pos] = self.simple_indices
                for index in product(*tocheck):
                    index = tuple(i for i in index if i != 'skip')
                    self._check_multi_index(self.a, index)
                    self._check_multi_index(self.b, index)
        # Check very simple item getting:
        self._check_multi_index(self.a, (0,0,0,0))
        self._check_multi_index(self.b, (0,0,0,0))
        # Also check (simple cases of) too many indices:
        assert_raises(IndexError, self.a.__getitem__, (0,0,0,0,0))
        assert_raises(IndexError, self.a.__setitem__, (0,0,0,0,0), 0)
        assert_raises(IndexError, self.a.__getitem__, (0,0,[1],0,0))
        assert_raises(IndexError, self.a.__setitem__, (0,0,[1],0,0), 0)

    def test_1d(self):
        a = np.arange(10)
        with warnings.catch_warnings():
            warnings.filterwarnings('error', '', DeprecationWarning)
            for index in self.complex_indices:
                self._check_single_index(a, index)

if __name__ == "__main__":
    run_module_suite()


# -*- coding: utf-8 -*-
# <nbformat>3.0</nbformat>

# <codecell>

import time
start_runtime = time.time()

# <markdowncell>

# ># IOOS System Test: [Extreme Events Theme:](https://github.com/ioos/system-test/wiki/Development-of-Test-Themes#theme-2-extreme-events) Coastal Inundation

# <markdowncell>

# ### Can we obtain observed current data at stations located within a bounding box?
# This notebook is based on IOOS System Test: Inundation

# <markdowncell>

# Methodology:
# * Define temporal and spatial bounds of interest, as well as
#   parameters of interest
# * Search for available service endpoints in the NGDC CSW catalog
#   meeting search criteria
# * Search for available OPeNDAP data endpoints
# * Obtain observation data sets from stations within the spatial
#   boundaries (from CO-OPS and NDBC)
# * Extract time series for identified stations
# * Plot time series data, current rose, annual max values per station
# * Plot observation stations on a map

# <markdowncell>

# #### import required libraries

# <codecell>

import os
import os.path
from datetime import datetime, timedelta

import uuid
import folium

import matplotlib.pyplot as plt
from owslib.csw import CatalogueServiceWeb
from owslib import fes

import numpy as np
from pandas import read_csv
from pyoos.collectors.ndbc.ndbc_sos import NdbcSos
from pyoos.collectors.coops.coops_sos import CoopsSos

from utilities import (fes_date_filter, service_urls, get_coordinates,
                       inline_map, css_styles, processStationInfo,
                       get_ncfiles_catalog, new_axes, set_legend)

css_styles()

# <markdowncell>

#
Temporal Bounds -
# Anything longer than one year kills the CO-OPS service

# <codecell>

bounding_box_type = "box"

# Bounding Box [lon_min, lat_min, lon_max, lat_max]
area = {'Hawaii': [-160.0, 18.0, -154., 23.0],
        'Gulf of Maine': [-72.0, 41.0, -69.0, 43.0],
        'New York harbor region': [-75., 39., -71., 41.5],
        'Puerto Rico': [-71, 14, -60, 24],
        'East Coast': [-77, 34, -70, 40],
        'North West': [-130, 38, -121, 50]}

bounding_box = area['North West']

# Temporal range.
jd_now = datetime.utcnow()
jd_start,  jd_stop = jd_now - timedelta(days=(365*10)), jd_now

start_date = jd_start.strftime('%Y-%m-%d %H:00')
stop_date = jd_stop.strftime('%Y-%m-%d %H:00')

jd_start = datetime.strptime(start_date, '%Y-%m-%d %H:%M')
jd_stop = datetime.strptime(stop_date, '%Y-%m-%d %H:%M')

print('%s to %s ' % (start_date, stop_date))

# <codecell>

# Put the names in a dict for ease of access.
data_dict = {}
sos_name = 'Currents'
data_dict['currents'] = {"names": ['currents',
                                   'surface_eastward_sea_water_velocity',
                                   '*surface_eastward_sea_water_velocity*'],
                         "sos_name": ['currents']}

# <markdowncell>

# CSW Search

# <codecell>

endpoint = 'http://www.ngdc.noaa.gov/geoportal/csw'  # NGDC Geoportal.
csw = CatalogueServiceWeb(endpoint, timeout=60)

# <markdowncell>

# Search

# <codecell>

# Convert User Input into FES filters.
start, stop = fes_date_filter(start_date, stop_date)
bbox = fes.BBox(bounding_box)

# Use the search name to create search filter.
kw = dict(propertyname='apiso:AnyText', escapeChar='\\',
          wildCard='*', singleChar='?')
or_filt = fes.Or([fes.PropertyIsLike(literal=('*%s*' % val), **kw) for
                  val in data_dict['currents']['names']])

val = 'Averages'
not_filt = fes.Not([fes.PropertyIsLike(literal=('*%s*' % val), **kw)])

filter_list = [fes.And([bbox, start, stop, or_filt, not_filt])]
# Connect to CSW, explore it's properties
# try request using multiple filters "and" syntax: [[filter1, filter2]]
csw.getrecords2(constraints=filter_list, maxrecords=1000, esn='full')
print("%s csw records found" % len(csw.records))
for rec, item in csw.records.items():
    print(item.title)

# <markdowncell>

# DAP

# <codecell>

dap_urls = service_urls(csw.records)
# Remove duplicates and organize.
dap_urls = sorted(set(dap_urls))
print("Total DAP: %s" % len(dap_urls))
# Print the first 5:
print("\n".join(dap_urls[:]))

# <markdowncell>

# Get SOS links, NDBC is not available so add it...

# <codecell>

sos_urls = service_urls(csw.records, service='sos:url')
# Remove duplicates and organize.
sos_urls = sorted(set(sos_urls))
print("Total SOS: %s" % len(sos_urls))
print("\n".join(sos_urls))

# <markdowncell>

# #### Update SOS time-date

# <codecell>

start_time = datetime.strptime(start_date, '%Y-%m-%d %H:%M')
end_time = datetime.strptime(stop_date, '%Y-%m-%d %H:%M')
iso_start = start_time.strftime('%Y-%m-%dT%H:%M:%SZ')
iso_end = end_time.strftime('%Y-%m-%dT%H:%M:%SZ')

# <markdowncell>

#
Get list of stations
# - we get a list of the available stations from NOAA and COOPS

# <markdowncell>

# #### Initialize Station Data List

# <codecell>

st_list = {}

# <markdowncell>

# #### Get CO-OPS Station Data

# <codecell>

coops_collector = CoopsSos()
coops_collector.start_time = start_time
coops_collector.end_time = end_time
coops_collector.variables = data_dict["currents"]["sos_name"]
coops_collector.server.identification.title

ofrs = coops_collector.server.offerings

print("%s:%s" % (coops_collector.start_time, coops_collector.end_time))
print(len(ofrs))

# <markdowncell>

# #### gets a list of the active stations from coops

# <codecell>

box_str = ','.join(str(e) for e in bounding_box)

url = (('http://opendap.co-ops.nos.noaa.gov/ioos-dif-sos/SOS?'
        'service=SOS&request=GetObservation&version=1.0.0&'
        'observedProperty=%s&bin=1&'
        'offering=urn:ioos:network:NOAA.NOS.CO-OPS:CurrentsActive&'
        'featureOfInterest=BBOX:%s&responseFormat=text/csv') %
       (sos_name, box_str))

obs_loc_df = read_csv(url)

print(url)
print("Date: %s to %s" % (iso_start, iso_end))
print("Lat/Lon Box: %s" % box_str)

# <markdowncell>

# #### COOPS Station Information

# <codecell>

st_list = processStationInfo(obs_loc_df, st_list, "coops")

# <codecell>

st_list

# <markdowncell>

# #### Get NDBC Station Data

# <codecell>

ndbc_collector = NdbcSos()
ndbc_collector.start_time = start_time
ndbc_collector.end_time = end_time
ndbc_collector.variables = data_dict["currents"]["sos_name"]
ndbc_collector.server.identification.title
print("%s:%s" % (ndbc_collector.start_time, ndbc_collector.end_time))
ofrs = ndbc_collector.server.offerings
print(len(ofrs))

# <codecell>

print("Date: %s to %s" % (iso_start, iso_end))
box_str = ','.join(str(e) for e in bounding_box)
print("Lat/Lon Box: %s" % box_str)

url = (('http://sdf.ndbc.noaa.gov/sos/server.php?'
        'request=GetObservation&service=SOS&'
        'version=1.0.0&'
        'offering=urn:ioos:network:noaa.nws.ndbc:all&'
        'featureofinterest=BBOX:%s&'
        'observedproperty=%s&'
        'responseformat=text/csv&') % (box_str, sos_name))

print(url)
obs_loc_df = read_csv(url)

# <markdowncell>

# #### NDBC Station information

# <codecell>

st_list = processStationInfo(obs_loc_df, st_list, "ndbc")
st_list

# <codecell>

print(st_list[st_list.keys()[0]]['lat'])
print(st_list[st_list.keys()[0]]['lon'])

# <markdowncell>

# #### The function only support who date time differences

# <markdowncell>

#

# Large Temporal Requests Need To Be Broken Down -
# When requesting a large temporal range outside the SOS limit, the sos
# request needs to be broken down.  See issues in
# [ioos](https://github.com/ioos/system-test/issues/81),
# [ioos](https://github.com/ioos/system-test/issues/101),
# [ioos](https://github.com/ioos/system-test/issues/116)
# and
# [pyoos](https://github.com/ioos/pyoos/issues/35).  Unfortunately currents
# is not available via DAP
# ([ioos](https://github.com/ioos/system-test/issues/116))

# <markdowncell>

#

# Large Temporal Requests Need To Be Broken Down -
# Obtaining long time series from COOPS via SOS is not ideal and the opendap
# links are not available, so we use the tides and currents api to get the
# currents in json format. The api response provides in default bin, unless a
# bin is specified (i.e bin=1)

# <markdowncell>

#
Pyoos -
# Should be able to use the collector, but does not work?

# <markdowncell>

#

# Use NDBC DAP endpoints to get time-series data -
# The DAP server for currents is available for NDBC data, we use that
# to get long time series data.

# <markdowncell>

#
Progress Information For Large Requests -
# Shows the user a progress bar for each stations as its processed.  Click
# [here]('http://www.tidesandcurrents.noaa.gov/cdata/StationList?type=Current+Data&filter=active')
# to show more information on the CO-OPS locations

# <markdowncell>

#
Processing long time series -
# The CO-OPS Server responds really slow (> 30 secs, for what should be
# a 5 sec request) to multiple requests, so getting long time series
# data is almost impossible.

# <markdowncell>

# #### get CO-OPS station data

# <codecell>

# Used to define the number of days allowable by the service.
coops_point_max_days = ndbc_point_max_days = 30
print("start & end dates: %s, %s\n" % (jd_start, jd_stop))

for station_index in st_list.keys():
    # Set it so we can use it later.
    st = station_index.split(":")[-1]
    print('[%s]: %s' % (st_list[station_index]['source'], station_index))
    divid = str(uuid.uuid4())

    if st_list[station_index]['source'] == 'coops':
        # Coops fails for large requests.
        master_df = []
    elif st_list[station_index]['source'] == 'ndbc':
        # Use the dap catalog to get the data.
        master_df = get_ncfiles_catalog(station_index, jd_start, jd_stop)
    if len(master_df) > 0:
        st_list[station_index]['hasObsData'] = True
    st_list[station_index]['obsData'] = master_df

# <codecell>

# Check theres data in there.
st_list[st_list.keys()[2]]

# <markdowncell>

# ### Plot the pandas data frames for the stations

# <markdowncell>

#
Station Data Plot -
# There might be an issue with some of the NDBC station data...

# <codecell>

for station_index in st_list.keys():
    df = st_list[station_index]['obsData']
    if len(df) > 1:
        st_list[station_index]['hasObsData'] = True
        print("num rows: %s" % len(df))
        fig = plt.figure(figsize=(18, 3))
        plt.scatter(df.index, df['sea_water_speed (cm/s)'])
        fig.suptitle('Station:'+station_index, fontsize=20)
        plt.xlabel('Date', fontsize=18)
        plt.ylabel('sea_water_speed (cm/s)', fontsize=16)
    else:
        st_list[station_index]['hasObsData'] = False

# <markdowncell>

# #### Find the min and max data values

# <markdowncell>

#
Station Data Plot -
# Some stations might not plot due to the data.

# <codecell>

# Build current roses.
filelist = [f for f in os.listdir("./images") if f.endswith(".png")]
for f in filelist:
    os.remove("./images/{}".format(f))

station_min_max = {}
for station_index in st_list.keys():
    all_spd_data = {}
    all_dir_data = {}
    all_time_spd = []
    all_time_dir = []
    df = st_list[station_index]['obsData']
    if len(df) > 1:
        try:
            spd_data = df['sea_water_speed (cm/s)'].values
            spd_data = np.array(spd_data)

            dir_data = df['direction_of_sea_water_velocity (degree)'].values
            dir_data = np.array(dir_data)

            time_data = df.index.tolist()
            time_data = np.array(time_data)

            # NOTE: This data cleanup can a vectorized function.
            for idx in range(0, len(spd_data)):
                if spd_data[idx] > 998:
                    continue
                elif np.isnan(spd_data[idx]):
                    continue
                elif dir_data[idx] == 0:
                    continue
                else:
                    dt_year = time_data[idx].year
                    dt_year = str(dt_year)
                    if dt_year not in all_spd_data.keys():
                        all_spd_data[dt_year] = []
                        all_dir_data[dt_year] = []
                    # Convert to knots.
                    knot_val = (spd_data[idx] * 0.0194384449)
                    knot_val = "%.4f" % knot_val
                    knot_val = float(knot_val)

                    all_spd_data[dt_year].append(knot_val)
                    all_dir_data[dt_year].append(dir_data[idx])

                    all_time_spd.append(knot_val)
                    all_time_dir.append(dir_data[idx])

            all_time_spd = np.array(all_time_spd, dtype=np.float)
            all_time_dir = np.array(all_time_dir, dtype=np.float)

            station_min_max[station_index] = {}
            for year in all_spd_data.keys():
                year_spd = np.array(all_spd_data[year])
                year_dir = np.array(all_dir_data[year])
                station_min_max[station_index][year] = {}
                station_min_max[station_index][year]['pts'] = len(year_spd)
                min_spd, max_spd = np.min(year_spd), np.max(year_spd)
                station_min_max[station_index][year]['spd_min'] = min_spd
                station_min_max[station_index][year]['spd_max'] = max_spd
                dir_min, dir_max = np.argmin(year_spd), np.argmax(year_spd)
                yr_dir_min, yr_dir_max = year_dir[dir_min], year_dir[dir_max]
                station_min_max[station_index][year]['dir_at_min'] = yr_dir_min
                station_min_max[station_index][year]['dir_at_max'] = yr_dir_max
            try:
                # A stacked histogram with normed
                # (displayed in percent) results.
                ax = new_axes()
                ax.set_title(station_index.split(":")[-1] +
                             " stacked histogram with normed (displayed in %)"
                             "\nresults (spd in knots), All Time.")
                ax.bar(all_time_dir, all_time_spd, normed=True,
                       opening=0.8, edgecolor='white')
                set_legend(ax)

                fig = plt.gcf()
                fig.set_size_inches(8, 8)
                fname = './images/%s.png' % station_index.split(":")[-1]
                fig.savefig(fname, dpi=100)
            except Exception as e:
                print("Error when plotting %s" % e)
                pass

        except Exception as e:  # Be specific here!
            print("Error: %s" % e)
            pass

# <codecell>

# Plot the min and max from each station.
fields = ['spd_']

for idx in range(0, len(fields)):
    d_field = fields[idx]
    fig, ax = plt.subplots(1, 1, figsize=(18, 5))
    for st in station_min_max:
        x, y_min, y_max = [], [], []
        for year in station_min_max[st]:
            x.append(year)
            y_max.append(station_min_max[st][year][d_field+'max'])
        marker_size = station_min_max[st][year]['pts'] / 80
        marker_size += 20
        station_label = st.split(":")[-1]

        ax.scatter(np.array(x), np.array(y_max),
                   label=station_label, s=marker_size,
                   c=np.random.rand(3, 1), marker="o")
        ax.set_xlim([2000, 2015])
        ax.set_title("Yearly Max Speed Per Station, Marker Scaled Per "
                     "Annual Pts (bigger = more pts per year)")
        ax.set_ylabel("speed (knots)")
        ax.set_xlabel("Year")
        ax.legend(loc='upper left')

# <markdowncell>

# #### Produce Interactive Map

# <codecell>

station = st_list[st_list.keys()[0]]
m = folium.Map(location=[station["lat"], station["lon"]], zoom_start=4)
m.line(get_coordinates(bounding_box, bounding_box_type),
       line_color='#FF0000', line_weight=5)

# Plot the obs station.
for st in st_list:
    hasObs = st_list[st]['hasObsData']
    if hasObs:
        fname = './images/%s.png' % st.split(":")[-1]
        if os.path.isfile(fname):
            popup = ('Obs Location:
%s
' %
                     (st, st.split(":")[-1]))
            m.simple_marker([st_list[st]["lat"], st_list[st]["lon"]],
                            popup=popup,
                            marker_color="green",
                            marker_icon="ok")
        else:
            popup = 'Obs Location:
%s' % st
            m.simple_marker([st_list[st]["lat"], st_list[st]["lon"]],
                            popup=popup,
                            marker_color="green",
                            marker_icon="ok")
    else:
        popup = 'Obs Location:
%s' % st
        m.simple_marker([st_list[st]["lat"], st_list[st]["lon"]],
                        popup=popup,
                        marker_color="red",
                        marker_icon="remove")
inline_map(m)

# <codecell>

elapsed = time.time() - start_runtime
print('{:.2f} minutes'.format(elapsed / 60.))


# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.

"""
.. _l-example-common-error:

Common errors with onnxruntime
==============================

This example looks into several common situations
in which *onnxruntime* does not return the model
prediction but raises an exception instead.
It starts by loading the model trained in example
:ref:`l-logreg-example` which produced a logistic regression
trained on *Iris* datasets. The model takes
a vector of dimension 2 and returns a class among three.
"""
import onnxruntime as rt
import numpy
from onnxruntime.datasets import get_example

example2 = get_example("logreg_iris.onnx")
sess = rt.InferenceSession(example2)

input_name = sess.get_inputs()[0].name
output_name = sess.get_outputs()[0].name

#############################
# The first example fails due to *bad types*.
# *onnxruntime* only expects single floats (4 bytes)
# and cannot handle any other kind of floats.

try:
    x = numpy.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=numpy.float64)
    sess.run([output_name], {input_name: x})
except Exception as e:
    print("Unexpected type")
    print("{0}: {1}".format(type(e), e))

#########################
# The model fails to return an output if the name
# is misspelled.

try:
    x = numpy.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=numpy.float32)
    sess.run(["misspelled"], {input_name: x})
except Exception as e:
    print("Misspelled output name")
    print("{0}: {1}".format(type(e), e))

###########################
# The output name is optional, it can be replaced by *None*
# and *onnxruntime* will then return all the outputs.

x = numpy.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=numpy.float32)
res = sess.run(None, {input_name: x})
print("All outputs")
print(res)

#########################
# The same goes if the input name is misspelled.

try:
    x = numpy.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], dtype=numpy.float32)
    sess.run([output_name], {"misspelled": x})
except Exception as e:
    print("Misspelled input name")
    print("{0}: {1}".format(type(e), e))

#########################
# *onnxruntime* does not necessarily fail if the input
# dimension is a multiple of the expected input dimension.

for x in [
        numpy.array([1.0, 2.0, 3.0, 4.0], dtype=numpy.float32),
        numpy.array([[1.0, 2.0, 3.0, 4.0]], dtype=numpy.float32),
        numpy.array([[1.0, 2.0], [3.0, 4.0]], dtype=numpy.float32),
        numpy.array([1.0, 2.0, 3.0], dtype=numpy.float32),
        numpy.array([[1.0, 2.0, 3.0]], dtype=numpy.float32),
        ]:
    r = sess.run([output_name], {input_name: x})
    print("Shape={0} and predicted labels={1}".format(x.shape, r))

for x in [
        numpy.array([1.0, 2.0, 3.0, 4.0], dtype=numpy.float32),
        numpy.array([[1.0, 2.0, 3.0, 4.0]], dtype=numpy.float32),
        numpy.array([[1.0, 2.0], [3.0, 4.0]], dtype=numpy.float32),
        numpy.array([1.0, 2.0, 3.0], dtype=numpy.float32),
        numpy.array([[1.0, 2.0, 3.0]], dtype=numpy.float32),
        ]:
    r = sess.run(None, {input_name: x})
    print("Shape={0} and predicted probabilities={1}".format(x.shape, r[1]))

#########################
# It does not fail either if the number of dimension
# is higher than expects but produces a warning.

for x in [
        numpy.array([[[1.0, 2.0], [3.0, 4.0]]], dtype=numpy.float32),
        numpy.array([[[1.0, 2.0, 3.0]]], dtype=numpy.float32),
        numpy.array([[[1.0, 2.0]], [[3.0, 4.0]]], dtype=numpy.float32),
        ]:
    r = sess.run([output_name], {input_name: x})
    print("Shape={0} and predicted labels={1}".format(x.shape, r))


# To add a new cell, type '# %%'
# To add a new markdown cell, type '# %% [markdown]'
# %% [markdown]
# ## Setup
#
# A célula abaixo carrega as bibliotecas e arquivos necessários para gerar os dataframes e suas respectivas explorações neste .ipynb.

# %%
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

online_pageviews = pd.read_json('dados/online_pageviews.json',lines='True')
offline_sales = pd.read_json('dados/offline_sales.json',lines='True')
online_orders = pd.read_json('dados/online_orders.json', lines='True')

# %% [markdown]
# ## 1. Qual foi o faturamento total no período?
#
# Pode-se determinar o faturamento total do presente período fiscal como a soma do valor total de todos os pedidos online e físicos listados, sendo o valor total de um pedido determinado por `total_order_value = quantity X price`.
#
# Assim, criando uma coluna adicional em cada dataframe de vendas, pode-se obter a soma do valor final de todas elas, online e físicas, conforme a céulula abaixo.

# %%
# gerar colunas de valor total de pedido para vendas físicas e online
offline_sales['total_order_value'] = offline_sales.apply(lambda row: (row['quantity'] * row['price']), axis=1)
online_orders['total_order_value'] = online_orders.apply(lambda row: (row['quantity'] * row['price']), axis=1)

# obter valor total de faturamento
total_value_from_sales_sum = offline_sales.total_order_value.sum() + online_orders.total_order_value.sum()
total_value_from_sales_sum

# %% [markdown]
# ## 2. Qual foi o produto mais vendido online?
#
# Para tanto, a forma mais direta é obter quais quantidades de produto estão associadas a determinado id de produto em cada pedido, somando-as e comparando de acordo.
#
# Buscando no dataframe 'online_orders', pode-se utilizar o método 'groupby' do pandas para agrupar todos os ids de produto e suas respectivas quantidades, que, conjuntamente com o método 'apply' e argumento 'sum', soma todos os valores encontrados de quantidade em função do id associado, retornando em formato de lista.

# %%
# agrupar por id e somar quantidades, ordenar por mais vendidos:
most_sold = online_orders.groupby('on_product_id')['quantity'].apply(sum).reset_index(drop=False)
most_sold.sort_values(by='quantity',ascending=False)

# caso queira apenas o resultado direto:
# most_sold.iloc[most_sold['quantity'].idxmax()]

# %% [markdown]
# Podemos ver então que o produto de id `626664333563363` é o mais vendido do período.
# %% [markdown]
# ## 3. Cariocas gostam de comprar fim de semana?
#
# "Gostam" é um critério muito subjetivo e **muito** dependente de contexto. Idealmente, 'gostar' envolve satisfação e prazer. No entanto, com os dados presentes, podemos fazer uma inferência de 'preferir', comparando o volume de vendas entre os dias de semana e o final de semana.
#
# Como o dataframe de entrada só possui datas em valor numérico no formato `date`, é melhor criar uma nova coluna que a partir desta data, nos dá o dia da semana, o que torna a comparação entre as vendas de sábado e domingo com as de segunda a sexta muito mais fáceis.

# %%
# gerar coluna de dias da semana de acordo com a data numérica fornecida
offline_sales['day_of_the_week'] = offline_sales['date'].dt.day_name()

# localizar todas as vendas do RJ
RJ_sales = offline_sales.loc[(offline_sales['state'] == 'RJ')]

# %% [markdown]
# Este tipo de questionamento conduz muito naturalmente a algum tipo de visualização. Como queremos saber a questão de preferência por volume de vendas, vamos comparar por dia da semana. No entanto, algumas necessidades específicas das ferramentas requerem certa manipulação.
#
# Primeiro, para organizar o gráfico, devemos gerar uma série categórica que nos dá a ordem da semana (para os fins daqui, segunda a domingo). Depois, agrupamos a soma de todos os valores de venda de cada pedido em função do dia da semana, mostrados em barras.
#
# (*NOTA TÉCNICA: importante notar que, ao gerar o gráfico via célula abaixo, o Pandas retorna um aviso. Isto ocorre devido a uma peculiaridade da linguagem Python e sua flexibilidade interpretativa, que para o Pandas, gera problemas por não saber se as engrenagens do Python estão retornando uma cópia ou uma 'view' (representação) do que foi atribuído. Como não serão feitas manipulações subsequentes dali, pode-se obter a visualização sem problemas.*)

# %%
# ordenar a semana em formato categórico para ter uma ordem de visualização
RJ_sales['day_of_the_week'] = pd.Categorical(RJ_sales['day_of_the_week'], categories=
    ['Monday','Tuesday','Wednesday','Thursday','Friday','Saturday', 'Sunday'],
    ordered=True)

# gerar gráfico de vendas físicas do RJ
RJ_sales_bar_plot = RJ_sales.groupby('day_of_the_week')['total_order_value'].apply(sum).reset_index(drop=False).plot(x='day_of_the_week',y='total_order_value', kind='bar')

# %% [markdown]
# Comparativamente falando, por meio do critério de volume de venda, cariocas têm preferência em comprar de segunda à sexta, no que diz respeito às lojas físicas.
# %% [markdown]
# ## 4 - *É comum escolher online e terminar a compra na loja física?*
#
# Primeiro, é necessário estabelecer o que é possível de analisar com os dados em mãos em relação aos fatores pedidos.
#
# Uma possível (e talvez mais direta abordagem) é, sabendo que temos um campo `customer_id` comum às vendas físicas e aos pageviews, encontrar esta população e comparar aos demais clientes. Assumindo que os ids são correspondentes e se tratam de fato do mesmo cliente, teríamos então o grupo de consumidores registrados na loja online - e logados, importante lembrar - que fizeram compras físicas. Entretanto, isso não nos dá ideia de outro grupo muito possivelmente mais significativo, o dos visitantes que compram presencialmente.
#
# Valores únicos de customer_id em offline_sales = 9901
#
# Valores nulos para customer_id em offline_sales = 3528
#
# Valores únicos de customer_id em online_pageviews = 5913
#
# Valores nulos de customer_id em online_pageviews = **3371775**
#
# Interseção de valores válidos entre os dois = 592
#
# Se considerarmos nossa população de teste somente os clientes cadastrados, então temos apenas 6% deste grupo como pessoas que finalizam suas compras em loja física após visualizar online.
#
#

# %%
# contagem de ids
offline_sales['customer_id'].value_counts()
online_pageviews['customer_id'].value_counts()

# conjuntos válidos
s1 = offline_sales['customer_id'].dropna()
s2 = online_pageviews['customer_id'].dropna()

# contar valores nulos
count_nan = len(online_pageviews['customer_id']) - online_pageviews['customer_id'].count()
# count_nan

# obter interseção dos conjuntos
pd.Series(list(set(s1)&set(s2)))

# %% [markdown]
# ## *5: O time de marketing desta rede quer fazer uma campanha oferecendo um cupom de 20% nas compras de loja física para quem visitou o site e abandonou um carrinho com produtos. Estime o resultado dessa campanha.*
#
# É tentador modelar uma regressão logo de início, mas, talvez, temos já o suficiente para uma extrapolação interessante considerando os fatores da pergunta.
#
# 1 -- Visualização online para compra física. Como observado na questão #4, é possível inferir que comparado ao restante dos clientes, a população que vê o site para comprar na loja é bem pequena. Embora tenha sido feito somente a partir dos clientes cadastrados, ainda assim é uma amostragem significativa. A porcentagem da interseção entre clientes cadastrados e visualizações pode ser extrapolada aos demais?
#
# 2 -- Abandono de carrinho. Como isso poderia ser deduzido? Existem questões aqui que ultrapassam o escopo dos dados fornecidos. O site permite que um visitante não-cadastrado construa um carrinho? Se sim, é necessário criar um id de cliente para checkout? O checkout é tratado como uma compra online ou só no ato da retirada, em que vira uma compra física?
#
# Assumindo um método semelhante à questão acima, ao localizarmos os clientes que possuem id e que visualizaram páginas de carrinho, temos 1476 de entradas de id único, de 5913 possíveis em `online_pageviews`. Entretanto, não temos como saber se os carrinhos tinham algo ou não. E/ou se houve abandono da compra,a partir disso.
#
# Inferindo a partir das observações da #4, seria uma parcela muito pequena da clientela total, e menor ainda presumindo ainda estas condições adicionais sendo satisfeitas, para ser uma campanha de efeito significativo. Mais interessante seria talvez um cupom de desconto menor para primeira compra de visitantes online, permitindo aumentar inclusive a base de dados para análise?

# %%
pageviews_customers_with_carts = online_pageviews.loc[(online_pageviews['pageType'] == 'cart') & (online_pageviews['customer_id'])]
pageviews_customers_with_carts.drop_duplicates()

online_pageviews['customer_id'].dropna().drop_duplicates()

# %% [markdown]
# # Outros dados:
# %% [markdown]
# ## Volume de venda por estado:
#
# RJ e SP se posicionam consideravelmente à frente dos demais. Considerando que o Sudeste é região mais rica do país, segue uma tendência natural. Interessante, no entanto, se mantivermos a análise por questão de PIB, é notar a vantagem de Pernambuco sobre o Paraná - 10a e 5a economias do país, respectivamente.

# %%
# obter volume de vendas por estado, gerar gráfico de barras:
total_sale_value_by_state = offline_sales.groupby('state')['total_order_value'].apply(sum)
total_sale_value_by_state.plot(x='state',y='total_order_value',kind='bar')

# %% [markdown]
# ## Quantidade de visualizações de página por tipo de dispositivo:
#
# Mantendo a par com a tendência global, compras em mobile são mais frequentes que em desktop.

# %%
online_pageviews.groupby('deviceType')['deviceType'].count().plot.pie(autopct='%.2f')

# %% [markdown]
# ## Descrição estatística dos pedidos online por cliente:
#
# De 3530 clientes, temos um gasto médio de $1056.82, um desvio-padrão de 1484.56, com ganhos galopantes de dispêndio no último quartil: 1232.50 a 25703.00! Do primeiro ao terceiro, temos de 29.00 a 646.00.

# %%
online_orders.groupby('customer_id')['total_order_value'].sum().sort_values(ascending=False).describe()

# %% [markdown]
# ## Descrição estatística dos pedidos em loja física por cliente:
#
# De 9901 clientes, temos um gasto médio superior ao online: 1246.50, mas um desvio-padrão muito mais acentuado (mais que o dobro, aliás). Os avanços nos quartis se demonstram razovalmente paralelos até, novamente, chegar no último: de 1156.00 a **106473.00**, uma diferença homérica na participação de gasto.

# %%
offline_sales.groupby('customer_id')['total_order_value'].sum().sort_values(ascending=False).describe()

# %% [markdown]
# ## *Bônus. O que mais de interessante tem nestes dados?*
#
# Certamente, para um varejista (conforme a descrição do teste), é a participação maciça no valor total de vendas vinda de poucos clientes. Sem outras informações à mão, trata-se de uma interessante e representativa característica que corrobora o perfil de consumo nacional: certa homogeneidade consistente até determinado nível de gasto para avanços mais dramáticos conforme os percentis avançam ao topo.

# %%


"""
Tests for deepreg/model/layer_util.py in
pytest style
"""
import numpy as np
import pytest
import tensorflow as tf

import deepreg.model.layer_util as layer_util

def check_equal(tensor_1, tensor_2):
    """
    Given two tf tensors return True/False (not tf tensor)
    Tolerate small errors (<1e-6)
    :param tensor_1: Tensor to check equality to against tensor_2.
    :param tensor_2: Tensor to check equality to against tensor_1.
    :return: True if difference less than 1e-6, False otherwise.
    """
    return tf.reduce_max(tf.abs(tensor_1 - tensor_2)).numpy() < 1e-6

def test_check_inputs():
    """
    Test check_inputs by confirming that it accepts proper
    types and handles a few simple cases.
    """
    # Check inputs list - Pass
    assert layer_util.check_inputs([], 0) is None

    # Check inputs tuple - Pass
    assert layer_util.check_inputs((), 0) is None

    # Check inputs int - Fail
    with pytest.raises(ValueError) as execinfo:
        layer_util.check_inputs(0, 0)
    msg = " ".join(execinfo.value.args[0].split())
    assert "Inputs should be a list or tuple" in msg

    # Check inputs float - Fail
    with pytest.raises(ValueError) as execinfo:
        layer_util.check_inputs(0.0, 0)
    msg = " ".join(execinfo.value.args[0].split())
    assert "Inputs should be a list or tuple" in msg

    # Check size float - Fail
    with pytest.raises(ValueError) as execinfo:
        layer_util.check_inputs([1], 0.5)
    msg = " ".join(execinfo.value.args[0].split())
    assert "Inputs should be a list or tuple of size" in msg

    # Check size 0 - Pass
    assert layer_util.check_inputs([], 0) is None
    assert layer_util.check_inputs((), 0) is None

    # Check size 0 - Fail
    with pytest.raises(ValueError) as execinfo:
        layer_util.check_inputs([0], 0)
    msg = " ".join(execinfo.value.args[0].split())
    assert "Inputs should be a list or tuple of size" in msg
    with pytest.raises(ValueError) as execinfo:
        layer_util.check_inputs((0,), 0)
    msg = " ".join(execinfo.value.args[0].split())
    assert "Inputs should be a list or tuple of size" in msg

    # Check size 1 - Pass
    assert layer_util.check_inputs([0], 1) is None
    assert layer_util.check_inputs((0,), 1) is None

    # Check size 1 - Fail
    with pytest.raises(ValueError) as execinfo:
        layer_util.check_inputs([], 1)
    msg = " ".join(execinfo.value.args[0].split())
    assert "Inputs should be a list or tuple of size" in msg
    with pytest.raises(ValueError) as execinfo:
        layer_util.check_inputs((), 1)
    msg = " ".join(execinfo.value.args[0].split())
    assert "Inputs should be a list or tuple of size" in msg

def test_get_reference_grid():
    """
    Test get_reference_grid by confirming that it generates
    a sample grid test case to check_equal's tolerance level.
    """
    want = tf.constant(
        np.array(
            [[[[0, 0, 0], [0, 0, 1], [0, 0, 2]], [[0, 1, 0], [0, 1, 1], [0, 1, 2]]]],
            dtype=np.float32,
        )
    )
    get = layer_util.get_reference_grid(grid_size=[1, 2, 3])
    assert check_equal(want, get)

def test_get_n_bits_combinations():
    """
    Test get_n_bits_combinations by confirming that it generates
    appropriate solutions for 1D, 2D, and 3D cases.
    """
    # Check n=1 - Pass
    assert layer_util.get_n_bits_combinations(1) == [[0], [1]]
    # Check n=2 - Pass
    assert layer_util.get_n_bits_combinations(2) == [[0, 0], [0, 1], [1, 0], [1, 1]]

    # Check n=3 - Pass
    assert layer_util.get_n_bits_combinations(3) == [
        [0, 0, 0],
        [0, 0, 1],
        [0, 1, 0],
        [0, 1, 1],
        [1, 0, 0],
        [1, 0, 1],
        [1, 1, 0],
        [1, 1, 1],
    ]

def test_pyramid_combinations():
    """
    Test pyramid_combinations by confirming that it generates
    appropriate solutions for simple 1D and 2D cases.
    """
    # Check numerical outputs are correct for a simple 1D pair of weights, values - Pass
    weights = tf.constant(np.array([[0.2]], dtype=np.float32))
    values = tf.constant(np.array([[1], [2]], dtype=np.float32))
    # expected = 1 * 0.2 + 2 * 2
    expected = tf.constant(np.array([1.8], dtype=np.float32))
    got = layer_util.pyramid_combination(values=values, weights=weights)
    assert check_equal(got, expected)

    # Check numerical outputs are correct for a 2D pair of weights, values - Pass
    weights = tf.constant(np.array([[0.2], [0.3]], dtype=np.float32))
    values = tf.constant(
        np.array(
            [
                [1],  # value at corner (0, 0), weight = 0.2 * 0.3
                [2],  # value at corner (0, 1), weight = 0.2 * 0.7
                [3],  # value at corner (1, 0), weight = 0.8 * 0.3
                [4],  # value at corner (1, 1), weight = 0.8 * 0.7
            ],
            dtype=np.float32,
        )
    )
    # expected = 1 * 0.2 * 0.3
    #          + 2 * 0.2 * 0.7
    #          + 3 * 0.8 * 0.3
    #          + 4 * 0.8 * 0.7
    expected = tf.constant(np.array([3.3], dtype=np.float32))
    got = layer_util.pyramid_combination(values=values, weights=weights)
    assert check_equal(got, expected)

def test_resample():
    """
    Test resample by confirming that it generates appropriate
    resampling on two test cases with outputs within check_equal's
    tolerance level, and one which should fail (incompatible shapes).
    """
    # linear, vol has no feature channel - Pass
    interpolation = "linear"
    vol = tf.constant(
        np.array([[[0, 1, 2], [3, 4, 5]]], dtype=np.float32)
    )  # shape = [1,2,3]
    loc = tf.constant(
        np.array(
            [
                [
                    [[0, 0], [0, 1], [0, 3]],  # outside frame
                    [[0.4, 0], [0.5, 1], [0.6, 2]],
                    [[0.4, 0.7], [0.5, 0.5], [0.6, 0.3]],
                ]
            ],  # resampled = 3x+y
            dtype=np.float32,
        )
    )  # shape = [1,3,3,2]
    want = tf.constant(
        np.array([[[0, 1, 2], [1.2, 2.5, 3.8], [1.9, 2, 2.1]]], dtype=np.float32)
    )  # shape = [1,3,3]
    get = layer_util.resample(vol=vol, loc=loc, interpolation=interpolation)
    assert check_equal(want, get)

    # linear, vol has feature channel - Pass
    interpolation = "linear"
    vol = tf.constant(
        np.array(
            [[[[0, 0], [1, 1], [2, 2]], [[3, 3], [4, 4], [5, 5]]]], dtype=np.float32
        )
    )  # shape = [1,2,3,2]
    loc = tf.constant(
        np.array(
            [
                [
                    [[0, 0], [0, 1], [0, 3]],  # outside frame
                    [[0.4, 0], [0.5, 1], [0.6, 2]],
                    [[0.4, 0.7], [0.5, 0.5], [0.6, 0.3]],
                ]
            ],  # resampled = 3x+y
            dtype=np.float32,
        )
    )  # shape = [1,3,3,2]
    want = tf.constant(
        np.array(
            [
                [
                    [[0, 0], [1, 1], [2, 2]],
                    [[1.2, 1.2], [2.5, 2.5], [3.8, 3.8]],
                    [[1.9, 1.9], [2, 2], [2.1, 2.1]],
                ]
            ],
            dtype=np.float32,
        )
    )  # shape = [1,3,3,2]
    get = layer_util.resample(vol=vol, loc=loc, interpolation=interpolation)
    assert check_equal(want, get)

    # Inconsistent shapes for resampling - Fail
    interpolation = "linear"
    vol = tf.constant(np.array([[0]], dtype=np.float32))  # shape = [1,1]
    loc = tf.constant(np.array([[0, 0], [0, 0]], dtype=np.float32))  # shape = [2,2]
    with pytest.raises(ValueError) as execinfo:
        layer_util.resample(vol=vol, loc=loc, interpolation=interpolation)
    msg = " ".join(execinfo.value.args[0].split())
    assert "vol shape inconsistent with loc" in msg

def test_random_transform_generator():
    """
    Test random_transform_generator by confirming that it generates
    appropriate solutions and output sizes for seeded examples.
    """
    # Check shapes are correct Batch Size = 1 - Pass
    batch_size = 1
    transforms = layer_util.random_transform_generator(batch_size, 0)
    assert transforms.shape == (batch_size, 4, 3)

    # Check numerical outputs are correct for a given seed - Pass
    batch_size = 1
    scale = 0.1
    seed = 0
    expected = tf.constant(
        np.array(
            [
                [
                    [9.4661278e-01, -3.8267835e-03, 3.6934228e-03],
                    [5.5613145e-03, 9.8034811e-01, -1.8044969e-02],
                    [1.9651605e-04, 1.4576728e-02, 9.6243286e-01],
                    [-2.5107686e-03, 1.9579126e-02, -1.2195010e-02],
                ]
            ],
            dtype=np.float32,
        )
    )  # shape = (1, 4, 3)
    got = layer_util.random_transform_generator(
        batch_size=batch_size, scale=scale, seed=seed
    )
    assert check_equal(got, expected)

def test_warp_grid():
    """
    Test warp_grid by confirming that it generates
    appropriate solutions for a simple precomputed case.
    """
    grid = tf.constant(
        np.array(
            [[[[0, 0, 0], [0, 0, 1], [0, 0, 2]], [[0, 1, 0], [0, 1, 1], [0, 1, 2]]]],
            dtype=np.float32,
        )
    )  # shape = (1, 2, 3, 3)
    theta = tf.constant(
        np.array(
            [
                [
                    [0.86, 0.75, 0.48],
                    [0.07, 0.98, 0.01],
                    [0.72, 0.52, 0.97],
                    [0.12, 0.4, 0.04],
                ]
            ],
            dtype=np.float32,
        )
    )  # shape = (1, 4, 3)
    expected = tf.constant(
        np.array(
            [
                [
                    [
                        [[0.12, 0.4, 0.04], [0.84, 0.92, 1.01], [1.56, 1.44, 1.98]],
                        [[0.19, 1.38, 0.05], [0.91, 1.9, 1.02], [1.63, 2.42, 1.99]],
                    ]
                ]
            ],
            dtype=np.float32,
        )
    )  # shape = (1, 1, 2, 3, 3)
    got = layer_util.warp_grid(grid=grid, theta=theta)
    assert check_equal(got, expected)

def test_resize3d():
    """
    Test resize3d by confirming the output shapes.
    """

    # Check resize3d for images with different size and without channel nor batch - Pass
    input_shape = (1, 3, 5)
    output_shape = (2, 4, 6)
    size = (2, 4, 6)
    got = layer_util.resize3d(image=tf.ones(input_shape), size=size)
    assert got.shape == output_shape

    # Check resize3d for images with different size and without channel - Pass
    input_shape = (1, 1, 3, 5)
    output_shape = (1, 2, 4, 6)
    size = (2, 4, 6)
    got = layer_util.resize3d(image=tf.ones(input_shape), size=size)
    assert got.shape == output_shape

    # Check resize3d for images with different size and with one channel - Pass
    input_shape = (1, 1, 3, 5, 1)
    output_shape = (1, 2, 4, 6, 1)
    size = (2, 4, 6)
    got = layer_util.resize3d(image=tf.ones(input_shape), size=size)
    assert got.shape == output_shape

    # Check resize3d for images with different size and with multiple channels - Pass
    input_shape = (1, 1, 3, 5, 3)
    output_shape = (1, 2, 4, 6, 3)
    size = (2, 4, 6)
    got = layer_util.resize3d(image=tf.ones(input_shape), size=size)
    assert got.shape == output_shape

    # Check resize3d for images with the same size and without channel nor batch - Pass
    input_shape = (1, 3, 5)
    output_shape = (1, 3, 5)
    size = (1, 3, 5)
    got = layer_util.resize3d(image=tf.ones(input_shape), size=size)
    assert got.shape == output_shape

    # Check resize3d for images with the same size and without channel - Pass
    input_shape = (1, 1, 3, 5)
    output_shape = (1, 1, 3, 5)
    size = (1, 3, 5)
    got = layer_util.resize3d(image=tf.ones(input_shape), size=size)
    assert got.shape == output_shape

    # Check resize3d for images with the same size and with one channel - Pass
    input_shape = (1, 1, 3, 5, 1)
    output_shape = (1, 1, 3, 5, 1)
    size = (1, 3, 5)
    got = layer_util.resize3d(image=tf.ones(input_shape), size=size)
    assert got.shape == output_shape

    # Check resize3d for images with the same size and with multiple channels - Pass
    input_shape = (1, 1, 3, 5, 3)
    output_shape = (1, 1, 3, 5, 3)
    size = (1, 3, 5)
    got = layer_util.resize3d(image=tf.ones(input_shape), size=size)
    assert got.shape == output_shape


import csv
import numpy as np
import torch
from torch.utils.data import Dataset, DataLoader, SubsetRandomSampler

class SimpleDataset(Dataset):
    """SimpleDataset [summary]

    [extended_summary]

    :param path_to_csv: [description]
    :type path_to_csv: [type]
    """
    def __init__(self, path_to_csv, transform=None):
        self.data = np.genfromtxt(path_to_csv, delimiter=",")

        self.transform = transform

    def __len__(self):
        """__len__ returns length of dataset"""
        return len(self.data)

    def __getitem__(self, index):
        """__getitem__ [summary]

        [extended_summary]

        :param index: [description]
        :type index: [type]
        """
        ## This returns only ONE sample from the dataset, for a given index.
        ## The returned sample should be a tuple (x, y) where x is your input
        ## vector and y is your label
        ## Before returning your sample, you should check if there is a transform
        ## sepcified, and apply that transform to your sample
        # Eg:
        # if self.transform:
        #   sample = self.transform(sample)
        ## Remember to convert the x and y into torch tensors.
        x, y = torch.from_numpy(self.data[:,:-1]).float(), torch.from_numpy(self.data[:,-1]).float()
        sample = x[index], y[ index]
        if self.transform is None:
            return sample
        else:
            return self.transform(sample)

def get_data_loaders(path_to_csv,
                     transform_fn=None,
                     train_val_test=[0.8, 0.2, 0.2],
                     batch_size=32):
    """get_data_loaders [summary]

    [extended_summary]

    :param path_to_csv: [description]
    :type path_to_csv: [type]
    :param train_val_test: [description], defaults to [0.8, 0.2, 0.2]
    :type train_val_test: list, optional
    :param batch_size: [description], defaults to 32
    :type batch_size: int, optional
    :return: [description]
    :rtype: [type]
    """
    # First we create the dataset given the path to the .csv file
    dataset = SimpleDataset(path_to_csv, transform=transform_fn)

    # Then, we create a list of indices for all samples in the dataset.
    dataset_size = len(dataset)
    indices = list(range(dataset_size))

    ## TODO: Rewrite this section so that the indices for each dataset split
    ## are formed.

    ## BEGIN: YOUR CODE
    test_size = int(train_val_test[-1]*dataset_size)
    train_val_size = dataset_size - test_size
    train_size = int(train_val_test[0]*train_val_size)

    train_indices = indices[:train_size]
    val_indices = indices[train_size:-1*test_size]
    test_indices = indices[-1*test_size:]
    ## END: YOUR CODE

    # Now, we define samplers for each of the train, val and test data
    train_sampler = SubsetRandomSampler(train_indices)
    train_loader = DataLoader(dataset, batch_size=batch_size, sampler=train_sampler)

    val_sampler = SubsetRandomSampler(val_indices)
    val_loader = DataLoader(dataset, batch_size=batch_size, sampler=val_sampler)

    test_sampler = SubsetRandomSampler(test_indices)
    test_loader = DataLoader(dataset, batch_size=batch_size, sampler=test_sampler)

    return train_loader, val_loader, test_loader


"""
Authors : Aditya Jain
Contact : https://adityajain.me
"""

import numpy as np

def train_test_split(X,Y,test_size=None,random_state=5):
	"""
	Train Test split function

	Parameters
	----------

	X : numpy array, independent variables

	y : numpy array, dependent variables

	test_size : float, percent of test samples

	random_state : integer, random seed

	Returns
	-------
	X_train, X_test, Y_train, Y_test

	"""
	assert test_size!=None, "test_size cannot be None"
	np.random.seed(random_state)
	indexes = np.random.choice([False,True],size=len(X),p=[test_size,1-test_size])
	return X[indexes],X[~indexes],Y[indexes],Y[~indexes]


"""
GENERAL INSTRUCTIONS
1. WHICH PART TO CHANGE?: Uncomment every line with  [YOUR CODE HERE] and replace it with your code.
Please don't change anything else other than these lines. In other parts, the code isnt commented,
so just replace the parts with YOUR CODE or YOUR CODE HEREE

2. USE OF JUPYTER NOTEBOOK: For those who would like to use Jupyter Notebook. You can copy and paste
each function in the notebook environment, test your code their. However,
remember to paste back your code in a .py file and ensure that its running
okay.

3. INDENTATION: Please make sure that you check your indentation

4. Returning things from function: Please dont forget to use the return statement
to return a value when applicable

5. HINTS: please read my comments for hints and instructions where applicable

6. DEFINING YOUR OWN FUNCTIONS: where I ask you to define your own function
please make sure that you name the function exactly as I said.

7. Please work on the following functions in the order provided:
    - add_date_and_filter
    - get_events_before_jul13
    - preprocess_cdrs_using_spark
    - explore_data_with_spark
    - generate_user_attributes_with_pandas
8. Dataset for this work: all the questions are based on the simulated_cdrs dataset,
please download it and have it on your machine
"""
from itertools import groupby
import random
import seaborn as sns
import numpy as np
from collections import namedtuple
from datetime import datetime
from pathlib import Path
import random
from pyspark.sql import *
from pyspark.sql.functions import *
from pyspark.sql.types import *
import pandas as pd
import matplotlib.pyplot as plt

def add_date_and_filter(csv_file, date_format, ref_date):
    """
    Create a dataframe, add date and filter out events based on ref date
    :return:
    """
    df = pd.read_csv(csv_file)
    str_time_col = 'cdr datetime'

    # convert date string to Python datetime
    # please extract only the Year, month and day from the date string using
    # indexing, please use solutions from first assignment to achieve this
    #f = lambda x: pd.to_datetime(x[:10], format=date_format)
    df['cdr_date'] = df[str_time_col].apply(lambda x: pd.to_datetime(str(x)[:9],format=date_format))     ########YOUR CODE

    # please retrieve all events older than or equal to the reference date
    # please use the query function for pandas DataFrame like below
    df.query('cdr_date <= @ref_date', inplace=True)

    if df.shape[0]:
        return df
    else:
        return 'NA'

def get_events_before_jul13(csv_folder, ref_date, date_format, num_csv_files):
    """
    In this function, we will use the map function to run the
    add_date_and_filter above on a list of CSV_files
    :param csv_folder:
    :return:
    """
    list_csv = [f for f in csv_folder.iterdir() if f.suffix == '.csv']
    # use the function random.choices() to select only a sample of the CSVs  for fast processing
    # set k to num_csv_files in the function
    list_csv =  random.choices(list_csv,k= num_csv_files)
    time_format = "%Y%m%d"

    # Use  the datetime.strptime() to convert the string ref_date to Python datetime object
    # This function requires two args, the other one is date format
    ref_date = datetime.strptime(ref_date,  date_format)

    # Run map with multiple iterables(r.g., two lists)
    # Note that input-1 is the list of CSV we generated above
    # Prepare input-2 and input-3 as below. Please use the  multiplication
    # operator on a single list item to repeat the same element multiple
    # times in a list as we did in the first assignment
    list_date_format = [date_format]*len(list_csv)
    list_date_ref_date = [ref_date]*len(list_csv)

    # now run the map function with the three input lists we have defined
    results = map(add_date_and_filter,list_csv,list_date_format,list_date_ref_date)

    # write code to identify elements in the list which arent s 'NA'
    # You can use a list comprehension with if condition. To identify elements
    # which are 'NA', you can use isinstance() function to detect data type
    no_na_results = []
    print('I found {} non NA results'.format(len(no_na_results)))

    return no_na_results

def rename_sdf(df, mapper=None):
    ''' Rename column names of a dataframe
        mapper: a dict mapping from the old column names to new names
        Usage:
            df.rename({'old_col_name': 'new_col_name', 'old_col_name2': 'new_col_name2'})
            df.rename(old_col_name=new_col_name)
    '''
    for before, after in mapper.items():
        df = df.withColumnRenamed(before, after)

    return df

def preprocess_cdrs_using_spark(file_or_folder=None, number_of_users_to_sample=None,
                                output_csv=None, date_format='%Y%m%d%H%M%S',
                                debug_mode=True, loc_file=None, save_to_csv=False,
                                userid_col="user_id"):
    """
    In this function, we perfom some basic preprocessing such as below:
    1. rename columns
    2. change some data types
    3. Add location details
    Eventually, we will sample the data to use for our analysis
    :param data_folder:
    :param output_csv_for_sample_users:
    :return:
    """

    # ==============================
    # LOAD DATA
    # ==============================
    # create SparkSession
    spark = SparkSession.builder \
    .master("local[*]") \
    .appName("Assignement Of Louis") \
    .getOrCreate()
    # read data with spark and ensure you set header to True
    df = spark.read.csv(file_or_folder ,header=True)
    # repartition to speed up
    df = df.repartition(10)

    # if just testing/debugging, pick only a small subset of the dataset
    # for instance, 0.01 percent of the data
    # by using the sample function of spark
    if debug_mode:
        dfs = df.sample(fraction = 0.1)
        df = dfs

    # ==================================================
    # RENAME, DROP COLUMNS, ADD DATETIME AND DROP NULLS
    # ===================================================
    # rename columns to remove space the names
    # replace space with underscore
    cols_to_rename = {}
    for c in df.columns:
        if " " in c:
            cols_to_rename[c] = c.replace(" ", "_")

    #  now call the rename_sdf function
    # using df and the dictionary above
    df = rename_sdf(df, mapper=cols_to_rename)

    # this approach is also okay but you are hardcoding the column names
    # so, lets avoid hardcoding at all times
    #     df = (df.withColumnRenamed("cdr datetime", "cdrDatetime")
    #         .withColumnRenamed("last calling cellid", "cellId")
    #         .withColumnRenamed("call duration", "cellDuration"))

    # change the column "last_calling_cellid" to just cell_id
    df = df.withColumnRenamed("last_calling_cellid","cell_id")
    # drop the cdr_type and call_duration because we dont really need them
    df = df.drop("cdr type","call_duration")
    # lets make sure we don't have any values in the user_id
    # and cdr_datetime columns
    # use spark filter() function and isNotNull() in a nested
    # fashion to achieve this
    df = df.filter(df['user_id'].isNotNull())
    df = df.filter(df.cdr_datetime.isNotNull())

    # Use Spark UDF to add date and datetime
    add_datetime = udf(lambda x: datetime.strptime(x, date_format), TimestampType())
    add_date = udf(lambda x: datetime.strptime(x, date_format), DateType())

    # create timestamp and date column
    df = df.withColumn('datetime', add_datetime(col('cdr_datetime')))
    df = df.withColumn('date', add_date(col('cdr_datetime')))

    # ==================================================
    # ADD LOCATION THROUGH JOIN
    # ===================================================
    # Lets merge with location details using cellId from CDRs and also
    # cellID on the other
    # read pandas dataframe of location details
    dfLoc = pd.read_csv(loc_file)
    # remove duplicates from the cell_id column to make sure we only
    # unique cell_ids.Use the drop_duplicates() in pandas
    dfLoc = dfLoc.drop_duplicates(subset='cell_id')
    # create spark dataframe from the pandas dataframe
    # using the function createDataFrame from the SparkSession object
    # and the pandas DataFrame created above
    sdfLoc = spark.createDataFrame(dfLoc)

    # join the cdrs dataframe with the location dataframe
    # When using the join function, make sure you choose the option which allows
    # to keep all records on the cdrs side which match with records on the right.
    # please use "inner"
    # best option here. Check the docs for details:
    # https://spark.apache.org/docs/latest/api/python/reference/api/pyspark.sql.DataFrame
    df = df.join(sdfLoc,on='cell_id',how = "inner")

    # Drop all records which didn't find a match in the location file because we cant do this analysis
    # without location. There are several ways to do this. For instance, you can drop all rows where
    # use spark filter and condition set to cell_id isNotNull()
    df = df.filter(df.cell_id.isNotNull())
    # ==================================================
    # SAMPLE USERS
    # ===================================================
    # first, create a list of unique user user_ids
    # using the distinct() function on the userid_col
    all_users =  df.select(userid_col).distinct().collect()

    # randomly select the required number of users
    # using the random.choices() function
    random_userid_list = [i[userid_col] for i in random.choices(all_users, k=number_of_users_to_sample)]

    # Now, for each of the selected users, leets get their data
    # select only our random user data using spark filter
    # use df.filter(). Inside it, use the isin() function on the
    # userid column
    dfu = df.filter(df['user_id'].isin(random_userid_list))

    # save to CSV if necessary
    if save_to_csv:
        dfu.coalesce(1).write.csv(path=output_csv, header=True)
    else:
        return dfu

def explore_data_with_spark(df=None, output_plot_file=None, output_heatmap=None):
    """
    Lets do a quick exploration of the data by generating the following:
    1. Number of days in the data
    2. User call count stats
    3. Weekday and hour calling patterns
    """
    # =====================================
    # CALCULATE THE NUMBER OF DAYS IN DATA
    # =====================================

    # use relevant spark function to generate
    # a list of unique dates, recall that the date
    # column is 'date
    dates_rows = df.select('date').distinct().collect()
    # sort the dates using sorted() function
    sorted_dates = sorted(dates_rows)
    # use list indexing to get the first element and last
    # element from the sorted list, substract them to get
    # time difference
    diff = sorted_dates[-1]['date']-sorted_dates[0]['date']
    # use days function to get the number of days
    num_days = diff.days
    # =====================================
    # GENERATE WEEKDAY AND HOUR CALL COUNT
    # =====================================

    # define UDF to calculate hour and weekday
    # for weekday use weekday() function while
    # for hour, use hour()
    add_hr = udf(lambda x :x.hour)              # Define Spark udf to get hour from the datetime column
    add_wkday = udf(lambda x: x.weekday())         # Define Spark udf to get hour from the datetime column
    # create a dictionary with keys as the weekday integers while the values
    # are the weekday name
    day_dict = {'1':'Monday', '2':'Tuesday', '3':'Wednesday', '4':'Thursday', '5':'Friday', '6':'Saturday','7':'Sunday'}

    # add hour column, lets call it 'hr
    # also add weekday column, we call it 'wkday'
    df = df.withColumn('hr',add_hr(col('datetime')))

    df = df.withColumn('wkday',add_wkday(col('date')))

    # create pandas DataFrame from Spark DataFrame above using toPandas()
    pdf =  df.groupBy('wkday','hr').count().toPandas()
    # use the Pandas.Series.map() function on the "wkday" column to convert
    # wkday as numbers to week day names and create  new column for weekday name
    # Lets call that column 'weekDay'
    pdf['weekDay']=pdf['wkday'].map(lambda x: day_dict[x])
    # drop the "wkday" column now
    pdf = pdf.drop("wkday", axis = 1)
    pdf = pdf.pivot(index='weekDay', columns='hr', values='count')

    # create and save heatmap
    # use plt.figure() to create figure with your chosen size
    plt.figure(figsize = (7,7))
    # use sns.heatmap() function to create heatmap using the Pandas DataFrame, pdf
    sns.heatmap(pdf)
    # save the figure to file
    plt.savefig(output_heatmap)

    # =====================================
    # NUMBER OF CALLS FOR EACH USER
    # =====================================
    # Use Spark groupBy function to group user and count number of events
    # convert resulting spark dataframe to pandas in the samee line of code
    df_grp_user =  df.groupby('user_id').count().toPandas()

    # create a distribution plot of user call count using
    # seaborn distplot() function and save the figure
    # first, use plt.figure() to create new figure environment
    # Next, create the plot and then finally save it
    plt.figure(figsize= (5,5))
    sns.distplot(df_grp_user)                                                        # A refaire
    plt.savefig(output_plot_file)

    # report average number calls per day for each user
    # first use spark groupBy on user_id and day, then
    # convert that object to pandas dataframe using toPandas()
    # function
    df_grp_day = df.groupby('user_id','date').count().toPandas()

    # get mean and median
    mean =  df_grp_day['count'].mean()
    median = df_grp_day['count'].median()

    # return results like this mean, median, number of days
    return mean, median, df_grp_day['date'].nunique()

if __name__ == '__main__':
    # ============================================
    # QUESTION 1: USING MAP FUNCTION
    # ============================================
    time_format = "%Y%m%d"
    reference_date = '20180713'
    num_csv_files = 20
    #  add full path to the CSV folder and the function Path() to convert to
    # a path object
    path_to_csv = Path('/home/user/Desktop/big data analytic/simulated_cdr')
    get_events_before_jul13(path_to_csv , reference_date, time_format,num_csv_files)

    # ============================================
    # QUESTION 2: PREPROCESS SIMULATED CDRS
    # ============================================
    cdrs_dir ='/home/user/Desktop/big data analytic/simulated_cdr'                              #ADD PATH TO FOLDER WITH MULTIPLE CSV FILE
    # please start  with a few number of users, e.g., 1000
    number_of_users_to_sample =  1000
    # Full path to CSV to write outputs
    output_csv = "/home/user/Desktop/output_csv"
    # Download the file called simulated_locs.csv
    # and add its full path below
    loc_file = "/home/user/Desktop/big data analytic/simulated_locs.csv"
    # finall call the function preprocess_cdrs_using_spark and with other options left
    # as default
    dfu =  preprocess_cdrs_using_spark(file_or_folder=cdrs_dir, number_of_users_to_sample=number_of_users_to_sample,
                                output_csv=output_csv, date_format='%Y%m%d%H%M%S',
                                debug_mode=True, loc_file=loc_file, save_to_csv=False,
                                userid_col="user_id")

    # ============================================
    # QUESTION 3: EXPLORE USER ACTIVITY PATTERNS
    # ============================================
    # 1. use the df you save from above as input here
    # 2. create a full path to save a plot with "png" extension
    # 3. create a full path to save the heatmap plot
    # Run the function explore_data_with_spark below
    results = explore_data_with_spark(df=dfu, output_plot_file='/home/user/Desktop/big_data_analytic.png', output_heatmap='/home/user/Desktop/big.png')


import time
import pandas as pd
import numpy as np

CITY_DATA = { 'chicago': 'chicago.csv',
              'new york city': 'new_york_city.csv',
              'washington': 'washington.csv' }

def get_filters():
    """
    Asks user to specify a city, month, and day to analyze.

    Returns:
        (str) city - name of the city to analyze
        (str) month - name of the month to filter by, or "all" to apply no month filter
        (str) day - name of the day of week to filter by, or "all" to apply no day filter
    """
    print('Hello! Let\'s explore some US bikeshare data!')
    # TO DO: get user input for city (chicago, new york city, washington). HINT: Use a while loop to handle invalid inputs

    city = input('Please choose one of the following cities: chicago, new york city, washington: ')
    city = city.lower()

    while city not in CITY_DATA.keys():
        city = input('Something went wrong. Please enter one of the following cities: \n chicago, new york city, washington \n')
        city = city.lower()

    print('You have chosen ',city)

    # TO DO: get user input for month (all, january, february, ... , june)
    month = input('Please choose \'all\' or one of the following months: january, february, march, april, may, june: ')
    month = month.lower()

    while month not in ['all','january', 'february', 'march', 'april', 'may', 'june']:
        month = input('Something went wrong. Please enter one of the following months: \n all, january, february, march, april, may, june:\n')
        month = month.lower()

    print('You have chosen ', month)

    # TO DO: get user input for day of week (all, monday, tuesday, ... sunday)
    day = input('Please choose \'all\' or one of the following day: monday, tuesday, wednesday, thursday, friday, saturday, sunday: ')
    day = day.lower()

    while day not in ['all', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']:
        day = input('Something went wront. Please enter one of the following days: \n all, mon, tue, wed, thu, fri, sat, sun\n')
        day.lower()

    print('You have chosen ', day)

    print('-'*40)
    return city, month, day

def load_data(city, month, day):
    """
    Loads data for the specified city and filters by month and day if applicable.

    Args:
        (str) city - name of the city to analyze
        (str) month - name of the month to filter by, or "all" to apply no month filter
        (str) day - name of the day of week to filter by, or "all" to apply no day filter
    Returns:
        df - Pandas DataFrame containing city data filtered by month and day
    """
    #Loading data
    df = pd.read_csv(CITY_DATA[city])

    #Convert the starttime column to Datetime
    df['Start Time'] = pd.to_datetime(df['Start Time'])

    #Extract month and day of week from Start Time to create new column
    df['month'] = df['Start Time'].dt.month
    df['day_of_week'] = df['Start Time'].dt.weekday_name
    df['hour'] = df['Start Time'].dt.hour

    # filter by month if applicable
    if month != 'all':
        months = ['january', 'february', 'march', 'april', 'may', 'june']
        month = months.index(month)+1
        df = df[df['month'] == month]

    # filter by day of week if applicable
    if day != 'all':
        df = df[df['day_of_week'] == day.title()]

    print(df.head())
    return df

def time_stats(df):
    """Displays statistics on the most frequent times of travel."""

    print('\nCalculating The Most Frequent Times of Travel...\n')
    start_time = time.time()

    # TO DO: display the most common month
    popular_month = df['month'].mode()[0]
    print('Most common month in index numbers: ', popular_month)

    # TO DO: display the most common day of week
    popular_weekday = df['day_of_week'].mode()[0]
    print('Most common day of week: ', popular_weekday)

    # TO DO: display the most common start hour
    popular_hour = df['hour'].mode()[0]
    print('Most common start hour: ', popular_hour)

    print('\nThis took %s seconds.' % (time.time() - start_time))
    print('-'*40)

def station_stats(df):
    """Displays statistics on the most popular stations and trip."""

    print('\nCalculating The Most Popular Stations and Trip...\n')
    start_time = time.time()

    # TO DO: display most commonly used start station
    popular_start = df['Start Station'].mode()[0]
    print('The most commonly used start station: ', popular_start)

    # TO DO: display most commonly used end station
    popular_end = df['End Station'].mode()[0]
    print('The most commonly used end station: ', popular_end)

    # TO DO: display most frequent combination of start station and end station trip
    df['start_to_end'] = 'Start Station' + ' ' + 'End Station'
    frequent_route = df['start_to_end'].mode()[0]
    print('The most frequent combination of start station and end station trip: ', frequent_route)

    print("\nThis took %s seconds." % (time.time() - start_time))
    print('-'*40)

def trip_duration_stats(df):
    """Displays statistics on the total and average trip duration."""

    print('\nCalculating Trip Duration...\n')
    start_time = time.time()

    # TO DO: display total travel time
    total_travel = df['Trip Duration'].sum()
    print('The total travel time: ', total_travel)

    # TO DO: display mean travel time
    mean_travel = df['Trip Duration'].mean()
    print('The mean travel time: ', mean_travel)

    print("\nThis took %s seconds." % (time.time() - start_time))
    print('-'*40)

def user_stats(df, city):
    """Displays statistics on bikeshare users."""

    print('\nCalculating User Stats...\n')
    start_time = time.time()

    # TO DO: Display counts of user types
    counts_user = df['User Type'].value_counts()
    print('Counts of user type: ',counts_user)

    # TO DO: Display counts of gender. Display earliest, most recent, and most common year of birth

    if city != 'washington':
        counts_gender = df['Gender'].value_counts()
        print('Counts of gender: ', counts_gender)
        print('Earliest year of birth: ', df['Birth Year'].min())
        print('Most recent year of birth: ', df['Birth Year'].max())
        print('Most common year of birth: ', df['Birth Year'].mode()[0])
    else:
        print('The list for Washington does not contain a Gender and Birth Year column.')

    print("\nThis took %s seconds." % (time.time() - start_time))
    print('-'*40)

def display_data(df):
    """ Asks whether the user wants to raw data. If 'yes' the script prints 5 rows and asks again if the user wants to see 5 more rows. Once the user chooses'no' the script goes to the next section of the script"""

    raw_data = input('Would you like to see raw_data? Enter yes or no\n').lower()
    count = 0
    while raw_data not in ['yes', 'no']:
        raw_data = input('Please try again: yes or no?').lower()

    if raw_data == 'yes':
        print(df.head())

    while raw_data == 'yes':
        raw_data = input('Would you like to see 5 more rows?\n').lower()
        if raw_data == 'yes':
            count +=5
            print(df[count:count+5])

def main():
    while True:
        city, month, day = get_filters()
        df = load_data(city, month, day)

        time_stats(df)
        station_stats(df)
        trip_duration_stats(df)
        user_stats(df, city)
        display_data(df)

        restart = input('\nWould you like to restart? Enter yes or no.\n')
        if restart.lower() != 'yes':
            break

if __name__ == "__main__":
    main()


#todo p.330 ~ p.333
#todo code x ~ code x
#todo 7.2.1 콜백을 사용하여 모델의 훈련 과정 제어하기

# ▣ ModelCheckPoint 와 EarlyStopping 콜백
# - ModelCheckPoint: 훈련하는 동안 모델을 계속 저장하는 콜백 함수
# - EarlyStopping: 정해진 에포크 동안 모니터링 지표가 향상되지 않을 때 훈련을 중지하는 콜백 함수

import os
import keras
from keras.models import Model

x, y = [], []
x_val, y_val = [], []

callback_list = [
    keras.callbacks.EarlyStopping(  # 성능 향상이 멈추면 훈련을 중지
        monitor='val_acc',  # 모델의 검증 정확도를 모니터링
        patience=1  # 1 에포크보다 더 길게(즉 2 에포크 동안) 정확도가 향상되지 않으면 훈련이 중지
    ),
    keras.callbacks.ModelCheckpoint(  # 에포크마다 현재 가중치를 저장
        filepath=os.path.join(os.path.dirname(os.path.abspath(__file__)), 'train_log', 'model.h5'),  # 모델 파일의 경로
        monitor='val_loss',  # 이 두 매개변수는 val_loss 가 좋아지지 않으면 모델 파일을 덮어쓰지 않는다는 뜻.
        save_best_only=True  # 훈련하는 동안 가장 좋은 모델이 저장.
    )
]

model = Model()

model.compile(optimizer='rmsprop',
              loss='binary_crossentropy',
              metrics=['acc'])

model.fit(
    x=x,
    y=y,
    epochs=10,
    batch_size=32,
    callbacks=callback_list,
    validation_data=(x_val, y_val)
)

# ▣ ReduceLROnPlateau 콜백
# - 검증 손실이 향상되지 않을 때 학습률을 작게 할 수 있는 콜백 함수

callback_list = [
    keras.callbacks.ReduceLROnPlateau(
        monitor='val_loss',  # 모델의 검증 손실을 모니터링
        factor=0.1,  # 콜백이 홀출될 때 학습률을 10배로 줄임
        patience=10  # 검증 손실이 10 에포크 동안 좋아지지 않으면 콜백이 호출
    )
]

model.fit(
    x=x,
    y=y,
    epochs=10,
    batch_size=32,
    callbacks=callback_list,
    validation_data=(x_val, y_val)
)

# ▣ 사용자 콜백 함수
# - 내장 콜백에서 제공하지 않는 특수한 행동이 훈련 도중 필요하면 자신만의 콜백을 만들 수 있음
# - 콜백은 keras.callbacks.Callback 클래스를 상속받아 구현
# - 훈련시 호출 지점
#  - on_epoch_begin: 각 에포크가 시작할 때 호출
#  - on_epoch_end: 각 에포크가 끝날 때 호출
#  - on_batch_begin: 각 배치 처리가 시작되기 전에 호출
#  - on_batch_end: 각 배치 처리가 끝난 후에 호출
#  - on_train_begin: 훈련이 시작될 때 호출
#  - on_train_end: 훈련이 끝날 때 호출

import numpy as np

class ActivationLogger(keras.callbacks.Callback):
    def set_model(self, model):
        self.model = model
        layer_outputs = [layer.output for layer in model.layers]
        self.activations_model = keras.models.Model(inputs=model.input,
                                                    outputs=layer_outputs)

    def on_epoch_end(self, epoch, logs=None):
        if self.validation_data is None:
            raise RuntimeError('Requires validation_data.')

        validation_sample = self.validation_data[0][0:1]  # validation_data 의 첫 번째 원소는 입력 데이터고, 두 번째 원소는 레이블
        activations = self.activations_model.predict(validation_sample)
        f = open('activations_at_epoch_' + str(epoch) + '.npz', 'wb')
        np.savez(f, activations)
        f.close()