"""Tools for creating and manipulating an HDF5 VDS for Tristan data."""

from contextlib import ExitStack
from itertools import chain, compress, cycle, zip_longest
from pathlib import Path
from typing import Dict, Iterable, List

import h5py
import numpy as np

from .data import cue_keys, event_keys, ts_key_regex

Sources = Dict[str, Iterable[h5py.VirtualSource]]
TimeSliceInfo = List[slice], np.ndarray, List[slice], List[slice]
VirtualSourceInfo = Sources, Sources, List[int], Dict[str, type]
Layouts = Dict[str, h5py.VirtualLayout]

def time_slice_info(meta_file: h5py.File) -> TimeSliceInfo:
    """
    Assemble information about the event data time slices from the metadata file.

    Args:
        meta_file:  Metadata ('_meta.h5') file.  Assumes metadata version 1.

    Returns:
        - List of slice objects used to select each time slice from the virtual source
          objects in order to populate the virtual layouts.  Length and order
          correspond to 'events_per_ts'.
        - List of the number of events in each time slice, in the order that the time
          slices will appear in the VDS.  Length is the number of time slices recorded.
    """
    fp_per_module = meta_file["fp_per_module"][()]

    ts_keys = sorted(filter(ts_key_regex.match, meta_file.keys()))
    ts_data = [meta_file[ts_key] for ts_key in ts_keys]

    time_slices = []
    num_events_per_ts = []
    # Loop through the modules, acting on the time slice metadata for each in turn.
    for num_files, ts_counts in zip(fp_per_module, ts_data):
        ts_counts = ts_counts[()]
        # Reshape the time slice metadata for a single module into a rectangular array
        # with shape (number of time slices per file, number of files), so as to be
        # able to generate file-specific slices.
        num_ts_per_fp = -(-ts_counts.size // num_files)
        ts_counts.resize(num_ts_per_fp * num_files)
        # Keep a separate record of each module's array of event counts per time slice.
        num_events_per_ts.append(ts_counts)
        ts_counts = ts_counts.reshape(num_ts_per_fp, num_files)
        # Generate the cumulative count of events per time slice for each file.
        ts_per_module = np.pad(np.cumsum(ts_counts, axis=0), ((1, 0), (0, 0)))
        # Turn these counts into slices to select from a virtual source for each file.
        time_slices.append(
            map(slice, ts_per_module[:-1].flatten(), ts_per_module[1:].flatten())
        )

    # Assemble all the source slices into a single list, ordered first
    # chronologically, then by module number.  Where modules have recorded different
    # numbers of time slices, zip_longest will pad with None.
    time_slices = list(chain.from_iterable(zip_longest(*time_slices)))

    # Resize each module's array of event counts per time slice so that their sizes
    # match.  This is achieved by zero-padding to match the None-padding of the list
    # of time slices by zip_longest.
    max_size = max(data.size for data in num_events_per_ts)
    num_events_per_ts = np.column_stack(
        [np.pad(data, (0, max_size - data.size)) for data in num_events_per_ts]
    ).flatten()

    return time_slices, num_events_per_ts

def virtual_sources(files: List[Path], meta_file: h5py.File) -> VirtualSourceInfo:
    """
    Create HDF5 virtual sources and collate ancillary information from raw data files.

    Args:
        files:      Lexicographically sorted list of raw file paths.
        meta_file:  Tristan detector metadata file object.

    Returns:
        - Dictionary of event data set names and iterators of corresponding HDF5 virtual
          sources.  The iterator of sources for each data set is based on
          itertools.cycle and so repeats indefinitely in the order in which successive
          event slices should be selected to build the virtual data set.
        - Dictionary of cue data set names and lists of corresponding HDF5 virtual
          sources.  The lists of sources have length and order as per the list of input
          files.
        - List of the number of cues in each data file after zero-padding has been
          stripped.  Length and order as per the list of input files.
        - Dictionary of data set names and corresponding data types.
    """
    event_sources = {key: [] for key in event_keys}
    cue_sources = {key: [] for key in cue_keys}
    num_cues_per_file = []

    with ExitStack() as stack:
        raw_files = [stack.enter_context(h5py.File(path)) for path in files]

        dtypes = {key: raw_files[0][key].dtype for key in event_keys + cue_keys}

        for raw_file in raw_files:
            # The cues are padded with zeroes.  Find the first so we can slice them off.
            num_cues_per_file.append(np.argmax(raw_file["cue_id"][()] == 0))
            for key in event_keys:
                event_sources[key].append(h5py.VirtualSource(raw_file[key]))
            for key in cue_keys:
                cue_sources[key].append(h5py.VirtualSource(raw_file[key]))

    # Make a list of slices with which to divide the lexicographically sorted list of
    # file paths into sub-lists, each slice corresponding to a different detector
    # module.  Ordered by module number.  Length is equal to the number of modules in
    # the detector.
    file_slices = np.pad(np.cumsum(meta_file["fp_per_module"]), (1, 0))
    file_slices = list(map(slice, file_slices[:-1], file_slices[1:]))

    # Construct a carousel to select time slices in the order in which they should
    # appear in the virtual layout.
    for key, sources in event_sources.items():
        carousel = zip(*(cycle(sources[file_slice]) for file_slice in file_slices))
        event_sources[key] = chain.from_iterable(carousel)

    return event_sources, cue_sources, num_cues_per_file, dtypes

def virtual_layouts(num_events: int, num_cues: int, dtypes: Dict[str, type]) -> Layouts:
    """Create a dictionary of data set names and corresponding HDF5 virtual layouts."""
    layouts = {}
    for key in event_keys:
        layouts[key] = h5py.VirtualLayout(shape=(num_events,), dtype=dtypes[key])
    for key in cue_keys:
        layouts[key] = h5py.VirtualLayout(shape=(num_cues,), dtype=dtypes[key])
    return layouts

def virtual_data_set(
    raw_files: List[Path],
    meta_file: h5py.File,
    time_slices: List[slice],
    events_per_ts: Iterable[int],
) -> Layouts:
    """
    Define a virtual data set in the form of virtual layouts linked to virtual sources.

    The time slices of events will appear in the VDS ordered chronologically, then by
    module number to resolve ties.  Cue data will simply be aggregated in
    lexicographical order of the raw files from which they are sourced.

    Args:
        raw_files:      Lexicographically sorted list of raw file paths.
        meta_file:      Tristan detector metadata file object.
        time_slices:    List of slice objects used to select each time slice from the
                        virtual source objects in order to populate the virtual
                        layouts.  Length and order correspond to 'events_per_ts'.
        events_per_ts:  List of the number of events in each time slice, in the order
                        that the time slices will appear in the VDS.  Length is the
                        number of time slices recorded.

    Returns:
        A dictionary of raw data set names and corresponding HDF5 virtual layouts.
    """
    event_sources, cue_sources, cues_per_file, dtypes = virtual_sources(
        raw_files, meta_file
    )

    # Generate the slices used to assign the cue data to their virtual layouts.
    cue_slices = np.pad(np.cumsum(cues_per_file), (1, 0))
    num_cues = cue_slices[-1]
    cue_slices = list(map(slice, cue_slices[:-1], cue_slices[1:]))

    # Generate the slices used to assign the event data to their virtual layouts.
    event_slices = np.pad(np.cumsum(events_per_ts), (1, 0))
    num_events = event_slices[-1]
    event_slices = list(map(slice, event_slices[:-1], event_slices[1:]))

    layouts = virtual_layouts(num_events, num_cues, dtypes)

    # Map virtual source slices to virtual layout slices.
    for key, sources in event_sources.items():
        layout = layouts[key]
        # HDF5 VDS doesn't like empty slices, so only assign a source time slice to a
        # layout slice if the corresponding number of events is non-zero. Trying to
        # assign empty slices to empty slices somehow results in a corrupted VDS.
        mapping = compress(zip(event_slices, sources, time_slices), events_per_ts)
        for event_slice, source, time_slice in mapping:
            layout[event_slice] = source[time_slice]
    for key, sources in cue_sources.items():
        layout = layouts[key]
        for cue_slice, source, count in zip(cue_slices, sources, cues_per_file):
            layout[cue_slice] = source[:count]

    return layouts


# -*- coding: utf-8 -*-
import numpy as np
from asm.create_data.get_opcode import *

from gensim.models.doc2vec import Doc2Vec, TaggedDocument

def my_doc2vec(asm_path, model_path):
    """
    Generate a doc2vec model and save it.
    :param asm_path: The path of all .asm files, which is used to generate the document of Doc2Vec model.
    :param model_path: The path of Doc2Vec model.
    :return: The doc2vec model.
    """
    # Generate the documents, which is used in Doc2Vec model.
    all_opcode_seqs = get_all_opcode_seqs(asm_path)
    documents = [TaggedDocument(doc, [i]) for i, doc in enumerate(all_opcode_seqs)]

    # Generate the Doc2Vec model.
    model = Doc2Vec(documents, workers=4, vector_size=256)

    # Save the Doc2Vec model.
    dir = "/".join(model_path.split('/')[:-1])
    if not os.path.exists(dir):
        os.mkdir(dir)
    model.save(model_path)

    return model

def vectorize_asm(asm_path, vec_path, model_path):
    """
    Use Doc2Vec model to vectorize all .asm files and save them as .asm.model file.
    The dimension of vectorized .asm files is 1*100.
    :param asm_path: The path of .asm files.
    :param model_path: The path of Doc2Vec model.
    :param vec_path: The path of .asm.vec files.
    """
    model = Doc2Vec.load(model_path)
    if not os.path.exists(vec_path):
        os.mkdir(vec_path)

    dirs = os.listdir(asm_path)
    for dir in dirs:
        if not os.path.exists(vec_path + dir):
            os.mkdir(vec_path + dir)
        asm_file_list = os.listdir(asm_path + dir)
        for file in asm_file_list:
            vector = model.infer_vector(get_opcode_sequence(asm_path + dir + '/' + file))
            np.savetxt(vec_path + dir + '/' + file + '.vec', vector, '%1.8f')


#########################################
###  container for various functions  ###
#########################################

import numpy as np
import eccodes
import netCDF4 as nc
import Ngl
import os
from sys import argv
from PIL import Image

def which_grid_point(pointname, model):
    ###################################################################################################
    ##### this is the database for the different grid point and city coordinates                  #####
    ##### city: point of actual city                                                              #####
    ##### icon-xyz: point that lies within desired corresponding grid point of the model icon-xyz #####
    ###################################################################################################

    if model == 'city':
        allpoints =[dict(lat = 49.014, lon =  8.404, name = 'Karlsruhe'),\
                    dict(lat = 50.001, lon =  8.265, name = 'Mainz'),\
                    dict(lat = 48.141, lon = 11.580, name = 'Munich'),\
                    dict(lat = 52.519, lon = 13.391, name = 'Berlin'),\
                    dict(lat = 53.549, lon =  9.990, name = 'Hamburg'),\
                    dict(lat = 50.098, lon =  8.761, name = 'Offenbach'),\
                    dict(lat = 48.477, lon =  8.934, name = 'Rottenburg_am_Neckar'),\
                    dict(lat = 52.373, lon =  4.899, name = 'Amsterdam'),\
                    dict(lat = 48.210, lon = 16.371, name = 'Vienna'),\
                    dict(lat = 41.898, lon = 12.491, name = 'Rome'),\
                    dict(lat = 37.976, lon = 23.725, name = 'Athens'),\
                    dict(lat = 40.418, lon = -3.704, name = 'Madrid'),\
                    dict(lat = 39.474, lon = -0.377, name = 'Valencia'),\
                    dict(lat = 38.718, lon = -9.139, name = 'Lisbon'),\
                    dict(lat = 51.511, lon = -0.118, name = 'London'),\
                    dict(lat = 53.347, lon = -6.264, name = 'Dublin'),\
                    dict(lat = 55.678, lon = 12.576, name = 'Copenhagen'),\
                    dict(lat = 52.233, lon = 21.009, name = 'Warsaw'),\
                    dict(lat = 48.857, lon =  2.344, name = 'Paris'),\
                    dict(lat = 43.602, lon =  1.443, name = 'Toulouse'),\
                    dict(lat = 53.798, lon = -1.545, name = 'Leeds'),\
                    dict(lat = 44.496, lon = 11.343, name = 'Bologna'),\
                    dict(lat = 50.848, lon =  4.351, name = 'Brussels'),\
                    dict(lat =-34.617, lon =-58.416, name = 'BuenosAires'),\
                    dict(lat =-34.617, lon =-58.416, name = 'Kapstadt'),\
                    dict(lat = 47.373, lon =  8.539, name = 'Zurich'),\
                    dict(lat =  6.375, lon =  2.388, name = 'Cotonou_sea'),\
                    dict(lat =  6.375, lon =  2.388, name = 'Cotonou_land'),\
                    dict(lat =  5.594, lon = -0.186, name = 'Accra'),\
                    dict(lat =  6.681, lon = -1.616, name = 'Kumasi'),\
                    dict(lat =  5.349, lon = -4.011, name = 'Abidjan'),\
                    dict(lat = 16.740, lon =-22.931, name = 'Sal'),\
                    dict(lat = 14.712, lon =-17.467, name = 'Dakar_sea'),\
                    dict(lat = 14.712, lon =-17.467, name = 'Dakar_land'),\
                    dict(lat = 12.625, lon = -7.990, name = 'Bamako'),\
                    dict(lat = 12.365, lon = -1.522, name = 'Ouagadougou'),\
                    dict(lat = 13.505, lon =  2.125, name = 'Niamey'),\
                    dict(lat =  4.046, lon =  9.758, name = 'Douala'),\
                    dict(lat =  7.330, lon = 13.571, name = 'Ngaoundere'),\
                    dict(lat =  9.079, lon =  7.424, name = 'Abuja'),\
                    dict(lat = 16.972, lon =  7.986, name = 'Agadez'),\
                    dict(lat =  9.343, lon =  2.610, name = 'Parakou'),\
                    dict(lat =  0.405, lon =  9.459, name = 'Libreville_sea'),\
                    dict(lat =  0.405, lon =  9.459, name = 'Libreville_land')]
    elif model == 'icon-eu-eps':
        allpoints =[dict(lat = 49.014, lon =  8.350, name = 'Karlsruhe'),\
                    dict(lat = 50.003, lon =  8.268, name = 'Mainz'),\
                    dict(lat = 48.109, lon = 11.594, name = 'Munich'),\
                    dict(lat = 52.519, lon = 13.407, name = 'Berlin'),\
                    dict(lat = 53.500, lon =  9.993, name = 'Hamburg'),\
                    dict(lat = 50.098, lon =  8.761, name = 'Offenbach'),\
                    dict(lat = 48.477, lon =  8.934, name = 'Rottenburg_am_Neckar'),\
                    dict(lat = 52.373, lon =  4.899, name = 'Amsterdam'),\
                    dict(lat = 48.256, lon = 16.438, name = 'Vienna'),\
                    dict(lat = 41.890, lon = 12.481, name = 'Rome'),\
                    dict(lat = 37.984, lon = 23.690, name = 'Athens'),\
                    dict(lat = 40.420, lon = -3.730, name = 'Madrid'),\
                    dict(lat = 39.550, lon = -0.490, name = 'Valencia'),\
                    dict(lat = 38.754, lon = -9.200, name = 'Lisbon'),\
                    dict(lat = 51.507, lon =  0.000, name = 'London'),\
                    dict(lat = 53.400, lon = -6.350, name = 'Dublin'),\
                    dict(lat = 55.710, lon = 12.480, name = 'Copenhagen'),\
                    dict(lat = 52.217, lon = 21.033, name = 'Warsaw'),\
                    dict(lat = 48.860, lon =  2.350, name = 'Paris'),\
                    dict(lat = 43.602, lon =  1.400, name = 'Toulouse'),\
                    dict(lat = 53.850, lon = -1.500, name = 'Leeds'),\
                    dict(lat = 44.490, lon = 11.380, name = 'Bologna'),\
                    dict(lat = 50.930, lon =  4.330, name = 'Brussels'),\
                    dict(lat = 46.949, lon =  7.448, name = 'Bern'),\
                    dict(lat = 47.373, lon =  8.539, name = 'Zurich')]
    elif model == 'icon-eps' or model == 'icon-global-eps':
        allpoints =[dict(lat = 49.014, lon =  8.350, name = 'Karlsruhe'),\
                    dict(lat = 50.003, lon =  8.268, name = 'Mainz'),\
                    dict(lat = 48.109, lon = 11.594, name = 'Munich'),\
                    dict(lat = 52.519, lon = 13.407, name = 'Berlin'),\
                    dict(lat = 53.500, lon =  9.993, name = 'Hamburg'),\
                    dict(lat = 50.098, lon =  8.761, name = 'Offenbach'),\
                    dict(lat = 48.477, lon =  8.934, name = 'Rottenburg_am_Neckar'),\
                    dict(lat = 52.373, lon =  4.899, name = 'Amsterdam'),\
                    dict(lat = 48.256, lon = 16.438, name = 'Vienna'),\
                    dict(lat = 41.890, lon = 12.481, name = 'Rome'),\
                    dict(lat = 37.984, lon = 23.690, name = 'Athens'),\
                    dict(lat = 40.420, lon = -3.730, name = 'Madrid'),\
                    dict(lat = 39.550, lon = -0.490, name = 'Valencia'),\
                    dict(lat = 38.754, lon = -8.930, name = 'Lisbon'),\
                    dict(lat = 51.507, lon =  0.000, name = 'London'),\
                    dict(lat = 53.400, lon = -6.350, name = 'Dublin'),\
                    dict(lat = 55.710, lon = 12.480, name = 'Copenhagen'),\
                    dict(lat = 52.217, lon = 21.033, name = 'Warsaw'),\
                    dict(lat = 48.860, lon =  2.350, name = 'Paris'),\
                    dict(lat = 43.602, lon =  1.400, name = 'Toulouse'),\
                    dict(lat = 53.850, lon = -1.500, name = 'Leeds'),\
                    dict(lat = 44.490, lon = 11.380, name = 'Bologna'),\
                    dict(lat = 50.930, lon =  4.330, name = 'Brussels'),\
                    dict(lat = 46.949, lon =  7.448, name = 'Bern'),\
                    dict(lat =-34.800, lon =-58.400, name = 'BuenosAires'),\
                    dict(lat = 47.373, lon =  8.539, name = 'Zurich'),\
                    dict(lat =  6.375, lon =  2.388, name = 'Cotonou_sea'),\
                    dict(lat =  6.475, lon =  2.388, name = 'Cotonou_land'),\
                    dict(lat =  5.594, lon = -0.186, name = 'Accra'),\
                    dict(lat =  6.681, lon = -1.616, name = 'Kumasi'),\
                    dict(lat =  5.349, lon = -4.011, name = 'Abidjan'),\
                    dict(lat = 16.740, lon =-22.931, name = 'Sal'),\
                    dict(lat = 14.712, lon =-17.467, name = 'Dakar_sea'),\
                    dict(lat = 14.712, lon =-17.067, name = 'Dakar_land'),\
                    dict(lat = 12.625, lon = -7.990, name = 'Bamako'),\
                    dict(lat = 12.365, lon = -1.522, name = 'Ouagadougou'),\
                    dict(lat = 13.505, lon =  2.125, name = 'Niamey'),\
                    dict(lat =  4.046, lon =  9.758, name = 'Douala'),\
                    dict(lat =  7.330, lon = 13.571, name = 'Ngaoundere'),\
                    dict(lat =  9.079, lon =  7.424, name = 'Abuja'),\
                    dict(lat = 16.972, lon =  7.986, name = 'Agadez'),\
                    dict(lat =  9.343, lon =  2.610, name = 'Parakou'),\
                    dict(lat =  0.405, lon =  9.459, name = 'Libreville_sea'),\
                    dict(lat =  0.405, lon =  9.559, name = 'Libreville_land')]
    elif model == 'icon-eu-det':
        allpoints =[dict(lat = 49.014, lon =  8.404, name = 'Karlsruhe'),\
                    dict(lat = 50.001, lon =  8.265, name = 'Mainz'),\
                    dict(lat = 48.141, lon = 11.580, name = 'Munich'),\
                    dict(lat = 52.519, lon = 13.391, name = 'Berlin'),\
                    dict(lat = 53.549, lon =  9.990, name = 'Hamburg'),\
                    dict(lat = 50.098, lon =  8.761, name = 'Offenbach'),\
                    dict(lat = 48.477, lon =  8.934, name = 'Rottenburg_am_Neckar'),\
                    dict(lat = 52.373, lon =  4.899, name = 'Amsterdam'),\
                    dict(lat = 48.210, lon = 16.371, name = 'Vienna'),\
                    dict(lat = 41.898, lon = 12.491, name = 'Rome'),\
                    dict(lat = 37.976, lon = 23.725, name = 'Athens'),\
                    dict(lat = 40.418, lon = -3.704, name = 'Madrid'),\
                    dict(lat = 39.474, lon = -0.377, name = 'Valencia'),\
                    dict(lat = 38.718, lon = -9.139, name = 'Lisbon'),\
                    dict(lat = 51.511, lon = -0.118, name = 'London'),\
                    dict(lat = 53.347, lon = -6.264, name = 'Dublin'),\
                    dict(lat = 55.678, lon = 12.576, name = 'Copenhagen'),\
                    dict(lat = 52.233, lon = 21.009, name = 'Warsaw'),\
                    dict(lat = 48.857, lon =  2.344, name = 'Paris'),\
                    dict(lat = 43.602, lon =  1.443, name = 'Toulouse'),\
                    dict(lat = 53.798, lon = -1.545, name = 'Leeds'),\
                    dict(lat = 44.496, lon = 11.343, name = 'Bologna'),\
                    dict(lat = 50.848, lon =  4.351, name = 'Brussels'),\
                    dict(lat = 46.949, lon =  7.448, name = 'Bern'),\
                    dict(lat = 47.373, lon =  8.539, name = 'Zurich')]
    elif model == 'icon' or model == 'icon-global-det':
        allpoints =[dict(lat = 49.014, lon =  8.404, name = 'Karlsruhe'),\
                    dict(lat = 50.001, lon =  8.265, name = 'Mainz'),\
                    dict(lat = 48.141, lon = 11.580, name = 'Munich'),\
                    dict(lat = 52.519, lon = 13.391, name = 'Berlin'),\
                    dict(lat = 53.549, lon =  9.990, name = 'Hamburg'),\
                    dict(lat = 50.098, lon =  8.761, name = 'Offenbach'),\
                    dict(lat = 48.477, lon =  8.934, name = 'Rottenburg_am_Neckar'),\
                    dict(lat = 52.373, lon =  4.899, name = 'Amsterdam'),\
                    dict(lat = 48.210, lon = 16.371, name = 'Vienna'),\
                    dict(lat = 41.898, lon = 12.491, name = 'Rome'),\
                    dict(lat = 37.976, lon = 23.725, name = 'Athens'),\
                    dict(lat = 40.418, lon = -3.704, name = 'Madrid'),\
                    dict(lat = 39.474, lon = -0.377, name = 'Valencia'),\
                    dict(lat = 38.718, lon = -9.139, name = 'Lisbon'),\
                    dict(lat = 51.511, lon = -0.118, name = 'London'),\
                    dict(lat = 53.347, lon = -6.264, name = 'Dublin'),\
                    dict(lat = 55.678, lon = 12.576, name = 'Copenhagen'),\
                    dict(lat = 52.233, lon = 21.009, name = 'Warsaw'),\
                    dict(lat = 48.857, lon =  2.344, name = 'Paris'),\
                    dict(lat = 43.602, lon =  1.443, name = 'Toulouse'),\
                    dict(lat = 53.798, lon = -1.545, name = 'Leeds'),\
                    dict(lat = 44.496, lon = 11.343, name = 'Bologna'),\
                    dict(lat = 50.848, lon =  4.351, name = 'Brussels'),\
                    dict(lat = 46.949, lon =  7.448, name = 'Bern'),\
                    dict(lat =-34.617, lon =-58.416, name = 'BuenosAires'),\
                    dict(lat = 47.373, lon =  8.539, name = 'Zurich')]
    else:
        print('Wrong model name!')
        exit()

    for point in allpoints:
        if point['name'] == pointname:
            return point

    print('no point found with this name: {}'.format(pointname))
    return None

#######################################################################################################################
#######################################################################################################################
#######################################################################################################################
#######################################################################################################################
#######################################################################################################################
#######################################################################################################################
#######################################################################################################################
#######################################################################################################################

def get_latlon_filter_distance(model):
    ################################################################################################
    ##### filter distance in zonal and meridional direction in degrees (1 deg = 111.2km)       #####
    ##### formula for icosahedral grid RnBk:                                                   #####
    #####     filter_distance = (2/3)*sqrt(sqrt(3)*1.05*A_avg) [km]                            #####
    #####     A_avg: average cell area, 1.05: cell area can be up to 5% bigger as average area #####
    #####     A_avg = (pi/5)*r_e^2/(n^2*4^k)                                                   #####
    #####  -> filter_distance = 40.83/(n*2^k) [deg]                                            #####
    ##### put value in deg below for new grid                                                  #####
    ################################################################################################

    if model == 'icon-eu-eps':
        filter_distance = 0.160         # R2B7 icosahedral grid
    elif model == 'icon-global-eps':
        filter_distance = 0.319         # R2B6 icosahedral grid
    elif model == 'icon-eu-det':
        filter_distance = 0.065         # 0.0625deg latlon grid
    elif model == 'icon-global-det':
        filter_distance = 0.107         # R3B7 icosahedral grid

    return  filter_distance

#######################################################################################################################
#######################################################################################################################
#######################################################################################################################
#######################################################################################################################
#######################################################################################################################
#######################################################################################################################
#######################################################################################################################
#######################################################################################################################

def plot_point_information_eu(pointname, models_type, no_title):
    path = dict(base = '/',
                plots = 'data/plots/point_information/')

    print('plot {} information about {}'.format(models_type, pointname))

    if models_type == 'eu-eps':
        filenames = dict()
        filenames['oro_icon-eu-eps']            = plot_orography(pointname, 'icon-eu-eps')
        filenames['oro_icon-global-eps']        = plot_orography(pointname, 'icon-global-eps')
        filenames['landsea_icon-eu-eps']        = plot_landsea(pointname, 'icon-eu-eps')
        filenames['landsea_icon-global-eps']    = plot_landsea(pointname, 'icon-global-eps')
        filenames['composite']                  = 'grid_information_{}_{}.png'.format(pointname, models_type)

        img_combined = Image.new('RGB',(1500, 1300),(255,255,255))

        #img_example.crop((39, 44, 601, 596))   # (left, upper, right, lower) corner

        text_title = Image.open(path['base'] + path['plots'] + filenames['landsea_icon-eu-eps'])
        img_combined.paste(text_title.crop((15, 620, 760, 663)), (750-310, 20))
        text_iconeueps = Image.open(path['base'] + path['plots'] + filenames['oro_icon-eu-eps'])
        img_combined.paste(text_iconeueps.crop((154, 624, 464, 655)), (200+271-136, 170-36-15))
        text_iconeps = Image.open(path['base'] + path['plots'] + filenames['oro_icon-global-eps'])
        img_combined.paste(text_iconeps.crop((154, 624, 530, 655)), (200+542+20+271-170, 170-36-15))

        text_oro = Image.open(path['base'] + path['plots'] + filenames['oro_icon-eu-eps'])
        img_combined.paste(text_oro.crop((154, 701, 302, 732)), (38, 170+266-15))
        map_oro_iconeueps = Image.open(path['base'] + path['plots'] + filenames['oro_icon-eu-eps'])
        img_combined.paste(map_oro_iconeueps.crop((37, 42, 579, 574)), (200, 170))
        map_oro_iconeps = Image.open(path['base'] + path['plots'] + filenames['oro_icon-global-eps'])
        img_combined.paste(map_oro_iconeps.crop((37, 42, 579, 574)), (200+542+20, 170))
        lb_oro = Image.open(path['base'] + path['plots'] + filenames['oro_icon-eu-eps'])
        img_combined.paste(lb_oro.crop((606, 44, 746, 574)), (200+542+20+542+20, 172))

        text_landsea1 = Image.open(path['base'] + path['plots'] + filenames['landsea_icon-eu-eps'])
        img_combined.paste(text_landsea1.crop((77, 730, 255, 755)), (13, 170+532+20+266-12-20))
        text_landsea2 = Image.open(path['base'] + path['plots'] + filenames['landsea_icon-eu-eps'])
        img_combined.paste(text_landsea2.crop((268, 730, 378, 755)), (50, 170+532+20+266-12+20))
        map_landsea_iconeueps = Image.open(path['base'] + path['plots'] + filenames['landsea_icon-eu-eps'])
        img_combined.paste(map_landsea_iconeueps.crop((37, 42, 579, 574)), (200, 170+532+20))
        map_landsea_iconeps = Image.open(path['base'] + path['plots'] + filenames['landsea_icon-global-eps'])
        img_combined.paste(map_landsea_iconeps.crop((37, 42, 579, 574)), (200+542+20, 170+532+20))
        lb_landsea = Image.open(path['base'] + path['plots'] + filenames['landsea_icon-eu-eps'])
        img_combined.paste(lb_landsea.crop((623, 44, 746, 574)), (200+542+20+542+20, 172+532+20))

        img_combined.save(path['base'] + path['plots'] + 'eu/' + filenames['composite'],'png')

        os.remove(path['base'] + path['plots'] + filenames['oro_icon-eu-eps'])
        os.remove(path['base'] + path['plots'] + filenames['oro_icon-global-eps'])
        os.remove(path['base'] + path['plots'] + filenames['landsea_icon-eu-eps'])
        os.remove(path['base'] + path['plots'] + filenames['landsea_icon-global-eps'])

    elif models_type == 'eu-det':
        filenames = dict()
        filenames['oro_icon-eu-det']            = plot_orography(pointname, 'icon-eu-det')
        filenames['oro_icon-global-det']        = plot_orography(pointname, 'icon-global-det')
        filenames['landsea_icon-eu-det']        = plot_landsea(pointname, 'icon-eu-det')
        filenames['landsea_icon-global-det']    = plot_landsea(pointname, 'icon-global-det')
        filenames['composite']                  = 'grid_information_{}_{}.png'.format(pointname, models_type)
        path['plots'] += 'eu/'

        img_combined = Image.new('RGB',(1500, 1300),(255,255,255))

        #img_example.crop((39, 44, 601, 596))   # (left, upper, right, lower) corner

        text_title = Image.open(path['base'] + path['plots'] + filenames['landsea_icon-eu-det'])
        img_combined.paste(text_title.crop((15, 620, 760, 663)), (750-340, 20))
        text_iconeueps = Image.open(path['base'] + path['plots'] + filenames['oro_icon-eu-det'])
        img_combined.paste(text_iconeueps.crop((154, 624, 490, 655)), (200+271-136, 170-36-15))
        text_iconeps = Image.open(path['base'] + path['plots'] + filenames['oro_icon-global-det'])
        img_combined.paste(text_iconeps.crop((154, 624, 530, 655)), (200+542+20+271-170, 170-36-15))

        text_oro = Image.open(path['base'] + path['plots'] + filenames['oro_icon-eu-det'])
        img_combined.paste(text_oro.crop((154, 701, 302, 732)), (38, 170+269-15))
        map_oro_iconeueps = Image.open(path['base'] + path['plots'] + filenames['oro_icon-eu-det'])
        img_combined.paste(map_oro_iconeueps.crop((37, 40, 579, 578)), (200, 170))
        map_oro_iconeps = Image.open(path['base'] + path['plots'] + filenames['oro_icon-global-det'])
        img_combined.paste(map_oro_iconeps.crop((37, 40, 579, 578)), (200+542+20, 170))
        lb_oro = Image.open(path['base'] + path['plots'] + filenames['oro_icon-eu-det'])
        img_combined.paste(lb_oro.crop((605, 42, 746, 574)), (200+542+20+542+20, 172))

        text_landsea1 = Image.open(path['base'] + path['plots'] + filenames['landsea_icon-eu-det'])
        img_combined.paste(text_landsea1.crop((77, 730, 255, 755)), (13, 170+538+20+269-12-20))
        text_landsea2 = Image.open(path['base'] + path['plots'] + filenames['landsea_icon-eu-det'])
        img_combined.paste(text_landsea2.crop((268, 730, 378, 755)), (50, 170+538+20+269-12+20))
        map_landsea_iconeueps = Image.open(path['base'] + path['plots'] + filenames['landsea_icon-eu-det'])
        img_combined.paste(map_landsea_iconeueps.crop((37, 40, 579, 578)), (200, 170+538+20))
        map_landsea_iconeps = Image.open(path['base'] + path['plots'] + filenames['landsea_icon-global-det'])
        img_combined.paste(map_landsea_iconeps.crop((37, 40, 579, 578)), (200+542+20, 170+538+20))
        lb_landsea = Image.open(path['base'] + path['plots'] + filenames['landsea_icon-eu-det'])
        img_combined.paste(lb_landsea.crop((622, 42, 748, 574)), (200+542+20+542+20, 172+538+20))

        img_combined.save(path['base'] + path['plots'] + 'eu/' + filenames['composite'],'png')

        os.remove(path['base'] + path['plots'] + filenames['oro_icon-eu-det'])
        os.remove(path['base'] + path['plots'] + filenames['oro_icon-global-det'])
        os.remove(path['base'] + path['plots'] + filenames['landsea_icon-eu-det'])
        os.remove(path['base'] + path['plots'] + filenames['landsea_icon-global-det'])

    if models_type == 'global-eps':
        filenames = dict()
        filenames['oro_icon-global-eps']        = plot_orography(pointname, 'icon-global-eps')
        filenames['landsea_icon-global-eps']    = plot_landsea(pointname, 'icon-global-eps')
        filenames['composite']                  = 'grid_information_{}_{}.png'.format(pointname, models_type)

        img_combined = Image.new('RGB',(920, 1300),(255,255,255))

        #img_example.crop((39, 44, 601, 596))   # (left, upper, right, lower) corner

        text_title = Image.open(path['base'] + path['plots'] + filenames['landsea_icon-global-eps'])
        img_combined.paste(text_title.crop((15, 620, 760, 663)), (500-360, 20))
        text_iconeps = Image.open(path['base'] + path['plots'] + filenames['oro_icon-global-eps'])
        img_combined.paste(text_iconeps.crop((154, 624, 530, 655)), (200+271-170, 170-36-15))

        text_oro = Image.open(path['base'] + path['plots'] + filenames['oro_icon-global-eps'])
        img_combined.paste(text_oro.crop((154, 701, 302, 732)), (38, 170+266-15))
        map_oro_iconeps = Image.open(path['base'] + path['plots'] + filenames['oro_icon-global-eps'])
        img_combined.paste(map_oro_iconeps.crop((37, 42, 579, 574)), (200, 170))
        lb_oro = Image.open(path['base'] + path['plots'] + filenames['oro_icon-global-eps'])
        img_combined.paste(lb_oro.crop((606, 44, 746, 574)), (200+542+20, 172))

        text_landsea1 = Image.open(path['base'] + path['plots'] + filenames['landsea_icon-global-eps'])
        img_combined.paste(text_landsea1.crop((77, 730, 255, 755)), (13, 170+532+20+266-12-20))
        text_landsea2 = Image.open(path['base'] + path['plots'] + filenames['landsea_icon-global-eps'])
        img_combined.paste(text_landsea2.crop((268, 730, 378, 755)), (50, 170+532+20+266-12+20))
        map_landsea_iconeps = Image.open(path['base'] + path['plots'] + filenames['landsea_icon-global-eps'])
        img_combined.paste(map_landsea_iconeps.crop((37, 42, 579, 574)), (200, 170+532+20))
        lb_landsea = Image.open(path['base'] + path['plots'] + filenames['landsea_icon-global-eps'])
        img_combined.paste(lb_landsea.crop((623, 44, 746, 574)), (200+542+20, 172+532+20))

        img_combined.save(path['base'] + path['plots'] + 'global/' + filenames['composite'],'png')

        os.remove(path['base'] + path['plots'] + filenames['oro_icon-global-eps'])
        os.remove(path['base'] + path['plots'] + filenames['landsea_icon-global-eps'])

    elif models_type == 'global-det':
        filenames = dict()
        filenames['oro_icon-global-det']        = plot_orography(pointname, 'icon-global-det')
        filenames['landsea_icon-global-det']    = plot_landsea(pointname, 'icon-global-det')
        filenames['composite']                  = 'grid_information_{}_{}.png'.format(pointname, models_type)

        img_combined = Image.new('RGB',(920, 1300),(255,255,255))

        #img_example.crop((39, 44, 601, 596))   # (left, upper, right, lower) corner

        text_title = Image.open(path['base'] + path['plots'] + filenames['landsea_icon-global-det'])
        img_combined.paste(text_title.crop((15, 620, 760, 663)), (500-360, 20))
        text_iconeps = Image.open(path['base'] + path['plots'] + filenames['oro_icon-global-det'])
        img_combined.paste(text_iconeps.crop((154, 624, 530, 655)), (200+271-170, 170-36-15))

        text_oro = Image.open(path['base'] + path['plots'] + filenames['oro_icon-global-det'])
        img_combined.paste(text_oro.crop((154, 701, 302, 732)), (38, 170+269-15))
        map_oro_iconeps = Image.open(path['base'] + path['plots'] + filenames['oro_icon-global-det'])
        img_combined.paste(map_oro_iconeps.crop((37, 40, 579, 578)), (200, 170))
        lb_oro = Image.open(path['base'] + path['plots'] + filenames['oro_icon-global-det'])
        img_combined.paste(lb_oro.crop((605, 42, 746, 574)), (200+542+20, 172))

        text_landsea1 = Image.open(path['base'] + path['plots'] + filenames['landsea_icon-global-det'])
        img_combined.paste(text_landsea1.crop((77, 730, 255, 755)), (13, 170+538+20+269-12-20))
        text_landsea2 = Image.open(path['base'] + path['plots'] + filenames['landsea_icon-global-det'])
        img_combined.paste(text_landsea2.crop((268, 730, 378, 755)), (50, 170+538+20+269-12+20))
        map_landsea_iconeps = Image.open(path['base'] + path['plots'] + filenames['landsea_icon-global-det'])
        img_combined.paste(map_landsea_iconeps.crop((37, 40, 579, 578)), (200, 170+538+20))
        lb_landsea = Image.open(path['base'] + path['plots'] + filenames['landsea_icon-global-det'])
        img_combined.paste(lb_landsea.crop((622, 42, 748, 574)), (200+542+20, 172+538+20))

        img_combined.save(path['base'] + path['plots'] + 'global/' + filenames['composite'],'png')

        os.remove(path['base'] + path['plots'] + filenames['oro_icon-global-det'])
        os.remove(path['base'] + path['plots'] + filenames['landsea_icon-global-det'])

    if no_title:
        if models_type == 'eu-eps' or models_type == 'eu-det':
            img_with_title = Image.open(path['base'] + path['plots'] + 'eu/' + filenames['composite'])
            img_no_title = img_with_title.crop((0, 100, 1500, 1300))
            img_no_title.save(path['base'] + path['plots'] + 'eu/' + filenames['composite'],'png')

        if models_type == 'global-eps' or models_type == 'global-det':
            img_with_title = Image.open(path['base'] + path['plots'] + 'global/' + filenames['composite'])
            img_no_title = img_with_title.crop((0, 100, 920, 1300))
            img_no_title.save(path['base'] + path['plots'] + 'global/' + filenames['composite'],'png')

    del filenames

    return

#######################################################################################################################
#######################################################################################################################
#######################################################################################################################
#######################################################################################################################
#######################################################################################################################
#######################################################################################################################
#######################################################################################################################
#######################################################################################################################

def plot_orography(pointname, model):

    point_city = which_grid_point(pointname, 'city')
    point_grid = which_grid_point(pointname, model)

    plot_name = '{}_orography_{}'.format(model, pointname)

    filenames = dict()
    path = dict(base = '/',
                plots = 'data/plots/point_information/')

    path['data'] = 'data/model_data/{}/invariant/'.format(model)
    path['grid'] = 'data/model_data/{}/grid/'.format(model)

    if model == 'icon-eu-eps':
        filenames['cells'] = 'icon_grid_0028_R02B07_N02.nc'
        filenames['oro'] = 'icon-eu-eps_europe_icosahedral_time-invariant_2021021700_hsurf.grib2'

    elif model == 'icon-global-eps':
        filenames['cells'] = 'icon_grid_0024_R02B06_G.nc'
        filenames['oro'] = 'icon-eps_global_icosahedral_time-invariant_2019010800_hsurf.grib2'

    elif model == 'icon-eu-det':
        filenames['lat'] = 'icon-eu_europe_regular-lat-lon_time-invariant_2019040800_RLAT.grib2'
        filenames['lon'] = 'icon-eu_europe_regular-lat-lon_time-invariant_2019040800_RLON.grib2'
        filenames['oro'] = 'icon-eu_europe_regular-lat-lon_time-invariant_2019061400_HSURF.grib2'

    elif model == 'icon-global-det':
        filenames['cells'] = 'icon_grid_0026_R03B07_G.nc'
        filenames['oro'] = 'icon_global_icosahedral_time-invariant_2019061400_HSURF.grib2'

    if model == 'icon-eu-det':
        with open(path['base'] + path['grid'] + filenames['lat'],'rb') as file:
            grib_id = eccodes.codes_grib_new_from_file(file)
            clat = eccodes.codes_get_array(grib_id, 'values')
            eccodes.codes_release(grib_id)
        with open(path['base'] + path['grid'] + filenames['lon'],'rb') as file:
            grib_id = eccodes.codes_grib_new_from_file(file)
            clon = eccodes.codes_get_array(grib_id, 'values')
            eccodes.codes_release(grib_id)
            #np.where(clon < 0.0, 360.0 - clon, clon)
    else:
        mpi_file = nc.Dataset(path['base'] + path['grid'] + filenames['cells'], 'r')
        vlat = mpi_file.variables['clat_vertices'][:].data * 180./np.pi
        vlon = mpi_file.variables['clon_vertices'][:].data * 180./np.pi
        clat = mpi_file.variables['clat'][:].data * 180./np.pi
        clon = mpi_file.variables['clon'][:].data * 180./np.pi
        mpi_file.close()

    with open(path['base'] + path['data'] + filenames['oro'],'rb') as file:
        grib_id = eccodes.codes_grib_new_from_file(file)
        oro_hgt = eccodes.codes_get_array(grib_id, 'values')
        eccodes.codes_release(grib_id)
    oro_hgt += 10    # offset of colormap

    if model == 'icon-eu-det':
        clon = np.reshape(clon,(1097, 657),'F')
        clat = np.reshape(clat,(1097, 657),'F')
        oro_hgt = np.reshape(oro_hgt,(1097, 657),'F')

    ########################################################################

    x_resolution        = 770
    y_resolution        = 770
    wks_res             = Ngl.Resources()
    wks_res.wkWidth     = x_resolution
    wks_res.wkHeight    = y_resolution

    wks_type    = 'png'
    wks         = Ngl.open_wks(wks_type, path['base'] + path['plots'] + plot_name, wks_res)
    resources   = Ngl.Resources()

    resources.mpProjection = 'Hammer'
    resources.mpCenterLonF = point_grid['lon']
    resources.mpCenterLatF = point_grid['lat']

    if model == 'icon-eu-det' or model == 'icon-global-det':
        radius = 100    # image radius in km around centered point
    else:
        radius = 200    # image radius in km around centered point

    cutout_plot = dict(
                        lat_min = point_grid['lat'] - radius / 111.2,
                        lat_max = point_grid['lat'] + radius / 111.2,
                        lon_min = point_grid['lon'] - radius / (111.2 * np.cos(point_grid['lat']*np.pi/180)),
                        lon_max = point_grid['lon'] + radius / (111.2 * np.cos(point_grid['lat']*np.pi/180)),
                       )

    resources.mpLimitMode   = 'latlon'
    resources.mpMinLonF     = cutout_plot['lon_min']
    resources.mpMaxLonF     = cutout_plot['lon_max']
    resources.mpMinLatF     = cutout_plot['lat_min']
    resources.mpMaxLatF     = cutout_plot['lat_max']

    resources.nglMaximize   = False
    resources.vpXF          = 0.05
    resources.vpYF          = 0.95
    resources.vpWidthF      = 0.7
    resources.vpHeightF     = 0.7

    ########################################################################

    # Turn on filled map areas:
    resources.mpFillOn = True

    # Set colors for [FillValue, Ocean, Land , InlandWater]:
    resources.mpFillColors = ['pink','blue','white','blue']

    resources.mpDataBaseVersion         = 'MediumRes'
    resources.mpDataSetName             = 'Earth..4'
    resources.mpOutlineBoundarySets     = 'national'

    resources.mpGeophysicalLineThicknessF   = 7.0 * x_resolution / 1000
    resources.mpNationalLineThicknessF      = 7.0 * x_resolution / 1000
    #resources.mpGridAndLimbDrawOrder        = 'postdraw'

    resources.mpGridAndLimbOn               = False
    #resources.mpLimbLineColor               = 'black'
    #resources.mpLimbLineThicknessF          = 10
    #resources.mpGridLineColor               = 'black'
    #resources.mpGridLineThicknessF          = 1.0
    #resources.mpGridSpacingF                = 1

    resources.mpPerimOn                     = True
    resources.mpPerimLineColor              = 'black'
    resources.mpPerimLineThicknessF         = 8.0 * x_resolution / 1000

    resources.tmXBOn = False
    resources.tmXTOn = False
    resources.tmYLOn = False
    resources.tmYROn = False

    resources.sfDataArray       = oro_hgt
    resources.sfXArray          = clon
    resources.sfYArray          = clat
    resources.sfMissingValueV   = 9999
    if model != 'icon-eu-det':
        resources.sfXCellBounds     = vlon
        resources.sfYCellBounds     = vlat

    resources.cnFillOn              = True
    if model == 'icon-eu-det':
        resources.cnFillMode            = 'RasterFill'
        resources.cnRasterSampleFactorF = 0.0
    else:
        resources.cnFillMode            = 'CellFill'
        resources.cnCellFillEdgeColor   = 'black'

    resources.cnMissingValFillColor = 'black'
    resources.cnFillPalette         = 'OceanLakeLandSnow' #'MPL_terrain'
    resources.cnLevelSelectionMode  = 'ManualLevels'

    minlevel                        = 0.0
    maxlevel                        = 1000.0
    numberoflevels                  = 250
    resources.cnMinLevelValF        = minlevel
    resources.cnMaxLevelValF        = maxlevel
    resources.cnLevelSpacingF       = (maxlevel - minlevel) / numberoflevels

    # Turn off contour lines and labels:
    resources.cnLinesOn         = False
    resources.cnLineLabelsOn    = False

    # Set resources for a nice label bar
    resources.lbLabelBarOn          = True
    resources.lbAutoManage          = False
    resources.lbOrientation         = 'vertical'
    resources.lbLabelOffsetF        = 0.05
    #resources.lbBoxMinorExtentF     = 0.2

    resources.lbLabelStride         = 50
    resources.lbLabelFontHeightF    = 0.02
    resources.lbBoxSeparatorLinesOn = False
    resources.lbBoxLineThicknessF   = 4.0
    #resources.lbBoxEndCapStyle     = 'TriangleBothEnds'
    resources.lbLabelAlignment      = 'BoxCenters'

    resources.lbTitleString         = 'meters above sea level'
    resources.lbTitleFontHeightF    = 0.02
    resources.lbTitlePosition       = 'Right'
    resources.lbTitleDirection      = 'Across'
    resources.lbTitleAngleF         = 90.0
    resources.lbTitleExtentF        = 0.1
    resources.lbTitleOffsetF        = 0.0

    resources.nglFrame = False
    plot = Ngl.contour_map(wks, oro_hgt, resources)

    ########################################################################

    text = 'Orography'
    x = 0.2
    y = 0.05

    text_res_1 = Ngl.Resources()
    text_res_1.txFontHeightF    = 0.03
    text_res_1.txJust           = 'BottomLeft'

    Ngl.text_ndc(wks, text, x, y, text_res_1)

    ########################################################################

    if model == 'icon-eu-eps':
        text = 'ICON-EU-EPS (20km)'
    elif model == 'icon-global-eps':
        text = 'ICON-Global-EPS (40km)'
    elif model == 'icon-eu-det':
        text = 'ICON-EU-DET (6.5km)'
    elif model == 'icon-global-det':
        text = 'ICON-Global-DET (13km)'

    x = 0.2
    y = 0.15

    text_res_2 = Ngl.Resources()
    text_res_2.txFontHeightF    = 0.03
    text_res_2.txJust           = 'BottomLeft'

    Ngl.text_ndc(wks, text, x, y, text_res_2)

    ########################################################################

    filter_distance = get_latlon_filter_distance(model)

    if model == 'icon-eu-det':
        lat_near = list(np.where(abs(clat.flatten('F') - point_grid['lat']) < filter_distance)[0])
        lon_near = list(np.where(abs(clon.flatten('F') - point_grid['lon']) < filter_distance)[0])
        id_near = list(set(lat_near).intersection(lon_near))
        id_near.sort()
        distances = np.sqrt( np.square(abs(clat.flatten('F')[id_near] - point_grid['lat']) * 111.2) \
                            + np.square(abs(clon.flatten('F')[id_near] - point_grid['lon']) * 111.2 \
                                                     * np.cos(point_grid['lat']*np.pi/180)) )
        index_nearest = id_near[np.argmin(distances)]

        linelats = np.zeros(5)
        linelats[0] = clat.flatten('F')[index_nearest] + 0.03125
        linelats[1] = clat.flatten('F')[index_nearest] + 0.03125
        linelats[2] = clat.flatten('F')[index_nearest] - 0.03125
        linelats[3] = clat.flatten('F')[index_nearest] - 0.03125
        linelats[4] = linelats[0]    # end point is start point
        linelons = np.zeros(5)
        linelons[0] = clon.flatten('F')[index_nearest] - 0.03125
        linelons[1] = clon.flatten('F')[index_nearest] + 0.03125
        linelons[2] = clon.flatten('F')[index_nearest] + 0.03125
        linelons[3] = clon.flatten('F')[index_nearest] - 0.03125
        linelons[4] = linelons[0]    # end point is start point

        polyline_res = Ngl.Resources()
        polyline_res.gsLineColor         = 'red'
        polyline_res.gsLineThicknessF    = 4.0
        Ngl.polyline(wks, plot, linelons, linelats, polyline_res)

    else:
        lat_near = list(np.where(abs(clat - point_grid['lat']) < filter_distance)[0])
        lon_near = list(np.where(abs(clon - point_grid['lon']) < filter_distance)[0])
        id_near = list(set(lat_near).intersection(lon_near))
        id_near.sort()
        distances = np.sqrt( np.square(abs(clat[id_near] - point_grid['lat']) * 111.2) \
                            + np.square(abs(clon[id_near] - point_grid['lon']) * 111.2 \
                                                     * np.cos(point_grid['lat']*np.pi/180)) )
        index_nearest = id_near[np.argmin(distances)]

        linelats = np.zeros(4)
        linelats[:3] = vlat[index_nearest]
        linelats[3] = vlat[index_nearest][0]    # end point is start point
        linelons = np.zeros(4)
        linelons[:3] = vlon[index_nearest]
        linelons[3] = vlon[index_nearest][0]

        polyline_res = Ngl.Resources()
        polyline_res.gsLineColor         = 'red'
        polyline_res.gsLineThicknessF    = 6.0
        Ngl.polyline(wks, plot, linelons, linelats, polyline_res)

    ########################################################################

    polymarker_res_1 = Ngl.Resources()
    polymarker_res_1.gsMarkerColor = 'black'
    polymarker_res_1.gsMarkerIndex = 16
    polymarker_res_1.gsMarkerSizeF = 0.015
    polymarker_res_1.gsMarkerThicknessF = 1
    Ngl.polymarker(wks, plot, point_city['lon'], point_city['lat'], polymarker_res_1)

    ########################################################################

    polymarker_res_2 = Ngl.Resources()
    polymarker_res_2.gsMarkerColor = 'white'
    polymarker_res_2.gsMarkerIndex = 16
    polymarker_res_2.gsMarkerSizeF = 0.010
    polymarker_res_2.gsMarkerThicknessF = 1
    Ngl.polymarker(wks, plot, point_city['lon'], point_city['lat'], polymarker_res_2)

    ########################################################################

    Ngl.frame(wks)

    del plot, wks_res, resources, polymarker_res_1, polymarker_res_2, text_res_1, text_res_2, clat, clon
    if model != 'icon-eu-det':
        del vlon, vlat, mpi_file
    Ngl.destroy(wks)

    plot_name += '.png'
    return plot_name

#######################################################################################################################
#######################################################################################################################
#######################################################################################################################
#######################################################################################################################
#######################################################################################################################
#######################################################################################################################
#######################################################################################################################
#######################################################################################################################

def plot_landsea(pointname, model):

    point_city = which_grid_point(pointname, 'city')
    point_grid = which_grid_point(pointname, model)

    plot_name = '{}_landsea_{}'.format(model, pointname)

    filenames = dict()
    path = dict(base = '/',
                plots = 'data/plots/point_information/')

    path['data'] = 'data/model_data/{}/invariant/'.format(model)
    path['grid'] = 'data/model_data/{}/grid/'.format(model)

    if model == 'icon-eu-eps':
        filenames['cells'] = 'icon_grid_0028_R02B07_N02.nc'
        filenames['landsea_fr'] = 'icon-eu-eps_europe_icosahedral_time-invariant_2018121312_fr_land.grib2'

    elif model == 'icon-global-eps':
        filenames['cells'] = 'icon_grid_0024_R02B06_G.nc'
        filenames['landsea_fr'] = 'icon-eps_global_icosahedral_time-invariant_2019010800_fr_land.grib2'

    elif model == 'icon-eu-det':
        filenames['lat'] = 'icon-eu_europe_regular-lat-lon_time-invariant_2019040800_RLAT.grib2'
        filenames['lon'] = 'icon-eu_europe_regular-lat-lon_time-invariant_2019040800_RLON.grib2'
        filenames['landsea_fr'] = 'icon-eu_europe_regular-lat-lon_time-invariant_2019061400_FR_LAND.grib2'

    elif model == 'icon-global-det':
        filenames['cells'] = 'icon_grid_0026_R03B07_G.nc'
        filenames['landsea_fr'] = 'icon_global_icosahedral_time-invariant_2019061400_FR_LAND.grib2'

    if model == 'icon-eu-det':
        with open(path['base'] + path['grid'] + filenames['lat'],'rb') as file:
            grib_id = eccodes.codes_grib_new_from_file(file)
            clat = eccodes.codes_get_array(grib_id, 'values')
            eccodes.codes_release(grib_id)
        with open(path['base'] + path['grid'] + filenames['lon'],'rb') as file:
            grib_id = eccodes.codes_grib_new_from_file(file)
            clon = eccodes.codes_get_array(grib_id, 'values')
            eccodes.codes_release(grib_id)
            #np.where(clon < 0.0, 360.0 - clon, clon)
    else:
        mpi_file = nc.Dataset(path['base'] + path['grid'] + filenames['cells'], 'r')
        vlat = mpi_file.variables['clat_vertices'][:].data * 180./np.pi
        vlon = mpi_file.variables['clon_vertices'][:].data * 180./np.pi
        clat = mpi_file.variables['clat'][:].data * 180./np.pi
        clon = mpi_file.variables['clon'][:].data * 180./np.pi
        mpi_file.close()

    with open(path['base'] + path['data'] + filenames['landsea_fr'],'rb') as file:
        grib_id = eccodes.codes_grib_new_from_file(file)
        fr_land = eccodes.codes_get_array(grib_id, 'values')
        eccodes.codes_release(grib_id)

    if model == 'icon-eu-det':
        clon = np.reshape(clon,(1097, 657),'F')
        clat = np.reshape(clat,(1097, 657),'F')
        fr_land = np.reshape(fr_land,(1097, 657),'F')

    ########################################################################

    x_resolution        = 770
    y_resolution        = 770
    wks_res             = Ngl.Resources()
    wks_res.wkWidth     = x_resolution
    wks_res.wkHeight    = y_resolution

    wks_type    = 'png'
    wks         = Ngl.open_wks(wks_type, path['base'] + path['plots'] + plot_name, wks_res)
    resources   = Ngl.Resources()

    resources.mpProjection = 'Hammer'
    resources.mpCenterLonF = point_grid['lon']
    resources.mpCenterLatF = point_grid['lat']

    if model == 'icon-eu-det' or model == 'icon-global-det':
        radius = 100    # image radius in km around centered point
    else:
        radius = 200    # image radius in km around centered point

    cutout_plot = dict(
                        lat_min = point_grid['lat'] - radius / 111.2,
                        lat_max = point_grid['lat'] + radius / 111.2,
                        lon_min = point_grid['lon'] - radius / (111.2 * np.cos(point_grid['lat']*np.pi/180)),
                        lon_max = point_grid['lon'] + radius / (111.2 * np.cos(point_grid['lat']*np.pi/180)),
                       )

    resources.mpLimitMode   = 'latlon'
    resources.mpMinLonF     = cutout_plot['lon_min']
    resources.mpMaxLonF     = cutout_plot['lon_max']
    resources.mpMinLatF     = cutout_plot['lat_min']
    resources.mpMaxLatF     = cutout_plot['lat_max']

    resources.nglMaximize   = False
    resources.vpXF          = 0.05
    resources.vpYF          = 0.95
    resources.vpWidthF      = 0.7
    resources.vpHeightF     = 0.7

    ########################################################################

    # Turn on filled map areas:
    resources.mpFillOn = True

    # Set colors for [FillValue, Ocean, Land , InlandWater]:
    resources.mpFillColors = ['pink','blue','white','blue']

    resources.mpDataBaseVersion         = 'MediumRes'
    resources.mpDataSetName             = 'Earth..4'
    resources.mpOutlineBoundarySets     = 'national'

    resources.mpGeophysicalLineThicknessF   = 7.0 * x_resolution / 1000
    resources.mpNationalLineThicknessF      = 7.0 * x_resolution / 1000
    #resources.mpGridAndLimbDrawOrder        = 'postdraw'

    resources.mpGridAndLimbOn               = False
    #resources.mpLimbLineColor               = 'black'
    #resources.mpLimbLineThicknessF          = 10
    #resources.mpGridLineColor               = 'black'
    #resources.mpGridLineThicknessF          = 1.0
    #resources.mpGridSpacingF                = 1

    resources.mpPerimOn                     = True
    resources.mpPerimLineColor              = 'black'
    resources.mpPerimLineThicknessF         = 8.0 * x_resolution / 1000

    resources.tmXBOn = False
    resources.tmXTOn = False
    resources.tmYLOn = False
    resources.tmYROn = False

    resources.sfDataArray       = 1.0 - fr_land
    resources.sfXArray          = clon
    resources.sfYArray          = clat
    resources.sfMissingValueV   = 9999
    if model != 'icon-eu-det':
        resources.sfXCellBounds     = vlon
        resources.sfYCellBounds     = vlat

    resources.cnFillOn              = True
    if model == 'icon-eu-det':
        resources.cnFillMode            = 'RasterFill'
        resources.cnRasterSampleFactorF = 0.0
    else:
        resources.cnFillMode            = 'CellFill'
        resources.cnCellFillEdgeColor   = 'black'

    resources.cnMissingValFillColor = 'black'
    resources.cnFillPalette         = 'MPL_GnBu'
    resources.cnLevelSelectionMode  = 'ManualLevels'

    minlevel                        = 0.0
    maxlevel                        = 1.0
    numberoflevels                  = 250
    resources.cnMinLevelValF        = minlevel
    resources.cnMaxLevelValF        = maxlevel
    resources.cnLevelSpacingF       = (maxlevel - minlevel) / numberoflevels

    # Turn off contour lines and labels:
    resources.cnLinesOn         = False
    resources.cnLineLabelsOn    = False

    # Set resources for a nice label bar
    resources.lbLabelBarOn          = True
    resources.lbAutoManage          = False
    resources.lbOrientation         = 'vertical'
    resources.lbLabelOffsetF        = 0.05
    #resources.lbBoxMinorExtentF     = 0.2

    resources.lbLabelStride         = 125
    resources.lbLabelFontHeightF    = 0.02
    resources.lbBoxSeparatorLinesOn = False
    resources.lbBoxLineThicknessF   = 4.0
    #resources.lbBoxEndCapStyle     = 'TriangleBothEnds'
    resources.lbLabelAlignment      = 'BoxCenters'

    resources.lbTitleString         = '0 = Land    1 = Sea'
    resources.lbTitleFontHeightF    = 0.02
    resources.lbTitlePosition       = 'Right'
    resources.lbTitleDirection      = 'Across'
    resources.lbTitleAngleF         = 90.0
    resources.lbTitleExtentF        = 0.1
    resources.lbTitleOffsetF        = 0.0

    resources.nglFrame = False
    plot = Ngl.contour_map(wks, 1.0-fr_land, resources)

    ########################################################################

    text = 'Ocean+Lake Fraction'
    x = 0.1
    y = 0.02

    text_res_1 = Ngl.Resources()
    text_res_1.txFontHeightF    = 0.03
    text_res_1.txJust           = 'BottomLeft'

    Ngl.text_ndc(wks, text, x, y, text_res_1)

    ########################################################################

    text = 'Point Information about {}'.format(pointname.replace('_', ' '))

    x = 0.02
    y = 0.14

    text_res_2 = Ngl.Resources()
    text_res_2.txFontHeightF    = 0.032
    text_res_2.txJust           = 'BottomLeft'

    Ngl.text_ndc(wks, text, x, y, text_res_2)

    ########################################################################

    filter_distance = get_latlon_filter_distance(model)

    if model == 'icon-eu-det':
        lat_near = list(np.where(abs(clat.flatten('F') - point_grid['lat']) < filter_distance)[0])
        lon_near = list(np.where(abs(clon.flatten('F') - point_grid['lon']) < filter_distance)[0])
        id_near = list(set(lat_near).intersection(lon_near))
        id_near.sort()
        distances = np.sqrt( np.square(abs(clat.flatten('F')[id_near] - point_grid['lat']) * 111.2) \
                            + np.square(abs(clon.flatten('F')[id_near] - point_grid['lon']) * 111.2 \
                                                     * np.cos(point_grid['lat']*np.pi/180)) )
        index_nearest = id_near[np.argmin(distances)]

        linelats = np.zeros(5)
        linelats[0] = clat.flatten('F')[index_nearest] + 0.03125
        linelats[1] = clat.flatten('F')[index_nearest] + 0.03125
        linelats[2] = clat.flatten('F')[index_nearest] - 0.03125
        linelats[3] = clat.flatten('F')[index_nearest] - 0.03125
        linelats[4] = linelats[0]    # end point is start point
        linelons = np.zeros(5)
        linelons[0] = clon.flatten('F')[index_nearest] - 0.03125
        linelons[1] = clon.flatten('F')[index_nearest] + 0.03125
        linelons[2] = clon.flatten('F')[index_nearest] + 0.03125
        linelons[3] = clon.flatten('F')[index_nearest] - 0.03125
        linelons[4] = linelons[0]    # end point is start point

        polyline_res = Ngl.Resources()
        polyline_res.gsLineColor         = 'red'
        polyline_res.gsLineThicknessF    = 4.0
        Ngl.polyline(wks, plot, linelons, linelats, polyline_res)

    else:
        lat_near = list(np.where(abs(clat - point_grid['lat']) < filter_distance)[0])
        lon_near = list(np.where(abs(clon - point_grid['lon']) < filter_distance)[0])
        id_near = list(set(lat_near).intersection(lon_near))
        id_near.sort()
        distances = np.sqrt( np.square(abs(clat[id_near] - point_grid['lat']) * 111.2) \
                            + np.square(abs(clon[id_near] - point_grid['lon']) * 111.2 \
                                                     * np.cos(point_grid['lat']*np.pi/180)) )
        index_nearest = id_near[np.argmin(distances)]

        linelats = np.zeros(4)
        linelats[:3] = vlat[index_nearest]
        linelats[3] = vlat[index_nearest][0]    # end point is start point
        linelons = np.zeros(4)
        linelons[:3] = vlon[index_nearest]
        linelons[3] = vlon[index_nearest][0]

        polyline_res = Ngl.Resources()
        polyline_res.gsLineColor         = 'red'
        polyline_res.gsLineThicknessF    = 6.0
        Ngl.polyline(wks, plot, linelons, linelats, polyline_res)

    ########################################################################

    polymarker_res_1 = Ngl.Resources()
    polymarker_res_1.gsMarkerColor = 'black'
    polymarker_res_1.gsMarkerIndex = 16
    polymarker_res_1.gsMarkerSizeF = 0.015
    polymarker_res_1.gsMarkerThicknessF = 1
    Ngl.polymarker(wks, plot, point_city['lon'], point_city['lat'], polymarker_res_1)

    ########################################################################

    polymarker_res_2 = Ngl.Resources()
    polymarker_res_2.gsMarkerColor = 'white'
    polymarker_res_2.gsMarkerIndex = 16
    polymarker_res_2.gsMarkerSizeF = 0.010
    polymarker_res_2.gsMarkerThicknessF = 1
    Ngl.polymarker(wks, plot, point_city['lon'], point_city['lat'], polymarker_res_2)

    ########################################################################

    Ngl.frame(wks)

    del plot, wks_res, resources, polymarker_res_1, polymarker_res_2, text_res_1, text_res_2, clat, clon
    if model != 'icon-eu-det':
        del vlon, vlat, mpi_file
    Ngl.destroy(wks)

    plot_name += '.png'
    return plot_name

########################################################################
########################################################################
########################################################################


from src import cvutils
from PIL import Image
import numpy as np
import pandas as pd
import skimage
import os

class CSConfig():
    '''
    Define your constants below.

    IS_CODEX_OUTPUT - CODEX output files have special filenames that allow outputs to contain more metadata about absolute positions, regs, and other things.
    Set this to false if not using the filename convention.  Follow the naming convention in the install/run page on the CellSeg website to output this metadata for non-CODEX images.
    input_path - path to directory containing image folders and config files.
    output_path - name of directory to save output in. If directory does not exist, CellSeg creates directory.
    DIRECTORY_PATH - directory that contains the images to be processed.
    CHANNEL_PATH - path to your channels file (usually called channelnames.txt). Required for tif images with more than 3 channels, or 4D TIF images.
    NUCLEAR_CHANNEL_NAME - name of nuclear stain (corresponding to channelnames.txt).  Case sensitive.  Only used for tif images with more than 3 channels, or 4D TIF images.
    GROWTH_PIXELS - number of pixels from which to grow out from the nucleus to define a cell boundary.  Change based on tissue types.
    BOOST - multiplier with which to boost the pixels of the nuclear stain before inference.
    OVERLAP - pixel overlap between adjacent subtiles during nuclei segmentation. Must be divisible by 2 and should be greater than expected average cell diameter.
    THRESHOLD - minimum size (in pixel area) of detected objects to keep.
    INCREASE_FACTOR - Upscale the image by this factor for segmentation. Default is 2.5x, decided by visual inspection after training on the Kaggle dataset.

    ---------OUTPUT PATHS-------------
    IMAGEJ_OUTPUT_PATH - path to output imagej .txt files
    QUANTIFICATION_OUTPUT_PATH - path to output .csv and .fcs quantifications
    VISUAL_OUTPUT_PATH - path to output visual masks as pngs
    '''

    # change these!
    IS_CODEX_OUTPUT = True
    NUCLEAR_CHANNEL_NAME = 'DRAQ5'
    NUCLEAR_CYCLE = 18
    NUCLEAR_CHANNEL = 4
    NUCLEAR_SLICE = 0

    if 0: # ADVANCED! compute a nuclear stain channel on-the-fly
      # synthesized nuclear image = (DRAQ5 * 1.5) - (cy02_ch02 * 0.5)
      synthesize_nuclear_image = lambda self, x: \
      x[:,:,(self.NUCLEAR_CYCLE-1)*4+(self.NUCLEAR_CHANNEL-1)] * 1.5 - x[:,:,(2-1)*4+(2-1)] * 0.5
    else:
      synthesize_nuclear_image = None # to disable the synthesized nuclear image

    # nuclei detection parameters
    BOOST = 1 # brightness multiplier for nuclear channel during segmentation
    MIN_AREA = 40 # minimum area in pixels for a detected object to be kept
    INCREASE_FACTOR = 3.0 # size scaling factor for object detection - increase if you have smaller cells or low magnification and vice-versa

    # quantification parameters:
    # dilate nuclear mask before quantification step
    GROWTH_PIXELS_PLANE = 0 # dilate cells on the plane_mask [0,1,1.5,2,2.5, ...]

    # adjacency quantification computes spillover between touching cells based on dilated cell masks
    output_adjacency_quant = True # 'uncompensated' and 'compensated' outputs.  The compensated version has reduced spillover between neighboring cells
    GROWTH_PIXELS_QUANT_A = 0 # dilate cells during adjacency quantification [0,1,1.5,2,2.5, ...]

    # morphological quantification of signal on full/nuclear/peri-nuclear regions of cells
    # nuclear masks are first dilated by the growth parameter to form the full area
    # the interior and border regions are defined by the erosion of the full mask by the border thickness
    output_morphological_quant = True # 'loose' and 'tight' quantification outputs.  The tight version has less spillover between neighboring cells
    GROWTH_PIXELS_QUANT_M = 2.5 # dilate cells during morphological quantification [0,0.5,1,1.5,2,2.5, ...]
    BORDER_PIXELS_QUANT_M = 2.0 # thickness of border during morphological quantification [1,1.5,2,2.5, ...]

    # don't change these values unless you know what you're doing!
    OUTPUT_METHOD = 'all'
    OVERLAP = 80 # pixels of overlap between subdivided tiles used in the nuclei detection step - should be more than the max diameter of a nucleus
    GROWTH_PIXELS_MASKS = 0 # initial erosion or dilation of masks [0,1,1.5,2,...] or negative

    # probably don't change this, except the valid image extensions when working with unique extensions
    def __init__(self, input_path, increase_factor=None, growth_plane=None, growth_quant_A=None, growth_quant_M=None, border_quant_M=None, nuclear_cycle=None, nuclear_channel=None, nuclear_slice=None):
      if increase_factor is not None: self.INCREASE_FACTOR = increase_factor
      if growth_plane is not None: self.GROWTH_PIXELS_PLANE = growth_plane
      if growth_quant_A is not None: self.GROWTH_PIXELS_QUANT_A = growth_quant_A
      if growth_quant_M is not None: self.GROWTH_PIXELS_QUANT_M = growth_quant_M
      if border_quant_M is not None: self.BORDER_PIXELS_QUANT_M = border_quant_M

      if nuclear_cycle   is not None: self.NUCLEAR_CYCLE   = nuclear_cycle
      if nuclear_channel is not None: self.NUCLEAR_CHANNEL = nuclear_channel
      if nuclear_slice   is not None: self.NUCLEAR_SLICE   = nuclear_slice

      if not os.path.exists(input_path):
        raise NameError("Error: input directory '{}' doesn't exist!".format(input_path))

      output_path = os.path.join(input_path, 'processed/segm/segm-1')

      self.DIRECTORY_PATH = os.path.join(input_path, 'stitched')
      self.CHANNEL_PATH = os.path.join(input_path, 'channelnames.txt')

      self.IMAGEJ_OUTPUT_PATH = os.path.join(output_path, 'imagej_files')
      self.QUANTIFICATION_OUTPUT_PATH = os.path.join(output_path, 'fcs')
      self.VISUAL_OUTPUT_PATH = os.path.join(output_path, 'masks')

      def trymakedirs(d):
        if not os.path.exists(d): os.makedirs(d, exist_ok=True)

      trymakedirs(self.IMAGEJ_OUTPUT_PATH)
      trymakedirs(self.QUANTIFICATION_OUTPUT_PATH + '/uncompensated')
      trymakedirs(self.QUANTIFICATION_OUTPUT_PATH + '/compensated')
      trymakedirs(self.QUANTIFICATION_OUTPUT_PATH + '/tight')
      trymakedirs(self.QUANTIFICATION_OUTPUT_PATH + '/loose')
      trymakedirs(self.VISUAL_OUTPUT_PATH)

      filename_filter = lambda filename: f'z{self.NUCLEAR_SLICE:02d}' in filename

      self.CHANNEL_NAMES = pd.read_csv(self.CHANNEL_PATH, sep='\t', header=None).values[:, 0]

      VALID_IMAGE_EXTENSIONS = (f'_z{self.NUCLEAR_SLICE:02d}_cy{self.NUCLEAR_CYCLE:02d}_ch{self.NUCLEAR_CHANNEL:02d}.tif')
      self.FILENAMES = sorted([f for f in os.listdir(self.DIRECTORY_PATH) if f.endswith(VALID_IMAGE_EXTENSIONS) and not f.startswith('.')])
      if len(self.FILENAMES) < 1:
        raise NameError("No image files found.  Make sure you are pointing to the right directory '{}'".format(self.DIRECTORY_PATH))

      reference_image_path = os.path.join(self.DIRECTORY_PATH, self.FILENAMES[0])

      self.N_DIMS, self.EXT, self.DTYPE, self.SHAPE, self.READ_METHOD = cvutils.meta_from_image(reference_image_path, filename_filter)


#!/usr/bin/env python

"""Generate bottom-up attention features as a tsv file. Can use multiple gpus, each produces a
   separate tsv file that can be merged later (e.g. by using merge_tsv function).
   Modify the load_image_ids script as necessary for your data location. """

# Example:
# ./tools/generate_tsv.py --gpu 0,1,2,3,4,5,6,7 --cfg experiments/cfgs/faster_rcnn_end2end_resnet.yml --def models/vg/ResNet-101/faster_rcnn_end2end/test.prototxt --out test2014_resnet101_faster_rcnn_genome.tsv --net data/faster_rcnn_models/resnet101_faster_rcnn_final.caffemodel --split coco_test2014

## N.B.: Several paths need to be in the PYTHONPATH, we add them here so we make sure that the scripts is going ton use the correct imports
import sys
sys.path.insert(0, './caffe/python/')
sys.path.insert(0, './lib/')
sys.path.insert(0, './tools/')

import argparse
import caffe
import csv
import cv2
import gzip
import h5py
import json
import numpy as np
import os
import pprint
import random
from fast_rcnn.config import cfg, cfg_from_file
from fast_rcnn.nms_wrapper import nms
from fast_rcnn.test import im_detect, _get_blobs
from fast_rcnn.config import cfg_from_list
import glob
from tqdm import tqdm

csv.field_size_limit(sys.maxsize)

guesswhat_files = {
    "train": "guesswhat.train.jsonl.gz",
    "valid": "guesswhat.valid.jsonl.gz",
    "test": "guesswhat.test.jsonl.gz"
}
FIELDNAMES = ['image_id', 'image_w', 'image_h', 'num_boxes', 'boxes', 'features']

# Settings for the number of features per image. To re-create pretrained features with 36 features
# per image, set both values to 36.
MIN_BOXES = 36
MAX_BOXES = 36
FEATURE_MAP_SIZE = 2048

def read_raw_dataset_games(file_path):
    if file_path.endswith(".gz"):
        with gzip.open(file_path) as f:
            for line in f:
                line = line.decode("utf-8")
                game = json.loads(line.strip("\n"))

                yield game
    else:
        with open(file_path) as f:
            for line in f:
                game = json.loads(line.strip("\n"))

                yield game

def load_image_ids(data_path=None, img_path=None):
    ''' Load a list of (path,image_id tuples). Modify this to suit your data locations. '''
    split = []

    loaded_images = set()

    for game in tqdm(read_raw_dataset_games(data_path)):
        image_id = game["image"]["id"]
        if image_id not in loaded_images:
            filepath = os.path.join(img_path, str(image_id) + '.jpg')
            split.append((filepath, image_id))
            loaded_images.add(image_id)

    return split

def get_detections_from_im(net, im_file, image_id, conf_thresh=0.2):
    im = cv2.imread(im_file)
    scores, boxes, attr_scores, rel_scores = im_detect(net, im)

    # Keep the original boxes, don't worry about the regresssion bbox outputs
    rois = net.blobs['rois'].data.copy()
    # unscale back to raw image space
    blobs, im_scales = _get_blobs(im, None)

    cls_boxes = rois[:, 1:5] / im_scales[0]
    cls_prob = net.blobs['cls_prob'].data
    pool5 = net.blobs['pool5_flat'].data

    # Keep only the best detections
    max_conf = np.zeros((rois.shape[0]))
    for cls_ind in range(1, cls_prob.shape[1]):
        cls_scores = scores[:, cls_ind]
        dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32)
        keep = np.array(nms(dets, cfg.TEST.NMS))
        max_conf[keep] = np.where(cls_scores[keep] > max_conf[keep], cls_scores[keep], max_conf[keep])

    keep_boxes = np.where(max_conf >= conf_thresh)[0]
    if len(keep_boxes) < MIN_BOXES:
        keep_boxes = np.argsort(max_conf)[::-1][:MIN_BOXES]
    elif len(keep_boxes) > MAX_BOXES:
        keep_boxes = np.argsort(max_conf)[::-1][:MAX_BOXES]

    return {
        'image_id': image_id,
        'image_h': np.size(im, 0),
        'image_w': np.size(im, 1),
        'num_boxes': len(keep_boxes),
        'boxes': cls_boxes[keep_boxes],
        'features': pool5[keep_boxes]
    }

def parse_args():
    """
    Parse input arguments
    """
    parser = argparse.ArgumentParser(description='Generate bbox output from a Fast R-CNN network')
    parser.add_argument('--gpu', dest='gpu_id', help='GPU id to use',
                        default=0, type=int)
    parser.add_argument('--def', dest='prototxt',
                        help='prototxt file defining the network',
                        required=True, type=str)
    parser.add_argument('--net', dest='caffemodel',
                        help='model to use',
                        required=True, type=str)
    parser.add_argument('--output_path', dest='output_path',
                        help='output filepath',
                        required=True, type=str)
    parser.add_argument('--guesswhat_folder', dest='guesswhat_folder',
                        required=True, help='Guesswhat folder',type=str)
    parser.add_argument('--img_dir', required=True, dest='img_dir', help='MSCOCO image directory', type=str)
    parser.add_argument('--cfg', dest='cfg_file',
                        help='optional config file', default=None, type=str)
    parser.add_argument('--set', dest='set_cfgs',
                        help='set config keys', default=None,
                        nargs=argparse.REMAINDER)

    if len(sys.argv) == 1:
        parser.print_help()
        sys.exit(1)

    args = parser.parse_args()
    return args

def generate_features(gpu_id, prototxt, weights, image_ids, boxes_dataset, features_dataset):
    if gpu_id >= 0:
        caffe.set_mode_gpu()
        caffe.set_device(gpu_id)
    net = caffe.Net(prototxt, caffe.TEST, weights=weights)

    for i, (im_file, image_id) in enumerate(tqdm(image_ids)):
        proc_image = get_detections_from_im(net, im_file, image_id)
        boxes_dataset[i] = proc_image["boxes"]
        features_dataset[i] = proc_image["features"]

if __name__ == '__main__':

    args = parse_args()

    print('Called with args:')
    print(args)

    if args.cfg_file is not None:
        cfg_from_file(args.cfg_file)
    if args.set_cfgs is not None:
        cfg_from_list(args.set_cfgs)

    gpu_id = args.gpu_id

    print('Using config:')
    pprint.pprint(cfg)
    assert cfg.TEST.HAS_RPN

    for split_name, split_file in guesswhat_files.items():
        subdir = "val" if split_name == "valid" else split_name
        feat_h5_file = h5py.File(os.path.join(args.output_path, split_name) + "_features.h5", 'w')
        idx2img = []
        image_index = 0

        image_ids = load_image_ids(os.path.join(args.guesswhat_folder, split_file), args.img_dir)
        num_images = len(image_ids)
        print("Number of images for {} set: {}".format(split_name, num_images))

        '''
        The original FastRCNN feature files contained the following fields:
        {
            'image_id': image_id,
            'image_h': np.size(im, 0),
            'image_w': np.size(im, 1),
            'num_boxes': len(keep_boxes),
            'boxes': base64.b64encode(cls_boxes[keep_boxes]),
            'features': base64.b64encode(pool5[keep_boxes])
        }
        '''

        boxes_dataset = feat_h5_file.create_dataset(
            name="boxes",
            dtype='float32',
            shape=(num_images, MAX_BOXES, 4)
        )

        features_dataset = feat_h5_file.create_dataset(
            name="features",
            dtype='float32',
            shape=(num_images, MAX_BOXES, FEATURE_MAP_SIZE)
        )

        feat_h5_file.create_dataset(name="idx2img", dtype='int64', data=np.array([image_id for (_, image_id) in image_ids]))

        random.seed(10)

        generate_features(
            gpu_id,
            args.prototxt,
            args.caffemodel,
            image_ids,
            boxes_dataset,
            features_dataset
        )


def selection_15():

    # Library import
    import numpy
    import matplotlib
    import matplotlib.pyplot   as plt
    import matplotlib.gridspec as gridspec

    # Library version
    matplotlib_version = matplotlib.__version__
    numpy_version      = numpy.__version__

    # Histo binning
    xBinning = numpy.linspace(0.0,8000.0,81,endpoint=True)

    # Creating data sequence: middle of each bin
    xData = numpy.array([50.0,150.0,250.0,350.0,450.0,550.0,650.0,750.0,850.0,950.0,1050.0,1150.0,1250.0,1350.0,1450.0,1550.0,1650.0,1750.0,1850.0,1950.0,2050.0,2150.0,2250.0,2350.0,2450.0,2550.0,2650.0,2750.0,2850.0,2950.0,3050.0,3150.0,3250.0,3350.0,3450.0,3550.0,3650.0,3750.0,3850.0,3950.0,4050.0,4150.0,4250.0,4350.0,4450.0,4550.0,4650.0,4750.0,4850.0,4950.0,5050.0,5150.0,5250.0,5350.0,5450.0,5550.0,5650.0,5750.0,5850.0,5950.0,6050.0,6150.0,6250.0,6350.0,6450.0,6550.0,6650.0,6750.0,6850.0,6950.0,7050.0,7150.0,7250.0,7350.0,7450.0,7550.0,7650.0,7750.0,7850.0,7950.0])

    # Creating weights for histo: y16_DELTAR_0
    y16_DELTAR_0_weights = numpy.array([1071.86803796,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0])

    # Creating weights for histo: y16_DELTAR_1
    y16_DELTAR_1_weights = numpy.array([364.169510928,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0])

    # Creating weights for histo: y16_DELTAR_2
    y16_DELTAR_2_weights = numpy.array([2253.73790982,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0])

    # Creating weights for histo: y16_DELTAR_3
    y16_DELTAR_3_weights = numpy.array([2038.92927727,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0])

    # Creating weights for histo: y16_DELTAR_4
    y16_DELTAR_4_weights = numpy.array([279.398832451,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0])

    # Creating weights for histo: y16_DELTAR_5
    y16_DELTAR_5_weights = numpy.array([47.9097622224,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0])

    # Creating weights for histo: y16_DELTAR_6
    y16_DELTAR_6_weights = numpy.array([12.0629169563,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0])

    # Creating weights for histo: y16_DELTAR_7
    y16_DELTAR_7_weights = numpy.array([0.677058371215,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0])

    # Creating weights for histo: y16_DELTAR_8
    y16_DELTAR_8_weights = numpy.array([0.0488740483587,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0])

    # Creating weights for histo: y16_DELTAR_9
    y16_DELTAR_9_weights = numpy.array([1767.26384769,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0])

    # Creating weights for histo: y16_DELTAR_10
    y16_DELTAR_10_weights = numpy.array([8038.27777846,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0])

    # Creating weights for histo: y16_DELTAR_11
    y16_DELTAR_11_weights = numpy.array([6955.8994415,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0])

    # Creating weights for histo: y16_DELTAR_12
    y16_DELTAR_12_weights = numpy.array([638.223054462,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0])

    # Creating weights for histo: y16_DELTAR_13
    y16_DELTAR_13_weights = numpy.array([89.9211594404,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0])

    # Creating weights for histo: y16_DELTAR_14
    y16_DELTAR_14_weights = numpy.array([21.8553471635,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0])

    # Creating weights for histo: y16_DELTAR_15
    y16_DELTAR_15_weights = numpy.array([1.31303018851,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0])

    # Creating weights for histo: y16_DELTAR_16
    y16_DELTAR_16_weights = numpy.array([0.0877527020769,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0])

    # Creating a new Canvas
    fig   = plt.figure(figsize=(12,6),dpi=80)
    frame = gridspec.GridSpec(1,1,right=0.7)
    pad   = fig.add_subplot(frame[0])

    # Creating a new Stack
    pad.hist(x=xData, bins=xBinning, weights=y16_DELTAR_0_weights+y16_DELTAR_1_weights+y16_DELTAR_2_weights+y16_DELTAR_3_weights+y16_DELTAR_4_weights+y16_DELTAR_5_weights+y16_DELTAR_6_weights+y16_DELTAR_7_weights+y16_DELTAR_8_weights+y16_DELTAR_9_weights+y16_DELTAR_10_weights+y16_DELTAR_11_weights+y16_DELTAR_12_weights+y16_DELTAR_13_weights+y16_DELTAR_14_weights+y16_DELTAR_15_weights+y16_DELTAR_16_weights,\
             label="$bg\_dip\_1600\_inf$", histtype="step", rwidth=1.0,\
             color=None, edgecolor="#e5e5e5", linewidth=1, linestyle="solid",\
             bottom=None, cumulative=False, normed=False, align="mid", orientation="vertical")

    pad.hist(x=xData, bins=xBinning, weights=y16_DELTAR_0_weights+y16_DELTAR_1_weights+y16_DELTAR_2_weights+y16_DELTAR_3_weights+y16_DELTAR_4_weights+y16_DELTAR_5_weights+y16_DELTAR_6_weights+y16_DELTAR_7_weights+y16_DELTAR_8_weights+y16_DELTAR_9_weights+y16_DELTAR_10_weights+y16_DELTAR_11_weights+y16_DELTAR_12_weights+y16_DELTAR_13_weights+y16_DELTAR_14_weights+y16_DELTAR_15_weights,\
             label="$bg\_dip\_1200\_1600$", histtype="step", rwidth=1.0,\
             color=None, edgecolor="#f2f2f2", linewidth=1, linestyle="solid",\
             bottom=None, cumulative=False, normed=False, align="mid", orientation="vertical")

    pad.hist(x=xData, bins=xBinning, weights=y16_DELTAR_0_weights+y16_DELTAR_1_weights+y16_DELTAR_2_weights+y16_DELTAR_3_weights+y16_DELTAR_4_weights+y16_DELTAR_5_weights+y16_DELTAR_6_weights+y16_DELTAR_7_weights+y16_DELTAR_8_weights+y16_DELTAR_9_weights+y16_DELTAR_10_weights+y16_DELTAR_11_weights+y16_DELTAR_12_weights+y16_DELTAR_13_weights+y16_DELTAR_14_weights,\
             label="$bg\_dip\_800\_1200$", histtype="step", rwidth=1.0,\
             color=None, edgecolor="#ccc6aa", linewidth=1, linestyle="solid",\
             bottom=None, cumulative=False, normed=False, align="mid", orientation="vertical")

    pad.hist(x=xData, bins=xBinning, weights=y16_DELTAR_0_weights+y16_DELTAR_1_weights+y16_DELTAR_2_weights+y16_DELTAR_3_weights+y16_DELTAR_4_weights+y16_DELTAR_5_weights+y16_DELTAR_6_weights+y16_DELTAR_7_weights+y16_DELTAR_8_weights+y16_DELTAR_9_weights+y16_DELTAR_10_weights+y16_DELTAR_11_weights+y16_DELTAR_12_weights+y16_DELTAR_13_weights,\
             label="$bg\_dip\_600\_800$", histtype="step", rwidth=1.0,\
             color=None, edgecolor="#ccc6aa", linewidth=1, linestyle="solid",\
             bottom=None, cumulative=False, normed=False, align="mid", orientation="vertical")

    pad.hist(x=xData, bins=xBinning, weights=y16_DELTAR_0_weights+y16_DELTAR_1_weights+y16_DELTAR_2_weights+y16_DELTAR_3_weights+y16_DELTAR_4_weights+y16_DELTAR_5_weights+y16_DELTAR_6_weights+y16_DELTAR_7_weights+y16_DELTAR_8_weights+y16_DELTAR_9_weights+y16_DELTAR_10_weights+y16_DELTAR_11_weights+y16_DELTAR_12_weights,\
             label="$bg\_dip\_400\_600$", histtype="step", rwidth=1.0,\
             color=None, edgecolor="#c1bfa8", linewidth=1, linestyle="solid",\
             bottom=None, cumulative=False, normed=False, align="mid", orientation="vertical")

    pad.hist(x=xData, bins=xBinning, weights=y16_DELTAR_0_weights+y16_DELTAR_1_weights+y16_DELTAR_2_weights+y16_DELTAR_3_weights+y16_DELTAR_4_weights+y16_DELTAR_5_weights+y16_DELTAR_6_weights+y16_DELTAR_7_weights+y16_DELTAR_8_weights+y16_DELTAR_9_weights+y16_DELTAR_10_weights+y16_DELTAR_11_weights,\
             label="$bg\_dip\_200\_400$", histtype="step", rwidth=1.0,\
             color=None, edgecolor="#bab5a3", linewidth=1, linestyle="solid",\
             bottom=None, cumulative=False, normed=False, align="mid", orientation="vertical")

    pad.hist(x=xData, bins=xBinning, weights=y16_DELTAR_0_weights+y16_DELTAR_1_weights+y16_DELTAR_2_weights+y16_DELTAR_3_weights+y16_DELTAR_4_weights+y16_DELTAR_5_weights+y16_DELTAR_6_weights+y16_DELTAR_7_weights+y16_DELTAR_8_weights+y16_DELTAR_9_weights+y16_DELTAR_10_weights,\
             label="$bg\_dip\_100\_200$", histtype="step", rwidth=1.0,\
             color=None, edgecolor="#b2a596", linewidth=1, linestyle="solid",\
             bottom=None, cumulative=False, normed=False, align="mid", orientation="vertical")

    pad.hist(x=xData, bins=xBinning, weights=y16_DELTAR_0_weights+y16_DELTAR_1_weights+y16_DELTAR_2_weights+y16_DELTAR_3_weights+y16_DELTAR_4_weights+y16_DELTAR_5_weights+y16_DELTAR_6_weights+y16_DELTAR_7_weights+y16_DELTAR_8_weights+y16_DELTAR_9_weights,\
             label="$bg\_dip\_0\_100$", histtype="step", rwidth=1.0,\
             color=None, edgecolor="#b7a39b", linewidth=1, linestyle="solid",\
             bottom=None, cumulative=False, normed=False, align="mid", orientation="vertical")

    pad.hist(x=xData, bins=xBinning, weights=y16_DELTAR_0_weights+y16_DELTAR_1_weights+y16_DELTAR_2_weights+y16_DELTAR_3_weights+y16_DELTAR_4_weights+y16_DELTAR_5_weights+y16_DELTAR_6_weights+y16_DELTAR_7_weights+y16_DELTAR_8_weights,\
             label="$bg\_vbf\_1600\_inf$", histtype="step", rwidth=1.0,\
             color=None, edgecolor="#ad998c", linewidth=1, linestyle="solid",\
             bottom=None, cumulative=False, normed=False, align="mid", orientation="vertical")

    pad.hist(x=xData, bins=xBinning, weights=y16_DELTAR_0_weights+y16_DELTAR_1_weights+y16_DELTAR_2_weights+y16_DELTAR_3_weights+y16_DELTAR_4_weights+y16_DELTAR_5_weights+y16_DELTAR_6_weights+y16_DELTAR_7_weights,\
             label="$bg\_vbf\_1200\_1600$", histtype="step", rwidth=1.0,\
             color=None, edgecolor="#9b8e82", linewidth=1, linestyle="solid",\
             bottom=None, cumulative=False, normed=False, align="mid", orientation="vertical")

    pad.hist(x=xData, bins=xBinning, weights=y16_DELTAR_0_weights+y16_DELTAR_1_weights+y16_DELTAR_2_weights+y16_DELTAR_3_weights+y16_DELTAR_4_weights+y16_DELTAR_5_weights+y16_DELTAR_6_weights,\
             label="$bg\_vbf\_800\_1200$", histtype="step", rwidth=1.0,\
             color=None, edgecolor="#876656", linewidth=1, linestyle="solid",\
             bottom=None, cumulative=False, normed=False, align="mid", orientation="vertical")

    pad.hist(x=xData, bins=xBinning, weights=y16_DELTAR_0_weights+y16_DELTAR_1_weights+y16_DELTAR_2_weights+y16_DELTAR_3_weights+y16_DELTAR_4_weights+y16_DELTAR_5_weights,\
             label="$bg\_vbf\_600\_800$", histtype="step", rwidth=1.0,\
             color=None, edgecolor="#afcec6", linewidth=1, linestyle="solid",\
             bottom=None, cumulative=False, normed=False, align="mid", orientation="vertical")

    pad.hist(x=xData, bins=xBinning, weights=y16_DELTAR_0_weights+y16_DELTAR_1_weights+y16_DELTAR_2_weights+y16_DELTAR_3_weights+y16_DELTAR_4_weights,\
             label="$bg\_vbf\_400\_600$", histtype="step", rwidth=1.0,\
             color=None, edgecolor="#84c1a3", linewidth=1, linestyle="solid",\
             bottom=None, cumulative=False, normed=False, align="mid", orientation="vertical")

    pad.hist(x=xData, bins=xBinning, weights=y16_DELTAR_0_weights+y16_DELTAR_1_weights+y16_DELTAR_2_weights+y16_DELTAR_3_weights,\
             label="$bg\_vbf\_200\_400$", histtype="step", rwidth=1.0,\
             color=None, edgecolor="#89a8a0", linewidth=1, linestyle="solid",\
             bottom=None, cumulative=False, normed=False, align="mid", orientation="vertical")

    pad.hist(x=xData, bins=xBinning, weights=y16_DELTAR_0_weights+y16_DELTAR_1_weights+y16_DELTAR_2_weights,\
             label="$bg\_vbf\_100\_200$", histtype="step", rwidth=1.0,\
             color=None, edgecolor="#829e8c", linewidth=1, linestyle="solid",\
             bottom=None, cumulative=False, normed=False, align="mid", orientation="vertical")

    pad.hist(x=xData, bins=xBinning, weights=y16_DELTAR_0_weights+y16_DELTAR_1_weights,\
             label="$bg\_vbf\_0\_100$", histtype="step", rwidth=1.0,\
             color=None, edgecolor="#adbcc6", linewidth=1, linestyle="solid",\
             bottom=None, cumulative=False, normed=False, align="mid", orientation="vertical")

    pad.hist(x=xData, bins=xBinning, weights=y16_DELTAR_0_weights,\
             label="$signal$", histtype="step", rwidth=1.0,\
             color=None, edgecolor="#7a8e99", linewidth=1, linestyle="solid",\
             bottom=None, cumulative=False, normed=False, align="mid", orientation="vertical")

    # Axis
    plt.rc('text',usetex=False)
    plt.xlabel(r"TET",\
               fontsize=16,color="black")
    plt.ylabel(r"$\mathrm{Events}$ $(\mathcal{L}_{\mathrm{int}} = 40.0\ \mathrm{fb}^{-1})$ ",\
               fontsize=16,color="black")

    # Boundary of y-axis
    ymax=(y16_DELTAR_0_weights+y16_DELTAR_1_weights+y16_DELTAR_2_weights+y16_DELTAR_3_weights+y16_DELTAR_4_weights+y16_DELTAR_5_weights+y16_DELTAR_6_weights+y16_DELTAR_7_weights+y16_DELTAR_8_weights+y16_DELTAR_9_weights+y16_DELTAR_10_weights+y16_DELTAR_11_weights+y16_DELTAR_12_weights+y16_DELTAR_13_weights+y16_DELTAR_14_weights+y16_DELTAR_15_weights+y16_DELTAR_16_weights).max()*1.1
    ymin=0 # linear scale
    #ymin=min([x for x in (y16_DELTAR_0_weights+y16_DELTAR_1_weights+y16_DELTAR_2_weights+y16_DELTAR_3_weights+y16_DELTAR_4_weights+y16_DELTAR_5_weights+y16_DELTAR_6_weights+y16_DELTAR_7_weights+y16_DELTAR_8_weights+y16_DELTAR_9_weights+y16_DELTAR_10_weights+y16_DELTAR_11_weights+y16_DELTAR_12_weights+y16_DELTAR_13_weights+y16_DELTAR_14_weights+y16_DELTAR_15_weights+y16_DELTAR_16_weights) if x])/100. # log scale
    plt.gca().set_ylim(ymin,ymax)

    # Log/Linear scale for X-axis
    plt.gca().set_xscale("linear")
    #plt.gca().set_xscale("log",nonposx="clip")

    # Log/Linear scale for Y-axis
    plt.gca().set_yscale("linear")
    #plt.gca().set_yscale("log",nonposy="clip")

    # Legend
    plt.legend(bbox_to_anchor=(1.05,1), loc=2, borderaxespad=0.)

    # Saving the image
    plt.savefig('../../HTML/MadAnalysis5job_0/selection_15.png')
    plt.savefig('../../PDF/MadAnalysis5job_0/selection_15.png')
    plt.savefig('../../DVI/MadAnalysis5job_0/selection_15.eps')

# Running!
if __name__ == '__main__':
    selection_15()