# -*- coding: utf-8 -*-
"""ALE plots."""
import math
from collections import defaultdict
from copy import deepcopy
from operator import attrgetter

import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
import numpy as np
import sklearn.base
from joblib import parallel_backend
from matplotlib.colors import SymLogNorm
from matplotlib.lines import Line2D
from matplotlib.patches import Polygon
from wildfires.cache.proxy_backend import HashProxy
from wildfires.qstat import get_ncpus
from wildfires.utils import shorten_features

from .. import variable
from ..cache import add_cached_shape, cache, get_proxied_estimator, process_proxy
from ..model import assign_n_jobs
from ..plotting import get_float_format, get_sci_format, update_label_with_exp
from ..utils import column_check, tqdm

orig_clone = sklearn.base.clone

@cache
def cached_clone(estimator, safe=True):
    """Adapted from sklearn.base.clone."""
    from wildfires.dask_cx1 import DaskRandomForestRegressor

    if not isinstance(estimator, DaskRandomForestRegressor):
        return orig_clone(estimator, safe=safe)
    estimator_params = estimator.get_params(deep=False)
    new_estimator = estimator.__class__(**deepcopy(estimator_params))
    new_params = new_estimator.get_params(deep=False)
    for name in estimator_params:
        param1 = estimator_params[name]
        param2 = new_params[name]
        assert param1 is param2
    return new_estimator

def delegate_clone(estimator, safe=True):
    from wildfires.dask_cx1 import DaskRandomForestRegressor

    if isinstance(estimator, HashProxy) or isinstance(
        estimator, DaskRandomForestRegressor
    ):
        cached_est = cached_clone(estimator, safe=safe)
        # Update n_jobs since this will be determined by the original call to the
        # cached function. Do this lazily so it is only updated when needed.
        return process_proxy((cached_est,), (assign_n_jobs,))[0]
    return orig_clone(estimator, safe=safe)

# Transparently cache estimator cloning.
sklearn.base.clone = delegate_clone

# Import after the line above in order for the `clone` caching to take effect within
# the module.
import alepython  # isort:skip
import alepython.ale  # isort:skip

# Transparently cache the ALE computations.
alepython.ale.first_order_ale_quant = cache(alepython.ale.first_order_ale_quant)
alepython.ale.second_order_ale_quant = cache(
    alepython.ale.second_order_ale_quant, ignore=["n_jobs"]
)
alepython.ale._mc_replicas = cache(alepython.ale._mc_replicas, ignore=["verbose"])

# Make use of proxied access to `predict` implicitly.
_orig_ale_plot = alepython.ale.ale_plot

def proxied_ale_plot(**kwargs):
    kwargs["model"] = get_proxied_estimator(kwargs["model"])
    # Cache the 'shape' attribute of `train_set`.
    kwargs["train_set"] = add_cached_shape(kwargs["train_set"])
    return _orig_ale_plot(**kwargs)

alepython.ale.ale_plot = proxied_ale_plot

# Implicitly handle getting array data from Series.
orig_asarray = np.asarray

def lazy_series_asarray(*args, **kwargs):
    if len(args) == 1 and not kwargs and isinstance(args[0], HashProxy):
        # Handle the specific case of a single lazy input argument to avoid
        # loading it from disk for as long as possible.
        return process_proxy((args[0],), (orig_asarray,))[0]

    return orig_asarray(*args, **kwargs)

np.asarray = lazy_series_asarray

def save_ale_1d(
    model,
    X_train,
    column,
    train_response=None,
    monte_carlo=True,
    monte_carlo_rep=100,
    monte_carlo_ratio=1000,
    monte_carlo_hull=True,
    verbose=True,
    center=False,
    figure_saver=None,
    sub_dir="ale",
    fig=None,
    ax=None,
    ale_factor_exp=0,
    x_factor_exp=0,
    x_ndigits=2,
    x_rotation=0,
    x_skip=4,
):
    assert monte_carlo

    if fig is None and ax is None:
        fig = plt.figure(figsize=(7.5, 4.5))
    elif fig is None:
        fig = ax.get_figure()
    if ax is None:
        ax = plt.axes()

    out = alepython.ale.ale_plot(
        model=model,
        train_set=X_train,
        features=column,
        bins=20,
        train_response=train_response,
        monte_carlo=monte_carlo,
        monte_carlo_rep=monte_carlo_rep,
        monte_carlo_ratio=monte_carlo_ratio,
        monte_carlo_hull=monte_carlo_hull,
        plot_quantiles=False,
        quantile_axis=True,
        rugplot_lim=0,
        scilim=0.6,
        return_data=True,
        return_mc_data=True,
        verbose=verbose,
        center=center,
        rng=np.random.default_rng(0),
        fig=plt.figure(),  # Create dummy figure.
        ax=None,
    )

    temp_fig, _, (quantiles, ale), mc_data = out
    plt.close(temp_fig)

    mc_hull_points = alepython.ale._compute_mc_hull_poly_points(
        mc_data,
        np.linspace(
            np.min([mc_quantiles[0] for mc_quantiles, mc_ale in mc_data]),
            np.max([mc_quantiles[-1] for mc_quantiles, mc_ale in mc_data]),
            150,
        ),
    )
    ax.add_patch(
        Polygon(
            mc_hull_points,
            facecolor="C0",
            alpha=0.7,
            label=str(column),
            zorder=2,
        )
    )
    min_x = np.min(mc_hull_points[:, 0])
    max_x = np.max(mc_hull_points[:, 0])
    min_y = np.min(mc_hull_points[:, 1])
    max_y = np.max(mc_hull_points[:, 1])

    x_margin = 0.02 * (max_x - min_x)
    y_margin = 0.04 * (max_y - min_y)

    ax.set_xlim(min_x - x_margin, max_x + x_margin)
    ax.set_ylim(min_y - y_margin, max_y + y_margin)

    x_factor = 10 ** x_factor_exp

    if callable(x_skip):
        ax.set_xticks(x_skip(np.arange(len(quantiles))))
        ax.xaxis.set_ticklabels(
            np.vectorize(
                get_float_format(factor=x_factor, ndigits=x_ndigits, atol=np.inf)
            )(x_skip(quantiles))
        )
    else:
        ax.set_xticks(np.arange(len(quantiles))[::x_skip])
        ax.xaxis.set_ticklabels(
            np.vectorize(
                get_float_format(factor=x_factor, ndigits=x_ndigits, atol=np.inf)
            )(quantiles[::x_skip])
        )
    ax.set_xlabel(
        f"{shorten_features(str(column))} ({variable.units[column.parent]})"
        if x_factor_exp == 0
        else f"{column} ($10^{{{x_factor_exp}}}$ {variable.units[column.parent]})"
    )

    ax.yaxis.set_major_formatter(
        get_float_format(factor=10 ** ale_factor_exp, ndigits=0)
    )
    ax.set_ylabel(
        "ALE (BA)" if ale_factor_exp == 0 else f"ALE ($10^{{{ale_factor_exp}}}$ BA)"
    )

    ax.xaxis.set_tick_params(rotation=x_rotation)

    ax.grid(True)

    if figure_saver is not None:
        figure_saver.save_figure(fig, str(column), sub_directory=sub_dir)

def save_ale_2d(
    model,
    train_set,
    features,
    n_jobs=1,
    include_first_order=True,
    figure_saver=None,
    plot_samples=True,
    figsize=(6.15, 3.17),
    ale_factor_exp=0,
    x_factor_exp=0,
    x_ndigits=2,
    y_factor_exp=0,
    y_ndigits=2,
    *,
    experiment,
    ale_x_skip=2,
    samples_x_skip=4,
    samples_y_skip=2,
):
    model.n_jobs = n_jobs

    cbar_width = 0.01

    x_coords = {}
    x_coords["ALE start"] = 0
    x_coords["ALE end"] = 0.42
    x_coords["ALE cbar start"] = x_coords["ALE end"] + 0.01
    x_coords["ALE cbar end"] = x_coords["ALE cbar start"] + cbar_width
    x_coords["Samples start"] = 0.6
    x_coords["Samples end"] = 0.9
    x_coords["Samples cbar start"] = x_coords["Samples end"] + 0.01
    x_coords["Samples cbar end"] = x_coords["Samples cbar start"] + cbar_width

    y_bottom = {
        "Samples": 0.22,  # Samples plot and cbar bottom.
    }
    cbar_height = {
        "ALE": 0.6,
        "Samples": 0.35,
    }

    top = 1

    fig = plt.figure(figsize=figsize)

    # ALE plot axes.
    ax = [
        fig.add_axes(
            [x_coords["ALE start"], 0, x_coords["ALE end"] - x_coords["ALE start"], top]
        )
    ]
    # ALE plot cbar axes.
    cax = [
        fig.add_axes(
            [
                x_coords["ALE cbar start"],
                top * (1 - cbar_height["ALE"]) / 2,
                x_coords["ALE cbar end"] - x_coords["ALE cbar start"],
                cbar_height["ALE"],
            ]
        )
    ]
    if plot_samples:
        # Samples plot axes.
        ax.append(
            fig.add_axes(
                [
                    x_coords["Samples start"],
                    y_bottom["Samples"],
                    x_coords["Samples end"] - x_coords["Samples start"],
                    top - y_bottom["Samples"],
                ]
            )
        )
        # Samples plot cbar axes.
        cax.append(
            fig.add_axes(
                [
                    x_coords["Samples cbar start"],
                    (y_bottom["Samples"] + top) / 2 - cbar_height["Samples"] / 2,
                    x_coords["Samples cbar end"] - x_coords["Samples cbar start"],
                    cbar_height["Samples"],
                ]
            )
        )

    with parallel_backend("threading", n_jobs=n_jobs):
        fig, axes, (quantiles_list, ale, samples) = alepython.ale.ale_plot(
            model=model,
            train_set=train_set,
            features=features,
            bins=20,
            fig=fig,
            ax=ax[0],
            plot_quantiles=False,
            quantile_axis=True,
            plot_kwargs={
                "kind": "grid",
                "cmap": "inferno",
                "colorbar_kwargs": dict(
                    format=get_float_format(factor=10 ** ale_factor_exp, ndigits=0),
                    cax=cax[0],
                    label=(
                        "ALE (BA)"
                        if ale_factor_exp == 0
                        else f"ALE ($10^{{{ale_factor_exp}}}$ BA)"
                    ),
                ),
            },
            return_data=True,
            n_jobs=n_jobs,
            include_first_order=include_first_order,
        )

    axes["ale"].set_aspect("equal")
    axes["ale"].set_title("")

    x_factor = 10 ** x_factor_exp
    y_factor = 10 ** y_factor_exp

    axes["ale"].xaxis.set_ticklabels(
        np.vectorize(get_float_format(factor=x_factor, ndigits=x_ndigits, atol=np.inf))(
            quantiles_list[0]
        )
    )
    for i, label in enumerate(axes["ale"].xaxis.get_ticklabels()):
        if i % ale_x_skip:
            label.set_visible(False)

    axes["ale"].yaxis.set_ticklabels(
        np.vectorize(get_float_format(factor=y_factor, ndigits=y_ndigits, atol=np.inf))(
            quantiles_list[1]
        )
    )

    axes["ale"].set_xlabel(
        f"{shorten_features(str(features[0]))} ({variable.units[features[0].parent]})"
        if x_factor_exp == 0
        else (
            f"{shorten_features(str(features[0]))} "
            f"($10^{{{x_factor_exp}}}$ {variable.units[features[0].parent]})"
        )
    )
    axes["ale"].set_ylabel(
        f"{shorten_features(str(features[1]))} ({variable.units[features[1].parent]})"
        if y_factor_exp == 0
        else (
            f"{shorten_features(str(features[1]))} "
            f"($10^{{{y_factor_exp}}}$ {variable.units[features[1].parent]})"
        )
    )

    if plot_samples:
        # Plotting samples.
        mod_quantiles_list = []
        for axis, quantiles, factor, ndigits in zip(
            ("x", "y"), quantiles_list, (x_factor, y_factor), (x_ndigits, y_ndigits)
        ):
            inds = np.arange(len(quantiles))
            mod_quantiles_list.append(inds)
            ax[1].set(**{f"{axis}ticks": inds})
            ax[1].set(
                **{
                    f"{axis}ticklabels": np.vectorize(
                        get_float_format(factor=factor, ndigits=ndigits, atol=np.inf)
                    )(quantiles)
                }
            )

        for i, label in enumerate(ax[1].xaxis.get_ticklabels()):
            if i % samples_x_skip:
                label.set_visible(False)

        for i, label in enumerate(ax[1].yaxis.get_ticklabels()):
            if i % samples_y_skip:
                label.set_visible(False)

        samples_img = ax[1].pcolormesh(
            *mod_quantiles_list, samples.T, norm=SymLogNorm(linthresh=1)
        )

        @ticker.FuncFormatter
        def integer_sci_format(x, pos):
            if np.log10(x).is_integer():
                return get_sci_format(ndigits=0, trim_leading_one=True)(x, pos)
            return ""

        fig.colorbar(
            samples_img,
            cax=cax[1],
            label="samples",
            format=integer_sci_format,
        )
        ax[1].set_aspect("equal")
        ax[1].set_xlabel(
            (
                f"{shorten_features(str(features[0]))} "
                f"({variable.units[features[0].parent]})"
            )
            if x_factor_exp == 0
            else (
                f"{shorten_features(str(features[0]))} "
                f"($10^{{{x_factor_exp}}}$ {variable.units[features[0].parent]})"
            )
        )
        ax[1].set_ylabel(
            (
                f"{shorten_features(str(features[1]))} "
                f"({variable.units[features[1].parent]})"
            )
            if y_factor_exp == 0
            else (
                f"{shorten_features(str(features[1]))} "
                f"($10^{{{y_factor_exp}}}$ {variable.units[features[1].parent]})"
            )
        )

        fig.set_constrained_layout_pads(
            w_pad=0.000, h_pad=0.000, hspace=0.0, wspace=0.015
        )

    if figure_saver is not None:
        name_root = (
            experiment.name
            + "_"
            + "__".join(map(shorten_features, map(str, features))).replace(" ", "_")
        )
        if plot_samples:
            figure_saver.save_figure(
                fig,
                name_root,
                sub_directory="2d_ale_first_order" if include_first_order else "2d_ale",
            )
        else:
            figure_saver.save_figure(
                fig,
                name_root + "_no_count",
                sub_directory="2d_ale_first_order_no_count"
                if include_first_order
                else "2d_ale_no_count",
            )

def multi_ale_1d(
    model,
    X_train,
    features,
    train_response=None,
    monte_carlo=True,
    monte_carlo_rep=100,
    monte_carlo_ratio=1000,
    verbose=True,
    center=False,
    fig=None,
    ax=None,
    rngs=None,
    legend=True,
    legend_kwargs=None,
    ale_factor_exp=0,
    x_factor_exp=0,
    x_ndigits=2,
    x_rotation=0,
    x_skip=4,
    x_margin_factor=0.015,
    y_margin_factor=0.025,
):
    assert monte_carlo

    if fig is None and ax is None:
        fig, ax = plt.subplots(
            figsize=(7, 3)
        )  # Make sure plot is plotted onto a new figure.
    elif fig is None:
        fig = ax.get_figure()
    if ax is None:
        ax = plt.axes()

    if rngs is None:
        rngs = [np.random.default_rng(i) for i in range(len(features))]

    if legend_kwargs is None:
        legend_kwargs = {}

    markers = ("o", "v", "^", "<", ">", "x", "+")
    colors = plt.rcParams["axes.prop_cycle"].by_key()["color"]

    zorders = list(range(2, 2 + len(features)))

    quantile_list = []
    ale_list = []
    mc_data_list = []
    for feature, rng in zip(
        tqdm(
            features,
            desc="Calculating feature ALEs",
            disable=not verbose,
        ),
        rngs,
    ):
        out = alepython.ale.ale_plot(
            model=model,
            train_set=X_train,
            train_response=train_response,
            features=feature,
            bins=20,
            rng=rng,
            quantile_axis=False,
            center=center,
            monte_carlo=True,
            monte_carlo_rep=monte_carlo_rep,
            monte_carlo_ratio=monte_carlo_ratio,
            return_data=True,
            return_mc_data=True,
            fig=plt.figure(),  # Create dummy figure.
            ax=None,
            verbose=verbose,
        )
        temp_fig, _, (quantiles, ale), mc_data = out
        # Close the unneeded temporary figure.
        plt.close(temp_fig)

        # Record the generated data for this feature.
        quantile_list.append(quantiles)
        ale_list.append(ale)
        mc_data_list.append(mc_data)

    # Construct quantiles from the individual quantiles, minimising the amount of interpolation.
    combined_quantiles = np.vstack([quantiles[None] for quantiles in quantile_list])

    final_quantiles = np.mean(combined_quantiles, axis=0)

    mod_quantiles = np.arange(len(quantiles))

    x_lims = [np.inf, -np.inf]
    y_lims = [np.inf, -np.inf]

    def update_lims(v, lims):
        v_min = np.min(v)
        v_max = np.max(v)
        if v_min < lims[0]:
            lims[0] = v_min
        if v_max > lims[1]:
            lims[1] = v_max

    for feature, quantiles, ale, marker, color, zorder, mc_data in zip(
        features,
        quantile_list,
        ale_list,
        markers,
        colors,
        zorders,
        mc_data_list,
    ):
        assert mc_data is not None

        # Compute the hull and plot it as a Polygon.
        mod_mc_data = tuple(
            (np.interp(mc_quantiles, final_quantiles, mod_quantiles), mc_ale)
            for mc_quantiles, mc_ale in mc_data
        )
        mc_hull_points = alepython.ale._compute_mc_hull_poly_points(
            mod_mc_data,
            np.linspace(
                np.min([mc_quantiles[0] for mc_quantiles, mc_ale in mod_mc_data]),
                np.max([mc_quantiles[-1] for mc_quantiles, mc_ale in mod_mc_data]),
                150,
            ),
        )
        ax.add_patch(
            Polygon(
                mc_hull_points,
                facecolor=color,
                zorder=zorder,
                label=f"{feature.shift}M",
                alpha=0.7,
            )
        )

        # Update plot limits.
        update_lims(mc_hull_points[:, 0], x_lims)
        update_lims(mc_hull_points[:, 1], y_lims)

    # Set plot limits.
    x_margin = x_margin_factor * (x_lims[1] - x_lims[0])
    ax.set_xlim(x_lims[0] - x_margin, x_lims[1] + x_margin)
    y_margin = y_margin_factor * (y_lims[1] - y_lims[0])
    ax.set_ylim(y_lims[0] - y_margin, y_lims[1] + y_margin)

    if legend:
        print({**{"loc": "best"}, **legend_kwargs})
        ax.legend(**{**{"loc": "best", "ncol": 2}, **legend_kwargs})

    x_factor = 10 ** x_factor_exp

    if callable(x_skip):
        ax.set_xticks(x_skip(mod_quantiles))
        ax.xaxis.set_ticklabels(
            np.vectorize(
                get_float_format(factor=x_factor, ndigits=x_ndigits, atol=np.inf)
            )(x_skip(final_quantiles))
        )
    else:
        ax.set_xticks(mod_quantiles[::x_skip])
        ax.xaxis.set_ticklabels(
            np.vectorize(
                get_float_format(factor=x_factor, ndigits=x_ndigits, atol=np.inf)
            )(final_quantiles[::x_skip])
        )

    ax.yaxis.set_major_formatter(
        get_float_format(factor=10 ** ale_factor_exp, ndigits=0)
    )
    ax.set_ylabel(
        "ALE (BA)" if ale_factor_exp == 0 else f"ALE ($10^{{{ale_factor_exp}}}$ BA)"
    )

    ax.xaxis.set_tick_params(rotation=x_rotation)

    ax.grid(True)

def get_model_predict(model):
    return model.predict

def single_ax_multi_ale_1d(
    ax,
    feature_data,
    feature,
    xlabel=None,
    ylabel=None,
    title=None,
    verbose=False,
    x_ndigits=2,
    x_factor=1,
    x_rotation=18,
):
    quantile_list = []
    ale_list = []

    for experiment, single_experiment_data in zip(
        tqdm(
            feature_data["experiment"],
            desc="Calculating feature ALEs",
            disable=not verbose,
        ),
        feature_data["single_experiment_data"],
    ):
        model = single_experiment_data["model"]
        X_train = single_experiment_data["X_train"]

        with parallel_backend("threading", n_jobs=get_ncpus()):
            quantiles, ale = alepython.ale.first_order_ale_quant(
                process_proxy((model,), (get_model_predict,))[0],
                X_train,
                feature,
                bins=20,
            )

        quantile_list.append(quantiles)
        ale_list.append(ale)

    # Construct quantiles from the individual quantiles, minimising the amount of interpolation.
    combined_quantiles = np.vstack([quantiles[None] for quantiles in quantile_list])

    final_quantiles = np.mean(combined_quantiles, axis=0)

    # The chosen variable for this plot comes from the same dataset (i.e. subsets of
    # the ALL-dataset), thus the quantiles, which are purely informed by the data,
    # should match.
    assert np.allclose(final_quantiles, combined_quantiles)

    mod_quantiles = np.arange(len(quantiles))

    for plot_kwargs, quantiles, ale in zip(
        feature_data["plot_kwargs"], quantile_list, ale_list
    ):
        # Interpolate each of the quantiles relative to the accumulated final quantiles.
        ax.plot(
            np.interp(quantiles, final_quantiles, mod_quantiles),
            ale,
            **{"marker": "o", "ms": 3, **plot_kwargs},
        )

        ax.set_xticks(mod_quantiles[::2])
        ax.set_xticklabels(
            map(
                lambda x: get_float_format(
                    ndigits=x_ndigits, factor=x_factor, atol=np.inf
                )(x, None),
                final_quantiles[::2],
            )
        )
        ax.xaxis.set_tick_params(rotation=x_rotation)

        ax.grid(True)

        ax.set_xlabel(xlabel + f"({x_factor})")
        ax.set_ylabel(ylabel)

    ax.set_title(title)

def multi_model_ale_1d(
    variable_factory,
    experiment_data,
    experiment_plot_kwargs,
    lags=(0, 1, 3, 6, 9),
    title=None,
    verbose=False,
    figure_saver=None,
    single_figsize=(5.4, 1.5),
    legend_bbox=(0.5, 0.5),
    fig=None,
    axes=None,
    legend=True,
    x_ndigits=2,
    x_factor=1,
    x_rotation=18,
    y_ndigits=1,
    y_factor=1e-3,
):
    plotted_experiments = set()

    # Compile data for later plotting.
    comp_data = {}

    for lag in lags:
        assert lag <= 9

        feature = variable_factory[lag]

        feature_data = defaultdict(list)

        experiment_count = 0
        for experiment, single_experiment_data in experiment_data.items():
            # Skip experiments that do not contain this feature.
            if not column_check(single_experiment_data["X_train"], feature):
                continue

            experiment_count += 1
            plotted_experiments.add(experiment)

            # Data required to calculate the ALEs.
            feature_data["experiment"].append(experiment)
            feature_data["single_experiment_data"].append(single_experiment_data)
            feature_data["plot_kwargs"].append(experiment_plot_kwargs[experiment])

        if experiment_count <= 1:
            # We need at least two models for a comparison.
            continue

        comp_data[feature] = feature_data

    n_plots = len(comp_data)
    n_cols = 2
    n_rows = math.ceil(n_plots / n_cols)

    if fig is None and axes is None:
        fig, axes = plt.subplots(
            n_rows,
            n_cols,
            figsize=np.array(single_figsize) * np.array([n_cols, n_rows]),
        )
    elif fig is not None and axes is not None:
        pass
    else:
        raise ValueError("Either both or none of fig and axes need to be given.")

    # Disable unused axes.
    if len(axes.flatten()) > n_plots:
        for ax in axes.flatten()[-(len(axes.flatten()) - n_plots) :]:
            ax.axis("off")

    for ax, feature, feature_data in zip(axes.flatten(), comp_data, comp_data.values()):
        single_ax_multi_ale_1d(
            ax,
            feature_data=feature_data,
            feature=feature,
            xlabel=shorten_features(str(feature)),
            verbose=verbose,
            x_ndigits=x_ndigits,
            x_factor=x_factor,
            x_rotation=x_rotation,
        )

    for ax in axes.flatten()[:n_plots]:
        ax.yaxis.set_major_formatter(
            get_float_format(factor=y_factor, ndigits=y_ndigits, atol_exceeded="adjust")
        )

    for row_axes in axes:
        row_axes[0].set_ylabel(update_label_with_exp("ALE (BA)", str(y_factor)))

    fig.tight_layout()

    lines = []
    labels = []
    for experiment in sorted(plotted_experiments, key=attrgetter("value")):
        lines.append(Line2D([0], [0], **experiment_plot_kwargs[experiment]))
        labels.append(experiment_plot_kwargs[experiment]["label"])

    if legend:
        fig.legend(
            lines,
            labels,
            loc="center",
            bbox_to_anchor=legend_bbox,
            ncol=len(labels) if len(labels) <= 6 else 6,
        )

    if figure_saver is not None:
        figure_saver.save_figure(
            fig,
            f"{shorten_features(variable_factory).replace(' ', '_').lower()}_ale_comp",
            sub_directory="ale_comp",
        )


#!/usr/bin/python
# -*- coding: UTF-8 -*-

import json
import os
import h5py
import numpy as np

class MicrosoftCocoDataset:
    """
    包装了 MicrosoftCoco 数据集,  我们通过此类来访问该数据集

    1.初始化完毕后, 结果放入 self.dataset (字典), 数据集的各个部分如下:

    训练集(train)
    key               shape        value
    train_captions    (400135, 17) 图片评论
    train_image_idxs  (400135,)    评论到图片的映射, 通过映射找到评论对应的图片向量
    train_features    (82783, 512) 向量化后的图片, 维度为 512
    train_urls        (82783,)     图片的 url 地址

    验证集(val)
    val_captions      (195954, 17)
    val_image_idxs    (195954,)
    val_features      (40504, 512)
    val_urls          (40504,)

    idx_to_word        1004        单词标号到单词的词典
    word_to_idx        1004        单词到单词标号的词典

    Author: xrh
    Date: 2021-9-30

    """

    def __init__(self, base_dir='../../dataset/ImageCaption/microsoft_coco', sample_N=None, use_pca_features=True):
        """

        :param base_dir: 数据集集的根路径
        :param sample_N: 训练数据集(采样后)的样本数
        :param use_pca_features: 是否使用 PCA 降维后的图像特征

        """

        self.base_dir = base_dir
        self.sample_N = sample_N
        self.use_pca_features = use_pca_features

        self.dataset = {}

        self.__load_data()  # 读取预处理后的数据集

        self.vocab_obj = Vocab(id_to_word=self.dataset['idx_to_word'], word_to_id=self.dataset['word_to_idx'])

        if sample_N is not None:

            # 对训练数据集进行采样
            np.random.seed(231)  # 控制随机数, 在程序的当前上下文有效

            N_train = np.shape(self.dataset['train_captions'])[0]  # 训练集的样本总数
            mask = np.random.randint(N_train, size=self.sample_N)
            self.dataset['train_captions'] = self.dataset['train_captions'][mask]
            self.dataset['train_image_idxs'] = self.dataset['train_image_idxs'][mask]

            # 对验证(测试)数据集进行采样, 采样个数为 训练集的 1/2

            N_val = np.shape(self.dataset['train_captions'])[0]  # 训练集的样本总数
            mask = np.random.randint(N_val, size=self.sample_N//2)
            self.dataset['val_captions'] = self.dataset['val_captions'][mask]
            self.dataset['val_image_idxs'] = self.dataset['val_image_idxs'][mask]

            self.N = sample_N

        else:
            self.N = np.shape(self.dataset['train_captions'])[0]

        self.feature_dim = self.dataset['train_features'].shape[1]  # feature_dim - 图片向量的维度
        self.caption_length = self.dataset['train_captions'].shape[1]  # caption_length - 图片描述的长度

    def __load_data(self):
        """
        载入数据集

        :return:
        """
        caption_file = os.path.join(self.base_dir, 'coco2014_captions.h5')
        with h5py.File(caption_file, 'r') as f:
            for k, v in f.items():
                self.dataset[k] = np.asarray(v)

        if self.use_pca_features:
            train_feat_file = os.path.join(self.base_dir, 'train2014_vgg16_fc7_pca.h5')
        else:
            train_feat_file = os.path.join(self.base_dir, 'train2014_vgg16_fc7.h5')
        with h5py.File(train_feat_file, 'r') as f:
            self.dataset['train_features'] = np.asarray(f['features'])

        if self.use_pca_features:
            val_feat_file = os.path.join(self.base_dir, 'val2014_vgg16_fc7_pca.h5')
        else:
            val_feat_file = os.path.join(self.base_dir, 'val2014_vgg16_fc7.h5')
        with h5py.File(val_feat_file, 'r') as f:
            self.dataset['val_features'] = np.asarray(f['features'])

        dict_file = os.path.join(self.base_dir, 'coco2014_vocab.json')
        with open(dict_file, 'r') as f:
            dict_data = json.load(f)
            for k, v in dict_data.items():
                self.dataset[k] = v

        train_url_file = os.path.join(self.base_dir, 'train2014_urls.txt')
        with open(train_url_file, 'r') as f:
            train_urls = np.asarray([line.strip() for line in f])
        self.dataset['train_urls'] = train_urls

        val_url_file = os.path.join(self.base_dir, 'val2014_urls.txt')
        with open(val_url_file, 'r') as f:
            val_urls = np.asarray([line.strip() for line in f])
        self.dataset['val_urls'] = val_urls

    def decode_captions(self, captions):
        """
        将由单词标号组成的句子 解码为 原始句子

        :param captions: 一个句子, 或者一个句子列表
        :return:
        """

        singleton = False

        if captions.ndim == 1:  # 说明 captions 只有一个句子
            singleton = True
            captions = captions[None]

        decoded = []
        N, T = captions.shape

        for i in range(N):
            words = []
            for t in range(T):
                word = self.dataset['idx_to_word'][captions[i, t]]
                if word != '<NULL>':
                    words.append(word)
                if word == '<END>':
                    break
            decoded.append(' '.join(words))

        if singleton:
            decoded = decoded[0]

        return decoded

    def sample_minibatch(self, batch_size=128, Type='train', return_url=False):
        """
        从数据集中采样 1个 batch 的样本用于训练

        :param batch_size:  1个 batch的样本个数
        :param Type: 数据集的类型, 'train' 为训练数据集
        :param return_url: 是否返回图片的 url
        :return:

        captions, image_features

        """

        split_size = self.dataset['%s_captions' % Type].shape[0]
        mask = np.random.choice(split_size, batch_size)  # 第二次调用此函数时, 类初始化(def __init__) 中的控制随机数就失效了

        captions = self.dataset['%s_captions' % Type][mask]
        image_idxs = self.dataset['%s_image_idxs' % Type][mask]

        image_features = self.dataset['%s_features' % Type][image_idxs]

        urls = self.dataset['%s_urls' % Type][image_idxs]

        if return_url:

            return captions, image_features, urls

        return captions, image_features

class Vocab:

    def __init__(self, word_to_id, id_to_word, _unk_str='<UNK>'):

        self.word_to_id = word_to_id
        self.id_to_word = id_to_word

        self._unk_str = _unk_str

    def map_id_to_word(self, id):
        """
        输入单词标号, 返回单词

        :param id:
        :return:
        """

        return self.id_to_word[id]

    def map_word_to_id(self, word):
        """
        输入单词, 返回单词标号

        考虑未登录词:
        1.若输入的单词不在词典中, 返回 '<UNK>' 的标号

        :param word: 单词
        :return:
        """

        if word not in self.word_to_id:
            return self.word_to_id[self._unk_str]
        else:
            return self.word_to_id[word]


# -*- coding: utf-8 -*-
"""
Created October 02, 11:54:08 2021

@author: Thien Le, Postdoc Researcher
"""
import cv2
import numpy as np
import pandas as pd
# Import libraries
import streamlit as st
import tensorflow as tf
from PIL import Image
from tensorflow.keras.applications.vgg19 import preprocess_input as vgg19_preprocess_input

@st.cache(allow_output_mutation=True)
def load_model():
    model = tf.keras.models.load_model("aimodels/strokedata3_model_vgg19_29August2021.h5")
    return model
@st.cache()
def convert_df(df):
    # IMPORTANT: Cache the conversion to prevent computation on every rerun
    return df.to_csv().encode('utf-8')

with st.spinner('Loading Model Into Memory...'):
    model = load_model()

def app():
    image = Image.open('chula_logo.png')
    st.image(image)
    st.title("""Multiple CT-Brain Images Abnormal Detection""")
    # Create header explaination
    my_expander = st.expander("See explanation", expanded=False)
    with my_expander:
        st.write("""
                 This web-apps receives many CT-brain images and predicts the results as table using backend AI engine.
                 The trained AI engine is based on the pre-trained VGG19 deep learning model.
                 """)
        image = Image.open('example_pics.jpg')
        st.image(image,
                 caption='Example CT-brain images. Source of these images from CQ500 publish dataset (http://headctstudy.qure.ai/dataset)')
    ### Create a table to store the predicted resutls

    df = pd.DataFrame({
        'The image name': [],
        'Predicted result': []
    })
    ###
    map_dict = {0: 'Non interest.',
                1: 'Abnormal at center brain layer',  # hge
                2: 'Abnormal at eyeball brain layer.',  # hge_eyeball
                3: 'Abnormal at top brain layer.',  # hge_top
                4: 'Normal at center brain layer.',  # nonehge
                5: 'Normal at eyeball brain layer.',  # nonehge_eyeball
                6: 'Normal at top brain layer.',  # nonehge_top
                }
    ###
    st.subheader('Choose CT-brain images and get the detection')
    # Allow multiple images uploading
    uploaded_files = st.file_uploader("Choose CT-brain images", type=["jpg", "png"], accept_multiple_files=True)
    if uploaded_files is not None:
        # Count the uploaded images
        for uploaded_file in uploaded_files:
            bytes_data = uploaded_file.read()
            # df.loc[len(df.index)]=[uploaded_file.name, 'Yes']

            # Convert the file to an opencv image.
            file_bytes = np.asarray(bytearray(bytes_data), dtype=np.uint8)
            opencv_image = cv2.imdecode(file_bytes, 1)
            opencv_image = cv2.cvtColor(opencv_image, cv2.COLOR_BGR2RGB)
            resized = cv2.resize(opencv_image, (224, 224))
            resized = vgg19_preprocess_input(resized) / 255
            img_reshape = resized[np.newaxis, ...]

            # Generate the prediction
            prediction = model.predict(img_reshape).argmax()
            if prediction == 0: df.loc[len(df.index)] = [uploaded_file.name, format(map_dict[prediction])]
            if prediction == 4: df.loc[len(df.index)] = [uploaded_file.name, format(map_dict[prediction])]
            if prediction == 5: df.loc[len(df.index)] = [uploaded_file.name, format(map_dict[prediction])]
            if prediction == 6: df.loc[len(df.index)] = [uploaded_file.name, format(map_dict[prediction])]
            if prediction == 1: df.loc[len(df.index)] = [uploaded_file.name, format(map_dict[prediction])]
            if prediction == 2: df.loc[len(df.index)] = [uploaded_file.name, format(map_dict[prediction])]
            if prediction == 3: df.loc[len(df.index)] = [uploaded_file.name, format(map_dict[prediction])]

    st.table(df)

    # Convert to csv file
    csv = convert_df(df)

    st.download_button(
        label="Download the table as CSV",
        data=csv,
        file_name="predicted_results.csv",
        mime="text/csv"
    )


import os
from . import DATA_DIR
from .activity_maps import InventorySet
from .geomap import Geomap
from wurst import searching as ws
from wurst.ecoinvent import filters
import csv
import numpy as np
import uuid
import wurst

PRODUCTION_PER_TECH = (DATA_DIR / "electricity_production_volumes_per_tech.csv")
LOSS_PER_COUNTRY = (DATA_DIR / "losses_per_country.csv")
LHV_FUELS = (DATA_DIR / "fuels_lower_heating_value.txt")

class Electricity:
    """
    Class that modifies electricity markets in ecoinvent based on REMIND output data.

    :ivar scenario: name of a Remind scenario
    :vartype scenario: str

    """

    def __init__(self, db, rmd, scenario, year):
        self.db = db
        self.rmd = rmd
        self.geo = Geomap()
        self.production_per_tech = self.get_production_per_tech_dict()
        self.losses = self.get_losses_per_country_dict()
        self.scenario = scenario
        self.year = year
        self.fuels_lhv = self.get_lower_heating_values()

        mapping = InventorySet(self.db)
        self.emissions_map = mapping.get_remind_to_ecoinvent_emissions()
        self.powerplant_map = mapping.generate_powerplant_map()

    def get_lower_heating_values(self):
        """
        Loads a csv file into a dictionary. This dictionary contains lower heating values for a number of fuel types.
        Taken from: https://www.engineeringtoolbox.com/fuels-higher-calorific-values-d_169.html

        :return: dictionary that contains lower heating values
        :rtype: dict
        """
        with open(LHV_FUELS) as f:
            return dict(filter(None, csv.reader(f, delimiter=";")))

    def get_suppliers_of_a_region(self, ecoinvent_regions, ecoinvent_technologies):
        """
        Return a list of electricity-producing datasets which location and name correspond to the region and name given,
        respectively.

        :param ecoinvent_regions: an ecoinvent region
        :type ecoinvent_regions: list
        :param ecoinvent_technologies: name of ecoinvent dataset
        :type ecoinvent_technologies: str
        :return: list of wurst datasets
        :rtype: list
        """

        return ws.get_many(
            self.db,
            *[
                ws.either(
                    *[
                        ws.equals("name", supplier)
                        for supplier in ecoinvent_technologies
                    ]
                ),
                ws.either(*[ws.equals("location", loc) for loc in ecoinvent_regions]),
                ws.equals("unit", "kilowatt hour"),
            ]
        )

    def get_losses_per_country_dict(self):
        """
        Create a dictionary with ISO country codes as keys and loss ratios as values.
        :return: ISO country code to loss ratio dictionary
        :rtype: dict
        """

        if not LOSS_PER_COUNTRY.is_file():
            raise FileNotFoundError(
                "The production per country dictionary file could not be found."
            )

        with open(LOSS_PER_COUNTRY) as f:
            csv_list = [[val.strip() for val in r.split(";")] for r in f.readlines()]

        (_, *header), *data = csv_list
        csv_dict = {}
        for row in data:
            key, *values = row
            csv_dict[key] = {key: float(value) for key, value in zip(header, values)}

        return csv_dict

    def get_production_per_tech_dict(self):
        """
        Create a dictionary with tuples (technology, country) as keys and production volumes as values.
        :return: technology to production volume dictionary
        :rtype: dict
        """

        if not PRODUCTION_PER_TECH.is_file():
            raise FileNotFoundError(
                "The production per technology dictionary file could not be found."
            )
        csv_dict = {}
        with open(PRODUCTION_PER_TECH) as f:
            input_dict = csv.reader(f, delimiter=";")
            for row in input_dict:
                csv_dict[(row[0], row[1])] = row[2]

        return csv_dict

    def get_production_weighted_share(self, supplier, suppliers):
        """
        Return the share of production of an electricity-producing dataset in a specific location,
        relative to the summed production of similar technologies in locations contained in the same REMIND region.

        :param supplier: electricity-producing dataset
        :type supplier: wurst dataset
        :param suppliers: list of electricity-producing datasets
        :type suppliers: list of wurst datasets
        :return: share of production relative to the total population
        :rtype: float
        """

        # Fetch the production volume of the supplier
        loc_production = float(self.production_per_tech.get((supplier["name"], supplier["location"]), 0))

        # Fetch the total production volume of similar technologies in other locations
        # contained within the REMIND region.

        total_production = 0
        for loc in suppliers:
            total_production += float(self.production_per_tech.get((loc['name'], loc["location"]), 0))

        # If a corresponding production volume is found.
        if total_production != 0:
            return loc_production / total_production
        else:
            # If not, we allocate an equal share of supply
            return 1 / len(suppliers)

    def get_production_weighted_losses(self, voltage, remind_region):
        """
        Return the transformation, transmission and distribution losses at a given voltage level for a given location.
        A weighted average is made of the locations contained in the REMIND region.

        :param voltage: voltage level (high, medium or low)
        :type voltage: str
        :param remind_region: Remind region
        :type remind_region: str
        :return: tuple that contains transformation and distribution losses
        :rtype: tuple
        """

        # Fetch locations contained in REMIND region
        locations = self.geo.remind_to_ecoinvent_location(remind_region)

        if voltage == 'high':

            cumul_prod, transf_loss = 0, 0
            for loc in locations:
                dict_loss = self.losses.get(loc, {
                    'Transformation loss, high voltage':0,
                    'Production volume':0
                })

                transf_loss += dict_loss['Transformation loss, high voltage']\
                        * dict_loss['Production volume']
                cumul_prod += dict_loss['Production volume']
            transf_loss /= cumul_prod
            return transf_loss

        if voltage == 'medium':

            cumul_prod, transf_loss, distr_loss = 0, 0, 0
            for loc in locations:
                dict_loss = self.losses.get(loc, {
                    'Transformation loss, medium voltage': 0,
                    'Transmission loss to medium voltage': 0,
                    'Production volume': 0
                })
                transf_loss += dict_loss['Transformation loss, medium voltage']\
                        * dict_loss['Production volume']
                distr_loss += dict_loss['Transmission loss to medium voltage']\
                        * dict_loss['Production volume']
                cumul_prod += dict_loss['Production volume']
            transf_loss /= cumul_prod
            distr_loss /= cumul_prod
            return transf_loss, distr_loss

        if voltage == 'low':

            cumul_prod, transf_loss, distr_loss = 0, 0, 0

            for loc in locations:
                dict_loss = self.losses.get(loc, {
                    'Transformation loss, low voltage': 0,
                    'Transmission loss to low voltage': 0,
                    'Production volume': 0
                })
                transf_loss += dict_loss['Transformation loss, low voltage']\
                        * dict_loss['Production volume']
                distr_loss += dict_loss['Transmission loss to low voltage']\
                        * dict_loss['Production volume']
                cumul_prod += dict_loss['Production volume']
            transf_loss /= cumul_prod
            distr_loss /= cumul_prod
            return transf_loss, distr_loss

    def create_new_markets_low_voltage(self):
        """
        Create low voltage market groups for electricity, by receiving medium voltage market groups as inputs
        and adding transformation and distribution losses.
        Contribution from solar power is added here as well.
        Does not return anything. Modifies the database in place.
        """
        # Loop through REMIND regions

        for region in self.rmd.electricity_markets.coords["region"].values:
            created_markets = []
            # Create an empty dataset
            new_dataset = {}
            new_dataset["location"] = region
            new_dataset["name"] = (
                "market group for electricity, low voltage"
            )
            new_dataset["reference product"] = "electricity, low voltage"
            new_dataset["unit"] = "kilowatt hour"
            new_dataset["database"] = self.db[1]["database"]
            new_dataset["code"] = str(uuid.uuid4().hex)
            new_dataset[
                "comment"
            ] = "Dataset produced from REMIND scenario output results"

            # First, add the reference product exchange
            new_exchanges = []
            new_exchanges.append(
                {
                    "uncertainty type": 0,
                    "loc": 1,
                    "amount": 1,
                    "type": "production",
                    "production volume": 0,
                    "product": "electricity, low voltage",
                    "name": "market group for electricity, low voltage",
                    "unit": "kilowatt hour",
                    "location": region,
                }
            )

            # Second, add an input to of sulfur hexafluoride emission to compensate the transformer's leakage
            # And an emission of a corresponding amount
            new_exchanges.append(
                {
                    "uncertainty type": 0,
                    "loc": 2.99e-9,
                    "amount": 2.99e-9,
                    "type": "technosphere",
                    "production volume": 0,
                    "product": "sulfur hexafluoride, liquid",
                    "name": "market for sulfur hexafluoride, liquid",
                    "unit": "kilogram",
                    "location": "RoW",
                }
            )
            new_exchanges.append(
                {
                    "uncertainty type": 0,
                    "loc": 2.99e-9,
                    "amount": 2.99e-9,
                    "type": "biosphere",
                    "input": ("biosphere3", "35d1dff5-b535-4628-9826-4a8fce08a1f2"),
                    "name": "Sulfur hexafluoride",
                    "unit": "kilogram",
                    "categories": ("air", "non-urban air or from high stacks"),
                }
            )

            # Third, transmission line
            new_exchanges.append(
                {
                    "uncertainty type": 0,
                    "loc": 8.74e-8,
                    "amount": 8.74e-8,
                    "type": "technosphere",
                    "production volume": 0,
                    "product": "distribution network, electricity, low voltage",
                    "name": "distribution network construction, electricity, low voltage",
                    "unit": "kilometer",
                    "location": "RoW",
                }
            )

            # Fourth, add the contribution of solar power
            solar_amount = 0
            gen_tech = list(
                (
                    tech
                    for tech in self.rmd.electricity_markets.coords["variable"].values
                    if "Solar" in tech
                )
            )
            for technology in gen_tech:
                # If the solar power technology contributes to the mix
                if self.rmd.electricity_markets.loc[technology, region, 0] != 0.0:
                    # Fetch ecoinvent regions contained in the REMIND region
                    ecoinvent_regions = self.geo.remind_to_ecoinvent_location(region)

                    # Contribution in supply
                    amount = self.rmd.electricity_markets.loc[
                        technology, region, 0
                    ].values
                    solar_amount += amount

                    # Get the possible names of ecoinvent datasets
                    ecoinvent_technologies = self.powerplant_map[
                        self.rmd.rev_electricity_market_labels[technology]
                    ]

                    # Fetch electricity-producing technologies contained in the REMIND region
                    suppliers = list(
                        self.get_suppliers_of_a_region(
                            ecoinvent_regions, ecoinvent_technologies
                        )
                    )

                    suppliers = self.check_for_production_volume(suppliers)

                    # If no technology is available for the REMIND region
                    if len(suppliers) == 0:
                        # We fetch European technologies instead
                        suppliers = list(
                            self.get_suppliers_of_a_region(
                                ["RER"], ecoinvent_technologies
                            )
                        )

                    suppliers = self.check_for_production_volume(suppliers)

                    # If, after looking for European technologies, no technology is available
                    if len(suppliers) == 0:
                        # We fetch RoW technologies instead
                        suppliers = list(
                            self.get_suppliers_of_a_region(
                                ["RoW"], ecoinvent_technologies
                            )
                        )

                    suppliers = self.check_for_production_volume(suppliers)

                    for supplier in suppliers:

                        share = self.get_production_weighted_share(supplier, suppliers)

                        new_exchanges.append(
                            {
                                "uncertainty type": 0,
                                "loc": (amount * share),
                                "amount": (amount * share),
                                "type": "technosphere",
                                "production volume": 0,
                                "product": supplier["reference product"],
                                "name": supplier["name"],
                                "unit": supplier["unit"],
                                "location": supplier["location"],
                            }
                        )
                        created_markets.append([
                            "low voltage, " + self.scenario + ", " + str(self.year),
                            "n/a",
                            region,
                            0,
                            0,
                            supplier['name'],
                            supplier['location'],
                            share,
                            (share * amount)
                        ])
            # Fifth, add:
            # * an input from the medium voltage market minus solar contribution, including distribution loss
            # * an self-consuming input for transformation loss

            transf_loss, distr_loss = self.get_production_weighted_losses('low', region)

            new_exchanges.append(
                {
                    "uncertainty type": 0,
                    "loc": 0,
                    "amount": (1 - solar_amount) * (1 + distr_loss),
                    "type": "technosphere",
                    "production volume": 0,
                    "product": "electricity, medium voltage",
                    "name": "market group for electricity, medium voltage",
                    "unit": "kilowatt hour",
                    "location": region,
                }
            )

            new_exchanges.append(
                {
                    "uncertainty type": 0,
                    "loc": 0,
                    "amount": transf_loss,
                    "type": "technosphere",
                    "production volume": 0,
                    "product": "electricity, low voltage",
                    "name": "market group for electricity, low voltage",
                    "unit": "kilowatt hour",
                    "location": region,
                }
            )

            created_markets.append([
                "low voltage, " + self.scenario + ", " + str(self.year),
                "n/a",
                region,
                transf_loss,
                distr_loss,
                "low voltage, " + self.scenario + ", " + str(self.year),
                region,
                1,
                (1 - solar_amount) * (1 + distr_loss)
            ])

            with open(DATA_DIR / "logs/log created markets.csv", "a") as csv_file:
                writer = csv.writer(csv_file,
                                    delimiter=';',
                                    lineterminator='\n')
                for line in created_markets:
                    writer.writerow(line)

            new_dataset["exchanges"] = new_exchanges
            self.db.append(new_dataset)

    def create_new_markets_medium_voltage(self):
        """
        Create medium voltage market groups for electricity, by receiving high voltage market groups as inputs
        and adding transformation and distribution losses.
        Contribution from solar power is added in low voltage market groups.
        Does not return anything. Modifies the database in place.
        """
        # Loop through REMIND regions
        gen_region = (
            region for region in self.rmd.electricity_markets.coords["region"].values
        )

        created_markets = []

        for region in gen_region:

            # Create an empty dataset
            new_dataset = {}
            new_dataset["location"] = region
            new_dataset["name"] = (
                "market group for electricity, medium voltage"
            )
            new_dataset["reference product"] = "electricity, medium voltage"
            new_dataset["unit"] = "kilowatt hour"
            new_dataset["database"] = self.db[1]["database"]
            new_dataset["code"] = str(uuid.uuid1().hex)
            new_dataset[
                "comment"
            ] = "Dataset produced from REMIND scenario output results"

            # First, add the reference product exchange
            new_exchanges = []
            new_exchanges.append(
                {
                    "uncertainty type": 0,
                    "loc": 1,
                    "amount": 1,
                    "type": "production",
                    "production volume": 0,
                    "product": "electricity, medium voltage",
                    "name": "market group for electricity, medium voltage",
                    "unit": "kilowatt hour",
                    "location": region,
                }
            )

            # Second, add:
            # * an input from the high voltage market, including transmission loss
            # * an self-consuming input for transformation loss

            transf_loss, distr_loss = self.get_production_weighted_losses('medium', region)
            new_exchanges.append(
                {
                    "uncertainty type": 0,
                    "loc": 0,
                    "amount": 1 + distr_loss,
                    "type": "technosphere",
                    "production volume": 0,
                    "product": "electricity, high voltage",
                    "name": "market group for electricity, high voltage",
                    "unit": "kilowatt hour",
                    "location": region,
                }
            )

            new_exchanges.append(
                {
                    "uncertainty type": 0,
                    "loc": 0,
                    "amount": transf_loss,
                    "type": "technosphere",
                    "production volume": 0,
                    "product": "electricity, medium voltage",
                    "name": "market group for electricity, medium voltage",
                    "unit": "kilowatt hour",
                    "location": region,
                }
            )

            # Third, add an input to of sulfur hexafluoride emission to compensate the transformer's leakage
            # And an emission of a corresponding amount
            new_exchanges.append(
                {
                    "uncertainty type": 0,
                    "loc": 5.4e-8,
                    "amount": 5.4e-8,
                    "type": "technosphere",
                    "production volume": 0,
                    "product": "sulfur hexafluoride, liquid",
                    "name": "market for sulfur hexafluoride, liquid",
                    "unit": "kilogram",
                    "location": "RoW",
                }
            )
            new_exchanges.append(
                {
                    "uncertainty type": 0,
                    "loc": 5.4e-8,
                    "amount": 5.4e-8,
                    "type": "biosphere",
                    "input": ("biosphere3", "35d1dff5-b535-4628-9826-4a8fce08a1f2"),
                    "name": "Sulfur hexafluoride",
                    "unit": "kilogram",
                    "categories": ("air", "non-urban air or from high stacks"),
                }
            )

            # Fourth, transmission line
            new_exchanges.append(
                {
                    "uncertainty type": 0,
                    "loc": 1.8628e-8,
                    "amount": 1.8628e-8,
                    "type": "technosphere",
                    "production volume": 0,
                    "product": "transmission network, electricity, medium voltage",
                    "name": "transmission network construction, electricity, medium voltage",
                    "unit": "kilometer",
                    "location": "RoW",
                }
            )

            new_dataset["exchanges"] = new_exchanges

            created_markets.append([
                "medium voltage, " + self.scenario + ", " + str(self.year),
                "n/a",
                region,
                transf_loss,
                distr_loss,
                "medium voltage, " + self.scenario + ", " + str(self.year),
                region,
                1,
                1 + distr_loss
            ])

            self.db.append(new_dataset)

        with open(DATA_DIR / "logs/log created markets.csv", "a") as csv_file:
            writer = csv.writer(csv_file,
                                delimiter=';',
                                lineterminator='\n')
            for line in created_markets:
                writer.writerow(line)

    def create_new_markets_high_voltage(self):
        """
        Create high voltage market groups for electricity, based on electricity mixes given by REMIND.
        Contribution from solar power is added in low voltage market groups.
        Does not return anything. Modifies the database in place.
        """
        # Loop through REMIND regions
        gen_region = (
            region for region in self.rmd.electricity_markets.coords["region"].values
        )
        gen_tech = list(
            (
                tech
                for tech in self.rmd.electricity_markets.coords["variable"].values
                if "Solar" not in tech
            )
        )

        created_markets = []

        for region in gen_region:

            # Fetch ecoinvent regions contained in the REMIND region
            ecoinvent_regions = self.geo.remind_to_ecoinvent_location(region)

            # Create an empty dataset
            new_dataset = {}
            new_dataset["location"] = region
            new_dataset["name"] = (
                "market group for electricity, high voltage"
            )
            new_dataset["reference product"] = "electricity, high voltage"
            new_dataset["unit"] = "kilowatt hour"
            new_dataset["database"] = self.db[1]["database"]
            new_dataset["code"] = str(uuid.uuid4().hex)
            new_dataset[
                "comment"
            ] = "Dataset produced from REMIND scenario output results"

            new_exchanges = []

            # First, add the reference product exchange
            new_exchanges.append(
                {
                    "uncertainty type": 0,
                    "loc": 1,
                    "amount": 1,
                    "type": "production",
                    "production volume": 0,
                    "product": "electricity, high voltage",
                    "name": "market group for electricity, high voltage",
                    "unit": "kilowatt hour",
                    "location": region,
                }
            )

            # Second, add transformation loss
            transf_loss = self.get_production_weighted_losses('high', region)
            new_exchanges.append(
                {
                    "uncertainty type": 0,
                    "loc": 1,
                    "amount": transf_loss,
                    "type": "technosphere",
                    "production volume": 0,
                    "product": "electricity, high voltage",
                    "name": "market group for electricity, high voltage",
                    "unit": "kilowatt hour",
                    "location": region,
                }
            )

            # Fetch solar contribution in the mix, to subtract it
            # as solar energy is an input of low-voltage markets

            index_solar = [ind for ind in self.rmd.rev_electricity_market_labels if "solar" in ind.lower()]
            solar_amount = self.rmd.electricity_markets.loc[index_solar, region, 0].values.sum()

            # Loop through the REMIND technologies
            for technology in gen_tech:

                # If the given technology contributes to the mix
                if self.rmd.electricity_markets.loc[technology, region, 0] != 0.0:

                    # Contribution in supply
                    amount = self.rmd.electricity_markets.loc[
                        technology, region, 0
                    ].values

                    # Get the possible names of ecoinvent datasets
                    ecoinvent_technologies = self.powerplant_map[
                        self.rmd.rev_electricity_market_labels[technology]
                    ]

                    # Fetch electricity-producing technologies contained in the REMIND region
                    suppliers = list(
                        self.get_suppliers_of_a_region(
                            ecoinvent_regions, ecoinvent_technologies
                        )
                    )

                    suppliers = self.check_for_production_volume(suppliers)

                    # If no technology is available for the REMIND region
                    if len(suppliers) == 0:
                        # We fetch European technologies instead
                        suppliers = list(
                            self.get_suppliers_of_a_region(
                                ["RER"], ecoinvent_technologies
                            )
                        )

                    suppliers = self.check_for_production_volume(suppliers)

                    # If, after looking for European technologies, no technology is available
                    if len(suppliers) == 0:
                        # We fetch RoW technologies instead
                        suppliers = list(
                            self.get_suppliers_of_a_region(
                                ["RoW"], ecoinvent_technologies
                            )
                        )

                    suppliers = self.check_for_production_volume(suppliers)

                    if len(suppliers) == 0:
                        print('no suppliers for {} in {} with ecoinvent names {}'.format(technology, region, ecoinvent_technologies))

                    for supplier in suppliers:
                        share = self.get_production_weighted_share(supplier, suppliers)

                        new_exchanges.append(
                            {
                                "uncertainty type": 0,
                                "loc": (amount * share) / (1 - solar_amount),
                                "amount": (amount * share) / (1 - solar_amount),
                                "type": "technosphere",
                                "production volume": 0,
                                "product": supplier["reference product"],
                                "name": supplier["name"],
                                "unit": supplier["unit"],
                                "location": supplier["location"],
                            }
                        )

                        created_markets.append([
                            "high voltage, " + self.scenario + ", " + str(self.year),
                            technology,
                            region,
                            transf_loss,
                            0.0,
                            supplier['name'],
                            supplier['location'],
                            share,
                            (amount * share) / (1 - solar_amount)
                            ])
            new_dataset["exchanges"] = new_exchanges

            self.db.append(new_dataset)

        # Writing log of created markets

        with open(DATA_DIR / "logs/log created markets.csv", "w") as csv_file:
            writer = csv.writer(csv_file,
                                delimiter=';',
                                lineterminator='\n')
            writer.writerow(['dataset name',
                             'energy type',
                             'REMIND location',
                             'Transformation loss',
                             'Distr./Transmission loss',
                             'Supplier name',
                             'Supplier location',
                             'Contribution within energy type',
                             'Final contribution'])
            for line in created_markets:
                writer.writerow(line)

    def check_for_production_volume(self, suppliers):

        # Remove suppliers that do not have a production volume
        return [supplier for supplier in suppliers
        if self.get_production_weighted_share(supplier, suppliers) != 0]

    def relink_activities_to_new_markets(self):
        """
        Links electricity input exchanges to new datasets with the appropriate REMIND location:
        * "market for electricity, high voltage" --> "market group for electricity, high voltage"
        * "market for electricity, medium voltage" --> "market group for electricity, medium voltage"
        * "market for electricity, low voltage" --> "market group for electricity, low voltage"
        Does not return anything.
        """

        # Filter all activities that consume high voltage electricity

        for ds in ws.get_many(
            self.db, ws.exclude(ws.contains("name", "market group for electricity"))
        ):

            for exc in ws.get_many(
                ds["exchanges"],
                *[
                    ws.either(
                        *[
                            ws.contains("unit", "kilowatt hour"),
                            ws.contains("name", "market for electricity"),
                            ws.contains("name", "electricity voltage transformation"),
                            ws.contains("name", "market group for electricity"),
                        ]
                    )
                ]
            ):
                if exc["type"] != "production" and exc["unit"] == "kilowatt hour":
                    if "high" in exc["product"]:
                        exc["name"] = (
                            "market group for electricity, high voltage"
                        )
                        exc["product"] = "electricity, high voltage"
                        exc["location"] = self.geo.ecoinvent_to_remind_location(
                            exc["location"]
                        )
                    if "medium" in exc["product"]:
                        exc["name"] = (
                            "market group for electricity, medium voltage"
                        )
                        exc["product"] = "electricity, medium voltage"
                        exc["location"] = self.geo.ecoinvent_to_remind_location(
                            exc["location"]
                        )
                    if "low" in exc["product"]:
                        exc["name"] = (
                            "market group for electricity, low voltage"
                        )
                        exc["product"] = "electricity, low voltage"
                        exc["location"] = self.geo.ecoinvent_to_remind_location(
                            exc["location"]
                        )
                if 'input' in exc:
                    exc.pop('input')

    def find_ecoinvent_fuel_efficiency(self, ds, fuel_filters):
        """
        This method calculates the efficiency value set initially, in case it is not specified in the parameter
        field of the dataset. In Carma datasets, fuel inputs are expressed in megajoules instead of kilograms.

        :param ds: a wurst dataset of an electricity-producing technology
        :param fuel_filters: wurst filter to to filter fule input exchanges
        :return: the efficiency value set by ecoinvent
        """

        if "ecoinvent" in ds["database"]:
            not_allowed = ["thermal"]
            key = list(
                key
                for key in ds["parameters"]
                if "efficiency" in key and not any(item in key for item in not_allowed)
            )
            if len(key) > 0:
                return ds["parameters"][key[0]]

            else:
                energy_input = np.sum(
                    np.sum(
                        np.asarray(
                            [
                                [
                                    float(self.fuels_lhv[k]) / 3.6 * exc["amount"]
                                    for exc in ws.technosphere(ds, *fuel_filters)
                                    if k in exc["name"]
                                ]
                                for k in self.fuels_lhv
                            ]
                        )
                    )
                )
                ds["parameters"]["efficiency"] = (
                    float(ws.reference_product(ds)["amount"]) / energy_input
                )
                return ds["parameters"]["efficiency"]

        else:

            # Carma inventories have their fuel inputs directly expressed in megajoules.
            energy_input = np.sum(
                [exc["amount"] / 3.6 for exc in ws.technosphere(ds, *fuel_filters)]
            )

            ds["parameters"] = {}
            ds["parameters"]["efficiency"] = (
                float(ws.reference_product(ds)["amount"]) / energy_input
            )

            return ds["parameters"]["efficiency"]

    def find_fuel_efficiency_scaling_factor(self, ds, fuel_filters, technology):
        """
        This method calculates a scaling factor to change the process efficiency set by ecoinvent
        to the efficiency given by REMIND.

        :param ds: wurst dataset of an electricity-producing technology
        :param fuel_filters: wurst filter to filter the fuel input exchanges
        :param technology: label of an electricity-producing technology
        :return: a rescale factor to change from ecoinvent efficiency to REMIND efficiency
        :rtype: float
        """

        ecoinvent_eff = self.find_ecoinvent_fuel_efficiency(ds, fuel_filters)
        remind_locations = self.geo.ecoinvent_to_remind_location(ds["location"])
        remind_eff = (
            self.rmd.electricity_efficiencies.loc[
                dict(
                    variable=self.rmd.electricity_efficiency_labels[technology],
                    region=remind_locations,
                    value=0,
                )
            ]
            .mean()
            .values
        )

        return ecoinvent_eff / remind_eff

    def update_ecoinvent_efficiency_parameter(self, ds, scaling_factor):
        """
        Update the old efficiency value in the ecoinvent dataset by the newly calculated one.
        :param ds: dataset
        :type ds: dict
        :param scaling_factor: scaling factor (new efficiency / old efficiency)
        :type scaling_factor: float
        """
        parameters = ds["parameters"]
        possibles = ["efficiency", "efficiency_oil_country", "efficiency_electrical"]

        for key in possibles:
            if key in parameters:
                ds["parameters"][key] /= scaling_factor

    def get_remind_mapping(self):
        """
        Define filter functions that decide which wurst datasets to modify.
        :return: dictionary that contains filters and functions
        :rtype: dict
        """
        generic_excludes = [
            ws.exclude(ws.contains("name", "aluminium industry")),
            ws.exclude(ws.contains("name", "carbon capture and storage")),
            ws.exclude(ws.contains("name", "market")),
        ]
        no_imports = [ws.exclude(ws.contains("name", "import"))]

        gas_open_cycle_electricity = [
            ws.equals(
                "name", "electricity production, natural gas, conventional power plant"
            )
        ]

        biomass_chp_electricity = [
            ws.either(ws.contains("name", " wood"), ws.contains("name", "bio")),
            ws.equals("unit", "kilowatt hour"),
            ws.contains("name", "heat and power co-generation"),
        ]

        coal_IGCC = [
            ws.either(ws.contains("name", "coal"), ws.contains("name", "lignite")),
            ws.contains("name", "IGCC"),
            ws.contains("name", "no CCS"),
            ws.equals("unit", "kilowatt hour"),
        ]

        coal_IGCC_CCS = [
            ws.either(ws.contains("name", "coal"), ws.contains("name", "lignite")),
            ws.contains("name", "storage"),
            ws.contains("name", "pre"),
            ws.equals("unit", "kilowatt hour"),
        ]

        coal_PC_CCS = [
            ws.either(ws.contains("name", "coal"), ws.contains("name", "lignite")),
            ws.contains("name", "storage"),
            ws.contains("name", "post"),
            ws.equals("unit", "kilowatt hour"),
        ]

        gas_CCS = [
            ws.contains("name", "natural gas"),
            ws.either(ws.contains("name", "post"), ws.contains("name", "pre")),
            ws.contains("name", "storage"),
            ws.equals("unit", "kilowatt hour"),
        ]

        biomass_IGCC_CCS = [
            ws.either(
                ws.contains("name", "SNG"),
                ws.contains("name", "wood"),
                ws.contains("name", "BIGCC"),
            ),
            ws.contains("name", "storage"),
            ws.equals("unit", "kilowatt hour"),
        ]

        biomass_IGCC = [
            ws.contains("name", "BIGCC"),
            ws.contains("name", "no CCS"),
            ws.equals("unit", "kilowatt hour"),
        ]

        return {
            "Coal IGCC": {
                "eff_func": self.find_fuel_efficiency_scaling_factor,
                "technology filters": coal_IGCC,
                "fuel filters": [
                    ws.either(
                        ws.contains("name", "Hard coal"), ws.contains("name", "Lignite")
                    ),
                    ws.equals("unit", "megajoule"),
                ],
                "technosphere excludes": [],
            },
            "Coal IGCC CCS": {
                "eff_func": self.find_fuel_efficiency_scaling_factor,
                "technology filters": coal_IGCC_CCS,
                "fuel filters": [
                    ws.either(
                        ws.contains("name", "Hard coal"), ws.contains("name", "Lignite")
                    ),
                    ws.equals("unit", "megajoule"),
                ],
                "technosphere excludes": [],
            },
            "Coal PC": {
                "eff_func": self.find_fuel_efficiency_scaling_factor,
                "technology filters": filters.coal_electricity + generic_excludes,
                "fuel filters": [
                    ws.either(
                        ws.contains("name", "hard coal"), ws.contains("name", "lignite")
                    ),
                    ws.doesnt_contain_any("name", ("ash", "SOx")),
                    ws.equals("unit", "kilogram"),
                ],
                "technosphere excludes": [],
            },
            "Coal PC CCS": {
                "eff_func": self.find_fuel_efficiency_scaling_factor,
                "technology filters": coal_PC_CCS,
                "fuel filters": [
                    ws.either(
                        ws.contains("name", "Hard coal"), ws.contains("name", "Lignite")
                    ),
                    ws.equals("unit", "megajoule"),
                ],
                "technosphere excludes": [],
            },
            "Coal CHP": {
                "eff_func": self.find_fuel_efficiency_scaling_factor,
                "technology filters": filters.coal_chp_electricity + generic_excludes,
                "fuel filters": [
                    ws.either(
                        ws.contains("name", "hard coal"), ws.contains("name", "lignite")
                    ),
                    ws.doesnt_contain_any("name", ("ash", "SOx")),
                    ws.equals("unit", "kilogram"),
                ],
                "technosphere excludes": [],
            },
            "Gas OC": {
                "eff_func": self.find_fuel_efficiency_scaling_factor,
                "technology filters": gas_open_cycle_electricity
                + generic_excludes
                + no_imports,
                "fuel filters": [
                    ws.either(
                        ws.contains("name", "natural gas, low pressure"),
                        ws.contains("name", "natural gas, high pressure"),
                    ),
                    ws.equals("unit", "cubic meter"),
                ],
                "technosphere excludes": [],
            },
            "Gas CC": {
                "eff_func": self.find_fuel_efficiency_scaling_factor,
                "technology filters": filters.gas_combined_cycle_electricity
                + generic_excludes
                + no_imports,
                "fuel filters": [
                    ws.either(
                        ws.contains("name", "natural gas, low pressure"),
                        ws.contains("name", "natural gas, high pressure"),
                    ),
                    ws.equals("unit", "cubic meter"),
                ],
                "technosphere excludes": [],
            },
            "Gas CHP": {
                "eff_func": self.find_fuel_efficiency_scaling_factor,
                "technology filters": filters.gas_chp_electricity
                + generic_excludes
                + no_imports,
                "fuel filters": [
                    ws.either(
                        ws.contains("name", "natural gas, low pressure"),
                        ws.contains("name", "natural gas, high pressure"),
                    ),
                    ws.equals("unit", "cubic meter"),
                ],
                "technosphere excludes": [],
            },
            "Gas CCS": {
                "eff_func": self.find_fuel_efficiency_scaling_factor,
                "technology filters": gas_CCS,
                "fuel filters": [
                    ws.contains("name", "Natural gas"),
                    ws.equals("unit", "megajoule"),
                ],
                "technosphere excludes": [],
            },
            "Oil": {
                "eff_func": self.find_fuel_efficiency_scaling_factor,
                "technology filters": (
                    filters.oil_open_cycle_electricity
                    + generic_excludes
                    + [ws.exclude(ws.contains("name", "nuclear"))]
                ),
                "fuel filters": [
                    ws.contains("name", "heavy fuel oil"),
                    ws.equals("unit", "kilogram"),
                ],
                "technosphere excludes": [],
            },
            "Biomass CHP": {
                "eff_func": self.find_fuel_efficiency_scaling_factor,
                "technology filters": biomass_chp_electricity + generic_excludes,
                "fuel filters": [
                    ws.either(
                        ws.contains("name", "wood pellet"),
                        ws.contains("name", "biogas"),
                    ),
                    ws.either(
                        ws.equals("unit", "kilogram"), ws.equals("unit", "cubic meter")
                    ),
                ],
                "technosphere excludes": [],
            },
            "Biomass IGCC CCS": {
                "eff_func": self.find_fuel_efficiency_scaling_factor,
                "technology filters": biomass_IGCC_CCS,
                "fuel filters": [
                    ws.either(
                        ws.contains("name", "100% SNG, burned in CC plant"),
                        ws.contains("name", "Wood chips"),
                        ws.contains("name", "Hydrogen"),
                    ),
                    ws.equals("unit", "megajoule"),
                ],
                "technosphere excludes": [],
            },
            "Biomass IGCC": {
                "eff_func": self.find_fuel_efficiency_scaling_factor,
                "technology filters": biomass_IGCC,
                "fuel filters": [
                    ws.contains("name", "Hydrogen"),
                    ws.equals("unit", "megajoule"),
                ],
                "technosphere excludes": [],
            },
        }

    def update_electricity_efficiency(self):
        """
        This method modifies each ecoinvent coal, gas,
        oil and biomass dataset using data from the REMIND model.
        Return a wurst database with modified datasets.

        :return: a wurst database, with rescaled electricity-producing datasets.
        :rtype: list
        """

        technologies_map = self.get_remind_mapping()

        for remind_technology in technologies_map:
            dict_technology = technologies_map[remind_technology]
            print("Rescale inventories and emissions for", remind_technology)

            datsets = list(ws.get_many(self.db, *dict_technology["technology filters"]))

            # no activities found? Check filters!
            assert (len(datsets) > 0), "No dataset found for {}".format(remind_technology)
            for ds in datsets:
                # Modify using remind efficiency values:
                scaling_factor = dict_technology["eff_func"](
                    ds, dict_technology["fuel filters"], remind_technology
                )
                self.update_ecoinvent_efficiency_parameter(ds, scaling_factor)

                # Rescale all the technosphere exchanges according to REMIND efficiency values
                wurst.change_exchanges_by_constant_factor(
                    ds,
                    float(scaling_factor),
                    dict_technology["technosphere excludes"],
                    [ws.doesnt_contain_any("name", self.emissions_map)],
                )

                # Update biosphere exchanges according to REMIND emission values
                for exc in ws.biosphere(
                    ds, ws.either(*[ws.contains("name", x) for x in self.emissions_map])
                ):
                    remind_emission_label = self.emissions_map[exc["name"]]

                    remind_emission = self.rmd.electricity_emissions.loc[
                        dict(
                            region=self.geo.ecoinvent_to_remind_location(ds["location"]),
                            pollutant=remind_emission_label,
                            sector=self.rmd.electricity_emission_labels[
                                remind_technology
                            ]
                        )
                    ].values.item(0)

                    if exc["amount"] == 0:
                        wurst.rescale_exchange(
                            exc, remind_emission / 1, remove_uncertainty=True
                        )
                    else:
                        wurst.rescale_exchange(exc, remind_emission / exc["amount"])

        return self.db

    def update_electricity_markets(self):
        """
        Delete electricity markets. Create high, medium and low voltage market groups for electricity.
        Link electricity-consuming datasets to newly created market groups for electricity.
        Return a wurst database with modified datasets.

        :return: a wurst database with new market groups for electricity
        :rtype: list
        """
        # We first need to delete 'market for electricity' and 'market group for electricity' datasets
        print("Remove old electricity datasets")
        list_to_remove = [
            "market group for electricity, high voltage",
            "market group for electricity, medium voltage",
            "market group for electricity, low voltage",
            "market for electricity, high voltage",
            "market for electricity, medium voltage",
            "market for electricity, low voltage",
            "electricity, high voltage, import",
            "electricity, high voltage, production mix",
        ]

        # Writing log of deleted markets
        markets_to_delete = [
            [i['name'], i['location']] for i in self.db
            if any(stop in i["name"] for stop in list_to_remove)
        ]

        if not os.path.exists(DATA_DIR / "logs"):
            os.makedirs(DATA_DIR / "logs")

        with open(DATA_DIR / "logs/log deleted markets.csv", "w") as csv_file:
            writer = csv.writer(csv_file,
                                delimiter=';',
                                lineterminator = '\n')
            writer.writerow(['dataset name', 'location'])
            for line in markets_to_delete:
                writer.writerow(line)

        self.db = [
            i for i in self.db if not any(stop in i["name"] for stop in list_to_remove)
        ]

        # We then need to create high voltage REMIND electricity markets
        print("Create high voltage markets.")
        self.create_new_markets_high_voltage()
        print("Create medium voltage markets.")
        self.create_new_markets_medium_voltage()
        print("Create low voltage markets.")
        self.create_new_markets_low_voltage()

        # Finally, we need to relink all electricity-consuming activities to the new electricity markets
        print("Link activities to new electricity markets.")
        self.relink_activities_to_new_markets()

        print('Log of deleted electricity markets saved in {}'.format(DATA_DIR / 'logs'))
        print('Log of created electricity markets saved in {}'.format(DATA_DIR / 'logs'))

        return self.db


"""Text encoders using neural network-inspired architectures.

Several encoders are defined, all using works as tokens (i.e. the atomic text
unit).

## Non-parametric encoders

@@EncoderAvg
@@EncoderMax

## CNN-based encoder

@@EncoderCNN

## RNN-based encoder

@@EncoderUniRNN
@@EncoderBiRNN
"""
import numpy as np
import tensorflow as tf
import tensorflow.compat.v1 as tf1
import anna.model.utils as utils
import anna.data.dataset.glove as glove
import anna.data.dataset.fasttext as fasttext

class Encoder:
    """
    Encoder that takes the input features, and produces a
    vector representation of them.
    """

    def __init__(self,
                 data_dir,
                 input_names=None,
                 input_limit=None,
                 voc_size=100000,
                 oov_size=1000,
                 fixed_embeddings=False,
                 pretrained_embeddings="glove",
                 *args,
                 **kwargs):
        """
        Creates an encoder with the given embeddings and maximum size
        for the input.

        Args:
            data_dir (str): path to the data folder
            input_names (list[str]): names of the string inputs to encode
            input_limit (int): maximum size to use from the input sequence
            voc_size (int): nr of embeddings to store (i.e. size of vocabulary)
            oov_size (int): nr of buckets to use for out-of-vocabulary words
            fixed_embeddings (bool): whether the embeddings should be trained
            pretrained_embeddings (str): which pretrained embeddings to use.
              options are "glove", "fasttext" (default "glove")
        """
        if not input_names:
            input_names = ["title", "text"]

        self.data_dir = data_dir
        self.input_names = input_names
        self.input_limit = input_limit
        self.oov_size = oov_size
        self.fixed_emb = fixed_embeddings

        # Fetch pre-trained word embeddings
        if pretrained_embeddings == "glove":
            self.words, self.emb = glove.fetch_and_parse(data_dir,
                                                         voc_size=voc_size)
        elif pretrained_embeddings == "fasttext":
            self.words, self.emb = fasttext.fetch_and_parse(data_dir,
                                                            voc_size=voc_size)
        else:
            raise ValueError("Unknown pretrained_embeddings: {}"
                             .format(pretrained_embeddings))

        if oov_size > 0:
            extra_emb = np.random.uniform(-.1, .1, size=[oov_size,
                                                         self.emb.shape[1]])
            self.emb = np.concatenate([self.emb, extra_emb])

    def __call__(self, features, mode):
        """
        Builds the encoder for a specific feature `name` in `features`.

        Args:
            features (dict): dictionary of input features
            mode (tf.estimator.ModeKeys): the mode we are on

        Returns:
            mem (tf.Tensor): sequential representation of the input.
              [batch, sum(len), size]
            mem_len (tf.Tensor): length of `mem`.
              [batch]
            mem_fixed (tf.Tensor): fixed-sized representation of the input.
              [batch, size]
        """
        emb = tf1.get_variable("word_embeddings",
                               self.emb.shape,
                               initializer=tf.constant_initializer(self.emb),
                               trainable=not self.fixed_emb)

        with tf.name_scope("encoder"):
            # Encode all inputs
            mem = []
            mem_len = []
            mem_fixed = []
            for name in self.input_names:
                with tf1.variable_scope("input_" + name):
                    x, x_len = get_input(features,
                                         name,
                                         self.words,
                                         emb,
                                         self.input_limit,
                                         self.oov_size)

                    m, m_len, m_fixed = self.encode(x, x_len, mode, name)

                    # (batch, seq_len, size)
                    mem.append(m)
                    # (batch)
                    mem_len.append(m_len)
                    # (batch, emb_size)
                    mem_fixed.append(m_fixed)

            # Concatenate variable memory:
            # (batch, sum(seq_len), size)
            final_mem, final_mem_len = utils.seq_concat(mem, mem_len)

            # Concatenate fixed memory:
            # (batch, len(input_names) * emb_size)
            final_mem_fixed = tf.concat(mem_fixed, -1)

            return final_mem, final_mem_len, final_mem_fixed

    def encode(self, x, x_len, mode, name):
        """
        Encode a given tensor `x` with length `x_len`.

        Args:
            x (tf.Tensor): the tensor we want to encode
            x_len (tf.Tensor): the length `x`
            mode (tf.estimator.ModeKeys): the mode we are on
            name (str): name of the tensor

        Returns:
            mem (tf.Tensor): sequential representation of the input.
              [batch, sum(len), size]
            mem_len (tf.Tensor): length of `mem`.
              [batch]
            mem_fixed (tf.Tensor): fixed-sized representation of the input.
              [batch, size]
        """
        raise NotImplementedError

def get_input(features, name, words, emb, input_limit=None, oov_size=0):
    """
    Gets the sequence feature `name` from the `features`,
    trims the size if necessary, and maps it to its list
    of embeddings.

    Args:
        features (dict): dictionary of input features
        name (str): name of the feature to encode
        words (list[str]): list of strings as vocabulary
        emb (tf.Tensor): initialization for the word embeddings
        input_limit (int): maximum size to use from the input sequence
        oov_size (int): nr of buckets to use for out-of-vocabulary words

    Returns:
        x (tf.Tensor): the tensor of embeddings for the feature `name`
        x_len (tf.Tensor): the length each `x`
    """
    x = features[name]
    x_mask = features[name + "_mask"]

    # Limit size
    # (batch, input_limit)
    if input_limit:
        with tf.name_scope("trim"):
            x = x[:, :input_limit]
            x_mask = x_mask[:, :input_limit]

    # Length of each sequence
    # (batch)
    with tf.name_scope("length"):
        x_len = tf.reduce_sum(x_mask, 1)
        x_len = tf.cast(x_len, tf.int32)

    with tf.name_scope("embed"):
        # Convert strings to ids
        # (batch, input_limit)
        x = tf.contrib.lookup.index_table_from_tensor(
            mapping=words,
            default_value=1,
            num_oov_buckets=oov_size).lookup(x)

        # Count how many out-of-vocabulary (OOV) words are in the input
        num_oov = tf.cast(x, tf.float32) * x_mask
        num_oov = tf.logical_or(tf.equal(num_oov, 1),
                                tf.greater_equal(num_oov, len(words)))
        num_oov = tf.reduce_sum(tf.cast(num_oov, tf.float32), axis=1)

        # Replace with embeddings
        # (batch, input_limit, emb_size)
        x = tf.nn.embedding_lookup(emb, x)

        # Clear embeddings for pads
        # (batch, input_limit, emb_size)
        x = x * x_mask[:, :, tf.newaxis]

    tf1.summary.scalar("n_words", tf.reduce_mean(x_len))
    tf1.summary.scalar("n_oov_words", tf.reduce_mean(num_oov))

    return x, x_len

class EncoderAvg(Encoder):
    """
    Encodes the input as an average of its word embeddings.
    """

    def encode(self, x, x_len, mode, name):
        # Average embeddings, avoiding zero division when the input is empty
        # (batch, emb_size)
        div = tf.cast(x_len[:, tf.newaxis], tf.float32)
        result = tf.reduce_sum(x, 1) / tf.maximum(div, tf.ones_like(div))
        return x, x_len, result

class EncoderMax(Encoder):
    """
    Encodes the input taking the max of each of its word embedding dimensions.
    """

    def encode(self, x, x_len, mode, name):
        # Average embeddings, avoiding zero division when the input is empty
        # (batch, emb_size)
        return x, x_len, tf.reduce_max(x, 1)

class EncoderCNN(Encoder):
    """
    Encodes the input using a simple CNN and max-over-time pooling.
    """

    def encode(self, x, x_len, mode, name):
        pools = []
        for size in [2, 3, 4]:
            # Run CNN over words
            # (batch, input_len, filters)
            pool = tf.layers.conv1d(
                name="{}_conv_{}".format(name, size),
                inputs=x,
                filters=256,
                kernel_size=size,
                padding="same",
                activation=tf.nn.relu)

            # Max over-time pooling
            # (batch, filters)
            pool = tf.reduce_max(pool, 1)

            pools.append(pool)

        # Max over-time pooling
        # (batch, cnns * filters)
        result = tf.concat(pools, 1)

        return x, x_len, result

class EncoderRNN(Encoder):
    """
    Encodes the input using a bidirectional RNN, returning the outputs for
    all steps.
    """

    def __init__(self,
                 data_dir,
                 rnn_type="gru",
                 hidden_size=1024,
                 dropout=.3,
                 *args,
                 **kwargs):
        super().__init__(data_dir, *args, **kwargs)

        self.hidden_size = hidden_size
        self.dropout = dropout
        self.rnn_type = rnn_type

    def encode(self, x, x_len, mode, name):
        raise NotImplementedError

class EncoderUniRNN(EncoderRNN):
    """
    Encodes the input using a unidirectional RNN.
    """

    def encode(self, x, x_len, mode, name):
        cell = utils.rnn_cell(self.rnn_type,
                              self.hidden_size,
                              mode,
                              self.dropout)

        # Runs encoding RNN
        # out: (batch_size, size, hidden_size)
        # states: (2 or 0, batch_size, hidden_size)
        outputs, state = tf.nn.dynamic_rnn(cell,
                                           x,
                                           sequence_length=x_len,
                                           swap_memory=True,
                                           dtype=tf.float32)

        # Get the right part of the state (useful for tuple states like in LSTM)
        # (batch_size, 2 * hidden_size)
        state = utils.rnn_state_trim(self.rnn_type, state)

        return outputs, x_len, state

class EncoderBiRNN(EncoderRNN):
    """
    Encodes the input using a bidirectional RNN.
    """

    def encode(self, x, x_len, mode, name):
        c_fw = utils.rnn_cell(self.rnn_type,
                              self.hidden_size // 2,
                              mode,
                              self.dropout)
        c_bw = utils.rnn_cell(self.rnn_type,
                              self.hidden_size // 2,
                              mode,
                              self.dropout)

        # Runs encoding RNN
        # out: 2 x (batch_size, size, hidden_size)
        # states: 2 x (2 or 0, batch_size, hidden_size)
        outputs, states = tf.nn.bidirectional_dynamic_rnn(c_fw,
                                                          c_bw,
                                                          x,
                                                          sequence_length=x_len,
                                                          swap_memory=True,
                                                          dtype=tf.float32)

        # Concatenates outputs
        # out: (batch_size, size, 2 * hidden_size)
        # states: (batch_size, 2 * hidden_size)
        outputs = tf.concat(outputs, axis=-1)
        states = tf.concat(states, axis=-1)

        # Get the right part of the state (useful for tuple states like in LSTM)
        # (batch_size, 2 * hidden_size)
        state = utils.rnn_state_trim(self.rnn_type, states)

        return outputs, x_len, state


#!/usr/bin/env python3
# Copyright 2016 Codethink Ltd.
# Apache 2.0 license

'''Generate a random build graph in 'node-link' JSON format.'''

import networkx
import networkx.readwrite.json_graph

import json
import sys

INPUT_NAMES = '/usr/share/dict/words'

# Remember Concourse's UI kills your browser if given more than a couple
# of hundred nodes... or if there's any cycles
N_NODES = 10
N_EDGES = 5

# Return a random graph with a fixed number of nodes and edges.
g = networkx.gnm_random_graph(N_NODES, N_EDGES, directed=True)

# Assign names
with open(INPUT_NAMES) as f:
    lines = f.readlines()

    for i in g.nodes():
        line_mult = len(lines) / g.number_of_nodes()
        name = lines[int(i * line_mult)].strip()
        g.node[i]['name'] = name

json.dump(networkx.readwrite.json_graph.node_link_data(g), sys.stdout)