import tensorflow as tf
import numpy as np
import os
import sys
import gflags

from keras.optimizers import Adam
from keras.callbacks import ModelCheckpoint, LearningRateScheduler
from keras.callbacks import ReduceLROnPlateau
from keras.callbacks import TensorBoard
from keras.utils import plot_model
from keras import backend as K

import logz
import nets
import cifar10_resnet
import utils
import data_utils
import log_utils
from common_flags import FLAGS
from time import time, strftime, localtime
import pdb

# Constants
TRAIN_PHASE = 1

os.environ["PATH"] += os.pathsep + 'C:/Program Files (x86)/Graphviz2.38/bin'
def getModel(img_width, img_height, img_channels, output_dim, weights_path):
    """
    Initialize model.

    # Arguments
       img_width: Target image widht.
       img_height: Target image height.
       img_channels: Target image channels. -----------FUTURE NUMBER OF IMAGES
       output_dim: Dimension of model output (number of classes).
       weights_path: Path to pre-trained model.

    # Returns
       model: A Model instance.

    ------------ FUTURE CNN?? TO FIT THE INPUT (MANY IMAGES)
    """
    model = nets.resnet50(img_width, img_height, img_channels, output_dim)

    if weights_path:
        try:
            model.load_weights(weights_path)
            print("Loaded model from {}".format(weights_path))
        except:
            print("Impossible to find weight path. Returning untrained model")

    return model

def getModelResnet(n, version, img_width, img_height, img_channels, output_dim, weights_path):
    """
    Initialize model.

    # Arguments
       n: parameter that determines the net depth.
       version: 1 for resnet v1 or 2 for v2.
       img_width: Target image widht.
       img_height: Target image height.
       img_channels: Target image channels.----- # IMAGES PER BLOCK
       output_dim: Dimension of model output (number of classes).
       weights_path: Path to pre-trained model.

    # Returns
       model: A Model instance.
    """

    # Model parameter
    # ----------------------------------------------------------------------------
    #           |      | 200-epoch | Orig Paper| 200-epoch | Orig Paper| sec/epoch
    # Model     |  n   | ResNet v1 | ResNet v1 | ResNet v2 | ResNet v2 | GTX1080Ti
    #           |v1(v2)| %Accuracy | %Accuracy | %Accuracy | %Accuracy | v1 (v2)
    # ----------------------------------------------------------------------------
    # ResNet20  | 3 (2)| 92.16     | 91.25     | -----     | -----     | 35 (---)
    # ResNet32  | 5(NA)| 92.46     | 92.49     | NA        | NA        | 50 ( NA)
    # ResNet44  | 7(NA)| 92.50     | 92.83     | NA        | NA        | 70 ( NA)
    # ResNet56  | 9 (6)| 92.71     | 93.03     | 93.01     | NA        | 90 (100)
    # ResNet110 |18(12)| 92.65     | 93.39+-.16| 93.15     | 93.63     | 165(180)
    # ResNet164 |27(18)| -----     | 94.07     | -----     | 94.54     | ---(---)
    # ResNet1001| (111)| -----     | 92.39     | -----     | 95.08+-.14| ---(---)
    # ---------------------------------------------------------------------------

    # Model version
    # Orig paper: version = 1 (ResNet v1), Improved ResNet: version = 2 (ResNet v2)

    # Computed depth from supplied model parameter n
    if version == 1:
        depth = n * 6 + 2
    elif version == 2:
        depth = n * 9 + 2

#    model = nets.resnet50(img_width, img_height, img_channels, output_dim)
    input_shape = (img_height, img_width, img_channels);

    if version == 2:
        model = cifar10_resnet.resnet_v2(input_shape=input_shape, depth=depth, num_classes=output_dim)
    else:
        model = cifar10_resnet.resnet_v1(input_shape=input_shape, depth=depth, num_classes=output_dim)

    # Model name, depth and version
    model_type = 'ResNet%dv%d' % (depth, version)
    print(model_type)
    print(model.summary())

    if weights_path:
        try:
            model.load_weights(weights_path)
            print("Loaded model from {}".format(weights_path))
        except:
            print("Impossible to find weight path. Returning untrained model")

    return model

def trainModel(train_data_generator, val_data_generator, model, initial_epoch):
    """
    Model training.

    # Arguments
       train_data_generator: Training data generated batch by batch.
       val_data_generator: Validation data generated batch by batch.
       model: A Model instance.
       initial_epoch: Epoch from which training starts.
    """
    # Configure training process
    model.compile(loss='categorical_crossentropy',
              optimizer=Adam(lr=cifar10_resnet.lr_schedule(0)),
              metrics=['categorical_accuracy'])

    # Save model with the lowest validation loss
    weights_path = os.path.join(FLAGS.experiment_rootdir, 'weights_{epoch:03d}.h5')
    writeBestModel = ModelCheckpoint(filepath=weights_path, monitor='val_loss',
                                     save_best_only=True, save_weights_only=True)

    # Save training and validation losses.
    logz.configure_output_dir(FLAGS.experiment_rootdir)
    saveModelAndLoss = log_utils.MyCallback(filepath=FLAGS.experiment_rootdir)

    # Train model
    steps_per_epoch = int(np.ceil(train_data_generator.samples / FLAGS.batch_size))
    validation_steps = int(np.ceil(val_data_generator.samples / FLAGS.batch_size))-1

    lr_scheduler = LearningRateScheduler(cifar10_resnet.lr_schedule)

    lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1),
                                   cooldown=0,
                                   patience=5,
                                   min_lr=0.5e-6)

    #tensorboard = TensorBoard(log_dir="logs/{}".format(time()))
    strTime = strftime("%Y%b%d_%Hh%Mm%Ss", localtime(time()))
#    tensorboard = TensorBoard(log_dir="logs/{}".format(strTime), histogram_freq=10,
#                              batch_size=32, write_graph=False, write_grads=True,
#                              write_images=False, embeddings_freq=0,
#                              embeddings_layer_names=None, embeddings_metadata=None)
    tensorboard = TensorBoard(log_dir="logs/{}".format(strTime), histogram_freq=0)
    # TENSORBOARD SIRVE PARA VISUALIZAR LOS RESULTADOS DE LAS ITERACIONES
    # HASTA AQUI SE HARÁ UN PROCESO ITERATIVO QUE GUARDARÁ DE TODOS LOS CASOS
    # EL MODELO CON EL MEJOR RESULTADO SOBRE LOS DATOS DE VALIDACIÓN

    callbacks = [writeBestModel, saveModelAndLoss, lr_reducer, lr_scheduler, tensorboard]

    model.fit_generator(train_data_generator,
                        epochs=FLAGS.epochs, steps_per_epoch = steps_per_epoch,
                        callbacks=callbacks,
                        validation_data=val_data_generator,
                        validation_steps = validation_steps,
                        initial_epoch=initial_epoch) # INITIAL EPOCH: POR SI EL ENTRENAMIENTO SE QUEDA A MEDIAS Y QUIERES QUE SIGA
    # CON EL MEJOR MODELO, SE ENTRENA

def _main():

    # Set random seed
    if FLAGS.random_seed:
        seed = np.random.randint(0,2*31-1)
    else:
        seed = 5
    np.random.seed(seed)
    tf.set_random_seed(seed)

    # Set training phase
    K.set_learning_phase(TRAIN_PHASE)

    # Create the experiment rootdir if not already there: create a model if the name of the one in the parameters doesn't exist
    if not os.path.exists(FLAGS.experiment_rootdir):
        os.makedirs(FLAGS.experiment_rootdir)

    # Input image dimensions: CAMBIAR EN COMMON_FLAGS : ESTA POR DEFECTO
    img_width, img_height = FLAGS.img_width, FLAGS.img_height

    # Image mode (RGB or grayscale)
    if FLAGS.img_mode=='rgb':
        img_channels = 3
    elif FLAGS.img_mode == 'grayscale':
        img_channels = 1
    else:
        raise IOError("Unidentified image mode: use 'grayscale' or 'rgb'")

    # Output dimension (7 classes/gestures): CAMBIAR SEGUN LAS CLASES DE LA BASE DE DATOS
    num_classes = 7

    # Generate training data with real-time augmentation: HABRÁ QUE CAMBIARLO
    # TODO LO QUE SE HAGA SOBRE LOS DATOS AQUI SE TENDRA QUE HACER EN LOS DATOS PARA EL TEST
    train_datagen = data_utils.DataGenerator(rescale = 1./255)

    # Iterator object containing training data to be generated batch by batch
    # ESTA ES LA FUNCIÓN QUE TENDREMOS QUE TOCAR. BATCH-SIZE: CUANTOS DATOS PASAMOS EN CADA "TANDA" POR PROBLEMAS DE MEMORIA
    train_generator = train_datagen.flow_from_directory(FLAGS.train_dir,
                                                        num_classes,
                                                        shuffle = True,
                                                        img_mode = FLAGS.img_mode,
                                                        target_size=(img_height, img_width),
                                                        batch_size = FLAGS.batch_size)

    # Check if the number of classes in dataset corresponds to the one specified
    assert train_generator.num_classes == num_classes, \
                        " Not macthing output dimensions in training data."

    # Generate validation data with real-time augmentation
    val_datagen = data_utils.DataGenerator(rescale = 1./255)

    # Iterator object containing validation data to be generated batch by batch
    val_generator = val_datagen.flow_from_directory(FLAGS.val_dir,
                                                    num_classes,
                                                    shuffle = False,
                                                    img_mode = FLAGS.img_mode,
                                                    target_size=(img_height, img_width),
                                                    batch_size = FLAGS.batch_size)

    # Check if the number of classes in dataset corresponds to the one specified
    assert val_generator.num_classes == num_classes, \
                        " Not matching output dimensions in validation data."

    # Weights to restore
    weights_path = FLAGS.weights_fname

    # Epoch from which training starts
    initial_epoch = 0
    if not FLAGS.restore_model:
        # In this case weights are initialized randomly
        weights_path = None
    else:
        # In this case weigths are initialized as specified in pre-trained model
        initial_epoch = FLAGS.initial_epoch

    # Define model: SE DEFINE LA RED CON LOS PARAMETROS QUE QUERAMOS O SOBRE LOS PESOS QUE YA TENGAMOS ANTES
    n = 1
    version = 1 # 1 o 2
    model = getModelResnet(n, version, img_width, img_height, img_channels,
                    num_classes, weights_path)

    # Save the architecture of the network as png: IMAGEN DE LA ESTRUCTURA DE LA RED
    plot_arch_path = os.path.join(FLAGS.experiment_rootdir, 'architecture.png')
    plot_model(model, to_file=plot_arch_path)

    # Serialize model into json: SE GUARDA EL "ESQUELETO" DE LA RED: NUMERO DE CAPAS, NEURONAS... LOS PESOS SE GUARDAN A PARTE
    json_model_path = os.path.join(FLAGS.experiment_rootdir, FLAGS.json_model_fname)
    utils.modelToJson(model, json_model_path)

    # Train model
    trainModel(train_generator, val_generator, model, initial_epoch)

    # Plot training and validation losses
    utils.plot_loss(FLAGS.experiment_rootdir)

def main(argv):
    # Utility main to load flags
    try:
      argv = FLAGS(argv)  # parse flags
    except gflags.FlagsError:
      print ('Usage: %s ARGS\\n%s' % (sys.argv[0], FLAGS))

      sys.exit(1)
    _main()

if __name__ == "__main__":
    main(sys.argv)


import numpy as np
import numpy.linalg as la
from matplotlib import pyplot as plt

import cv2
import tifffile as tiff
from skimage.morphology import closing, medial_axis, disk
import sknw
from simplification.cutil import simplify_coords
from itertools import combinations, permutations
from shapely.geometry import LineString
import geopandas as gpd
import pandas as pd
from osgeo import gdal
import re

from loguru import logger
from tqdm import tqdm

from multiprocessing import Pool, cpu_count

import torch
from torch import Tensor

from mercantile import bounds, Tile, tile, parent
from tilelib import OSMRoadEngine, TileEngine

from pathlib import Path

from sys import argv

image_directory = Path(argv[1])
assert image_directory.exists(), "Path must exist!"
assert image_directory.is_dir(), "File must be a directory!"

model = torch.load("models/resnet34.pt")

def read_image(file):
    logger.info(f"Opening {file}...")
    file = str(file.absolute())
    try:
        img = tiff.imread(file)
    except Exception as e:
        logger.critical(f"{file} is corrupt or otherwise inaccessible! Returning black 1024x1024 image...")
        return np.zeros((1024,1024,3))
    logger.info(f"Tonemapping {file}...")
    img_colors = np.array([x for x in img.reshape(-1, 3) if x.sum()>0])
    lo = np.array([0,0,0])
    hi = np.array([0,0,0])
    for i in range(3):
        try:
            lo[i] = np.percentile(img_colors[:,i], 2)
            hi[i] = np.percentile(img_colors[:,i], 98)
        except Exception as e:
            logger.critical(f"{file} is all black on channel {i}! Refusing to tonemap...")
            lo[i] = 0
            hi[i] = 1
    img = (img - lo) / (hi-lo)
    img = np.clip(img, 0, 1)
    return 1-(1-img)**2

def np_to_torch(x):
    if len(x.shape) == 3:
        h, w, c = x.shape
        x = x.reshape(1, h, w, c).copy()
    if type(x) == np.ndarray:
        x = Tensor(x).cuda()
    if not x.is_cuda:
        x = x.cuda()
    return x.permute([0, 3, 1, 2]).float()

def make_d8_plot(img):
    logger.info(f"Segmenting image...")

    h, w, c = img.shape
    new_arr = np.zeros((7, c, h, w))
    c_img = img.swapaxes(1,2).swapaxes(0,1)
    new_arr[0] = c_img
    new_arr[1] = c_img[:, ::-1, :]
    new_arr[2] = c_img[:, :, ::-1]
    new_arr[3] = c_img[:, ::-1, ::-1]
    new_arr[4] = np.rot90(img, k=1).swapaxes(1,2).swapaxes(0,1)
    new_arr[5] = np.rot90(img, k=2).swapaxes(1,2).swapaxes(0,1)
    new_arr[6] = np.rot90(img, k=3).swapaxes(1,2).swapaxes(0,1)
    img = Tensor(new_arr).cuda()
    img_pile = np.zeros((7, 1, h ,w))
    for i in range(7):
        img_pile[i, 0, :, :] = torch.sigmoid(model(img[i:i+1])).cpu().numpy()

    img_pile[1] = img_pile[1, :, ::-1, :]
    img_pile[2] = img_pile[2, :, :, ::-1]
    img_pile[3] = img_pile[3, :, ::-1, ::-1]
    img_pile[4] = np.rot90(img_pile[4,:,:,:].swapaxes(0,1).swapaxes(1,2), k=-1).swapaxes(1,2).swapaxes(0,1)
    img_pile[5] = np.rot90(img_pile[5,:,:,:].swapaxes(0,1).swapaxes(1,2), k=-2).swapaxes(1,2).swapaxes(0,1)
    img_pile[6] = np.rot90(img_pile[6,:,:,:].swapaxes(0,1).swapaxes(1,2), k=-3).swapaxes(1,2).swapaxes(0,1)

    return img_pile.max(0)

def segment(basemap):
    delta = (1024-750)//2
    def make_nn_input(basemap):
        basemap = basemap.copy()
        basemap = cv2.resize(basemap, (750, 750))
        basemap = cv2.copyMakeBorder(basemap,delta,delta,delta,delta,cv2.BORDER_REFLECT)
        return basemap
    basemap = make_nn_input(basemap)
    with torch.no_grad():
        mask = make_d8_plot(basemap)
    mask = mask[:, delta:-delta, delta:-delta]
    return mask

def skeletonize(mask):
    m_img = mask[0,:,:] > 0.3
    m_img = closing(m_img)
    skel, dist = medial_axis(m_img, return_distance=True)
    return skel, dist

def get_geotiff_bbox(file):
    file = str(file.absolute())
    src = gdal.Open(file)
    if src is None:
        return (0,0),(1,1)
    ulx, xres, xskew, uly, yskew, yres  = src.GetGeoTransform()
    lrx = ulx + (src.RasterXSize * xres)
    lry = uly + (src.RasterYSize * yres)
    return (uly, ulx), (lry, lrx)

def vectorize(blob):
    (skel, dist), path = blob

    logger.info(f"Vectorizing {path}...")
    graph = sknw.build_sknw(skel)
    df = gpd.GeoDataFrame(columns=['geometry', 'width'])

    ul, lr = get_geotiff_bbox(path)
    dy = ul[0]-lr[0]
    dx = lr[1]-ul[1]

    for s,e in graph.edges():
        H, W = dist.shape
        widths = [dist[tuple(pt)] for pt in graph[s][e]['pts'].astype(np.int32)]
        width = np.percentile(widths, 33)/H*dy

    graph[s][e]['pts'] = simplify_coords(graph[s][e]['pts'], 2.0)

    # Pop short stubs
    pops = []
    for (s,e) in graph.edges():
        if graph[s][e]['weight'] < 10 and (len(graph[s]) == 1 or len(graph[e]) == 1):
            pops += [(s,e)]
    for s, e in pops:
        graph.remove_edge(s,e)

    # Pop orphaned nodes
    pops = []
    for node in graph.nodes():
        if len(graph[node]) == 0:
            pops += [node]
    for node in pops:
        graph.remove_node(node)

    # Bind nearby point pairs
    for n1, n2 in combinations(graph.node, 2):
        diff = la.norm(graph.node[n1]['o']-graph.node[n2]['o'])
        if diff < 20 and n2 not in graph[n1]:
            pts = np.zeros((3,2))
            pts[0,:] = graph.node[n1]['o']
            pts[1,:] = (graph.node[n1]['o']+graph.node[n2]['o']) / 2
            pts[2,:] = graph.node[n2]['o']
            graph.add_edge(n1, n2, pts=pts, weight=diff)

    # Bind points along a line
    for n1 in graph.node:
        pushes = []
        for n2 in graph[n1]:
            for n3 in graph.node:
                vec1 = graph.node[n1]['o']-graph.node[n2]['o']
                vec2 = graph.node[n3]['o']-graph.node[n1]['o']
                diff = la.norm(vec2)
                vec1/= la.norm(vec1)
                vec2/= diff
                try:
                    angle = np.arccos(vec1@vec2)*180/np.pi
                except:
                    logger.critical("Could not compute points on vector!")
                    angle = 90
                if abs(angle % 360) < 2 and diff < 150:
                    pts = np.zeros((3,2))
                    pts[0,:] = graph.node[n1]['o']
                    pts[1,:] = (graph.node[n1]['o']+graph.node[n3]['o']) / 2
                    pts[2,:] = graph.node[n3]['o']
                    pushes += [(n1, n3, pts, diff)]
        for n1, n2, pts, diff in pushes:
            graph.add_edge(n1, n2, pts=pts, weight=diff)

    for s,e in graph.edges():
        N = len(graph[s][e]['pts'])+2
        pts = np.zeros((N,2), dtype=np.float64)
        pts[0] = graph.node[s]['o']
        pts[-1] = graph.node[e]['o']
        pts[1:-1] = graph[s][e]['pts']

        pts[:, 0] = 1-pts[:, 0]/H
        pts[:, 0] *= dy
        pts[:, 0] += lr[0]

        pts[:, 1] = pts[:, 1]/W
        pts[:, 1] *= dx
        pts[:, 1] += ul[1]

        df = df.append({
            'geometry': LineString(pts[:, ::-1]),
            'width': width
        }, ignore_index=True)
    return df

p = Pool(cpu_count()//2+1)

paths = [f for f in image_directory.iterdir() if "tif" == f.name[-3:]]
imgs = p.imap(read_image, paths)
masks = map(segment, imgs)
skels = map(skeletonize, masks)
dfs = map(vectorize, zip(skels, paths))

csv = pd.DataFrame(columns=['ImageId', 'WKT_Pix'])
def to_wkt(df):
    lss = []
    for elem in df.geometry:
        coords = []
        if type(elem) == LineString:
            for x, y in zip(*elem.coords.xy):
                coords += [f"{x} {y}"]
        css = ", ".join(coords)
        lss += [f"LINESTRING ({css})"]
    return " ".join(lss)

for df, path in tqdm(zip(dfs, paths)):
    name = path.name[:-4]+".geojson"
    image_id = re.search("([0-9]*)\.tif$", path.name).group(1)
    wkt = to_wkt(df)
    csvdd = pd.DataFrame(
        {
            "ImageId": [image_id],
            "WKT_Pix": [wkt]
        }
    )
    csv = csv.append(csvdd)
    if not df.empty:
        df.to_file(image_directory/name, driver="GeoJSON")

csv.to_csv(argv[1]+".csv")


from django.template import Library, Node, RequestContext, Variable
from django.template.loader import select_template

register = Library()

class PromotionNode(Node):
    def __init__(self, promotion):
        self.promotion_var = Variable(promotion)

    def render(self, context):
        promotions = self.promotion_var.resolve(context)
        template = select_template(['homepage/gallery.html'])
        args = {'promotions': promotions, 'products': []}
        for promotion in promotions:
            if hasattr(promotion, 'product'):
                args['products'].append(promotion.product)
            # args.update(**promotion.template_context(request=context['request']))
        # ctx = RequestContext(context['request'], args)
        # return template.render(ctx)
        return template.render(args, context['request'])

def get_promotion_html(parser, token):
    _, promotion = token.split_contents()
    return PromotionNode(promotion)

register.tag('render_promotion', get_promotion_html)


"""
This module wrapps the plotting of the VV&UQ results to call the internal methods based on the user config.

It includes a main function called create_vvuq_plots. See details in its own documentations.

Contact person: Stefan Riedmaier
Creation date: 20.08.2020
Python version: 3.8
"""

# -- IMPORTS -----------------------------------------------------------------------------------------------------------
# -- built-in imports --
import os
import pathlib

# -- third-party imports --
import numpy as np

# -- custom imports --
from src.plots import vvuq_plots

# -- FUNCTIONS ---------------------------------------------------------------------------------------------------------

def create_vvuq_plots(vvuq, config, plot_path=None):
    """
    This function wraps the internal plot methods based on the user config.

    :param vvuq: vvuq object with all results
    :param dict config: config dictionary
    :param str plot_path: path where the plots shall be stored (otherwise automatically generated in results folder)
    """
    if 'analysis' not in config or 'plots' not in config['analysis']:
        return

    cfgpl = config['analysis']['plots']

    if not plot_path:
        # -- create a folder to the store the plots
        # replace the last part of the simulator path with "Plots"
        simulator_path = pathlib.Path(config['cross_domain']['simulator']['result_folder'])
        plot_path_parts = simulator_path.parts[:-1] + ('Plots',)
        plot_path = str(pathlib.Path('').joinpath(*plot_path_parts))

    # create the folder
    if not os.path.exists(plot_path):
        os.makedirs(plot_path)

    # -- set negative distance to line values to zero for plotting
    # this can happen during the error integration, since it subtracts the inferred errors from the model responses
    # the VVUQ results are the same, but the distance to line plots look more natural without negative values
    nested_break_flag = False
    for plot_type_list in cfgpl.values():
        for plot_type_dict in plot_type_list:
            if 'qois' in plot_type_dict and plot_type_dict['qois'] in {'D2LL', 'D2RL', 'D2L'}:
                d2l_qoi = plot_type_dict['qois']

                d2l_data = vvuq.application.qois_kpi_system_estimated_da.loc[{'qois': d2l_qoi}].data
                negative_mask = (d2l_data < 0) & (d2l_data != -np.inf)
                vvuq.application.qois_kpi_system_estimated_da.loc[{'qois': d2l_qoi}].data[negative_mask] = 0

                if hasattr(vvuq.application.error_integration, 'kpi_system_estimated_validation_da') and \
                        vvuq.application.error_integration.kpi_system_estimated_validation_da.dims != ():
                    d2l_data = vvuq.application.error_integration.kpi_system_estimated_validation_da.loc[
                        {'qois': d2l_qoi}].data
                    negative_mask = (d2l_data < 0) & (d2l_data != -np.inf)
                    vvuq.application.error_integration.kpi_system_estimated_validation_da.loc[
                        {'qois': d2l_qoi}].data[negative_mask] = 0

                nested_break_flag = True
                break
        if nested_break_flag:
            break

    # -- Scenario Space Plots --
    for i, plot_type_dict in enumerate(cfgpl['scenario_space_plots']):

        if vvuq.application.scenarios_model.samples_da.ndim == 2:
            # in case of a deterministic simulation w/o uncertainties (2D data), there are only space samples to plot
            vvuq_plots.plot_scenario_space(
                config=config,
                cfgpl=plot_type_dict,
                scenarios_space_validation_da=vvuq.validation.scenarios_model.space_samples_da,
                scenarios_space_application_da=vvuq.application.scenarios_model.space_samples_da,
                save_path=plot_path + '/' + 'scenario_space_' + str(i)
            )
        else:
            vvuq_plots.plot_scenario_space(
                config=config,
                cfgpl=plot_type_dict,
                scenarios_verification_da=vvuq.verification.scenarios.samples_da,
                scenarios_validation_model_da=vvuq.validation.scenarios_model.samples_da,
                scenarios_validation_system_da=vvuq.validation.scenarios_system.samples_da,
                scenarios_application_da=vvuq.application.scenarios_model.samples_da,
                scenarios_space_validation_da=vvuq.validation.scenarios_model.space_samples_da,
                scenarios_space_application_da=vvuq.application.scenarios_model.space_samples_da,
                save_path=plot_path + '/' + 'scenario_space_' + str(i)
            )

    if config['cross_domain']['assessment']['method'] != 'read_csv':

        # -- Time Series Plots --
        for i, plot_type_dict in enumerate(cfgpl['time_series_plots']):
            vvuq_plots.plot_timeseries(
                config=config,
                cfgpl=plot_type_dict,
                qois_ts_da=vvuq.validation.assessment_system.qois_ts_da,
                qois_kpi_da=vvuq.validation.assessment_system.qois_kpi_raw_da,
                save_path=plot_path + '/' + 'timeseries_validation_system_' + str(i)
            )

        # -- Time Series UNECE-R79 Plots --
        for i, plot_type_dict in enumerate(cfgpl['time_series_unecer79_plots']):

            if plot_type_dict['domain'] == 'application' and plot_type_dict['instance'] == 'simulator':
                vvuq_obj = vvuq.application.assessment_model
                save_path = plot_path + '/R79_TimeSeries/' + 'r79_timeseries_application_model_' + str(i)

            elif plot_type_dict['domain'] == 'application' and plot_type_dict['instance'] == 'experiment':
                vvuq_obj = vvuq.application.assessment_system
                save_path = plot_path + '/R79_TimeSeries/' + 'r79_timeseries_application_system_' + str(i)

            elif plot_type_dict['domain'] == 'validation' and plot_type_dict['instance'] == 'simulator':
                vvuq_obj = vvuq.validation.assessment_model
                save_path = plot_path + '/R79_TimeSeries/' + 'r79_timeseries_validation_model_' + str(i)

            elif plot_type_dict['domain'] == 'validation' and plot_type_dict['instance'] == 'experiment':
                vvuq_obj = vvuq.validation.assessment_system
                save_path = plot_path + '/R79_TimeSeries/' + 'r79_timeseries_validation_system_' + str(i)

            else:
                raise ValueError("This plot configuration should not be reachable.")

            fct_dict = {
                'cfgpl': plot_type_dict,
                'qois_ts_da': vvuq_obj.qois_ts_da,
                'quantities_ts_da': vvuq_obj.quantities_ts_da,
                'qois_ts_untrimmed_da': vvuq_obj.qois_ts_untrimmed_da,
                'qois_kpi_da': vvuq_obj.qois_kpi_raw_da,
                'save_path': save_path
            }

            if config['cross_domain']['assessment']['is_event_finder']:
                fct_dict['start_idx'] = vvuq_obj.unece_event_finder.start_idx_longest
                fct_dict['stop_idx'] = vvuq_obj.unece_event_finder.stop_idx_longest
                fct_dict['ay_lower_bound'] = vvuq_obj.unece_event_finder.ay_lower_bounds
                fct_dict['ay_upper_bound'] = vvuq_obj.unece_event_finder.ay_upper_bounds

            vvuq_plots.plot_timeseries_unecer79(**fct_dict)

    # -- KPI Surface Plots --
    for i, plot_type_dict in enumerate(cfgpl['kpi_surface_plots']):

        if plot_type_dict['domain'] == 'application' and plot_type_dict['instance'] == 'simulator':
            # plot the KPI surface of the simulation model in the application domain
            vvuq_plots.plot_kpi_surface(
                config=config,
                cfgpl=plot_type_dict,
                scenarios_da=vvuq.application.scenarios_model.samples_da,
                qois_kpi_raw_da=vvuq.application.assessment_model.qois_kpi_raw_da,
                plot_type=plot_type_dict['type'],
                save_path=plot_path + '/' + 'kpi_surface_application_model_' + str(i)
            )

        elif plot_type_dict['domain'] == 'application' and plot_type_dict['instance'] == 'experiment':
            # plot the KPI surface of the system in the application domain
            vvuq_plots.plot_kpi_surface(
                config=config,
                cfgpl=plot_type_dict,
                scenarios_da=vvuq.application.scenarios_system.samples_da,
                qois_kpi_raw_da=vvuq.application.assessment_system.qois_kpi_raw_da,
                plot_type=plot_type_dict['type'],
                save_path=plot_path + '/' + 'kpi_surface_application_system_' + str(i)
            )

        elif plot_type_dict['domain'] == 'validation' and plot_type_dict['instance'] == 'simulator':
            # plot the KPI surface of the simulation model in the validation domain
            vvuq_plots.plot_kpi_surface(
                config=config,
                cfgpl=plot_type_dict,
                scenarios_da=vvuq.validation.scenarios_model.samples_da,
                qois_kpi_raw_da=vvuq.validation.assessment_model.qois_kpi_raw_da,
                plot_type=plot_type_dict['type'],
                save_path=plot_path + '/' + 'kpi_surface_validation_model_' + str(i)
            )

        elif plot_type_dict['domain'] == 'validation' and plot_type_dict['instance'] == 'experiment':
            # plot the KPI surface of the system in the validation domain
            vvuq_plots.plot_kpi_surface(
                config=config,
                cfgpl=plot_type_dict,
                scenarios_da=vvuq.validation.scenarios_system.samples_da,
                qois_kpi_raw_da=vvuq.validation.assessment_system.qois_kpi_raw_da,
                plot_type=plot_type_dict['type'],
                save_path=plot_path + '/' + 'kpi_surface_validation_system_' + str(i)
            )

    # -- CDF Plots --
    for i, plot_type_dict in enumerate(cfgpl['cdf_plots']):

        vvuq_plots.plot_cdf(
            config=config,
            cfgpl=plot_type_dict,
            qois_kpi_raw_da=vvuq.validation.assessment_model.qois_kpi_raw_da,
            fill_flag=True,
            qois_kpi_da=vvuq.validation.assessment_model.qois_kpi_da,
            save_path=plot_path + '/CDFs/' + 'cdf_' + str(i)
        )

    # -- Metric Plots --
    for i, plot_type_dict in enumerate(cfgpl['metric_plots']):

        vvuq_plots.plot_area_metrics(
            config=config,
            cfgpl=plot_type_dict,
            pbox_y_model=vvuq.validation.metric.pbox_y_model,
            pbox_y_system=vvuq.validation.metric.pbox_y_system,
            pbox_x_model_list=vvuq.validation.metric.pbox_x_model_list,
            pbox_x_system_list=vvuq.validation.metric.pbox_x_system_list,
            metric_da=vvuq.validation.metric_da,
            scenarios_da=vvuq.validation.scenarios_model.samples_da,
            save_path=plot_path + '/Area_Metric/' + 'area_metric_' + str(i)
        )

    # -- Extrapolation Surface Plots --
    for i, plot_type_dict in enumerate(cfgpl['extrapolation_surface_plots']):
        vvuq_plots.plot_extrapolation_surface(
            config=config,
            cfgpl=plot_type_dict,
            scenarios_validation_da=vvuq.validation.scenarios_model.space_samples_da,
            metric_validation_da=vvuq.validation.metric_da,
            scenarios_application_da=vvuq.application.scenarios_model.space_samples_da,
            error_validation_da=vvuq.validation.error_model.error_full_da,
            save_path=plot_path + '/' + 'extrapolation_surface_' + str(i)
        )

    # -- Uncertainty Expansion Plots --
    # non-deterministic uncertainty expansion
    for i, plot_type_dict in enumerate(cfgpl['nondeterministic_uncertainty_expansion_plots']):
        vvuq_plots.plot_uncertainty_expansion_nondeterministic(
            config=config,
            cfgpl=plot_type_dict,
            qois_kpi_model_da=vvuq.application.qois_kpi_model_da,
            qois_kpi_system_estimated_da=vvuq.application.qois_kpi_system_estimated_da,
            qois_kpi_system_estimated_validation_da=
            vvuq.application.error_integration.kpi_system_estimated_validation_da,
            scenarios_model_da=vvuq.application.scenarios_model.samples_da,
            qois_kpi_system_da=vvuq.application.qois_kpi_system_da,
            save_path=plot_path + '/Uncertainty_Expansion/' + 'uncertainty_expansion_' + str(i)
        )

    # deterministic uncertainty expansion
    for i, plot_type_dict in enumerate(cfgpl['deterministic_error_integration_plots']):

        vvuq_plots.plot_error_integration_deterministic(
            config=config,
            cfgpl=plot_type_dict,
            qois_kpi_model_da=vvuq.application.qois_kpi_model_da,
            qois_kpi_system_estimated_da=vvuq.application.qois_kpi_system_estimated_da,
            scenarios_model_da=vvuq.application.scenarios_model.samples_da,
            qois_kpi_system_da=vvuq.application.qois_kpi_system_da,
            save_path=plot_path + '/Uncertainty_Expansion/' + 'uncertainty_expansion_' + str(i)
        )

    # -- Decision Space Plots --
    for i, plot_type_dict in enumerate(cfgpl['decision_space_plots']):

        fct_dict = {
            'config': config,
            'cfgpl': plot_type_dict,
            'scenarios_application_da': vvuq.application.scenarios_model.space_samples_da,
            'decision_application_system_estimated_da': vvuq.application.decision_system_estimated_da,
            'scenarios_validation_da': vvuq.validation.scenarios_model.space_samples_da,
            'save_path': plot_path + '/' + 'scenario_decision_' + str(i)
        }

        # add data to function args depending on user selections
        if plot_type_dict['application_decisions'] == 'uncertainty_model':
            fct_dict['decision_application_model_da'] = vvuq.application.decision_model_da
        elif plot_type_dict['application_decisions'] == 'uncertainty_system':
            fct_dict['decision_application_system_da'] = vvuq.application.decision_system_da

        if plot_type_dict['validation_decisions'] == 'accuracy':
            fct_dict['decision_validation_da'] = vvuq.validation.decision_da
        else:
            if plot_type_dict['validation_decisions'] in {'model_safety', 'model_system_safety'}:
                fct_dict['decision_validation_model_da'] = vvuq.validation.decision_model_da

            if plot_type_dict['validation_decisions'] in {'system_safety', 'model_system_safety'}:
                fct_dict['decision_validation_system_da'] = vvuq.validation.decision_system_da

        vvuq_plots.plot_decision_space(**fct_dict)

    return


from .vector_base import VectorBase
from ..object import ObjectID
import chromadb
import logging
import os

class ChromaVectorStore(VectorBase):
    def __init__(self, root_dir, model_name: str) -> None:
        super().__init__(model_name)

        logging.info(
            "will init chroma vector store, model={}".format(model_name)
        )

        directory = os.path.join(root_dir, "vector")
        logging.info("will use vector store: {}".format(directory))

        client = chromadb.PersistentClient(
            path=directory, settings=chromadb.Settings(anonymized_telemetry=False)
        )
        # client = chromadb.Client()

        collection_name = "coll_{}".format(model_name)
        logging.info("will init chroma colletion: %s", collection_name)

        collection = client.get_or_create_collection(collection_name)
        self.collection = collection

    async def insert(self, vector: [float], id: ObjectID):
        logging.info(f"will insert vector: {len(vector)} id: {str(id)}")
        logging.debug(f"vector is {vector}")
        self.collection.add(
            embeddings=vector,
            ids=str(id),
        )

    async def query(self, vector: [float], top_k: int) -> [ObjectID]:
        ret = self.collection.query(
            query_embeddings=vector,
            n_results=top_k,
        )
        logging.info(f"query result {ret}")
        if len(ret['ids']) == 0:
            return []
        return list(map(ObjectID.from_base58, ret["ids"][0]))

    async def delete(self, id: ObjectID):
        self.collection.delete(
            ids=id,
        )


import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import glob
from uncertainties import unumpy as unp
from uncertainties import ufloat

def get_means(x):
    return(ufloat(x.mean(),x.std()/np.sqrt(x.shape[0])))

#def get_ster(x):
    #return(x.std()/np.sqrt(5))

class point:

    def __init__(self,data):
        self.data=data#nx2 array n is number of trials
        self.means=np.apply_along_axis(get_means,0,data)#1x2 array [x,y]
        #self.ster=np.apply_along_axis(get_ster,0,data)

class Fids:

    def __init__(self,filename):
        self.filename=filename
        self.data=pd.read_csv(filename, sep=',',header=None)#read csv
        self.data=np.array(self.data)#Turn into numpy array
        self.data=np.delete(self.data,2,1)#delete z coordinate column - not needed and saves computation time

        self.points=[]#All points are stored in this array
        for i in range(4):
            x=self.data[i]

            for j in range(1,self.data.shape[0]//4):
                x=np.vstack((x,self.data[i+j*4]))
            self.points.append(point(x))
            x=np.array([])

        #self.dist_data=np.zeros([4,4])
        #self.dist_data_error=np.zeros([4,4])
        self.dist_data=np.array([[ufloat(0,0),ufloat(0,0),ufloat(0,0),ufloat(0,0)],[ufloat(0,0),ufloat(0,0),ufloat(0,0),ufloat(0,0)],[ufloat(0,0),ufloat(0,0),ufloat(0,0),ufloat(0,0)],[ufloat(0,0),ufloat(0,0),ufloat(0,0),ufloat(0,0)]])
        self.dist_data_error=np.array([[ufloat(0,0),ufloat(0,0),ufloat(0,0),ufloat(0,0)],[ufloat(0,0),ufloat(0,0),ufloat(0,0),ufloat(0,0)],[ufloat(0,0),ufloat(0,0),ufloat(0,0),ufloat(0,0)],[ufloat(0,0),ufloat(0,0),ufloat(0,0),ufloat(0,0)]])

    #This function calculates the distance between two points
    def pointDist(self):
        for i in range(4):
            for j in range(i+1,4):
                #self.dist_data[i][j]=self.dist_data[j][i]=unp.linalg.norm(self.points[i].means-self.points[j].means)
                x1=self.points[i].means[0]
                #x1_err=self.points[i].ster[0]
                x2=self.points[j].means[0]
                #x2_err=self.points[j].ster[0]
                y1=self.points[i].means[1]
                #y1_err=self.points[i].ster[1]
                y2=self.points[j].means[1]
                #print(x1,x2,y1,y2)
                self.dist_data[i][j]=unp.sqrt((x1-x2)**2+(y1-y2)**2)
                #y2_err=self.points[j].ster[1]
                #num=((x1-x2)**2)*(x1_err**2+x2_err**2)+((y1-y2)**2)*(y1_err**2+y2_err**2)
                #den=(x1-x2)**2+(y1-y2)**2
                #self.dist_data_error[i][j]=np.sqrt(num/den)

    def printPointDist(self):
        print("Distances Between:\n")
        print("0 & 3: " + str(self.dist_data[0][3]))# + " +/- " + str(self.dist_data_error[0][3]))
        print("1 & 2: " + str(self.dist_data[1][2]))# + " +/- " + str(self.dist_data_error[1][2]))
        print()
        print("0 & 1: " + str(self.dist_data[0][1]))# + " +/- " + str(self.dist_data_error[0][1]))
        print("2 & 3: " + str(self.dist_data[2][3]))# + " +/- " + str(self.dist_data_error[2][3]))
        print()
        print("0 & 2: " + str(self.dist_data[0][2]))# + " +/- " + str(self.dist_data_error[0][2]))
        print("1 & 3: " + str(self.dist_data[1][3]))# + " +/- " + str(self.dist_data_error[1][3]))

    def angles(self):
        diff=self.points[0].means-self.points[3].means
        #diff=[x0-x1,y0-y1]
        a1=unp.arctan(diff[1]/diff[0])
        diff=self.points[1].means-self.points[2].means
        a2=unp.arctan(diff[1]/diff[0])

        print("\n{} {}\n".format(a1,a2))
        return np.array([a1,a2])

    def cornerFidDist(self, cornerData):
        angle=np.mean(self.angles())
        diff=[]
       # rotMat=np.array([[unp.cos(angle),unp.sin(angle)],[-unp.sin(angle),unp.cos(angle)]])

        #v1=np.matmul(rotMat,self.points[0].means.T)
        #v2=np.matmul(rotMat,cornerData[0].T)
        #diff=abs(v2-v1)

        for i in range(4):
            #v1=unp.matmul(rotMat,self.points[i].means.T)
            #v2=unp.matmul(rotMat,cornerData[i].T)
            v1=np.array([self.points[i].means[0]*unp.cos(angle)+self.points[i].means[1]*unp.sin(angle),
                          -self.points[i].means[0]*unp.sin(angle)+self.points[i].means[1]*unp.cos(angle)])
            v2=np.array([cornerData[i][0]*unp.cos(angle)+cornerData[i][1]*unp.sin(angle),
                -cornerData[i][0]*unp.sin(angle)+cornerData[i][1]*unp.cos(angle)])
            diff.append(abs(v2-v1))
        return diff
            #print(abs(diff-np.array([.1275,.1535])))

    def printCornerFidDist(self, diff):
        print("Corner Distances:")
        print()
        for i in range(4):
            print("Corner #"+str(i)+":")
            print("x: "+str(diff[i][0]))
            print("y: "+str(diff[i][1]))
            print()

    def plotFid(self,point_data):
        plt.figure()

        #plot fiducial points
        x=np.array([])
        y=np.array([])
        for i in self.points:
            x=np.append(x,i.means[0].nominal_value)
            y=np.append(y,i.means[1].nominal_value)

        plt.plot(x,y,'o',label='Fiducial Points')

        #now plot corner points
        xc,yc=point_data.T
        plt.plot(xc,yc,'r^',label='Corner')
        plt.legend()

        plt.show()

    def plot_diff_histo(self,point_data):
        x_data,y_data=np.array([]),np.array([])

        for i in range(4):
            x_data=np.append(x_data,abs(point_data[i][0]-self.points[i].data[0]))
            y_data=np.append(y_data,abs(point_data[i][1]-self.points[i].data[1]))

        plt.figure()
        plt.hist(x_data,ec='black')
        plt.title("x")
        plt.figure()
        plt.hist(y_data,ec='black')
        plt.title('y')

#~~~MAIN~~~#
F=Fids('6-13-PatternMatchPosition-BetterF.csv')
F.pointDist()
F.printPointDist()
#F.angles()
print("\n")

data=pd.read_csv("6-13-Coords.csv", sep=',',header=None)#read csv
data=np.array(data)#Turn into numpy array
data=np.delete(data,2,1)#delete z coordinate column - not needed and saves computation time
#print(data)
#diff=F.cornerFidDist(data)
#F.printCornerFidDist(diff)
F.plot_diff_histo(data)
#angle1,angle2=data[0]-data[3],data[2]-data[1]
#angle1,angle2=np.arctan(angle1[1]/angle1[0]),np.arctan(angle2[1]/angle2[0])
#print("~~~Corner angles~~~\n{} {}\n".format(angle1,angle2))
"""
corner_dist=np.zeros([4,4])
for i in range(4):
    for j in range(i+1,4):
        corner_dist[i][j]=corner_dist[j][i]=np.linalg.norm(data[i]-data[j])
print(corner_dist)
F.plotFid(data)
"""


import numpy as np
import matplotlib.pyplot as plt

class CPPN:
    def __init__(self, num_input, num_output, num_hidden_layers=3,
                 num_hidden_units=32, activation_functions=[np.sin, np.tanh]):
        self.num_input = num_input
        self.num_output = num_output
        self.num_hidden_layers = num_hidden_layers
        self.num_hidden_units = num_hidden_units
        self.activation_functions = activation_functions

        # Initialize weights and biases
        self.weights = []
        self.biases = []
        input_to_hidden = np.random.uniform(low=-1, high=1, size=(self.num_input, self.num_hidden_units))
        self.weights.append(input_to_hidden)
        self.biases.append(np.zeros(self.num_hidden_units))

        for _ in range(self.num_hidden_layers - 1):
            hidden_to_hidden = np.random.uniform(low=-1, high=1, size=(self.num_hidden_units, self.num_hidden_units))
            self.weights.append(hidden_to_hidden)
            self.biases.append(np.zeros(self.num_hidden_units))

        hidden_to_output = np.random.uniform(low=-1, high=1, size=(self.num_hidden_units, self.num_output))
        self.weights.append(hidden_to_output)
        self.biases.append(np.zeros(self.num_output))

    def activate(self, x):
        for layer in range(len(self.weights)):
            x = np.dot(x, self.weights[layer]) + self.biases[layer]
            activation_func = self.activation_functions[layer % len(self.activation_functions)]
            x = activation_func(x)
        return x

    def compute_image(self, width, height):
        x = np.linspace(-1, 1, width)
        y = np.linspace(-1, 1, height)
        xs, ys = np.meshgrid(x, y)
        rs = np.sqrt(xs**2 + ys**2)
        thetas = np.arctan2(ys, xs)

        inputs = []
        for i in range(self.num_input):
            if i % 4 == 0:
                inputs.append(rs)
            elif i % 4 == 1:
                inputs.append(thetas)
            elif i % 4 == 2:
                inputs.append(np.sin(self.c[i] * rs + self.d[i] * thetas))
            elif i % 4 == 3:
                inputs.append(np.cos(self.c[i] * rs + self.d[i] * thetas))

        inputs = np.array(inputs).transpose()

        # apply the hidden layers
        hiddens = inputs
        for i in range(self.num_hidden_layers):
            hiddens = np.maximum(np.dot(hiddens, self.weights[i]) + self.biases[i], 0)

        # apply the output layer
        rgba = np.dot(hiddens, self.weights[-1]) + self.biases[-1]
        rgba = np.clip(rgba, -1, 1)
        rgba = (rgba + 1) / 2 # scale to 0-1 range
        rgba = (rgba * 255).astype(int) # scale to 0-255 range
        rgba = np.reshape(rgba, (height, width, 4))

        return rgba

# Example usage
cppn = CPPN(num_input=2, num_output=4, num_hidden_layers=8, num_hidden_units=16, activation_functions=[np.cos, np.sin, np.square, np.sin, np.tanh])
image = cppn.compute_image(width=480, height=480)
plt.imshow(image)
plt.show()


import configparser
import os
import yaml
from exception import ConfigNotFoundError

class DefaultConfig:
    """
    Configuration items that all users may need to modify
    """

    # **********
    # Log related
    # **********
    # level of LOG
    LOGGER_LEVEL = "INFO"
    # Log file path
    LOGGER_PATH = "/var/log/cve_tracking"
    # Log file name
    LOGGER_FILE_NAME = "cve_tracking.log"
    # Log file size, the default is 5 M
    LOGGER_BUFFER = 5242880
    # Number of log files saved
    LOGGER_FILE_COUNT = 20

    # **********
    # Path related
    # **********
    # Patch file download and save path.
    PATCH_SAVE_PATH = "/opt/cve_tracking/patches"
    # Source code package and patch file storage path,
    # used to download gitee source code package and package verification.
    SOURCE_CODE_PATH = "/opt/cve_tracking/source_code"

    # **********
    # Feedback issue related configuration
    # **********

    FEEDBACK_PLATFORM = "github"
    # Feedback to the owner of the issue
    FEEDBACK_ISSUE_OWNER = "liheavy"
    # Feedback to the repo of the issue
    FEEDBACK_ISSUE_REPO = "cve_tracking"

    # The owner of the comment function submits the comment
    DEFAULT_OWNER = "src-openeuler"

class Config:
    """
    Configuration item load and get value
    """

    config_file = os.path.join(
        os.path.dirname(os.path.abspath(os.path.dirname(__file__))), "config.ini"
    )

    def __init__(self):
        self._setattr_default()
        self._setattr_customize()

    def _setattr_default(self):
        """
        Set default configuration items and their values
        :return: self
        """
        for attr in dir(DefaultConfig):
            if not str(attr).startswith("_"):
                setattr(self, attr, getattr(DefaultConfig, attr))

    def _setattr_customize(self):
        """
        Set customize configuration items and their values
        :return: self
        """
        if not os.path.exists(self.config_file):
            raise ConfigNotFoundError(self.config_file)

        conf_parser = configparser.ConfigParser()
        conf_parser.read(self.config_file)
        for section in conf_parser.sections():
            for key_value in conf_parser.items(section):
                if key_value[1]:
                    setattr(self, key_value[0].upper(), key_value[1])

    def __getattr__(self, item):
        """
        When the configuration cannot be found, reload the configuration file
        :param item: config key
        :return: value
        """
        self._setattr_default()
        self._setattr_customize()
        try:
            return self.__dict__[item]
        except KeyError:
            return None

class YamlConfiguration:
    """
    yaml file parsing class
    """
    yaml = os.path.join(
        os.path.dirname(os.path.abspath(os.path.dirname(__file__))), "cve-tracking.yaml"
    )

    def __init__(self) -> None:
        if not os.path.exists(self.yaml):
            raise FileNotFoundError("file not found: %s" % self.yaml)
        self.__dict__.update(self._parse_yaml())

    def _parse_yaml(self):
        with open(self.yaml, "r", encoding="utf-8") as file:
            try:
                configs = yaml.load(file.read(), Loader=yaml.FullLoader)
            except yaml.YAMLError as error:
                raise ValueError(
                    "The format of the yaml configuration "
                    "file is wrong please check and try again:{0}".format(error)
                )
            else:
                return configs

    def __getattr__(self, name):
        if name not in self.__dict__:
            return None

        return self.__dict__[name]

    def get_platform(self, name=None):
        """
        Get the configured cve platform
        :param name: specify platform name
        :return: platform name
        """
        if name is None:
            return self.platform
        platform = filter(lambda x: x["name"] == name, self.platform)
        if platform:
            return list(platform)[-1]

    def get_regex(self, label=None):
        """
        Regular expression to get patch
        :param label: issue or pr or commit
        :return: regex
        """
        if label is None:
            regulars = [reg for reg in self.regex]
        else:
            regulars = filter(lambda x: x["label"] == label, self.regex)
        return [regx for reg in regulars for regx in reg["regular"]]

    def token(self, name):
        token = filter(lambda x: x["name"] == name, self.authentication)
        if token:
            return list(token)[-1]["token"]

    @property
    def configuration(self):
        """
        All configuration items
        :return: dict
        """
        return self.__dict__


from __future__ import print_function
from builtins import str  # pylint: disable=redefined-builtin
from builtins import map  # pylint: disable=redefined-builtin
import os
os.environ['DJANGO_SETTINGS_MODULE'] = 'openremproject.settings'

from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.shortcuts import redirect
from django.core.urlresolvers import reverse_lazy
from django.views.decorators.csrf import csrf_exempt
import remapp

@csrf_exempt
@login_required
def run_store(request, pk):
    from django.shortcuts import redirect
    from remapp.models import DicomStoreSCP
    from remapp.netdicom.storescp import web_store
    if request.user.groups.filter(name="admingroup"):
        store = DicomStoreSCP.objects.get(pk__exact = pk)
        store.run = True
        store.save()
        storetask = web_store(store_pk=pk)
    return redirect(reverse_lazy('dicom_summary'))

@csrf_exempt
@login_required
def stop_store(request, pk):
    from django.shortcuts import redirect
    from remapp.models import DicomStoreSCP
    if request.user.groups.filter(name="admingroup"):
        store = DicomStoreSCP.objects.filter(pk__exact = pk)
        if store:
            store[0].run = False
            store[0].save()
            store[0].status = u"Quit signal sent"
            store[0].save()
        else:
            print(u"Can't stop store SCP: Invalid primary key")
    return redirect(reverse_lazy('dicom_summary'))

import json
from django.http import HttpResponseRedirect, HttpResponse
from django.http import Http404

@csrf_exempt
def status_update_store(request):
    from remapp.models import DicomStoreSCP
    from remapp.netdicom.tools import echoscu

    resp = {}
    data = request.POST
    scp_pk = data.get('scp_pk')

    echo = echoscu(scp_pk=scp_pk, store_scp=True)

    store = DicomStoreSCP.objects.get(pk=scp_pk)

    if echo is "Success":
        resp['message'] = u"
Last status: {0}
".format(store.status)
        resp['statusindicator'] = u"
" \
                                  u"" \
                                  u"OK: Server is alive
"
        resp['delbutton'] = u"<button type='button' class='btn btn-primary' disabled='disabled'>Delete</button>"
        resp['startbutton'] = u""
        resp['stopbutton'] = u"Stop server".format(
            reverse_lazy('stop_store', kwargs={'pk': scp_pk}))
    elif echo is "AssocFail":
        resp['message'] = u"
Last status: {0}
".format(store.status)
        resp['statusindicator'] = u"
" \
                                  u"" \
                                  u"Error: Association fail - server not running?
"
        resp['delbutton'] = u"Delete".format(
            reverse_lazy('dicomstore_delete', kwargs={'pk': scp_pk}))
        resp['startbutton'] = u"Start server".format(
            reverse_lazy('run_store', kwargs={'pk': scp_pk}))
        resp['stopbutton'] = u""

    return HttpResponse(json.dumps(resp), content_type="application/json")

@csrf_exempt
def q_update(request):
    from django.core.exceptions import ObjectDoesNotExist
    from django.db.models import Count
    from remapp.models import DicomQuery

    resp = {}
    data = request.POST
    query_id = data.get('queryID')
    resp['queryID'] = query_id
    try:
        query = DicomQuery.objects.get(query_id=query_id)
    except ObjectDoesNotExist:
        resp['status'] = u'not complete'
        resp['message'] = u'
Query {0} not yet started
'.format(query_id)
        return HttpResponse(json.dumps(resp), content_type='application/json')

    if query.failed:
        resp['status'] = u'failed'
        resp['message'] ='
Query Failed
 {0}'.format(query.message)
        return HttpResponse(json.dumps(resp), content_type='application/json')

    study_rsp = query.dicomqrrspstudy_set.all()
    if not query.complete:
        modalities = study_rsp.values('modalities_in_study').annotate(count=Count('pk'))
        table = [u'

Modalities in study	Number of responses
']
        for m in modalities:
            table.append(u'
')
            if m['modalities_in_study']:
                table.append(u', '.join(json.loads(m['modalities_in_study'])))
            else:
                table.append(u'Unknown')
            table.append(u'	')
            table.append(str(m['count']))
            table.append(u'
	')
        table.append(u'
')
        tablestr = ''.join(table)
        resp['status'] = u'not complete'
        resp['message'] = u'
{0}

Responses so far:
 {1}'.format(query.stage, tablestr)
    else:
        modalities = study_rsp.values('modality').annotate(count=Count('pk'))
        table = [u'

Modality	Number of responses
']
        for m in modalities:
            table.append(u'
')
            if m['modality']:
                table.append(m['modality'])
            else:
                table.append(u'Unknown - SR only study?')
            table.append(u'	')
            table.append(str(m['count']))
            table.append(u'
	')
        table.append(u'
')
        tablestr = u''.join(table)
        resp['status'] = u'complete'
        query_details_text = u"
" \
                             u"
" \
                             u"
  "\
                             u"
" \
                             u"" \
                             u"Query details
" \
                             u"

" \
                             u"
" \
                             u"
" \
                             u"
{0}

".format(query.stage)
        not_as_expected_help_text = u"
" \
                                    u"
" \
                                    u"
  "\
                                    u"
" \
                                    u"" \
                                    u"Not what you expected?
" \
                                    u"

" \
                                    u"
" \
                                    u"
" \
                                    u"
For DX and mammography, the query will look for Radiation Dose Structured " \
                                    u"Reports, or images if the RDSR is not available. For Fluoroscopy, RDSRs are " \
                                    u"required. For CT RDSRs are preferred, but Philips dose images can be used and " \
                                    u"for some scanners, particularly older Toshiba scanners that can't create RDSR " \
                                    u"OpenREM can process the data to create an RDSR to import.
" \
                                    u"
If you haven't got the results you expect, it may be that the imaging system" \
                                    u" is not creating RDSRs or not sending them to the PACS you are querying. In " \
                                    u"either case you will need to have the system reconfigured to create and/or send" \
                                    u" them. If it is a CT scanner that can't create an RDSR (it is too old), it is " \
                                    u"worth trying the 'Toshiba' option, but you will need to be using Orthanc and " \
                                    u"configure your scanner in the " \
                                    u"" \
                                    u"toshiba_extractor_systems list" \
                                    u". You will need to verify the resulting data to confirm accuracy.
" \
                                    u"

".format(remapp.__docs_version__)
        resp['message'] = u'
Query complete - there are {1} studies we can move
 {0} {2} {3}'.format(
            tablestr, study_rsp.count(), query_details_text, not_as_expected_help_text)

    return HttpResponse(json.dumps(resp), content_type='application/json')

@csrf_exempt
@login_required
def q_process(request, *args, **kwargs):
    import uuid
    import datetime
    from django.shortcuts import render_to_response
    from django.template import RequestContext
    from remapp.netdicom.qrscu import qrscu
    from remapp.models import DicomRemoteQR
    from remapp.forms import DicomQueryForm

    if request.method == 'POST':
        form = DicomQueryForm(request.POST)
        if form.is_valid():
            rh_pk = form.cleaned_data.get('remote_host_field')
            store_pk = form.cleaned_data.get('store_scp_field')
            date_from = form.cleaned_data.get('date_from_field')
            date_until = form.cleaned_data.get('date_until_field')
            modalities = form.cleaned_data.get('modality_field')
            inc_sr = form.cleaned_data.get('inc_sr_field')
            remove_duplicates = form.cleaned_data.get('duplicates_field')
            desc_exclude = form.cleaned_data.get('desc_exclude_field')
            desc_include = form.cleaned_data.get('desc_include_field')
            stationname_exclude = form.cleaned_data.get('stationname_exclude_field')
            stationname_include = form.cleaned_data.get('stationname_include_field')
            get_toshiba_images = form.cleaned_data.get('get_toshiba_images_field')
            get_empty_sr = form.cleaned_data.get('get_empty_sr_field')

            query_id = str(uuid.uuid4())

            if date_from:
                date_from = date_from.isoformat()
            if date_until:
                date_until = date_until.isoformat()

            if desc_exclude:
                study_desc_exc = list(map(str.lower, list(map(str.strip, desc_exclude.split(',')))))
            else:
                study_desc_exc = None
            if desc_include:
                study_desc_inc = list(map(str.lower, list(map(str.strip, desc_include.split(',')))))
            else:
                study_desc_inc = None
            if stationname_exclude:
                stationname_exc = list(map(str.lower, list(map(str.strip, stationname_exclude.split(',')))))
            else:
                stationname_exc = None
            if stationname_include:
                stationname_inc = list(map(str.lower, list(map(str.strip, stationname_include.split(',')))))
            else:
                stationname_inc = None

            filters = {
                'stationname_inc': stationname_inc,
                'stationname_exc': stationname_exc,
                'study_desc_inc': study_desc_inc,
                'study_desc_exc': study_desc_exc,
            }

            task = qrscu.delay(qr_scp_pk=rh_pk, store_scp_pk=store_pk, query_id=query_id, date_from=date_from,
                               date_until=date_until, modalities=modalities, inc_sr=inc_sr,
                               remove_duplicates=remove_duplicates, filters=filters,
                               get_toshiba_images=get_toshiba_images, get_empty_sr=get_empty_sr,
                               )

            resp = {}
            resp['message'] = u'Request created'
            resp['status'] = u'not complete'
            resp['queryID'] = query_id

            return HttpResponse(json.dumps(resp), content_type='application/json')
        else:
            print(u"Bother, form wasn't valid")
            errors = form.errors
            print(errors)
            print(form)

            # Need to find a way to deal with this event
#            render_to_response('remapp/dicomqr.html', {'form': form}, context_instance=RequestContext(request))
            resp = {}
            resp['message'] = errors
            resp['status'] = 'not complete'

            admin = {'openremversion': remapp.__version__, 'docsversion': remapp.__docs_version__}

            for group in request.user.groups.all():
                admin[group.name] = True

            return render_to_response(
                'remapp/dicomqr.html',
                {'form': form, 'admin': admin},
                context_instance=RequestContext(request)
            )
            # return HttpResponse(
            #     json.dumps(resp),
            #     content_type="application/json"
            # )

@login_required
def dicom_qr_page(request, *args, **kwargs):
    from django.shortcuts import render_to_response
    from django.template import RequestContext
    from remapp.forms import DicomQueryForm
    from remapp.models import DicomStoreSCP, DicomRemoteQR
    from remapp.netdicom.tools import echoscu

    if not request.user.groups.filter(name="importqrgroup"):
        messages.error(request, u"You are not in the importqrgroup - please contact your administrator")
        return redirect(reverse_lazy('home'))

    form = DicomQueryForm

    storestatus = {}
    stores = DicomStoreSCP.objects.all()
    for store in stores:
        echo = echoscu(scp_pk=store.pk, store_scp=True)
        if echo is "Success":
            storestatus[store.name] = u"OK: responding to DICOM echo"
        else:
            storestatus[store.name] = u"Error: not responding to DICOM echo"

    qrstatus = {}
    qr = DicomRemoteQR.objects.all()
    for scp in qr:
        echo = echoscu(scp_pk=scp.pk, qr_scp=True)
        if echo is "Success":
            qrstatus[scp.name] = u"OK: responding to DICOM echo"
        else:
            qrstatus[scp.name] = u"Error: not responding to DICOM echo"

    admin = {'openremversion': remapp.__version__, 'docsversion': remapp.__docs_version__}

    for group in request.user.groups.all():
        admin[group.name] = True

    return render_to_response(
        'remapp/dicomqr.html',
        {'form': form, 'storestatus': storestatus, 'qrstatus': qrstatus, 'admin': admin},
        context_instance=RequestContext(request)
    )

@csrf_exempt
@login_required
def r_start(request):
    from remapp.netdicom.qrscu import movescu
    resp = {}
    data = request.POST
    query_id = data.get('queryID')
    resp['queryID'] = query_id

    movescu.delay(query_id)

    return HttpResponse(json.dumps(resp), content_type='application/json')

@csrf_exempt
def r_update(request):
    from django.core.exceptions import ObjectDoesNotExist
    from django.db.models import Count
    from remapp.models import DicomQuery

    resp = {}
    data = request.POST
    query_id = data.get('queryID')
    resp['queryID'] = query_id
    try:
        query = DicomQuery.objects.get(query_id=query_id)
    except ObjectDoesNotExist:
        resp['status'] = u'not complete'
        resp['message'] = u'
Move request {0} not yet started
'.format(query_id)
        return HttpResponse(json.dumps(resp), content_type='application/json')

    if query.failed:
        resp['status'] = u'failed'
        resp['message'] = u'
Move request failed
 {0}'.format(query.message)
        return HttpResponse(json.dumps(resp), content_type='application/json')

    if not query.move_complete:
        resp['status'] = u'not complete'
        resp['message'] = u'
{0}
'.format(query.stage)
    else:
        resp['status'] = u'move complete'
        resp['message'] = u'
Move request complete
'

    return HttpResponse(json.dumps(resp), content_type='application/json')


# -*- coding: utf-8 -*-
# 爬取动漫之家 pixiv 图片，参考页面：https://news.dmzj.com/article/48293.html
# TODO: 建立数据表与数据关联，图片上传至第三方云服务器存储

import requests
from bs4 import BeautifulSoup
import os
import time
import hashlib
import re
from db import Mysql

Mysql = Mysql()

class DmzjCrawler():

    # 标识下载数量
    x = 0
    taskNum = 0
    # 爬取第几页，进程运行时自增
    page = 60
    # 请求头
    headers = {
        'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
        'accept-encoding': 'gzip, deflate, br',
        'accept-language': 'zh-CN,zh;q=0.9',
        'cache-control': 'max-age=0',
        'cookie': 'UM_distinctid=165d8713f431c-03be91a8ec41a-54103715-1fa400-165d8713f4532b; show_tip_1=0',
        'referer': 'https://news.dmzj.com/article/12875.html',
        'upgrade-insecure-requests': '1',
        'if-modified-since': 'Thu, 04 Jan 2018 01:57:35 GMT',
        'if-none-match': "5a4d8a0f-a939e",
        'user-agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36',
    }

    def __init__(self, config):
        self.taskMaxNum = config['taskMaxNum']
        self.sleepTime = config['sleepTime']
        self.downloadPath = config['downloadPath']

    # 创建文件夹
    def mkdir(self, path):
        folder = os.path.exists(path)
        if not folder: # 判断是否存在文件夹，如果不存在则创建为文件夹
            os.makedirs(path)

    # 下载图片
    def downloadImage(self, url, downloadPath, name):
        self.mkdir(downloadPath)
        image = self.get(url)
        with open(os.path.join(downloadPath, name), 'wb') as f:
            f.write(image.content)
            f.close()
            self.x += 1
            # print('已下载第', self.x ,'张')

    # 获得图片地址
    def getImages(self, url, title):
        try:
            r = self.get(url)
            html = r.text
            soup = BeautifulSoup(html, 'html.parser')
            p_images = soup.find_all('p', style='text-align:center')
            if len(p_images) > 0:
                # 新增专题
                now = int(time.time())
                topic_id = Mysql.create("INSERT INTO lab_topic (title, create_time, update_time) VALUES ('%s', %s, %s)" % (title, now, now))
                images_values = []
                for p_image in p_images:
                    url = p_image.img['src']
                    p_name = p_image.img['alt']
                    p_object = re.search( r'id=(\d*)\..*', p_name, re.I)
                    p_id = int(p_object.group(1)) if p_object else 0
                    # 获取文件后缀名
                    etc = os.path.splitext(url)[1]
                    date = time.strftime('%Y%m%d',time.localtime(time.time()))
                    old_name = date + str(round(time.time() * 1000)) + p_name
                    name = hashlib.md5(old_name.encode(encoding='UTF-8')).hexdigest() + etc
                    downloadPath = os.path.join(self.downloadPath, date)
                    self.downloadImage(url, downloadPath, name)
                    save_path = date + '/' + name
                    images_values.append("('%s', %s, '%s', %s, %s, %s)" % (save_path, topic_id, p_name, p_id, now, now))
                create_sql = 'INSERT INTO lab_image (url, topic_id, name, p_id, create_time, update_time) VALUES ' + (','.join(images_values))
                Mysql.execute(create_sql)
                print('\033[1;32m--------------------已创建：', title, '\033[0m')
        except Exception as e:
            # TODO: log
            print(e)
            return
    # 获得图片页面
    def getImagesViewLinks(self, url):
        try:
            r = self.get(url)
            html = r.text
            soup = BeautifulSoup(html, 'html.parser')
            # 获取相关美图文章的链接
            links = soup.find_all('a', class_='dec_img')
            if len(links) == 0:
                self.page = -1
            for link in links:
                url = link['href']
                title = link['title']
                self.getImages(url, title)
            self.page -= 1
        except Exception as e:
            # TODO: log
            return

    def get(self, url):
        r = requests.get(url, headers = self.headers)
        return r

    def start(self):
        while (self.page > 0 and self.page < 61):
           url = 'https://news.dmzj.com/meituxinshang/p{}.html'.format(self.page)
           print('\033[1;35m--------------------当前页数：', self.page, '\033[0m')
           self.getImagesViewLinks(url)
           time.sleep(self.sleepTime)
        print('下载完成！')


from flask import Flask, render_template, request

import config
import os
import json
import ee
import time
import calendar
import datetime
import threading

import logging, logging.config, yaml

from pymemcache.client.hash import Client
#from google.appengine.api import memcache as mc

##################################################################################################
# CONSTANT DEFINITIONS
##################################################################################################

EE_CREDENTIALS = ee.ServiceAccountCredentials(config.EE_ACCOUNT, config.EE_PRIVATE_KEY_FILE)

HIGH_COLLECTION_ID = 'srtm90_v4'
LIGHTS_COLLECTION_ID = 'NOAA/DMSP-OLS/NIGHTTIME_LIGHTS'
TEMPERATURE_COLLECTION_ID = 'MODIS/MOD11A2'

REDUCTION_SCALE_METERS = 20000

COUNTRIES_FILE = 'static/countries.txt'
COUNTRIES_PATH = 'static/countries/'
DETAILS_PATH = 'static/details/'

CACHE = 0 # When use memcache
SAVE = 0 # When we save data to precompute it in details folder
INITIAL_MAP = '5'

#logging
#LOGGER_TYPE = 'file'
LOGGER_TYPE = 'console'
logging.config.dictConfig(yaml.load(open('logging.conf')))
logger = logging.getLogger(LOGGER_TYPE)

##################################################################################################
# FUNCTIONS
##################################################################################################
def json_serializer(key, value):
  if type(value) == str:
     return value, 1
  return json.dumps(value), 2

def json_deserializer(key, value, flags):
  if flags == 1:
      return value
  if flags == 2:
      return json.loads(value)
  raise Exception('Unknown serialization format')

mc = Client(('localhost', 11211), serializer = json_serializer, deserializer = json_deserializer)

def createCountries(filename):
  file  = open(filename, 'r')
  return  file.read().splitlines()

COUNTRIES_ID = createCountries(COUNTRIES_FILE)

def change_dict(dct):
  if 'features' in dct:
    return dct['features'][0]
  return dct

# Add a band containing image date as years since 1991.
def CreateTimeBand(img):
  year = ee.Date(img.get('system:time_start')).get('year').subtract(1991)
  return ee.Image(year).byte().addBands(img)

def CreateTimeBandTemp(img):
  year = ee.Date(img.get('system:time_start')).get('year').subtract(2013)
  return ee.Image(year).byte().addBands(img)

def GetFeature(country_id):
  """Returns an ee.Feature for the country with the given ID."""
  if CACHE:
    try:
      geojson = mc.get('geojson_' + country_id)
      if geojson is not None:
        return ee.Feature(geojson)
    except Exception as e:
      logger.debug('Error GetFeature cache: '  + str(e))

  # Note: The country IDs are read from the filesystem in the initialization
  path = COUNTRIES_PATH + country_id + '.geo.json'
  path = os.path.join(os.path.split(__file__)[0], path)
  try:
    with open(path, 'r') as f:
      t = f.read()
      elem = json.loads(t, object_hook = change_dict)
      if CACHE:
        mc.set('geojson_' + country_id, elem)
      f.close()
      return ee.Feature(elem)
  except Exception as e:
    logger.debug('Error GetFeature reading file ' + path)

def coordsToFeature(coords):
  feature = json.loads(coords, object_hook = change_dict)
  return ee.Feature(feature)

def GetMapFromId(id):
  if id == '0':
    return GetHighMap()
  elif id == '1':
    return GetLightsMap()
  elif id == '2':
    return GetTemperatureMap()
  elif id == '3':
    return GetWaterOccurrenceMap()
  elif id == '4':
    return GetWaterChangeMap()
  elif id == '5':
    return GetForestChangeMap()
  elif id == '6':
    return GetVegetationMap()
  else:
    raise Exception("Map does not exists")

##################################################################################################
# HIGH MAP
##################################################################################################

def GetHighMap():
  return ee.Image(HIGH_COLLECTION_ID).getMapId({
    'min': '0',
    'max': '1000',
    'palette': '0000ff, 008000, ff0000'
  })

def ComputeCountryTimeSeriesHigh(country_id, feature = None, zoom = 1):
  """Returns mean elevation for the country."""
  img = ee.Image(HIGH_COLLECTION_ID)
  img = img.select('elevation')

  scale = 50000
  if feature is None:
    feature = GetFeature(country_id)
  else:
    scale = scale / zoom

  reduction = img.reduceRegion(ee.Reducer.mean(), feature.geometry(), scale)

  feature = ee.Feature(None, {
    'elevation': reduction.get('elevation')
  })
  chart_data = feature.getInfo()

  return chart_data['properties']

##################################################################################################
# LIGHTS MAP
##################################################################################################

def GetLightsMap():
  """Returns the MapID for the night-time lights trend map."""
  collection = ee.ImageCollection(LIGHTS_COLLECTION_ID)

  collection = collection.select('stable_lights').map(CreateTimeBand)
  # Fit a linear trend to the nighttime lights collection.
  fit = collection.reduce(ee.Reducer.linearFit())

  return fit.getMapId({
      'min': '0',
      'max': '0.18,20,-0.18',
      'bands': 'scale,offset,scale',
  })

def ComputeCountryTimeSeriesLights(country_id, feature = None, zoom = 1):
  """Returns a series of brightness over time for the country."""
  collection = ee.ImageCollection(LIGHTS_COLLECTION_ID)
  collection = collection.select('stable_lights').sort('system:time_start')

  scale = REDUCTION_SCALE_METERS
  if feature is None:
    feature = GetFeature(country_id)
  else:
    scale = scale / zoom

  # Compute the mean brightness in the region in each image.
  def ComputeMeanLights(img):
    reduction = img.reduceRegion(ee.Reducer.mean(), feature.geometry(), scale)
    return ee.Feature(None, {
        'stable_lights': reduction.get('stable_lights'),
        'system:time_start': img.get('system:time_start')
    })

  chart_data = collection.map(ComputeMeanLights).getInfo()

  # Extract the results as a list of lists.
  def ExtractMeanLights(feature):
    if 'stable_lights' in feature['properties'] and feature['properties']['stable_lights'] is not None:
      return [
          feature['properties']['system:time_start'],
          feature['properties']['stable_lights']
      ]

  return map(ExtractMeanLights, chart_data['features'])

##################################################################################################
# TEMPERATURE MAP
##################################################################################################

def GetTemperatureMap():
  """Returns the MapID for the temperature map"""
  collection = ee.ImageCollection(TEMPERATURE_COLLECTION_ID)
  collection = collection.select('LST_Day_1km')#.map(CreateTimeBand)

  fit = collection.median().toFloat().multiply(ee.Image(0.02)).subtract(ee.Image(273.15))

  return fit.getMapId({
    'min': '0',
    'max': '40',
    'palette':'0000ff,32cd32,ffff00,ff8c00,ff0000'
  })

def ComputeCountryTimeSeriesTemp(country_id, feature = None, zoom = 1):
  """Returns a series of surface temperature over time for the country."""
  collection = ee.ImageCollection(TEMPERATURE_COLLECTION_ID)
  collection = collection.select('LST_Day_1km')

  scale = 30000
  if feature is None:
    feature = GetFeature(country_id)
  else:
    scale = scale / zoom
  # Compute the mean temperature in the region in each image.
  def ComputeMeanTemp(img):
    reduction = img.reduceRegion(ee.Reducer.mean(),feature.geometry(), scale)
    return ee.Feature(None, {
        'temperature': reduction.get('LST_Day_1km'),
        'system:time_start': img.get('system:time_start')
    })

  chart_data = collection.map(ComputeMeanTemp).getInfo()

  def toCelsius(kelvin):
    scale = 0.02
    return kelvin * scale - 273.15

  # Extract the results as a list of lists.
  def ExtractMeanTemp(feature):
    if 'temperature' in feature['properties'] and feature['properties']['temperature'] is not None:
      tempInCelsius = toCelsius(feature['properties']['temperature'])
      return [
        feature['properties']['system:time_start'],
        tempInCelsius
      ]

  return map(ExtractMeanTemp, chart_data['features'])

##################################################################################################
# WATER OCCURRENCE MAP
##################################################################################################

def GetWaterOccurrenceMap():
  img = ee.Image('JRC/GSW1_0/GlobalSurfaceWater')
  img = img.select('occurrence')
  return img.getMapId({
    'min': '0',
    'max': '100',
    'palette': 'ff0000,0000ff'
  })

def ComputeCountryTimeSeriesWaterOccurence(country_id, feature = None, zoom = 1):
  """Returns a series of water occurrence over time for the country."""
  image = ee.Image('JRC/GSW1_0/GlobalSurfaceWater')
  image = image.select('change_abs')

  scale = REDUCTION_SCALE_METERS
  if feature is None:
    feature = GetFeature(country_id)
  else:
    scale = scale / zoom

  # Compute the mean temperature in the region in each image.
  def ComputeMeanWaterOccurence(img):
    reduction = img.reduceRegion(ee.Reducer.histogram(), feature.geometry(), scale)

    return ee.Feature(None, {
        'system:time_start' : img.get('system:time_start'),
        'water': reduction.get('change_abs')
    })

  chart_data = ComputeMeanWaterOccurence(image).getInfo()

  return chart_data['properties']['water']

##################################################################################################
# WATER CHANGE MAP
##################################################################################################

def GetWaterChangeMap():
  img = ee.Image('JRC/GSW1_0/GlobalSurfaceWater')
  img = img.select('change_abs')
  return img.getMapId({
    'min': '-50',
    'max': '50',
    'palette': 'ff0000,000000,00ff00'
  })
def ComputeCountryTimeSeriesWaterChange(country_id, feature = None, zoom = 1):
  """Returns a series of water change over time for the country."""
  collection = ee.ImageCollection('JRC/GSW1_0/YearlyHistory')
  collection = collection.select('waterClass')

  scale = REDUCTION_SCALE_METERS
  if feature is None:
    feature = GetFeature(country_id)
  else:
    scale = scale / zoom
  # Compute the mean temperature in the region in each image.
  def ComputeMeanWaterChange(img):
    reduction = img.reduceRegion(ee.Reducer.mean(), feature.geometry(), scale)

    return ee.Feature(None, {
        'system:time_start' : img.get('system:time_start'),
        'water': reduction.get('waterClass')
    })

  chart_data = collection.map(ComputeMeanWaterChange).getInfo()

  # Extract the results as a list of lists.
  def ExtractMeanWaterChange(feature):
    if 'water' in feature['properties'] and feature['properties']['water'] is not None:
      return [
        feature['properties']['system:time_start'],
        feature['properties']['water']
      ]

  return map(ExtractMeanWaterChange, chart_data['features'])

##################################################################################################
# FOREST CHANGE MAP
##################################################################################################

def GetForestChangeMap():
  img = ee.Image('UMD/hansen/global_forest_change_2015')
  return img.getMapId({
    'bands': 'loss, treecover2000, gain',
    'max': '1, 255, 1'
  })

def ComputeCountryTimeSeriesForestChange(country_id, feature = None, zoom = 1):
  """Returns country forest change."""
  collection = ee.Image('UMD/hansen/global_forest_change_2015')

  scale = 14000
  if feature is None:
    feature = GetFeature(country_id)
  else:
    scale = scale / zoom
  # Compute the mean temperature in the region in each image.
  def ComputeMeanForestChange(img):
    reduction = img.reduceRegion(ee.Reducer.mean(), feature.geometry(), scale)

    return ee.Feature(None, {
        'treecover2000' : reduction.get('treecover2000'),
        'loss': reduction.get('loss'),
        'gain': reduction.get('gain')
    })

  chart_data = ComputeMeanForestChange(collection).getInfo()

  # Extract the results as a list of lists.
  def ExtractMeanForestChange(feature):
    if 'loss' in feature['properties'] and feature['properties']['loss'] is not None:
      return [
        feature['properties']['treecover2000'],
        feature['properties']['gain'],
        feature['properties']['loss']
      ]

  return ExtractMeanForestChange(chart_data)

##################################################################################################
# VEGETATION INDEX MAP
##################################################################################################

def GetVegetationMap():
  img = ee.Image(ee.ImageCollection('MODIS/MCD43A4_NDVI').mean())
  return img.getMapId({
    'min': '0',
    'max': '1',
    'palette' : 'FFFFFF,CC9966,CC9900,996600,33CC00,009900,006600,000000'
  })

def ComputeCountryTimeSeriesVegetation(country_id, feature = None, zoom = 1):
  """Returns a series of vegetation over time for the country."""
  collection = ee.ImageCollection('MODIS/MCD43A4_NDVI')
  collection = collection.select('NDVI')

  scale = REDUCTION_SCALE_METERS
  if feature is None:
    feature = GetFeature(country_id)
  else:
    scale = scale / zoom

  # Compute the mean temperature in the region in each image.
  def ComputeMeanVegetation(img):
    reduction = img.reduceRegion(ee.Reducer.mean(), feature.geometry(), scale)

    return ee.Feature(None, {
        'system:time_start' : img.get('system:time_start'),
        'NDVI': reduction.get('NDVI')
    })

  chart_data = collection.map(ComputeMeanVegetation).getInfo()

  # Extract the results as a list of lists.
  def ExtractMeanVegetation(feature):
    if 'NDVI' in feature['properties'] and feature['properties']['NDVI'] is not None:
      return [
        feature['properties']['system:time_start'],
        feature['properties']['NDVI']
      ]

  return map(ExtractMeanVegetation, chart_data['features'])

def ComputeCountryTimeSeries(map_id, country_id, feature = None, zoom = 1):
  """Returns a series of the specific map over time for the country."""
  if map_id == '0':
    return ComputeCountryTimeSeriesHigh(country_id, feature, zoom)
  elif map_id == '1':
    return ComputeCountryTimeSeriesLights(country_id, feature, zoom)
  elif map_id == '2':
    return ComputeCountryTimeSeriesTemp(country_id, feature, zoom)
  elif map_id == '3':
    return ComputeCountryTimeSeriesWaterOccurence(country_id, feature, zoom)
  elif map_id == '4':
    return ComputeCountryTimeSeriesWaterChange(country_id, feature, zoom)
  elif map_id == '5':
    return ComputeCountryTimeSeriesForestChange(country_id, feature, zoom)
  elif map_id == '6':
    return ComputeCountryTimeSeriesVegetation(country_id, feature, zoom)
  else:
    raise Exception("Map type does not exists")

##################################################################################################
# APP ROUTES
##################################################################################################

# API
app = Flask(__name__)

app.jinja_env.auto_reload = True
app.config.update(
  DEBUG=False,
  TEMPLATES_AUTO_RELOAD=False
)

@app.before_request
def before_request():
  # When you import jinja2 macros, they get cached which is annoying for local
  # development, so wipe the cache every request.
  #app.jinja_env.cache = dict()
  # Initialize Earth Engine
  ee.Initialize(EE_CREDENTIALS)
  #ee.Initialize()

# Define root route
@app.route('/')
def main():
  mapid = GetMapFromId(INITIAL_MAP)

  # Add variables to the template
  template_values = {
    'map': INITIAL_MAP,
    'mapid': mapid['mapid'],
    'token': mapid['token'],
    'API_KEY': config.API_KEY,
    'countries' : json.dumps(COUNTRIES_ID)
  }

  # Render the template index.html
  return render_template('index.html', values = template_values)

@app.route('/map/<id>')
def get(id):
  mapid = GetMapFromId(id)
  # Add variables to the template
  template_values = {
    'map' : id,
    'mapid': mapid['mapid'],
    'token': mapid['token'],
    'API_KEY': config.API_KEY,
    'countries' : json.dumps(COUNTRIES_ID)
  }
  # Render the template reload.html
  return render_template('reload.html', values = template_values)

@app.route('/save/<map_id>')
def saveCountryTimeSeries(map_id):
  error = 0
  message = ''
  if CACHE:
    elems = dict()
    if SAVE:
      path = DETAILS_PATH + 'mapid_' + map_id + '.json'
      details_file = open(path, 'w')
    else:
      thread = threading.Thread()
      thread.start()
    for country_id in COUNTRIES_ID:
      key = 'details' + '_'+ map_id + '_' + country_id
      details = dict()
      try:
        if map_id == '0' or map_id == '3':
          details = ComputeCountryTimeSeries(map_id, country_id)
          if details is not None:
            mc.set(key, json.dumps(details))
        elif map_id == '5':
          details['forestChange'] = ComputeCountryTimeSeries(map_id, country_id)
          if details['forestChange'] is not None:
            mc.set(key, json.dumps(details))
        else:
          details['timeSeries'] = list(ComputeCountryTimeSeries(map_id, country_id))
          if details['timeSeries'] is not None:
            mc.set(key, json.dumps(details))
        elems[country_id] = details
      except Exception as e:
        error = 1
        message = str(e)
        logger.debug('Error saveCountryTimeSeries: ' + message)
    if SAVE and not error:
      json.dump(elems, details_file , separators = (',', ':'))
      details_file.close()
  return json.dumps({'status' : error, 'message' : message})

@app.route('/static/<map_id>')
def staticCountryTimeSeries(map_id):
  error = 0
  message = ''
  if SAVE:
    elems = dict()
    path = DETAILS_PATH + 'mapid_' + map_id + '.json'
    details_file = open(path, 'w')
    for country_id in COUNTRIES_ID:
      key = 'details' + '_'+ map_id + '_' + country_id
      details = dict()
      try:
        if map_id == '0' or map_id == '3':
          details = ComputeCountryTimeSeries(map_id, country_id)
          if details is not None:
            elems[country_id] = details
        elif map_id == '5':
          details['forestChange'] = ComputeCountryTimeSeries(map_id, country_id)
          if details['forestChange'] is not None:
            elems[country_id] = details
        else:
          details['timeSeries'] = list(ComputeCountryTimeSeries(map_id, country_id))
          if details['timeSeries'] is not None:
            elems[country_id] = details
      except Exception as e:
        error = 1
        message = str(e)
        logger.debug('Error saveCountryTimeSeries: ' + message)
    if not error:
      json.dump(elems, details_file , separators = (',', ':'))
      details_file.close()
  return json.dumps({'status' : error, 'message' : message})

@app.route('/details/<map_id>/<country_id>')
def GetCountryTimeSeries(map_id, country_id):
  if CACHE:
    try:
      key = 'details' + '_'+ map_id + '_' + country_id
      details = mc.get(key)
      if details is not None:
        return details
    except Exception as e:
      logger.debug('Error cache GetCountryTimeSeries: ' + str(e))

  details = dict()
  try:
    if map_id == '0' or map_id == '3':
      details = ComputeCountryTimeSeries(map_id, country_id)
      if CACHE and details is not None:
        mc.set(key, json.dumps(details))
    elif map_id == '5':
      details['forestChange'] = ComputeCountryTimeSeries(map_id, country_id)
      if CACHE and details['forestChange'] is not None:
        mc.set(key, json.dumps(details))
    else:
      details['timeSeries'] = list(ComputeCountryTimeSeries(map_id, country_id))
      if CACHE and details['timeSeries'] is not None:
        mc.set(key, json.dumps(details))

  except ee.EEException as e:
    # Handle exceptions from the EE client library.
    details['error'] = str(e)
    logger.debug('Error GetCountryTimeSeries: ' + details['error'])
  # Send the results to the browser.
  return json.dumps(details)

@app.route('/details/<map_id>')
def GetAllCountriesDetails(map_id):
  if CACHE:
    try:
     key = 'details' + '_' + map_id
     countries = mc.get(key)
     if countries is not None:
       return countries
    except Exception as e:
      logger.debug('Error cache GetAllCountriesDetails' + str(e))
    countries = list()
    for country_id in COUNTRIES_ID:
      try:
        key = 'details' + '_'+ map_id + '_' + country_id
        country = mc.get(key)
      except Exception as e:
        logger.debug('Error cache GetAllCountriesDetails: ' + country_id)
      try:
        if country is None:
          elem = {'name' : getCountryName(country_id), 'data' : ComputeCountryTimeSeries(map_id, country_id) }
        else:
          elem = {'name' : getCountryName(country_id), 'data' : country }
        countries.append(json.dumps(elem))
      except Exception as e:
        logger.debug('Error GetAllCountriesDetails, country: ' + country_id + ' : ' + str(e))
    key = 'details' + '_'+ map_id
    mc.set(key, json.dumps(countries))
  else:
    countries = dict()
    countries['error'] = 'Not implemented yet'
  return json.dumps(countries)

@app.route('/custom/<map_id>/<zoom>', methods = ['POST'])
def GetCustomSeries(map_id, zoom):
  """Get time series from a custom polygon"""
  key = list(request.form.keys())[0]
  details = dict()
  try:
    feature = coordsToFeature(key)
    if feature is not None:
      try:
        zoom = int(zoom)
        if map_id == '0' or map_id == '3':
          details = ComputeCountryTimeSeries(map_id, None, feature, zoom)
        elif map_id == '5':
          details['forestChange'] = ComputeCountryTimeSeries(map_id, None, feature, zoom)
        else:
          details['timeSeries'] = list(ComputeCountryTimeSeries(map_id, None, feature, zoom))
      except ee.EEException as e:
        # Handle exceptions from the EE client library.
        details['error'] = str(e)
        logger.debug('Error GetCustomSeries: ' + details['error'])
  except Exception as e:
    details['error'] = str(e)
    logger.debug('Error GetCustomSeries: ' + details['error'])
  # Send the results to the browser.
  return json.dumps(details)

@app.route('/country/<country_id>')
def getCountryName(country_id):
  """Get country name from country id"""
  if CACHE:
    try:
      key = 'name_' + country_id
      name = mc.get(key)
      if name is not None:
        return name
    except Exception as e:
      logger.debug('Error cache getCountryName:' + str(e))
  path = COUNTRIES_PATH + country_id + '.geo.json'
  path = os.path.join(os.path.split(__file__)[0], path)
  try:
    with open(path, 'r') as f:
      t = f.read()
      elem = json.loads(t, object_hook = change_dict)
      name = elem['properties']['name']
      if CACHE and name is not None:
        mc.set(key, name)
      f.close()
      return name
  except Exception as e:
    logger.debug('Error getCountryName reading file ' + path + ' : ' + str(e))

# Run application in selected port
if __name__ == '__main__':
  app.run('0.0.0.0', 8080, threaded=True)