import torch
import torchvision
import numpy as np
import math
import cv2
# # from ultralytics.yolo.utils.general import xywh2xyxy
# try:
#     from ultralytics.yolo.utils.metrics import box_iou
#     from ultralytics.yolo.utils.ops import xywh2xyxy
#     from ultralytics.nn.autoshape import scale_boxes, Detections
# except ImportError:
def xywh2xyxy(x):
    # Convert nx4 boxes from [x, y, w, h] to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right
    y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)
    y[..., 0] = x[..., 0] - x[..., 2] / 2  # top left x
    y[..., 1] = x[..., 1] - x[..., 3] / 2  # top left y
    y[..., 2] = x[..., 0] + x[..., 2] / 2  # bottom right x
    y[..., 3] = x[..., 1] + x[..., 3] / 2  # bottom right y
    return y

def box_iou(box1, box2, eps=1e-7):
    """
    Calculate intersection-over-union (IoU) of boxes.
    Both sets of boxes are expected to be in (x1, y1, x2, y2) format.
    Based on https://github.com/pytorch/vision/blob/master/torchvision/ops/boxes.py

    Args:
        box1 (torch.Tensor): A tensor of shape (N, 4) representing N bounding boxes.
        box2 (torch.Tensor): A tensor of shape (M, 4) representing M bounding boxes.
        eps (float, optional): A small value to avoid division by zero. Defaults to 1e-7.

    Returns:
        (torch.Tensor): An NxM tensor containing the pairwise IoU values for every element in box1 and box2.
    """

    # inter(N,M) = (rb(N,M,2) - lt(N,M,2)).clamp(0).prod(2)
    (a1, a2), (b1, b2) = box1.unsqueeze(1).chunk(2, 2), box2.unsqueeze(0).chunk(2, 2)
    inter = (torch.min(a2, b2) - torch.max(a1, b1)).clamp_(0).prod(2)

    # IoU = inter / (area1 + area2 - inter)
    return inter / ((a2 - a1).prod(2) + (b2 - b1).prod(2) - inter + eps)

def exif_transpose(image):
    """
    Transpose a PIL image accordingly if it has an EXIF Orientation tag.
    Inplace version of https://github.com/python-pillow/Pillow/blob/master/src/PIL/ImageOps.py exif_transpose()

    :param image: The image to transpose.
    :return: An image.
    """
    exif = image.getexif()
    orientation = exif.get(0x0112, 1)  # default 1
    if orientation > 1:
        method = {
            2: Image.FLIP_LEFT_RIGHT,
            3: Image.ROTATE_180,
            4: Image.FLIP_TOP_BOTTOM,
            5: Image.TRANSPOSE,
            6: Image.ROTATE_270,
            7: Image.TRANSVERSE,
            8: Image.ROTATE_90}.get(orientation)
        if method is not None:
            image = image.transpose(method)
            del exif[0x0112]
            image.info['exif'] = exif.tobytes()
    return image

def make_divisible(x, divisor):
    # Returns nearest x divisible by divisor
    if isinstance(divisor, torch.Tensor):
        divisor = int(divisor.max())  # to int
    return math.ceil(x / divisor) * divisor

def letterbox(im, new_shape=(640, 640), color=(114, 114, 114), auto=True, scaleFill=False, scaleup=True, stride=32):
    # Resize and pad image while meeting stride-multiple constraints
    shape = im.shape[:2]  # current shape [height, width]
    if isinstance(new_shape, int):
        new_shape = (new_shape, new_shape)

    # Scale ratio (new / old)
    r = min(new_shape[0] / shape[0], new_shape[1] / shape[1])
    if not scaleup:  # only scale down, do not scale up (for better val mAP)
        r = min(r, 1.0)

    # Compute padding
    ratio = r, r  # width, height ratios
    new_unpad = int(round(shape[1] * r)), int(round(shape[0] * r))
    dw, dh = new_shape[1] - new_unpad[0], new_shape[0] - new_unpad[1]  # wh padding
    if auto:  # minimum rectangle
        dw, dh = np.mod(dw, stride), np.mod(dh, stride)  # wh padding
    elif scaleFill:  # stretch
        dw, dh = 0.0, 0.0
        new_unpad = (new_shape[1], new_shape[0])
        ratio = new_shape[1] / shape[1], new_shape[0] / shape[0]  # width, height ratios

    dw /= 2  # divide padding into 2 sides
    dh /= 2

    if shape[::-1] != new_unpad:  # resize
        im = cv2.resize(im, new_unpad, interpolation=cv2.INTER_LINEAR)
    top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1))
    left, right = int(round(dw - 0.1)), int(round(dw + 0.1))
    im = cv2.copyMakeBorder(im, top, bottom, left, right, cv2.BORDER_CONSTANT, value=color)  # add border
    return im, ratio, (dw, dh)

def non_max_suppression_obj(
        prediction,
        conf_thres=0.25,
        iou_thres=0.45,
        classes=None,
        agnostic=False,
        multi_label=False,
        labels=(),
        max_det=1000,
        nm=0,  # number of masks
):
    """Non-Maximum Suppression (NMS) on inference results to reject overlapping detections

    Returns:
         list of detections, on (n,7) tensor per image [xyxy, conf, cls, obj_conf]
    """

    # Checks
    assert 0 <= conf_thres <= 1, f'Invalid Confidence threshold {conf_thres}, valid values are between 0.0 and 1.0'
    assert 0 <= iou_thres <= 1, f'Invalid IoU {iou_thres}, valid values are between 0.0 and 1.0'
    if isinstance(prediction, (list, tuple)):  # YOLOv5 model in validation model, output = (inference_out, loss_out)
        prediction = prediction[0]  # select only inference output

    device = prediction.device
    mps = 'mps' in device.type  # Apple MPS
    if mps:  # MPS not fully supported yet, convert tensors to CPU before NMS
        prediction = prediction.cpu()
    bs = prediction.shape[0]  # batch size
    nc = prediction.shape[2] - nm - 5  # number of classes
    xc = prediction[..., 4] > conf_thres  # candidates

    # Settings
    # min_wh = 2  # (pixels) minimum box width and height
    max_wh = 7680  # (pixels) maximum box width and height
    max_nms = 30000  # maximum number of boxes into torchvision.ops.nms()
    time_limit = 0.5 + 0.05 * bs  # seconds to quit after
    redundant = True  # require redundant detections
    multi_label &= nc > 1  # multiple labels per box (adds 0.5ms/img)
    merge = False  # use merge-NMS

    # t = time.time()
    mi = 5 + nc  # mask start index
    output = [torch.zeros((0, 8 + nm), device=prediction.device)] * bs
    for xi, x in enumerate(prediction):  # image index, image inference
        # Apply constraints
        # x[((x[..., 2:4] < min_wh) | (x[..., 2:4] > max_wh)).any(1), 4] = 0  # width-height
        x = x[xc[xi]]  # confidence

        # Cat apriori labels if autolabelling
        if labels and len(labels[xi]):
            lb = labels[xi]
            v = torch.zeros((len(lb), nc + nm + 5), device=x.device)
            v[:, :4] = lb[:, 1:5]  # box
            v[:, 4] = 1.0  # conf
            v[range(len(lb)), lb[:, 0].long() + 5] = 1.0  # cls
            x = torch.cat((x, v), 0)

        # If none remain process next image
        if not x.shape[0]:
            continue

        # Compute conf
        obj_conf = x[:, 4:5]
        x[:, 5:] *= x[:, 4:5]  # conf = obj_conf * cls_conf

        # Box/Mask
        box = xywh2xyxy(x[:, :4])  # center_x, center_y, width, height) to (x1, y1, x2, y2)
        mask = x[:, mi:]  # zero columns if no masks

        # Detections matrix nx6 (xyxy, conf, cls)
        if multi_label:
            i, j = (x[:, 5:mi] > conf_thres).nonzero(as_tuple=False).T
            x = torch.cat((box[i], x[i, 5 + j, None], j[:, None].float(), obj_conf[i], x[i, 5 + j, None] / obj_conf[i], mask[i]), 1)
        else:  # best class only
            conf, j = x[:, 5:mi].max(1, keepdim=True)
            x = torch.cat((box, conf, j.float(), obj_conf, conf / obj_conf, mask), 1)[conf.view(-1) > conf_thres]

        # Filter by class
        if classes is not None:
            x = x[(x[:, 5:6] == torch.tensor(classes, device=x.device)).any(1)]

        # Apply finite constraint
        # if not torch.isfinite(x).all():
        #     x = x[torch.isfinite(x).all(1)]

        # Check shape
        n = x.shape[0]  # number of boxes
        if not n:  # no boxes
            continue
        x = x[x[:, 4].argsort(descending=True)[:max_nms]]  # sort by confidence and remove excess boxes

        # Batched NMS
        c = x[:, 5:6] * (0 if agnostic else max_wh)  # classes
        boxes, scores = x[:, :4] + c, x[:, 4]  # boxes (offset by class), scores
        i = torchvision.ops.nms(boxes, scores, iou_thres)  # NMS
        i = i[:max_det]  # limit detections
        if merge and (1 < n < 3E3):  # Merge NMS (boxes merged using weighted mean)
            # update boxes as boxes(i,4) = weights(i,n) * boxes(n,4)
            iou = box_iou(boxes[i], boxes) > iou_thres  # iou matrix
            weights = iou * scores[None]  # box weights
            x[i, :4] = torch.mm(weights, x[:, :4]).float() / weights.sum(1, keepdim=True)  # merged boxes
            if redundant:
                i = i[iou.sum(1) > 1]  # require redundancy

        output[xi] = x[i]
        if mps:
            output[xi] = output[xi].to(device)
        # if (time.time() - t) > time_limit:
        #     LOGGER.warning(f'WARNING ⚠ NMS time limit {time_limit:.3f}s exceeded')
        #     break  # time limit exceeded

    return output

from pathlib import Path
import pickle as pkl
import pandas as pd
from tqdm import tqdm, trange
from torchvision import transforms
from PIL import Image
def yolo_process(yolomodel, imgpathlist, batch_size=100, size=256, savename=None, sumdir=Path("")):
    """Process images with yolo model and return results in a list of dataframes"""
    results_dfs = []
    for i in trange(0, len(imgpathlist), batch_size):
        results = yolomodel(imgpathlist[i:i+batch_size], size=size)
        results_dfs.extend(results.pandas().xyxy)
        # yolo_results[i] = results

    yolo_stats = {}
    for i, single_df in tqdm(enumerate(results_dfs)):
        yolo_stats[i] = {"confidence": single_df.confidence.max(),
                        "class": single_df["class"][single_df.confidence.argmax()] if len(single_df) > 0 else None,
                        "n_objs": len(single_df),
                        "img_path": imgpathlist[i]}
    yolo_stats_df = pd.DataFrame(yolo_stats).T
    if savename is not None:
        yolo_stats_df.to_csv(sumdir / f"{savename}_yolo_stats.csv")
        pkl.dump(results_dfs, open(sumdir / f"{savename}_dfs.pkl", "wb"))
        print(f"Saved to {sumdir / f'{savename}_dfs.pkl'}")
        print(f"Saved to {sumdir / f'{savename}_yolo_stats.csv'}")
    print("Fraction of images with objects", (yolo_stats_df.n_objs > 0).mean())
    print("confidence", yolo_stats_df.confidence.mean(), "confidence with 0 filled",
          yolo_stats_df.confidence.fillna(0).mean())
    if len(yolo_stats_df) > 0:
        print("most common class", yolo_stats_df["class"].value_counts().index[0])
    print("n_objs", yolo_stats_df.n_objs.mean(), )
    return results_dfs, yolo_stats_df

def nms_output2df(nms_results, class_names=None):
    """
    :param nms_results: list of torch.tensor shape (n, 8)
            x1, y1, x2, y2, conf, cls, obj_conf, cls_conf
    :return:
        list of pandas.DataFrame
    """
    df_list = []
    for nms_result in nms_results:
        df = pd.DataFrame(nms_result.numpy(), columns=\
                        ['xmin', 'ymin', 'xmax', 'ymax',
                         'confidence', 'class', 'obj_conf', 'cls_conf'])
        df['class'] = df['class'].astype(int)
        if class_names is not None:
            df['name'] = df['class'].apply(lambda x: class_names[x]) if not df.empty else None
        df_list.append(df)
    return df_list

def load_batch_imgpaths(imgpaths, size=256, ):
    # https://github.com/ultralytics/yolov5/blob/2334aa733872bc4bb3e1a1ba90e5fd319399596f/models/common.py#LL679C9-L679C9
    # transforms.ToTensor()  # normalizes to 0-1
    import matplotlib.pyplot as plt
    import torch.nn.functional as F
    transform = transforms.Compose([
        transforms.Resize(size),
        transforms.ToTensor(),
    ])
    imglist = []
    for imgpath in imgpaths:
        im_pil = Image.open(imgpath).convert('RGB')
        imtsr = transform(im_pil)
        imglist.append(imtsr)
    imgtsrs = torch.stack(imglist, 0)
    return imgtsrs

def load_batch_imgs_resize(imgpaths, size=256, stride=32):
    import cv2
    if isinstance(size, int):  # expand
        size = (size, size)
    imglist = []
    n, ims = (len(imgpaths), list(imgpaths)) if isinstance(imgpaths, (list, tuple)) \
        else (1, [imgpaths])  # number, list of images
    shape0, shape1 = [], []
    for imgpath in imgpaths:
        if isinstance(imgpath, (str, Path)):
            im_pil = Image.open(imgpath)
            im = np.asarray(exif_transpose(im_pil))
        elif isinstance(imgpath, Image.Image):
            im = np.asarray(exif_transpose(imgpath))
        elif isinstance(imgpath, np.ndarray):
            im = imgpath

        if im.shape[0] < 5:  # image in CHW
            im = im.transpose((1, 2, 0))  # reverse dataloader .transpose(2, 0, 1)
        im = im[..., :3] if im.ndim == 3 else cv2.cvtColor(im, cv2.COLOR_GRAY2BGR)  # enforce 3ch input
        s = im.shape[:2]  # HWC
        s = im.shape[:2]  # HWC
        shape0.append(s)  # image shape
        g = max(size) / max(s)  # gain
        shape1.append([int(y * g) for y in s])
        if not im.data.contiguous:
            im = np.ascontiguousarray(im)  # update
        imglist.append(im)
    shape1 = [make_divisible(x, stride) for x in np.array(shape1).max(0)]  # inf shape
    x = [letterbox(im, shape1, auto=False)[0] for im in imglist]  # pad
    x = np.ascontiguousarray(np.array(x).transpose((0, 3, 1, 2)))  # stack and BHWC to BCHW
    imgtsrs = torch.from_numpy(x) / 255  # uint8 to fp16/32
    return imgtsrs

def yolo_process_objconf(yolomodel, imgpathlist, batch_size=100, size=256, savename=None, sumdir=Path(""),
                         use_letterbox_resize=False):
    """Process images with yolo model and return results in a list of dataframes"""
    results_dfs = []
    for i in trange(0, len(imgpathlist), batch_size):
        # results = yolomodel(imgpathlist[i:i+batch_size], size=size)
        if use_letterbox_resize:
            imtsrs = load_batch_imgs_resize(imgpathlist[i:i + batch_size], size=size, stride=yolomodel.stride)
        else:
            imtsrs = load_batch_imgpaths(imgpathlist[i:i+batch_size], size=size)
        outtsr = yolomodel(imtsrs.cuda()).cpu()
        nms_result = non_max_suppression_obj(outtsr,
                                             yolomodel.conf, yolomodel.iou,
                                             yolomodel.classes, yolomodel.agnostic,
                                             yolomodel.multi_label,
                                             max_det=yolomodel.max_det)  # NMS
        result_df = nms_output2df(nms_result, yolomodel.names)
        results_dfs.extend(result_df)
        # yolo_results[i] = results

    yolo_stats = {}
    for i, single_df in tqdm(enumerate(results_dfs)):
        if len(single_df) > 0:
            max_row = single_df.loc[single_df.confidence.argmax()]
            yolo_stats[i] = {"xmin": max_row.xmin,
                             "ymin": max_row.ymin,
                             "xmax": max_row.xmax,
                             "ymax": max_row.ymax,
                             "confidence": max_row.confidence,
                             'obj_confidence': max_row.obj_conf,
                             'cls_confidence': max_row.cls_conf,
                             "class": max_row["class"],
                             "n_objs": len(single_df),
                             "img_path": imgpathlist[i]}
        else:
            yolo_stats[i] = { "xmin": None, "ymin": None,
                              "xmax": None, "ymax": None,
                              "confidence": None,
                              'obj_confidence': None,
                              'cls_confidence': None,
                              "class": None,
                              "n_objs": len(single_df),
                              "img_path": imgpathlist[i]}

    yolo_stats_df = pd.DataFrame(yolo_stats).T
    if savename is not None:
        yolo_stats_df.to_csv(sumdir / f"{savename}_yolo_objconf_stats.csv")
        pkl.dump(results_dfs, open(sumdir / f"{savename}_objconf_dfs.pkl", "wb"))
        print(f"Saved to {sumdir / f'{savename}_objconf_dfs.pkl'}")
        print(f"Saved to {sumdir / f'{savename}_yolo_objconf_stats.csv'}")
    print("Fraction of images with objects", (yolo_stats_df.n_objs > 0).mean())
    print("confidence", yolo_stats_df.confidence.mean(), "confidence with 0 filled",
          yolo_stats_df.confidence.fillna(0).mean())
    print("obj_confidence", yolo_stats_df.obj_confidence.mean(), "obj_confidence with 0 filled",
          yolo_stats_df.obj_confidence.fillna(0).mean())
    print("cls_confidence", yolo_stats_df.cls_confidence.mean(), "cls_confidence with 0 filled",
          yolo_stats_df.cls_confidence.fillna(0).mean())
    print("n_objs", yolo_stats_df.n_objs.mean(), )
    if len(yolo_stats_df) > 0:
        most_common_class = yolo_stats_df["class"].value_counts().index[0]
        print("most common class", most_common_class, " name:", yolomodel.names[most_common_class])
    return results_dfs, yolo_stats_df

def yolo_process_objconf_arrs(yolomodel, imgarr, batch_size=100, size=256, savename=None, sumdir=Path(""), ):
    """Process images with yolo model and return results in a list of dataframes"""
    results_dfs = []
    for i in trange(0, len(imgarr), batch_size):
        # results = yolomodel(imgpathlist[i:i+batch_size], size=size)
        imtsrs = load_batch_imgs_resize(imgarr[i:i+batch_size],
                                        size=size, stride=yolomodel.stride)
        # TODO use size
        outtsr = yolomodel(imtsrs.cuda()).cpu()
        nms_result = non_max_suppression_obj(outtsr,
                                             yolomodel.conf, yolomodel.iou,
                                             yolomodel.classes, yolomodel.agnostic,
                                             yolomodel.multi_label,
                                             max_det=yolomodel.max_det)  # NMS
        result_df = nms_output2df(nms_result, yolomodel.names)
        results_dfs.extend(result_df)
        # yolo_results[i] = results

    yolo_stats = {}
    for i, single_df in tqdm(enumerate(results_dfs)):
        if len(single_df) > 0:
            max_row = single_df.loc[single_df.confidence.argmax()]
            yolo_stats[i] = {"xmin": max_row.xmin,
                             "ymin": max_row.ymin,
                             "xmax": max_row.xmax,
                             "ymax": max_row.ymax,
                             "confidence": max_row.confidence,
                             'obj_confidence': max_row.obj_conf,
                             'cls_confidence': max_row.cls_conf,
                             "class": max_row["class"],
                             "n_objs": len(single_df),
                             "imgnum": i}
        else:
            yolo_stats[i] = { "xmin": None, "ymin": None,
                              "xmax": None, "ymax": None,
                              "confidence": None,
                              'obj_confidence': None,
                              'cls_confidence': None,
                              "class": None,
                              "n_objs": len(single_df),
                              "imgnum": i}

    yolo_stats_df = pd.DataFrame(yolo_stats).T
    if savename is not None:
        yolo_stats_df.to_csv(sumdir / f"{savename}_yolo_objconf_stats.csv")
        pkl.dump(results_dfs, open(sumdir / f"{savename}_objconf_dfs.pkl", "wb"))
        print(f"Saved to {sumdir / f'{savename}_objconf_dfs.pkl'}")
        print(f"Saved to {sumdir / f'{savename}_yolo_objconf_stats.csv'}")
    print("Fraction of images with objects", (yolo_stats_df.n_objs > 0).mean())
    print("confidence", yolo_stats_df.confidence.mean(), "confidence with 0 filled",
          yolo_stats_df.confidence.fillna(0).mean())
    print("obj_confidence", yolo_stats_df.obj_confidence.mean(), "obj_confidence with 0 filled",
          yolo_stats_df.obj_confidence.fillna(0).mean())
    print("cls_confidence", yolo_stats_df.cls_confidence.mean(), "cls_confidence with 0 filled",
          yolo_stats_df.cls_confidence.fillna(0).mean())
    print("n_objs", yolo_stats_df.n_objs.mean(), )
    if len(yolo_stats_df) > 0:
        most_common_class = yolo_stats_df["class"].value_counts().index[0]
        print("most common class", most_common_class, " name:", yolomodel.names[most_common_class])
    return results_dfs, yolo_stats_df
# im_np = plt.imread(r"F:\insilico_exps\GAN_sample_fid\BigGAN_std_008\sample0310.png")
# im_np = plt.imread(r"F:\insilico_exps\GAN_sample_fid\BigGAN_std_008\sample0398.png")
# im_np = plt.imread(r"F:\insilico_exps\GAN_sample_fid\DeePSim_4std\sample0039.png")
# im_np = plt.imread(r"F:\insilico_exps\GAN_sample_fid\BigGAN_std_008\sample0709.png")
#
# # im_np = plt.imread(r"F:\insilico_exps\GAN_sample_fid\DeePSim_4std\sample0390.png")
# imtsr = torch.from_numpy(im_np.transpose(2, 0, 1)).unsqueeze(0).cuda()
# outtsr = yolomodel(imtsr).cpu()
# nms_result = non_max_suppression_obj(outtsr,
#                          yolomodel.conf, yolomodel.iou,
#                          yolomodel.classes, yolomodel.agnostic,
#                          yolomodel.multi_label,
#                          max_det=yolomodel.max_det)  # NMS
# Detections(imtsr, nms_result, None, None, None, None)


import os
import numpy as np
from astropy.io import fits
import matplotlib.pyplot as plt
import pynbody

from simifucube.util.doppler_shift import dopplerShift

import logging

def setup_logger(logger_level=logging.INFO, logger_name=None):

    # formatter = logging.Formatter('%(asctime)s %(levelname)s %(name)s - %(message)s')
    # log_file_handler = logging.FileHandler(os.path.join(config_logger['log_dir'], config_logger['log_file']))
    # log_file_handler.setFormatter(formatter)

    stream_formatter = logging.Formatter('%(asctime)s %(levelname)s - %(message)s')
    log_stream_handler = logging.StreamHandler()
    log_stream_handler.setFormatter(stream_formatter)

    logger = logging.getLogger(logger_name)
    # logger.addHandler(log_file_handler)
    logger.addHandler(log_stream_handler)
    logger.setLevel(logger_level)

    return logger

logger = setup_logger(logging.INFO)

# Metallicity [M/H] (Z)
bins_metallicity = [-2.27, -1.79, -1.49, -1.26, -0.96, -0.66, -0.35, -0.25, +0.06, +0.15, +0.26, +0.40]

# Age (Gyr)
bins_age = [00.03, 00.04, 00.05, 00.06, 00.07, 00.08, 00.09, 00.10, 00.15, 00.20,
00.25, 00.30, 00.35, 00.40, 00.45, 00.50, 00.60, 00.70, 00.80, 00.90, 01.00,
01.25, 01.50, 01.75, 02.00, 02.25, 02.50, 02.75, 03.00, 03.25, 03.50, 03.75, 04.00,
04.50, 05.00, 05.50, 06.00, 06.50, 07.00, 07.50, 08.00, 08.50, 09.00, 09.50, 10.00,
10.50, 11.00, 11.50, 12.00, 12.50, 13.00, 13.50, 14.00]

SPECTRA_DIR = '/home/michele/sim/IFU_images/library/EMILES_BASTI_BASE_CH_FITS'
ELEMENT_LIMIT = 10000  # For MUSE it's fine (10000 spectral pixel = 10678.9 Angstrom)

# -- From http://www.iac.es/proyecto/miles/pages/ssp-models.php
# The stellar population synthesis model predictions and are based on the code presented in Vazdekis et al. (2010).
# The spectra are stored in air wavelengths.
# The total mass of the SSP is 1Mo. The SSP spectra are given in units of L_λ / Lo Mo^{-1} Å^{-1}.

# -- From http://www.iac.es/proyecto/miles/pages/ssp-models/name-convention.php
# Please note that the relation between the total metallicity and iron metallicity is
#      [Fe/H]=[M/H]-0.75[Mg/Fe]
# Thus when selecting a scaled-solar model with total metallicity [M/H] then [Fe/H]=[M/H],
# whereas for the alpha-enhanced models with the same metallicity [Fe/H]=[M/H]-0.3.
# Finally for the "base" models it is assumed that [Fe/H]=[M/H], although this is only true at high metallicities.
# For low metallicities the input stars are alpha-enhanced and therefore when selecting a model with
# total metallicity [M/H] its [Fe/H] is lower.
# To summarize the base models follow the abundance pattern of The Galaxy.
# Note that these models are inconsistent at low metallicities because the adopted isochrones are always scaled-solar.

# As an example, if only models with total metallicity [M/H]=+0.06, 10 Gyr and bimodal IMF shape with slope 1.30 are requested, then the following set of models computed with BaSTI isochrones can  be downloaded:
# * Mbi1.30Zp0.06T10.0000_iTp0.00_Ep0.00   scaled-solar abundance ratio ([M/H]=[Fe/H]=+0.06)
# * Mbi1.30Zp0.06T10.0000_iTp0.40_Ep0.40   alpha-enhanced abundance ratio ([M/H]=+0.06; [Fe/H]=-0.24)
# * Mbi1.30Zp0.06T10.0000_iTp0.00_baseFe   base models where no consideration about the abundance ratio is being made.
#                                          The stars feeding the models are picked on the basis of their [Fe/H] metallicity,
#                                          and therefore these models follow the abundance pattern of The Galaxy.
# as well as base models computed with the Padova00 isochrones with nearly similar metallicity:
# * Mbi1.30Zp0.00T10.0000_iPp0.00_baseFe

def get_nearest(x, bins):
    return bins[np.abs(np.array(bins) - x).argmin()]

def get_bin(x, bins):
    n = np.digitize(x, bins)
    return bins[n-1], bins[n]

def get_filename(metallicity, age):
    """
    Get filename of the correct SSP by parsing the E-MILES library.

    I use E-MILES with Chabrier IMF with 1.3 slope and BaSTI isochrones.
    The metallicity is [Fe/H] of the star particle
    Age is in Gyr
    """
    met = get_nearest(metallicity, bins_metallicity)
    _age = get_nearest(age, bins_age)
    met_sign = 'p' if met >= 0 else 'm'
    name = 'Ech1.30Z{}{:3.2f}T{:07.4f}_iTp0.00_baseFe.fits'.format(met_sign, abs(met), _age)
    filename = os.path.join(SPECTRA_DIR, name)
    return filename

def get_spectrum(filename, n_limit=ELEMENT_LIMIT):
    """
    Get spectrum from the given filename. Limit the read to the first `n_limit` elements

    Parameters
    ----------
    filename: str
        The filename
    n_limit: int (default 10000)
        Number of elements to read from the spectrum from the beginning. This reduces the memory footprint.
        This useful because last spectral channels are usually outside of range for practical purpose.
        For MUSE the default of 10000 is fine. 10000 spectral pixels corresponds to an upper limit of 10678.9 Angstrom, MUSE arrives at 9600 Angstroms

    Returns
    -------
    wavelength:np.ndarray
        Wavelength in Angstrom
    flux: np.ndarray
        Spectral flux per unit mass (Luminosity per unit wavelength per unit mass) in Lo Mo^{-1} Å^{-1}
    """
    with fits.open(filename) as hdul:
        hdu = hdul[0]

        low_limit = hdu.header['CRVAL1']
        N = hdu.header['NAXIS1']
        delt = hdu.header['CDELT1']
        # logger.debug("CRVAL1 = {}".format(low_limit))
        # logger.debug("NAXIS1 = {}".format(N))
        # logger.debug("CDELT1 = {}".format(delt))
        high_limit = low_limit + N * delt
        flux = hdu.data[:n_limit].copy()  #* u.angstrom # FIXME

        del hdu.data  # allow the file to close

    wavelength = np.linspace(low_limit, high_limit, N)[:n_limit]  # FIXME * u.angstrom

    return wavelength, flux

# Obsolete
# def get_ML(age, metallicity, band='V'):
#     df = mass_to_light()
#     df['MH_rounded'] = df['[M/H]'].round(2)
#     row = df.query('Age == {} & "MH_rounded" == {}'.format(age, metallicity))
#     ml = row['(M/L){}'.format(band)].values
#     return ml

# # FIXME be sure to having taken the correct one. This seems not the correct one
# def mass_to_light():
#     return pd.read_csv('out_phot_CH_BASTI.txt', delim_whitespace=True)

# To avoid log(0). np.nan are possible too, but checks when interpolating have to be changed too.
NA_VALUE_FEH = -98.0
NA_VALUE_MGFE = 0.471782

MgFe_corr = -0.261299  # from Hyplot: log10(M_mgsol/M_fesol) = -0.261299, source: Grevesse et al. 2007 en 2010
FeH_corr = -2.756433

def _get_ftype(snap):
    if snap._unifamily is pynbody.family.star:
        return 'st'
    elif snap._unifamily is pynbody.family.gas:
        return 'sp'
    else:
        return None

def feh_template(snap, na_value):
    ftype = _get_ftype(snap)
    if not ftype:
        raise RuntimeError("Derived array 'feh' is available only for family gas or star")
    name = 'fe' + ftype
    # arr = np.log10(snap[name]/snap['mass']) - FeH_corr
    out = np.ones_like(snap[name].view(np.ndarray))  # ones_like so that log10 does not complain
    na_value_condition = np.logical_or(snap[name] == 0.0, snap['mass'] == 0.0)
    arr = np.log10(np.divide(snap[name], snap['mass'], out=out, where=np.logical_not(na_value_condition))) - FeH_corr
    arr[na_value_condition] = na_value  # -98.0
    return arr

@pynbody.derived_array
def feh(snap):
    return feh_template(snap, na_value=NA_VALUE_FEH)

@pynbody.derived_array
def feh_2_27(snap):
    return feh_template(snap, na_value=-2.27)

@pynbody.derived_array
def mgfe(snap, na_value=NA_VALUE_MGFE):
    ftype = _get_ftype(snap)
    if not ftype:
        raise RuntimeError("Derived array 'mgfe' is available only for family gas or star")
    name = 'mg' + ftype
    # arr = np.log10(snap[name]/snap['fe' + ftype]) - MgFe_corr
    out = np.ones_like(snap[name].view(np.ndarray))  # ones_like so that log10 does not complain
    na_value_condition = np.logical_or(snap[name] == 0.0, snap['fe' + ftype] == 0.0)
    arr = np.log10(np.divide(snap[name], snap['fe' + ftype], out=out, where=np.logical_not(na_value_condition))) - MgFe_corr
    arr[na_value_condition] = na_value # 0.471782
    return arr

class Spectrum:
    filename = None
    v_doppler_shift = 0

    def __init__(self, wavelength, flux):
        """wavelength in Angstrom
        Flux in L_sol Msol-1 A-1
        """
        self.wavelength = wavelength
        self.flux = flux

    @classmethod
    def from_file(cls, filename):
        cls.filename = filename
        logger.debug(os.path.basename(filename))
        wvl, f = get_spectrum(filename)
        my_sp = cls(wvl, f)
        my_sp.filename = filename
        return my_sp

    @classmethod
    def from_met_age(cls, metallicity, age):
        filename = get_filename(metallicity, age)
        return cls.from_file(filename)

    @classmethod
    def from_star(cls, star):
        # FIXME .in_units('Gyr') causes a problem in pynbody
        gadget_time_in_gyr = 0.9778139512067809
        metallicity = star['feh']
        age = star['age'] * gadget_time_in_gyr
        # print("ID: {} [Fe/H] = {}, age = {}".format(star['iord'], metallicity, age))
        return cls.from_met_age(metallicity, age)

    def __len__(self):
        return len(self.wavelength)

    def plot(self, ax=None, **kwargs):
        if ax is None:
            fig, ax = plt.subplots()
        ax.plot(self.wavelength, self.flux, **kwargs)
        ax.set_xlabel('$\lambda$ [$\AA$]')
        ax.set_ylabel('$L_λ$')

    def doppler_shifted(self, v, **kwargs):
        # Return a new doppler shifted spectrum
        # v in km/s
        self.v_doppler_shift = v
        wvl, flux = dopplerShift(self.wavelength, self.flux, v, **kwargs)
        my_sp = Spectrum(wvl, flux)
        my_sp.filename = self.filename
        return my_sp

    def convolve(self, filter):
        return filter.convolve_spectrum(self)

Spectrum.doppler_shifted.__doc__ = dopplerShift.__doc__

if __name__ == '__main__':
    import sys
    snap_name = '/home/michele/sim/MySimulations/np_glass/mb.62002_p200_a800_r600/out/snapshot_0048'
    s = pynbody.load(snap_name)
    if len(sys.argv) > 1:
        istar = int(sys.argv[1])
    else:
        istar=23

    my_star = s.s[istar]

    fig, (ax1, ax2) = plt.subplots(2)
    ax1.hist(s.s['feh'], bins=100)
    ax1.set_title('[Fe/H]')
    ax1.axvline(-5, color='r')
    ax2.hist(s.s['age'], bins=100)
    ax2.set_title('Age [{}]'.format(s.s['age'].units))
    # filename = get_filename(my_star['metals'], my_star['age'])
    # flux, wavelength = get_spectrum(filename)
    print("my_star ({}):".format(istar))
    print("Z = {}, age = {}".format(my_star['metals'], my_star['age']))
    sp = Spectrum.from_star(my_star)
    print('len(sp)=', len(sp))
    fig, ax = plt.subplots()
    sp.plot(ax=ax)

    guilty = s.s[s.s['iord'] == 1315277]

    # for i in tqdm.tqdm(range(len(s.s[600:800]))):
    #     fig, ax = plt.subplots()
    #     # if i % 100 == 0:
    #     #   print(i)
    #     star = s.s[i]
    #     sp = Spectrum.from_star(star)
    #     sp.plot(ax=ax)
    #     ax.set_title("({}) ID: {} Z = {}, age = {}".format(i, star['iord'], star['metals'], star['age']))

    # fig.legend()
    # sp1 = sp.doppler_shifted(10000)
    # sp1.plot(ax=ax)
    plt.show()


import aiohttp
import json
from flask import Flask, request, render_template

app = Flask(__name__)

# Define your configuration settings here
app.config['ACCESS_TOKEN'] = 'EAAN3bcQzzWoBO2zAXSK5yuKuG5JnibrwVNI6vfaHZAIbqkOVFTn7zZADevUL1XZAJIO4ZBHbV1G4XcidbWppxYILPkoZBx8p6zZCddzzoOtlFTyMaYnwcPS8ZAba1qvDSfZAHSR9XKBgA1UYgOZCslWFZBb7cMDTskKiA3161oAOL4QX7bct5cYwHgUafbT3C0ZBruPidQ1ptZCigCcTLhsyAjktZB9wNNMV2ff4xBH8XgodjWltY4GaCk29t'
app.config['VERSION'] = 'v17.0'
app.config['PHONE_NUMBER_ID'] = '136827409506486'

@app.route('/', methods=['GET', 'POST'])
async def send_template_message():
    if request.method == 'POST':
        recipients = request.form.get('recipients')
        if recipients:
            recipient_list = recipients.split(',')  # Split the numbers by comma or your preferred delimiter
            success_messages = []
            error_messages = []

            for recipient in recipient_list:
                recipient = recipient.strip()  # Remove leading/trailing spaces
                template_data = get_templated_message_input(recipient)
                result = await send_message(template_data)

                if result == 'Message sent successfully.':
                    success_messages.append(f'Message sent to {recipient} successfully.')
                else:
                    error_messages.append(f'Failed to send message to {recipient}.')

            return render_template('index.html', success_messages=success_messages, error_messages=error_messages, show_form=True)
        else:
            return render_template('index.html', error_message='Recipient phone numbers are required.', show_form=True)
    return render_template('index.html', show_form=True)

async def send_message(data):
    headers = {
        "Content-type": "application/json",
        "Authorization": f"Bearer {app.config['ACCESS_TOKEN']}",
    }

    async with aiohttp.ClientSession() as session:
        url = f"https://graph.facebook.com/{app.config['VERSION']}/{app.config['PHONE_NUMBER_ID']}/messages"
        try:
            async with session.post(url, data=data, headers=headers) as response:
                if response.status == 200:
                    return 'Message sent successfully.'
                else:
                    return 'Failed to send message.'
        except aiohttp.ClientConnectorError as e:
            return f'Connection Error: {str(e)}'

def get_templated_message_input(recipient):
    return json.dumps({
        "messaging_product": "whatsapp",
        "to": recipient,
        "type": "template",
        "template": {
            "name": "apartment_rental",
            "language": {
                "code": "en_US"
            }
        }
    })

if __name__ == '__main__':
    app.run(debug=True)


import discord #Importing discord.
from discord.ext import commands #Importing the commands extension from discord
from colorama import Fore #Importing Fore (A library used to print colored text in terminal) Yea, I like fancy stuff
import asyncio
from discord import utils

class Webhooks(commands.Cog): #initiate the class

	def __init__(self, bot): #initializing the class (ofc, how is the bot supposed to know that this class will contain code for bot to run lol)
		self.bot = bot #self.bot = bot :D> Simply assigning the bot variable to the class :D>

	@commands.Cog.listener() #Creating a listener
	async def on_ready(self): #Function which is invoked when the bot is "READY"
		print(f"{Fore.GREEN}{self.__class__.__name__} cog has been loaded\n{Fore.WHITE}-----") #Prints, cog is loaded

	@commands.command(name = "setname", aliases = ["sn"], description = "Set a name for yourself.", usage = "<setname> <yourname>")
	async def set_name(self, ctx, profile: int = None,  *, name:str = None):

		if profile >3:
			await ctx.send("Profiles can be only 1 to 3")
			return
		if profile < 1:
			await ctx.send("Profiles can be only 1 to 3")
			return
		if name == None:
			await ctx.send('Please enter a name for yourself. I am not supposed to just "assume" any name.')
			return
		filter_dict = {"user_id":ctx.author.id, "profile": profile}
		check = await self.bot.settings.find_many_by_custom(filter_dict)
		if not check:
			data = {"user_id": ctx.author.id, "name": name, "avatar": "", "is_enabled":"true", "profile": profile}
			await self.bot.settings.insert(data)
			await ctx.send("Success! Your brand new profile has been created!")
		if check:
			pfp = check[0]["avatar"]
			enable_status = check[0]["is_enabled"]
			profile_count = check[0]["profile"]
			new_data = {"user_id":ctx.author.id, "name":name, "avatar":pfp, "is_enabled": enable_status, "profile":profile_count}
			await self.bot.settings.upsert_custom(filter_dict, new_data)
			await ctx.send("Your profile has been updated successfully!")

	@commands.command(name= "setavatar", aliases = ["sa"], description = "Set an avatar for your character!", usage = "<setvatar> <image>") #creating a second command , for users to set their image. yeah their own image hurray !
	@commands.dm_only() #Esnures this command can be run onlyon DM ONLY.
	async def set_avatar(self,ctx, profile:int):
		if profile >3:
			await ctx.send("Profiles can be only 1 to 3")
			return
		if profile <1:
			await ctx.send("Profiles can be only 1 to 3")
			return
		em = discord.Embed(description = "Please send an image for your avatar in next 18 seconds.", color = ctx.author.colour)
		await ctx.send(embed = em, delete_after = 18)
		def check(m):
			return len(m.attachments) != 0 and m.channel == ctx.channel and m.author == ctx.author
		try: #Try and error :D>
			msg = await self.bot.wait_for("message", check = check, timeout = 15)
			x = str(msg.attachments).split()
			t = x[3]
			d = t.strip("'>]")
			f = d.strip("url='")
			filter_dict = {"user_id":ctx.author.id, "profile": profile}
			checks = await self.bot.settings.find_many_by_custom(filter_dict)

			if checks:
				name = checks[0]["name"]
				enable_status = checks[0]["is_enabled"]
				profile_count = checks[0]["profile"]
				new_data = {"user_id":ctx.author.id, "name":name, "avatar":f, "is_enabled": enable_status, "profile":profile_count}
				await self.bot.settings.upsert_custom(filter_dict, new_data)
				await ctx.send("Your pfp has been updated successfully.")
				return
			await ctx.send("Hey, you have not set any name. Expecting to set a avatar for yourself hmm?")
		except asyncio.TimeoutError:
			await ctx.send("This request has timed out. Please try again.")

	@commands.command(name = "activate", description = "Set the current active profile for the bot!")
	@commands.guild_only()
	async def activation(self, ctx, profile: int):
		filter_dict = {"user_id": ctx.author.id, "profile": profile}
		check = await self.bot.settings.find_many_by_custom(filter_dict)
		if not check:
			await ctx.send("Nope. No command 4u, Check the profile number ;-;")
			return
		#checking = await self.bot.profile.get_by_id(ctx.author.id)
		#if not checking:
			#data = {"_id":ctx.author.id, "current_profile": profile}
			#await self.bot.profile.upsert(data)
			#await ctx.send(f"Your current profile has been set to {profile}")
			#return
		info = {"_id":ctx.author.id, "current_profile": profile}
		await self.bot.profile.upsert(info)
		await ctx.send(f"Your current profile has been set to {profile}")

	@commands.Cog.listener()
	async def on_message(self, msg):
		if msg.content.startswith("!"):
			return
		search = await self.bot.profile.get_by_id(msg.author.id)
		if not search:
			return
		profilevalue = search["current_profile"]
		filter_dict = {"user_id": msg.author.id, "profile":profilevalue}
		checks = await self.bot.settings.find_many_by_custom(filter_dict)
		if not checks:
			return
		enable_status = checks[0]["is_enabled"]
		if enable_status == "false":
			return
		name = checks[0]["name"]
		pfp = checks[0]["avatar"]
		hook = await msg.channel.webhooks()
		webhook = utils.get(hook, name = "Proxy")
		if webhook is None:
			webhook = await msg.channel.create_webhook(name = "Proxy")
		await webhook.send(msg.content, username = name, avatar_url = pfp)
		await msg.delete()

	@commands.command(name = "toggle", description = "Choose if you wish to switch off the webhooks!")
	async def toggle_command(self, ctx, profile:int):
		if profile >3:
			await ctx.send("Profiles can be set from 1 to 3")
			return
		if profile <1:
			await ctx.send("Profiles can be set from 1 to 3")
			return
		filter_dict = {"user_id":ctx.author.id, "profile":profile}
		check = await self.bot.settings.find_many_by_custom(filter_dict)
		if not check:
			await ctx.send("This profile does not exists. Please create one by typing `!setname <your name>`")
			return
		name = check[0]["name"]
		pfp = check[0]["avatar"]
		profile_count = check[0]["profile"]
		enable_status = check[0]["is_enabled"]
		if enable_status == "true":
			data = {"user_id":ctx.author.id, "name":name, "avatar":pfp, "is_enabled": "false", "profile":profile_count}
			await self.bot.settings.upsert_custom(filter_dict, data)
			await ctx.send(f"You have successfully switched off Webhook proxy for `{profile}`")
			return
		data = {"user_id":ctx.author.id, "name":name, "avatar":pfp, "is_enabled": "true", "profile":profile_count}
		await self.bot.settings.upsert_custom(filter_dict, data)
		await ctx.send(f"You have successfully toggled the webhook proxy for `{profile}`")

def setup(bot):
	bot.add_cog(Webhooks(bot))


__author__ = 'Christian Dansereau'

import pandas as pd
from scipy.spatial.distance import pdist, squareform
from scipy.cluster.hierarchy import linkage
from scipy.cluster.hierarchy import fcluster
import numpy as np

def hclustering(data, t):
    row_dist = pd.DataFrame(squareform(pdist(data, metric='euclidean')))
    row_clusters = linkage(row_dist, method='ward')
    ind = fcluster(row_clusters, t, criterion='maxclust')
    return ind

def part(m,ind):
    n_cluster = np.max(ind)
    a = pd.DataFrame(m)
    new_m = np.zeros((n_cluster, n_cluster))
    for i1 in range(0, n_cluster):
        for i2 in range(0, n_cluster):
            new_m[i1, i2] = a.loc[ind == i1+1, ind == i2+1].mean().mean()
    return new_m

def ind2matrix(ind):
    part = pd.DataFrame(np.zeros([len(ind), len(ind)]))  # init partition
    for i in range(0,max(ind)):
        l = (ind == i+1)
        part.loc[l, l] = i+1
    return part

def order(ind):
    order_idx = []
    for i in range(0,max(ind)):
        l = (ind == i+1)
        order_idx = order_idx + np.where(l)[0].tolist()
    return order_idx

def ordermat(m,ind):
    order_idx = order(ind)
    new_m = pd.DataFrame(m)  # init
    return new_m.loc[order_idx, order_idx].values

# Test functions
def test_ind2matrix():
    ind = np.array([1,2,3,1])
    assert np.all(ind2matrix(ind) == np.array([[ 1.,  0.,  0.,  1.],
       [ 0.,  2.,  0.,  0.],
       [ 0.,  0.,  3.,  0.],
       [ 1.,  0.,  0.,  1.]]))


from builtins import object
import logging
import sys

from django.core.cache import cache

from desktop.lib.rest.http_client import RestException, HttpClient
from desktop.lib.rest.resource import Resource
from desktop.lib.i18n import smart_unicode

from metadata.conf import PROMETHEUS

if sys.version_info[0] > 2:
  from django.utils.translation import gettext as _
else:
  from django.utils.translation import ugettext as _

LOG = logging.getLogger()
VERSION = 'v1'

class PrometheusApiException(Exception):
  def __init__(self, message=None):
    self.message = message or _('No error message, please check the logs.')

  def __str__(self):
    return str(self.message)

  def __unicode__(self):
    return smart_unicode(self.message)

class PrometheusApi(object):

  def __init__(self, user=None, ssl_cert_ca_verify=False):
    self._api_url = '%s/%s' % (PROMETHEUS.API_URL.get().strip('/'), VERSION)

    self.user = user
    self._client = HttpClient(self._api_url, logger=LOG)

    self._client.set_verify(ssl_cert_ca_verify)
    self._root = Resource(self._client)

  def query(self, query):
    try:
      return self._root.get('query', {
        'query': query,
      })['data']
    except RestException as e:
      raise PrometheusApiException(e)

  def range_query(self, query, start, end, step):
    # e.g. /api/v1/query_range?query=up&start=2015-07-01T20:10:30.781Z&end=2015-07-01T20:11:00.781Z&step=15s
    try:
      return self._root.get('query_range', {
        'query': query,
        'start': start,
        'end': end,
        'step': step
      })['data']
    except RestException as e:
      raise PrometheusApiException(e)


import cv2
import numpy as np
import requests
import imutils
import sys
import re
import time
import urllib3

from threading import Thread as th
from imutils.video import FPS
from queue import Queue

def run1():
    if len(sys.argv) <= 1:
        print("Please enter ip with port")
    else:
        match = None
        for i in sys.argv:
            match = re.findall(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+', i)
            if len(match) > 0:
                url = f"http://{match[0]}/shot.jpg"
                break
        # print(match)
        if len(match) == 0:
            print("IP:PORT not found. Please enter ip with port in given format: IP:PORT")
            return
    while url is not None:
        img_resp = requests.get(url)
        img_arr = np.array(bytearray(img_resp.content),dtype=np.uint8)
        img = cv2.imdecode(img_arr,-1)
        img = imutils.resize(img, width=400, height = 720)
        cv2.imshow("Android_cam",img)

        if cv2.waitKey(1) == 27:
            break
    cv2.destroyAllWindows()

def run2():
    if len(sys.argv) <= 1:
        print("Please enter ip with port")
    else:
        match = None
        for i in sys.argv:
            match = re.findall(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+', i)
            if len(match) > 0:
                url = f"http://{match[0]}/video"
                break
        # print(match)
        if len(match) == 0:
            print("IP:PORT not found. Please enter ip with port in given format: IP:PORT")
            return
    if url is not None:
        cap = cv2.VideoCapture(url)
        while(True):
            ret, frame = cap.read()
            cv2.imshow('frame', frame)

            if cv2.waitKey(1) == 27:
                break
        cv2.destroyAllWindows()

class FileVideoStream:
    def __init__(self,path,queueSize=64):
        self.stream = cv2.VideoCapture(path)
        self.stopped = False
        self.Q = Queue(maxsize=queueSize)
    def start(self):
        t = th(target=self.update, args=())
        t.daemon = True
        t.start()
        return self
    def update(self):
        while(True):
            if self.stopped:
                return
            if not self.Q.full():
                ret, frame = self.stream.read()

                if not ret:
                    self.stop()
                    return;
                self.Q.put(frame)
    def read(self):
        return self.Q.get()
    def more(self):
        return self.Q.qsize() > 0
    def stop(self):
        self.stopped = True

def callFVS(url):
    match = re.findall(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+',url)

    if len(match) != 0:
        cap = cv2.VideoCapture(url)
        #print(cap)
        if cap is not None:
            fvs = FileVideoStream(url)
            return fvs
        else:
            print("[ERROR] invalid url, cannot be of type None")
        return None

#    if len(sys.argv) <= 1:
#        print("Please enter ip with port")
#    else:
#        match = None
#        for i in sys.argv:
#            match = re.findall(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+', i)
#            if len(match) > 0:
#                url = f"http://{match[0]}/video"
#                break
#        if len(match) == 0:
#            print("[ERROR] IP:PORT not found. Please enter ip with port in given format: IP:PORT")
#            return
#    if url is not None:
#        fvs = FileVideoStream(url)
#        return fvs
#    else:
#        print("[ERROR] invalid url, cannot be of type None")
#        return None

def startCapture(url):
#if __name__ == '__main__':
    #run1()
    #run2()
    print("[INFO] starting video file thread")
    fvs = callFVS(url)
    print(url)
    print(fvs)
    if fvs is None:
        return None

    fvs.start()
    #fps = FPS().start()
    while(True):
        frame= fvs.read()
        #if frame is None:
        #    break
        #frame = imutils.resize(frame)#,width=800)
        #cv2.putText(frame,"QSize: {}".format(fvs.Q.qsize()),(10,30), cv2.FONT_HERSHEY_PLAIN, 0.6, (0,255,100),2)

        cv2.imshow("Frame",frame)
        if cv2.waitKey(1) == 27:
            break
        #fps.update()
    #fps.stop()
    #print("[INFO] elapsed time: {:.2f}".format(fps.elapsed()))
    #print("[INFO] approx time: {:.2f}".format(fps.fps()))
    cv2.destroyAllWindows()
    fvs.stop()

# See PyCharm help at https://www.jetbrains.com/help/pycharm/


'''
docstr
'''
import multiprocessing as mp
import numpy as np
import timeit
import datetime
import re
import os
import traceback
import csv
from pathlib import Path
import pandas as pd
from performance_review_db import db_add_excel_file, db_add_performance_entry
# pylint: disable=W0703

class TWExportExcelFile():
    '''
    docstr
    '''
    def __init__(self, old_filepath):
        '''
        docstr
        '''
        self.old_filepath = old_filepath
        self._filepath = self.fix_file_ext()
        self.date_range_list = []
        self.process_data()
        self.process_file()
        os.remove(self._filepath)

    def fix_file_ext(self):
        '''
        doctr
        '''
        base_file = self.old_filepath
        name = base_file.split('.')
        new_file = f'{name[0]}.csv'

        try:
            with open(base_file, encoding="utf_16") as f_1:
                with open(new_file, 'w', encoding="utf_8") as f_2:
                    f_2.write(f_1.read())
        except Exception:
            pass

        return new_file

    def get_team_names(self, team_names_lst):
        '''
        docstr
        '''
        if len(team_names_lst) == 0:
            return []

        fst_lst = team_names_lst.split('&')
        fst_lst[-1] = fst_lst[-1].split('/')[0].strip()

        for name in fst_lst:
            name = re.sub('[\"\']', '', name)

        return fst_lst

    def process_file(self):
        '''
        docstr
        '''
        filename = self._filepath.split('/')[-1].split('.')[0]

        date_range_list = self.get_date_range()
        date_range = date_range_list[0] + ' - ' + date_range_list[1]

        db_add_excel_file([self._filepath, filename, date_range])

    def process_data(self):
        '''
        docstr
        '''
        try:
            data_frame = self.get_data_frame()
            col_index_lst = self.get_data_frame_cols(data_frame)
            self.get_values_from_row(data_frame, col_index_lst)
        except Exception:
            traceback.print_exc()

    def get_data_frame(self):
        '''
        docstr
        '''
        file_path = Path(self._filepath)
        file_extension = file_path.suffix.lower()[1:]

        if file_extension == 'xlsx':
            data_frame = pd.read_excel(self._filepath, engine='openpyxl')
        elif file_extension == 'xls':
            data_frame = pd.read_excel(self._filepath)
        elif file_extension == 'csv':
            data_frame = pd.read_csv(self._filepath, engine='python', sep=',',
                                     quoting=csv.QUOTE_NONE, quotechar='"',
                                     on_bad_lines='skip')
        else:
            raise Exception("File not supported")

        return data_frame

    def get_data_frame_cols(self, data_frame):
        '''
        docstr
        '''
        column_pd = list(data_frame.columns)
        column_pd = column_pd[0]
        column_list = column_pd.split('\t')
        formatted_col_list = []

        for column_name in column_list:
            col_name = re.sub('[\"\']', '', column_name)
            col_name.strip()
            formatted_col_list.append(col_name)

        team_names_column = formatted_col_list.index('SL - Remarks')
        booking_date_column = formatted_col_list.index('SL - Date and time of booking')
        job_type_column = formatted_col_list.index('SL - Resource name')
        transport_refrence_column = formatted_col_list.index('SL - Transport reference')
        job_start_time_column = formatted_col_list.index('SL - Custom timestamp 2 date and time ')
        job_end_time_column = formatted_col_list.index('SL - Custom timestamp 3 date and time ')
        in_or_out_column = formatted_col_list.index('SL - Location')

        return [team_names_column, booking_date_column, job_type_column, transport_refrence_column,
                job_start_time_column, job_end_time_column, in_or_out_column]

    def get_values_from_row(self, data_frame, col_index_lst):
        '''
        docstr
        '''
        data_frame = data_frame.reset_index()
        total_job_time = 0

        for row_set in data_frame.itertuples():
            row_str = row_set[2]
            row = row_str.split('\t')
            if len(row) > 6:
                team_names = self.get_team_names(row[col_index_lst[0]])

                booking_date = row[col_index_lst[1]]
                if len(booking_date) > 10:
                    booking_date = booking_date.split(' ')[0]

                job_type = row[col_index_lst[2]]
                job_type = re.sub('[\"\']', '', job_type)
                job_type = job_type.strip()

                transport_refrence = row[col_index_lst[3]]
                try:
                    if (row[col_index_lst[4]].find('AM') != -1
                       or row[col_index_lst[4]].find('PM') != -1):
                        row[col_index_lst[4]] = row[col_index_lst[4]][:-2].strip()

                    if (row[col_index_lst[5]].find('AM') != -1
                       or row[col_index_lst[5]].find('PM') != -1):
                        row[col_index_lst[5]] = row[col_index_lst[5]][:-2].strip()
                except Exception:
                    pass

                try:
                    job_start_time = datetime.datetime.strptime(row[col_index_lst[4]],
                                                                '%m/%d/%Y %H:%M')
                    job_end_time = datetime.datetime.strptime(row[col_index_lst[5]],
                                                              '%m/%d/%Y %H:%M')

                    if ((job_end_time < job_start_time) and
                            (job_end_time.date() == job_start_time.date())):

                        job_end_time = job_end_time + datetime.timedelta(hours=12)

                    total_job_time = job_end_time - job_start_time
                    total_job_time = int(total_job_time.total_seconds() / 60)
                    converted_total_job_time = str(total_job_time)
                except Exception:
                    converted_total_job_time = 'error'
                    # traceback.print_exc()

                try:
                    in_or_out_val = row[col_index_lst[6]]
                    if in_or_out_val.find('delivery') != -1:
                        in_or_out = 'Inbound'
                    else:
                        in_or_out = 'Outbound'
                except Exception:
                    pass

            if len(team_names) > 1 and job_type.find('20') != -1:
                job_type = 'Ocean Containers - 40 HQ'

            row_vals = [team_names, booking_date, job_type, converted_total_job_time,
                        transport_refrence, in_or_out]

            if ((total_job_time <= 480 and total_job_time >= 1)
               and converted_total_job_time != 'error'):
                self.add_entry_to_db(row_vals)

    def add_entry_to_db(self, row_vals):
        '''
        docstr
        '''
        team_names = row_vals[0]
        booking_date = row_vals[1]
        job_type = row_vals[2]
        converted_total_job_time = row_vals[3]
        transport_refrence = row_vals[4]
        in_or_out = row_vals[5]

        if isinstance(team_names, list) and len(team_names) != 0:
            db_add_performance_entry([team_names, booking_date, job_type, converted_total_job_time,
                                      transport_refrence, self._filepath, in_or_out], 'team')

            for person_name in team_names:
                if team_names.index(person_name) == 0:
                    worker_job = 'Driver'
                else:
                    worker_job = 'Unloader'

                db_add_performance_entry([person_name.strip(), booking_date, job_type,
                                          converted_total_job_time, transport_refrence,
                                          worker_job, len(team_names), self._filepath, in_or_out],
                                         'individual')
                self.date_range_list.append(booking_date)

    def get_date_range(self):
        '''
        docstr
        '''
        fin_date_range_list = ['', '']

        i = 0
        while i < len(self.date_range_list):
            try:
                self.date_range_list[i] = datetime.datetime.strptime(self.date_range_list[i],
                                                                     '%m/%d/%Y')
            except Exception:
                pass
            i += 1

        try:
            earliest_date = min(self.date_range_list)
            latest_date = max(self.date_range_list)

            fin_date_range_list[0] = datetime.datetime.strftime(earliest_date, '%m/%d/%Y')
            fin_date_range_list[1] = datetime.datetime.strftime(latest_date, '%m/%d/%Y')
        except Exception:
            print(self.date_range_list)

        return fin_date_range_list


"""IMPORT PACKAGES"""
import os
import json
import numpy as np
import pandas as pd
from skimage.measure import label
from scipy import ndimage
from PIL import Image
from numba import jit
import shutil

"""SPECIFY EXTENSIONS AND DATA ROOTS"""
EXT_VID = ['.mp4', '.m4v',  '.avi']
EXT_IMG = ['.jpg', '.png', '.tiff', '.tif', '.bmp', '.jpeg']

""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
"""DEFINE SET OF FUNCTIONS FOR SELECTING ROI IN RAW ENDOSCOPE IMAGES"""
""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""

# Define function for minimum pooling the images
@jit(nopython=True)
def min_pooling(img, g=8):

    # Copy Image
    out = img.copy()

    # Determine image shape and compute step size for pooling
    h, w = img.shape
    nh = int(h / g)
    nw = int(w / g)

    # Perform minimum pooling
    for y in range(nh):
        for x in range(nw):
            out[g * y:g * (y + 1), g * x:g * (x + 1)] = np.min(out[g * y:g * (y + 1), g * x:g * (x + 1)])

    return out

# Define function for finding largest connected region in images
def getlargestcc(segmentation):

    # Use built-in label method, to label connected regions of an integer array
    labels = label(segmentation)

    # Assume at least 1 CC
    assert (labels.max() != 0)  # assume at least 1 CC

    # Find the largest of connected regions, return as True and False
    largestcc = labels == np.argmax(np.bincount(labels.flat)[1:]) + 1

    return largestcc

# Define function for finding bounding box coordinates for ROI in images
def bbox(img):

    # Find rows and columns where a True Bool is encountered
    rows = np.any(img, axis=1)
    cols = np.any(img, axis=0)

    # Find the first and last row/column for the bounding box coordinates
    # cmin = left border, cmax = right border, rmin = top border, rmax = bottom border
    # Usage Image.crop((left, top, right, bottom))
    rmin, rmax = np.where(rows)[0][[0, -1]]
    cmin, cmax = np.where(cols)[0][[0, -1]]

    return rmin, rmax, cmin, cmax

# Define Complete function for finding ROI bounding box coordinates, by combining previous functions
def find_roi(img):

    # Open image as numpy array
    image = np.array(img, dtype=np.uint8)

    # Compute L1 norm of the image
    norm_org = np.linalg.norm(image, axis=-1)

    # Use Gaussian Filter to capture low-frequency information
    img_gauss = ndimage.gaussian_filter(norm_org, sigma=5)

    # Scale pixel values
    img_scaled = ((norm_org - np.min(img_gauss)) / (np.max(img_gauss) - np.min(img_gauss))) * 255

    # Use minimum pooling
    img_norm = min_pooling(img_scaled, g=8)

    # Find largest connected region with threshold image as input
    th = 10
    largestcc = getlargestcc(img_norm >= th)

    # Obtain cropping coordinates
    rmin, rmax, cmin, cmax = bbox(largestcc)

    return rmin, rmax, cmin, cmax

# Define a function for finding which expert readers to ignore when > 4 delineations are available
def find_ignore_readers(path):

    # Find targets
    maskdict = dict()
    for root, dirs, files in os.walk(path):

        # Loop over filenames in files
        for file in files:

            # Extract filename before .ext
            maskcase = os.path.splitext(file)[0]

            # Append filename to mask dictionary if already existing key; otherwise create key and append to list
            if maskcase in maskdict.keys():
                maskdict[maskcase].append(os.path.join(root, file))
            else:
                maskdict[maskcase] = list()
                maskdict[maskcase].append(os.path.join(root, file))

    target_list = list()
    for key in maskdict:
        if len(maskdict[key]) > 4:
            target_list.append(key)

    # Instantiate list
    folders_oi = dict()

    # Loop over all folders
    for roots, dirs, files in os.walk(path):
        # Loop over roots
        for dir in dirs:
            # Loop over target list
            for i in range(len(target_list)):
                # Find target list variables
                if target_list[i] == dir:
                    if os.path.exists(os.path.join(roots, dir, 'Higherlikelihood')) and os.path.exists(os.path.join(roots, dir, 'Lowerlikelihood')):
                        folders_oi[target_list[i]] = [os.path.join(roots, dir, 'Higherlikelihood'), os.path.join(roots, dir, 'Lowerlikelihood')]

    # Loop over targets and corresponding folders of interest
    for target in folders_oi:

        # Find HL delineations
        for roots, dirs, files in os.walk(folders_oi[target][0]):
            for file in files:
                if 'Expert_0' in os.path.join(roots, file):
                    exp0 = np.array(Image.open(os.path.join(roots, file)).convert('1'))*1
                    print('\nExpert 0: {}'.format(os.path.join(roots, file)))
                elif 'Expert_1' in os.path.join(roots, file):
                    exp1 = np.array(Image.open(os.path.join(roots, file)).convert('1'))*1
                    print('Expert 1: {}'.format(os.path.join(roots, file)))
                elif 'Expert_2' in os.path.join(roots, file):
                    exp2 = np.array(Image.open(os.path.join(roots, file)).convert('1'))*1
                    print('Expert 2: {}'.format(os.path.join(roots, file)))
                else:
                    raise Exception('No predefined expert found...')

        # Compute dice score for the combinations
        intersection01 = np.sum(np.multiply(exp0, exp1)) * 2.0
        intersection02 = np.sum(np.multiply(exp0, exp2)) * 2.0
        intersection12 = np.sum(np.multiply(exp1, exp2)) * 2.0
        dice01 = (intersection01 + 1e-7) / (np.sum(exp0) + np.sum(exp1) + 1e-7)
        dice02 = (intersection02 + 1e-7) / (np.sum(exp0) + np.sum(exp2) + 1e-7)
        dice12 = (intersection12 + 1e-7) / (np.sum(exp1) + np.sum(exp2) + 1e-7)

        print('file: {}, dice01: {:.4f}, dice02: {:.4f}, dice12: {:.4f}'.
              format(os.path.split(os.path.join(roots, file))[1], dice01, dice02, dice12))

        # Compute the highest dice score and delete folders
        if (dice01 >= dice02) and (dice01>= dice12):
            largest = dice01
            string = 'Highest Similarity between Expert 0 and Expert 1, Delete Expert 2'
            delete = 'Expert_2'
        elif (dice02 >= dice01) and (dice02 >= dice12):
            largest = dice02
            string = 'Highest Similarity between Expert 0 and Expert 2, Delete Expert 1'
            delete = 'Expert_1'
        else:
            largest = dice12
            string = 'Highest Similarity between Expert 1 and Expert 2, Delete Expert 0'
            delete = 'Expert_0'
        print(string)

        # Delete corresponding folders for Higher and Lower likelihoods
        if os.path.exists(os.path.join(folders_oi[target][0], delete)):
            shutil.rmtree(os.path.join(folders_oi[target][0], delete))
        if os.path.exists(os.path.join(folders_oi[target][1], delete)):
            shutil.rmtree(os.path.join(folders_oi[target][1], delete))

""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
"""DEFINE FUNCTION FOR CREATING CACHE WITH METADATA FOR AVAILABLE IMAGES"""
""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""

def cache_wle(root_dir, mask_dir, subtlety_dir, quality_dir, storing_folder):

    # Create directory
    print('Generating cache...')
    os.makedirs(os.path.join(os.getcwd(), '..', storing_folder))

    # Create empty dictionary for  image files
    img_files = list()

    # Loop over roots (folders in root_dir), dirs (folders in roots), files (files in dirs)
    for root, dirs, files in os.walk(root_dir):

        # Loop over filenames in files
        for name in files:

            # Check for .extension of the files; append to video files or image files accordingly
            # os.path.splitext splits name in filename and .ext; [1] will correspond to .ext
            if os.path.splitext(name.lower())[1] in EXT_IMG:
                img_files.append(os.path.join(root, name))
            elif name == 'Thumbs.db':
                os.remove(os.path.join(root, name))
            else:
                print('FILE NOT SUPPORTED: {}'.format(os.path.join(root, name)))

    # Create empty dictionary for image files
    maskdict = dict()

    # Loop over roots (folders in OTMASKDIR), dirs (folders in roots), files (files in dirs)
    for root, dirs, files in os.walk(mask_dir):
        if 1:  # 'Lowerlikelihood' in root:

            # Loop over filenames in files
            for file in files:

                # Extract filename before .ext
                maskcase = os.path.splitext(file)[0]

                # Append filename to mask dictionary if already existing key; otherwise create key and append to list
                if maskcase in maskdict.keys():
                    maskdict[maskcase].append(os.path.join(root, file))
                else:
                    maskdict[maskcase] = list()
                    maskdict[maskcase].append(os.path.join(root, file))

    # Read Subtlety database as excel file; Create empty dictionary for subtlety
    subtlety_df = pd.read_excel(subtlety_dir)
    subtlety_dict = dict()

    # Iterate over rows in excel file; initialize img name key in subtlety dict, set value based on subtlety
    for idx, frame in subtlety_df.iterrows():
        subtlety_dict[os.path.splitext(frame['Imagename'])[0]] = frame['Subtlety (0=easy, 1=medium, 2=hard)']

    # Read Quality database as excel file; Create empty dictionary for quality
    quality_df = pd.read_excel(quality_dir)
    quality_dict = dict()

    # Iterate over rows in excel file; initialize img name key in quality dict, set value based on quality
    # It can happen that multiple same key occurs multiple times in the dict
    for idx, frame in quality_df.iterrows():
        if not os.path.splitext(frame['Imagename'])[0] in quality_dict.keys():
            quality_dict[os.path.splitext(frame['Imagename'])[0]] = list()
            quality_dict[os.path.splitext(frame['Imagename'])[0]].append(frame['Quality'])
        elif os.path.splitext(frame['Imagename'])[0] in quality_dict.keys():
            quality_dict[os.path.splitext(frame['Imagename'])[0]].append(frame['Quality'])

    """"""""""""""""""""""""""""""""""""""
    """CREATE JSON FILE FOR EVERY IMAGE """
    """"""""""""""""""""""""""""""""""""""

    # Loop over list of img_files, which was created by appending the roots of all images in root_dir
    for img in img_files:
        print('Reading image: ', img)

        # Extract imagename from the path to image (img)
        imgname = os.path.split(img)[1]

        # Create empty dictionary for metadata
        data = dict()

        # Extract information from filename and place in dictionary with corresponding key
        data['patient'] = '_'.join(os.path.split(img)[1].split('_')[:2])  # hospital_patID
        data['file'] = img  # path to file
        data['dataset'] = os.path.split(os.path.split(os.path.split(os.path.split(os.path.split(img)[0])[0])[0])[0])[1]  # training/validatie
        data['source'] = os.path.split(os.path.split(img)[0])[1]  # image/frames
        data['class'] = os.path.split(os.path.split(os.path.split(img)[0])[0])[1]  # ndbe/neo
        data['protocol'] = os.path.split(os.path.split(os.path.split(os.path.split(img)[0])[0])[0])[1]
        data['modality'] = os.path.split(img)[1].split('_')[2]  # modality
        data['clinic'] = os.path.split(img)[1].split('_')[0]  # hospital
        data['subtlety'] = subtlety_dict[os.path.splitext(os.path.split(img)[1])[0]] if \
                           os.path.splitext(os.path.split(img)[1])[0] in subtlety_dict.keys() else 0
        if 'subtle' in img:
            data['subtlety'] = 2
        data['quality'] = quality_dict[os.path.splitext(os.path.split(img)[1])[0]] if \
                          os.path.splitext(os.path.split(img)[1])[0] in quality_dict.keys() else []

        # Open the image as numpy array; extract height and width and place in data dictionary
        frame = np.array(Image.open(img))
        data['height'] = frame.shape[0]
        data['width'] = frame.shape[1]
        print('Frame shape: ', frame.shape)

        # Extract bounding box coordinates, store in dictionary as rmin, rmax, cmin, cmax
        roi = find_roi(frame)
        data['roi'] = [float(x) for x in roi]

        # Create new key in data dictionary for masks and initialize as list; instantiate with maskdict list
        data['masks'] = list()
        if os.path.splitext(os.path.split(img)[1])[0] in maskdict.keys():
            data['masks'] = maskdict[os.path.splitext(os.path.split(img)[1])[0]]
        print('Number of masks: {}'.format(len(data['masks'])))

        # Print final version of data dictionary with all keys and values in there
        print('Data: ', data)

        # Check whether there is already a json file for this particular image; otherwise create file
        if not os.path.exists(os.path.join(os.getcwd(), '..', storing_folder, os.path.splitext(imgname)[0] + '.json')):
            jsonfile = os.path.join(os.getcwd(), '..', storing_folder, os.path.splitext(imgname)[0] + '.json')
        elif not os.path.exists(os.path.join(os.getcwd(), '..', storing_folder, os.path.splitext(imgname)[0] + '_2.json')):
            jsonfile = os.path.join(os.getcwd(), '..', storing_folder, os.path.splitext(imgname)[0] + '_2.json')
        elif not os.path.exists(os.path.join(os.getcwd(), '..', storing_folder, os.path.splitext(imgname)[0] + '_3.json')):
            jsonfile = os.path.join(os.getcwd(), '..', storing_folder, os.path.splitext(imgname)[0] + '_3.json')
        else:
            raise ValueError

        # For every img in img_files write dictionary data into corresponding json file
        with open(jsonfile, 'w') as outfile:
            json.dump(data, outfile, indent=4)

""""""""""""""""""""""""""
"""EXECUTION OF FUNCTIONS"""
""""""""""""""""""""""""""
if __name__ == '__main__':

    # Define paths to the folders for WLE (Images, Masks, Subtlety, Quality)
    wle_dir = ''
    wle_mask_dir_plausible = ''
    wle_mask_dir = ''
    wle_subtle_dir = ''
    wle_quality_dir = ''

    # Define storing folders for NBI and WL
    wle_plausible_store = 'cache'

    # Execute functions
    cache_wle(root_dir=wle_dir, mask_dir=wle_mask_dir_plausible, subtlety_dir=wle_subtle_dir,
              quality_dir=wle_quality_dir, storing_folder=wle_plausible_store)


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import keras
import os
import numpy as np
from skimage import io
from skimage.transform import resize
from keras.callbacks import ModelCheckpoint
from PIL import Image
import tensorflow as tf

class Segment(object):

    path = ""

    def __init__(self, data_dir):
        """
            data_directory : path like /home/rajat/nnproj/dataset/
                             includes the dataset folder with '/'
            Initialize all your variables here
        """
        self.img_size = 128
        self.path = data_dir
        inputs = keras.layers.Input((self.img_size, self.img_size, 3))
        c_layer_1 = keras.layers.Conv2D(32, (3, 3), activation='relu', padding='same')(inputs)
        c_layer_1 = keras.layers.Conv2D(32, (3, 3), activation='relu', padding='same')(c_layer_1)
        p_layer_1 = keras.layers.MaxPooling2D(pool_size=(2, 2))(c_layer_1)
        c_layer_2 = keras.layers.Conv2D(64, (3, 3), activation='relu', padding='same')(p_layer_1)
        c_layer_2 = keras.layers.Conv2D(64, (3, 3), activation='relu', padding='same')(c_layer_2)
        p_layer_2 = keras.layers.MaxPooling2D(pool_size=(2, 2))(c_layer_2)
        c_layer_3 = keras.layers.Conv2D(96, (3, 3), activation='relu', padding='same')(p_layer_2)
        c_layer_3 = keras.layers.Conv2D(96, (3, 3), activation='relu', padding='same')(c_layer_3)
        p_layer_3 = keras.layers.MaxPooling2D(pool_size=(2, 2))(c_layer_3)
        c_layer_4 = keras.layers.Conv2D(128, (3, 3), activation='relu', padding='same')(p_layer_3)
        c_layer_4 = keras.layers.Conv2D(128, (3, 3), activation='relu', padding='same')(c_layer_4)
        p_layer_4 = keras.layers.MaxPooling2D(pool_size=(2, 2))(c_layer_4)
        c_layer_5 = keras.layers.Conv2D(160, (3, 3), activation='relu', padding='same')(p_layer_4)
        c_layer_5 = keras.layers.Conv2D(160, (3, 3), activation='relu', padding='same')(c_layer_5)
        u_layer_6 = keras.layers.concatenate([keras.layers.UpSampling2D(size=(2, 2))(c_layer_5), c_layer_4], axis=3)
        d_layer_6 = keras.layers.Conv2D(128, (3, 3), activation='relu', padding='same')(u_layer_6)
        d_layer_6 = keras.layers.Conv2D(128, (3, 3), activation='relu', padding='same')(d_layer_6)
        u_layer_7 = keras.layers.concatenate([keras.layers.UpSampling2D(size=(2, 2))(d_layer_6), c_layer_3], axis=3)
        d_layer_7 = keras.layers.Conv2D(96, (3, 3), activation='relu', padding='same')(u_layer_7)
        d_layer_7 = keras.layers.Conv2D(96, (3, 3), activation='relu', padding='same')(d_layer_7)
        u_layer_8 = keras.layers.concatenate([keras.layers.UpSampling2D(size=(2, 2))(d_layer_7), c_layer_2], axis=3)
        d_layer_8 = keras.layers.Conv2D(64, (3, 3), activation='relu', padding='same')(u_layer_8)
        d_layer_8 = keras.layers.Conv2D(64, (3, 3), activation='relu', padding='same')(d_layer_8)
        u_layer_9 = keras.layers.concatenate([keras.layers.UpSampling2D(size=(2, 2))(d_layer_8), c_layer_1], axis=3)
        d_layer_9 = keras.layers.Conv2D(32, (3, 3), activation='relu', padding='same')(u_layer_9)
        d_layer_9 = keras.layers.Conv2D(32, (3, 3), activation='relu', padding='same')(d_layer_9)
        u_layer_10 = keras.layers.Conv2D(1, (1, 1), activation='sigmoid')(d_layer_9)

        self.model = keras.models.Model(inputs=[inputs], outputs=[u_layer_10])
        self.model.compile(optimizer=keras.optimizers.Adam(lr=1e-4), loss=self.loss_func, metrics=[self.metric_func])

    def metric_func(self,val_true, val_pred):
        val_t = keras.backend.flatten(val_true)
        val_p = keras.backend.flatten(val_pred)
        acc = keras.backend.sum(val_t * val_p)
        return (acc/(keras.backend.sum(val_t) + keras.backend.sum(val_p) - acc))

    def loss_func(self,true_val, pred_val):
        return keras.losses.binary_crossentropy(y_true=keras.backend.flatten(true_val), y_pred=keras.backend.flatten(pred_val))

    def preprocess_img(self, imgs, flag=True):
        if flag:
            imgs_p = np.ndarray((imgs.shape[0], self.img_size, self.img_size, 3), dtype=np.uint8)
        else:
            imgs_p = np.ndarray((imgs.shape[0], self.img_size, self.img_size), dtype=np.uint8)

        for i in range(imgs.shape[0]):
            if flag:
                imgs_p[i] = resize(imgs[i], (self.img_size, self.img_size, 3), preserve_range=True, mode='constant')
            else:
                imgs_p[i] = resize(imgs[i], (self.img_size, self.img_size), preserve_range=True, mode='constant')
        imgs_p = imgs_p[..., np.newaxis]
        return imgs_p

    def load_images_and_masks(self):
        image_files = []
        for i in os.listdir('traindata'):
            image_files.append(i)
        input_images = np.ndarray((len(image_files), self.img_size, self.img_size, 3), dtype=np.uint8)
        mask_images = np.ndarray((len(image_files), self.img_size, self.img_size), dtype=np.uint8)
        # input_images = np.ndarray((164, self.img_size, self.img_size, 3), dtype=np.uint8)
        # mask_images = np.ndarray((164, self.img_size, self.img_size), dtype=np.uint8)
        image_files.sort()
        files_masks = [image_files[i] for i in range(1, len(image_files), 2)]
        files_images = [image_files[i] for i in range(0, len(image_files), 2)]
        for i in range(len(files_images)):
            temp = io.imread('./traindata/'+files_images[i])
            input_images[i] = np.asarray([temp])
            mask_images[i] = np.array([io.imread('./traindata/'+files_masks[i], as_grey=True)])
        return input_images, mask_images

    def train(self):
        """
            Trains the model on data given in path/train.csv
            No return expected
        """
        imgs, imgs_mask = self.load_images_and_masks()
        imgs_mask = self.preprocess_img(imgs_mask, False)
        imgs = imgs.astype('float32')
        imgs -= np.mean(imgs)
        imgs /= np.std(imgs)
        imgs_mask = imgs_mask.astype('float32')
        imgs_mask /= 255
        model_checkpoint = ModelCheckpoint('avyav_weights.h5', monitor='loss', save_best_only=True)
        self.model.fit(imgs, imgs_mask, verbose=1, shuffle=True,epochs=20,validation_split=0.1, callbacks=[model_checkpoint])

    def compute_prediction(self,img):
        height = np.int(np.ceil(np.shape(img)[0]/128))
        width = np.int(np.ceil(np.shape(img)[1]/128))
        mat_with_pad = np.zeros([height*128, width*128, 3])
        # mat_with_pad_t = np.zeros([1,np.shape(img)[1],3])
        # x = img
        # for i in range(height*128-np.shape(img)[0]):
        #     x = np.vstack((x,mat_with_pad_t))
        # mat_with_pad_t = np.zeros([height*128,1,3])
        # for i in range(width*128-np.shape(img)[1]):
        #     x = np.hstack((x,mat_with_pad_t))
        # mat_with_pad = x
        mat_with_pad[0:np.shape(img)[0], 0:np.shape(img)[1], :] = img[:,:,:]
        image_set = np.ndarray([height*width, 128, 128, 3])
        index=0
        for i in range(height):
            for j in range(width):
                image_set[index,:,:,:] = mat_with_pad[i*128:(i+1)*128,j*128:(j+1)*128,:]
                index+=1

        final_image = np.zeros([height*128, width*128])
        index=0
        for i in range(height):
            for j in range(width):
                temp = self.predict_this_image(image_set[index])
                final_image[i*128:(i+1)*128,j*128:(j+1)*128] = temp
                index+=1

        return final_image[0:np.shape(img)[0], 0:np.shape(img)[1]].astype(np.uint8)

    def predict_this_image(self,img):
        imgs_t = np.array(np.resize(img,(1,128,128,3)))
        imgs_t = imgs_t.astype('float32')
        imgs_t -= np.mean(imgs_t)
        imgs_t /= np.std(imgs_t)
        np.resize(imgs_t, (1,128,128,3))
        img_mask_test = self.model.predict(imgs_t, verbose=0)
        image_t = (img_mask_test[0][:,:,0] * 255.0).astype(np.uint8)
        image_t[image_t < 128] = 0
        image_t[image_t >= 128] = 1
        return image_t

    def get_mask(self, image):
        """
            image : a variable resolution RGB image in the form of a numpy array
            return: A list of lists with the same 2d size as of the input image with either 0 or 1 as each entry
        """
        return self.compute_prediction(image)

    def save_model(self, **params):
        """
            saves model on the disk
            no return expected
        """
        self.model.save(params['name'])
        # file_name = params['name']
        # pickle.dump(self, gzip.open(file_name, 'wb'))

    @staticmethod
    def load_model(**params):
        """
            returns a pre-trained instance of Segment class
        """
        seg_class=Segment('traindata')
        seg_class.model.load_weights('avyav_weights.h5')
        return seg_class
        # file_name = params['name']
        # return pickle.load(gzip.open(file_name, 'rb'))

# if __name__ == "__main__":
#     # pass
#     obj = Segment('traindata')
#     # obj.train()
#     obj = Segment.load_model(name='avyav_weights.h5')
#     temp = io.imread('./validationdata/valid-4.jpg')
#     temp = np.asarray([temp]).astype('float32')
#     t = obj.get_mask(temp[0])
#     m=Image.fromarray(t)
#     m.save('predvf.jpg')
#     # obj.train()
#     # obj.save_model(name="segment.gz")


from django.conf.urls import url
from longclaw.basket import api
from longclaw.basket import views
from longclaw.settings import API_URL_PREFIX

basket_list = api.BasketViewSet.as_view({
    'get': 'list',
    'post': 'create',
    'put': 'bulk_update'
})

basket_detail = api.BasketViewSet.as_view({
    'delete': 'destroy'
})

item_count = api.BasketViewSet.as_view({
    'get': 'item_count'
})

total_items = api.BasketViewSet.as_view({
    'get': 'total_items'
})

urlpatterns = [

    url(API_URL_PREFIX + r'basket/$',
        basket_list,
        name='longclaw_basket_list'),
    url(API_URL_PREFIX + r'basket/count/$',
        total_items,
        name="longclaw_basket_total_items"),
    url(API_URL_PREFIX + r'basket/(?P<variant_id>[0-9]+)/$',
        basket_detail,
        name='longclaw_basket_detail'),
    url(API_URL_PREFIX + r'basket/(?P<variant_id>[0-9]+)/count/$',
        item_count,
        name='longclaw_basket_item_count'),

    url(r'basket/$',
        views.BasketView.as_view(),
        name="longclaw_basket")
]


from selenium.webdriver.common.by import By

from .base_element import BaseElement

class PageWithOpens(object):
    url = None

    def __init__(self, driver, automation_nr):
        self.driver = driver
        self.url ="https://thethirdwave.activehosted.com/report/#/campaign/" + str(automation_nr) + "/opens"

    def go(self):
        self.driver.get(self.url)

    @property
    def prop_total_number(self):
        locator = (By.CSS_SELECTOR, '#open_total_t')
        return BaseElement(
            driver = self.driver,
            by = locator[0],
            value = locator[1])

    @property
    def prop_unique_opens(self):
        locator = (By.CSS_SELECTOR, '#open_unique_t')
        return BaseElement(
            driver = self.driver,
            by = locator[0],
            value = locator[1])


from trytond.model import fields
from trytond.pool import Pool, PoolMeta
from trytond.pyson import Eval

__all__ = ['Production']

class Production(metaclass=PoolMeta):
    __name__ = 'production'

    origin = fields.Reference('Origin', selection='get_origin',
        states={
            'readonly': ~Eval('state').in_(['request', 'draft']),
            },
        depends=['state'])

    @classmethod
    def _get_origin(cls):
        'Return list of Model names for origin Reference'
        return []

    @classmethod
    def get_origin(cls):
        pool = Pool()
        Model = pool.get('ir.model')
        models = cls._get_origin()
        models = Model.search([
                ('model', 'in', models),
                ])
        return [('', '')] + [(m.model, m.name) for m in models]


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Jun 12 10:01:20 2017

@author: viche
"""

# Data Preprocessing
# Structure the folders for the pictures - readable by by Keras library.
# Folder structure: parent -> training_set & test_set -> individual categories

# ---------------------------------
# Build the CNN model --- starts ---

# Import the Keras libraries and packages
import keras
from keras.models import Sequential
from keras.layers import Convolution2D
from keras.layers import MaxPooling2D
from keras.layers import Flatten
from keras.layers import Dense
from keras.preprocessing.image import ImageDataGenerator

# Initialize the CNN
classifier = Sequential()

# Add Layer - Convolution
# convo: 32=number of filters, 3*3 pixel grid filter
# input_shape: 3=color pic; 256*256 pixels
classifier.add(Convolution2D(32,kernel_size=(3,3),input_shape=(64,64,3),activation='relu'))

# Add Layer - Max Pooling
# pool size: 2*2 pixel grid
classifier.add(MaxPooling2D(pool_size=(2,2)))

# Add Layer - Convolution - 2nd
# convo: 32=number of filters, 3*3 pixel grid filter
# input_shape: 3=color pic; 256*256 pixels
classifier.add(Convolution2D(32,kernel_size=(3,3),activation='relu'))

# Add Layer - Max Pooling - 2nd
# pool size: 2*2 pixel grid
classifier.add(MaxPooling2D(pool_size=(2,2)))

# Add Layer - Flattern
classifier.add(Flatten())

# Add Layer - Full Connected
# Hidden Layer
classifier.add(Dense(128, activation='relu'))
# Output Layer
classifier.add(Dense(1, activation='sigmoid'))

# Compile the model
classifier.compile(optimizer='adam',loss='binary_crossentropy',metrics=['accuracy'])

train_datagen = ImageDataGenerator(
        rescale=1./255,
        shear_range=0.2,
        zoom_range=0.2,
        horizontal_flip=True)

test_datagen = ImageDataGenerator(rescale=1./255)

train_generator = train_datagen.flow_from_directory('cnn_dataset/training_set',
        target_size=(64, 64),
        batch_size=32,
        class_mode='binary')

validation_generator = test_datagen.flow_from_directory('cnn_dataset/test_set',
                    target_size=(64, 64),
                    batch_size=32,
                    class_mode='binary')

classifier.fit_generator(train_generator,
                        steps_per_epoch=8000,
                        epochs=25,
                        validation_data=validation_generator,
                        validation_steps=2000)


import sys
import os
import logging
import argparse
from pathlib import Path
import shutil

import torch
import numpy as np

from simple_sfm.cameras import CameraMultiple
from simple_sfm.colmap_utils import ColmapBdManager
from simple_sfm.dataset import simple_sfm_dataset_utils
from simple_sfm.matchers import Matcher
from simple_sfm.scene_utils import OneVideoSceneProcesser
from simple_sfm.scene_utils.colmap_scene_converters import colmap_sparse_to_simple_sfm_json_views
from simple_sfm.utils.io import is_video_file
from simple_sfm.utils.video_streamer import VideoStreamer
from simple_sfm.utils.visualise import plotly_plot_cameras_to_images

root = logging.getLogger()
root.setLevel(logging.DEBUG)

handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
root.addHandler(handler)
logger = logging.getLogger(__name__)

def main():
    # TODO add mode in which render use folder with images

    # logger = logging.getLogger('root')
    parser = argparse.ArgumentParser()
    parser.add_argument('--input-path', type=str, default=None,
                        help='Path to the scene input data. It can be as a folder with images as a video file.')
    parser.add_argument('--output-dir-path', type=str, default=None,
                        help='Path to output dir.')
    parser.add_argument('--scene-name', type=str, default=None,
                        help='Name of scene dataset that will be generated.')

    parser.add_argument('--max-num-frames', type=int, default=30,
                        help='Max number of frames')
    parser.add_argument('--num-skip-frames', type=int, default=None,
                        help='Number frames that will be skipped before frame will be sampled.')
    parser.add_argument('--image-scale-factor', type=float, default=1,
                        help='Image rescale factor. If equal to 0.5  - will resize image in to half of it size.')
    parser.add_argument('--do-center-crop', type=bool, default=False,
                        help='Images will be central cropped before processing.')

    parser.add_argument('--nms-radius', type=int, default=4,
                        help='Non Maximum Suppression (NMS) radius for matching.')
    parser.add_argument('--keypoint-threshold', type=float, default=0.001,
                        help='Detector confidence threshold.')
    parser.add_argument('--match-threshold', type=float, default=0.9,
                        help='Threshold value for matching.')
    parser.add_argument('--match-nearest-radius', type=int, default=None,
                        help='Radius in which frames will be matched with each other. '
                             'It makes sense only for sequential data.')
    parser.add_argument('--sinkhorn-iterations', type=int, default=20,
                        help='Num sinkhorn iterations for super glue matching.')
    parser.add_argument('--super-glue-batch', type=int, default=5,
                        help='Batch size for super glue inference.')

    opts = parser.parse_args()

    input_data_path = opts.input_path
    output_dir_path = opts.output_dir_path
    scene_name = opts.scene_name

    simple_sfm_path = Path(os.path.abspath(__file__)).parent.parent

    modnet_weigths_path = Path(simple_sfm_path, 'weights/modnet_photographic_portrait_matting.torchscript')
    # ptf_segm_model_weigths_path = '/home/palsol/projects/SimpleSfm/notebooks/data/mma_multi_211122_l_cuda_0.torchscript.pt'

    capture_work_dir = Path(output_dir_path, scene_name)
    frames_path = Path(capture_work_dir, 'frames')

    is_video = is_video_file(opts.input_path)
    if is_video:
        logger.info(f' Data {opts.input_path} is video.')
    else:
        logger.info(f' Data {opts.input_path} is not video.')

    if is_video:
        raw_video_dir_path = Path(capture_work_dir, 'raw_video')
        video_name = input_data_path.split('/')[-1]
        os.makedirs(raw_video_dir_path, exist_ok=True)
        raw_video_path = Path(raw_video_dir_path, video_name)
        shutil.copy(input_data_path, raw_video_path)

        video_to_frames = OneVideoSceneProcesser(
            video_path=str(raw_video_path),
            dataset_output_path=str(capture_work_dir),
            skip=opts.num_skip_frames,
            center_crop=opts.do_center_crop,
            scale_factor=opts.image_scale_factor,
            max_len=opts.max_num_frames,
            img_prefix='jpg',
            filter_with_sharpness=True,
            #     force_rotate_code=cv2.ROTATE_180
        )

        video_to_frames.run()

    else:
        os.makedirs(capture_work_dir, exist_ok=True)
        shutil.copytree(input_data_path, frames_path)

    matcher = Matcher(
        super_point_extractor_weights_path=Path(simple_sfm_path, 'weights/superpoint_v1.pth'),
        super_glue_weights_path=Path(simple_sfm_path, 'weights/superglue_indoor.pth'),
        nms_radius=opts.nms_radius,
        keypoint_threshold=opts.keypoint_threshold,
        matcher_type='super_glue',
        match_threshold=opts.match_threshold,
        sinkhorn_iterations=opts.sinkhorn_iterations,
        super_glue_batch=int(opts.super_glue_batch)
    )

    vs = VideoStreamer(
        str(frames_path),
        height=None,
        width=None,
        max_len=None,
        img_glob='*.jpg'
    )

    camera_size = vs.get_resolution()

    colmap = ColmapBdManager(
        db_dir=str(Path(capture_work_dir, 'colmap')),
        images_folder_path=str(frames_path),
        camera_type='OPENCV',
        camera_params=None,
        camera_size=camera_size
    )

    if opts.match_nearest_radius is not None:
        pairs_filter_func = lambda x: np.abs(x[0] - x[1]) < opts.match_nearest_radius
    else:
        pairs_filter_func = lambda x: True

    processed_frames, match_table, images_names = matcher.match_video_stream(vs, pairs_filter_func)
    torch.cuda.empty_cache()

    colmap.replace_images_data(images_names)
    colmap.replace_keypoints(images_names, processed_frames)
    colmap.replace_and_verificate_matches(match_table, images_names)
    num_sparse_points = colmap.run_mapper()
    print(num_sparse_points)

    colmap_sparse_to_simple_sfm_json_views(
        scene_colmap_sparse_path=os.path.join(capture_work_dir, 'colmap', 'sparse'),
        work_dir_path=os.path.join(capture_work_dir),
        relative_frames_path='frames',
    )

    views_data_path = Path(capture_work_dir, 'views_data.json')

    """Extracts and saves people masks from all views"""
    simple_sfm_dataset_utils.generate_modnet_masks(capture_work_dir, modnet_weigths_path)
    """Takes colmap sparse pointcloud and saves sparse depth for each view"""
    simple_sfm_dataset_utils.generate_sparse_depth_from_colmap(capture_work_dir)

    views_data_oriented_path = Path(capture_work_dir, 'views_data_oriented.json')
    simple_sfm_dataset_utils.center_and_orient(views_data_path,
                                               views_data_oriented_path,
                                               orient_method='up')

    result_cameras = CameraMultiple.from_simple_sfm_json(views_data_oriented_path)
    plotly_plot_cameras_to_images(cameras=result_cameras, output_path=Path(capture_work_dir, 'cameras_plot'))

if __name__ == '__main__':
    main()