import logging
import os
from typing import Sequence, Dict, Any, Tuple, Set, List, Iterable, Optional, \
    Union, Callable

import PIL.Image
import PIL.ImageDraw
import numpy as np
import torch
from tqdm import tqdm

from . import keypoints_processing as kpt_proc
from .base import COCODataset
from .body_parts import BodyParts
from ... import transforms as trafos, DatasetSplit

LOGGER = logging.getLogger(__name__)

class ConceptDataset(COCODataset):
    # pylint: disable=line-too-long
    r"""Data handle for ground truth segmentations of visual concepts
    (body parts) generated from COCO keypoints. The concepts that can be
    handled are combinations of objects (body parts) marked by keypoints.

    The input images the dataset contains are the original COCO images.
    The annotations are segmentation masks masking the desired concept which are
    generated from the keypoint annotations. The translation of keypoint
    annotations to masks can be found in :py:meth:`annotations_to_mask`.

    The ground truth masks are generated and stored in a sub-directory
    :py:attr:`masks_root`:

    .. parsed-literal::

        ``<root root>``
        |
        +---annotations
        |   +---``<annotations file>.json``
        |
        +---images
        |   +---:py:attr:`~hybrid_learning.datasets.base.BaseDataset.dataset_root`
        |
        +---masks
            +---:py:attr:`masks_root`
                (default: ``<dataset_root>_<concept desc>``)
    """
    # pylint: enable=line-too-long
    MASKS_ROOT_ROOT = "masks"
    """Usual parent to all masks folders for all body parts; sibling to
    ``images`` folder"""

    def __init__(self,
                 body_parts: Sequence[Union[BodyParts, Sequence[str]]],
                 force_rebuild: bool = False,
                 pt_radius: float = None,
                 masks_root: str = None,
                 masks_root_root: str = None,
                 lazy_mask_generation: bool = True,
                 img_size: Tuple[int, int] = None,
                 mask_size: Optional[Tuple[int, int]] = None,
                 device: Optional[Union[str, torch.device]] = None,
                 **kwargs):
        """Init.

        :param body_parts: see
            :py:attr:`~hybrid_learning.datasets.custom.coco.mask_dataset.ConceptDataset.body_parts`
        :param force_rebuild: Whether to regenerate all masks during init,
            or to skip existing ones.
        :param pt_radius: see
            :py:attr:`~hybrid_learning.datasets.custom.coco.mask_dataset.ConceptDataset.pt_radius`
        :param masks_root: set the root directory under which to generate the
            masks; note that one should prefer setting ``masks_root_root``
            instead
        :param masks_root_root: the path to the root under which the masks
            folder is created; the masks folder's name is uniquely determined
            by the mask generation settings;
            defaults to ``coco_root/MASKS_ROOT_ROOT``
            where :py:attr:`MASKS_ROOT_ROOT` is the class default
        :param lazy_mask_generation: whether to generate the masks lazily or
            directly during init;
            overwritten by ``force_rebuild``: if this is true,
            masks are directly regenerated.
        :param img_size: see super class
        :param mask_size: if given, the size of the generated masks
            (annotations are scaled and padded to this size before mask
            generation)
        :param kwargs: Any other arguments for the super class
        """
        self.mask_size: Optional[Tuple[int, int]] = mask_size
        """If given, annotations are scaled and padded to this size before
        generating the masks (of this size).
        If left ``None``, the generated masks have the same size as the
        original image."""

        # Application of general spec
        # Apply custom default transforms:
        self._default_transforms: Callable[[PIL.Image.Image, PIL.Image.Image],
                                           Tuple[torch.Tensor, torch.Tensor]] \
            = self.get_default_transforms(img_size, self.mask_size,
                                          device=device)
        if img_size is not None:
            kwargs.update(img_size=img_size)
        super().__init__(**kwargs)

        if pt_radius is not None and pt_radius <= 0:
            raise ValueError(("pt_radius to generate masks must be > 0 "
                              "but was {}").format(pt_radius))
        self.pt_radius: float = pt_radius or 10 / 400
        """Radius of white circles to draw centered at the keypoint coordinates
        in the masks. Also used as half of the width of link lines between
        keypoints in the masks.
        Unit: proportion of height of annotated person in the image
        (defaults to the image height if person height cannot be estimated)."""

        # Concepts
        self.body_parts: Sequence[Union[BodyParts, Sequence[str]]] = body_parts
        """List of concepts to mask in selected images.
        See :py:meth:`annotations_to_mask` for details."""

        # The masks root folder
        self.masks_root = masks_root or self.default_masks_root(
            dataset_root=self.dataset_root, masks_root_root=masks_root_root,
            body_parts=self.body_parts, pt_radius=self.pt_radius,
            mask_size=self.mask_size,
        )
        """Folder name of the directory in which all generated masks are
        stored."""
        os.makedirs(self.masks_root, exist_ok=True)

        # Generate and save masks to root folder
        if not lazy_mask_generation or force_rebuild:
            self.generate_masks(force_rebuild=force_rebuild)

    @property
    def settings(self) -> Dict[str, Any]:
        """Settings to reproduce the instance."""
        return dict(body_parts=self.body_parts,
                    pt_radius=self.pt_radius,
                    masks_root=self.masks_root,
                    **({'mask_size': self.mask_size} if self.mask_size else {}),
                    **super().settings)

    @classmethod
    def settings_to_str(cls,
                        body_parts: Iterable[Iterable[str]] = None,
                        pt_radius: float = None,
                        person_rel_size_range: Tuple[
                            Optional[float], Optional[float]] = None,
                        img_size: Tuple[int, int] = None,
                        mask_size: Tuple[int, int] = None,
                        dataset_root: str = None,
                        split: Union[str, DatasetSplit] = None,
                        ) -> str:
        """Return a string representation of the given mask
        generation settings.
        Intended to serve as unique folder name.
        If optional specifiers are not given, they are omitted.

        :param body_parts: see
            :py:attr:`~hybrid_learning.datasets.custom.coco.mask_dataset.ConceptDataset.body_parts`
        :param pt_radius: see
            :py:attr:`~hybrid_learning.datasets.custom.coco.mask_dataset.ConceptDataset.pt_radius`
        :param person_rel_size_range: the minimum and maximum height of persons
            included, relative to the image height;
            provide as tuple ``(min, max)`` and set ``None`` to unset values
        :param mask_size: the size of the created masks as ``(height, width)``;
            defaults to ``img_size``
        :param img_size: the image size for which the mask is created;
            should be given if one cannot assume the mask fits the original
            image size, or if e.g. ``person_rel_size_range`` is dependent on the
            image size; should be given as ``(height, width)``
        :param dataset_root: if given, used instead of split
        :param split: the dataset split
        :return: string that is unique for the given settings and can be used
            as folder name
        """
        # concepts
        body_parts = body_parts or []
        concept_hash: str = '-'.join(sorted({kpt for body_part in body_parts
                                             for kpt in body_part}))
        # radii
        min_size, max_size = person_rel_size_range or (None, None)
        radius_info: str = "_rad{:.3f}".format(pt_radius) \
            if pt_radius is not None else ""
        rel_height_info: str = "_relsize{fromv}to{tov}".format(
            fromv="{:.2f}".format(min_size if min_size is not None else 0),
            tov="{:.2f}".format(max_size) if max_size is not None else "+inf"
        ) if max_size is not None or min_size is not None else ""

        # size
        mask_size = mask_size if mask_size is not None else img_size
        mask_size_info: str = "_{}x{}".format(*mask_size) if mask_size else ""

        # split
        split: Optional[str] = split if not isinstance(split, DatasetSplit) \
            else split.value
        img_base: Optional[str] = os.path.basename(dataset_root) \
            if dataset_root is not None else split
        img_base: str = img_base + "_" if img_base is not None else ""

        return "{img_base}{concept_hash}{rad}{height}{size}".format(
            img_base=img_base,
            concept_hash=concept_hash,
            rad=radius_info,
            height=rel_height_info,
            size=mask_size_info
        )

    # noinspection PyIncorrectDocstring
    @classmethod
    def default_masks_root(cls, *,
                           dataset_root: str = None,
                           body_parts: Iterable[Iterable[str]] = None,
                           pt_radius: float = None,
                           person_rel_size_range: Tuple[
                               Optional[float], Optional[float]] = None,
                           img_size: Tuple[int, int] = None,
                           mask_size: Tuple[int, int] = None,
                           masks_root_root: str = None,
                           split: str = None
                           ) -> str:
        """Return default mask root directory for given settings.
        By default, a directory structure as follows is assumed:

        .. parsed-literal::

            coco_root
             |
             +-images_root_root
             |  +-os.path.basename(dataset_root)
             |
             +-masks_root_root
               +-os.path.basename(dataset_root)+spec_specifier

        If optional specifiers are not given, they are omitted.
        For the details on the parameters, see
        :py:meth:`~hybrid_learning.datasets.custom.coco.mask_dataset.ConceptDataset.settings_to_str`.

        :param dataset_root: the root directory of the coco images;
            basename is used to prefix the mask folder;
            the folder is used to determine the default for ``masks_root_root``
        :param masks_root_root: the root folder under which to put the masks;
            defaults to ``dataset_root/../../``:py:attr:`MASKS_ROOT_ROOT`;
        """
        if masks_root_root is None and dataset_root is None:
            raise ValueError("No root directory given.")
        masks_dirname = masks_root_root or os.path.join(
            os.path.dirname(os.path.dirname(dataset_root)),
            cls.MASKS_ROOT_ROOT
        )
        masks_basename: str = cls.settings_to_str(
            body_parts=body_parts,
            pt_radius=pt_radius,
            person_rel_size_range=person_rel_size_range,
            img_size=img_size,
            mask_size=mask_size,
            dataset_root=dataset_root,
            split=split,
        )
        return os.path.join(masks_dirname, masks_basename)

    def getitem(self, i: int) -> Tuple[PIL.Image.Image, PIL.Image.Image]:
        """Get image and mask by index ``i`` in dataset.
        Used for
        :py:meth:`~hybrid_learning.datasets.base.BaseDataset.__getitem__`."""
        img: PIL.Image.Image = self.load_orig_image(i)

        # generate and save mask if it does not exist
        if not self.mask_exists(i):
            # obtain image keypoint annotations
            anns = self.raw_anns(i)
            mask = self.annotations_to_mask(orig_img_wh=img.size,
                                            annotations=anns)
            self.save_mask(i, mask)

        mask: PIL.Image.Image = self.load_orig_mask(i)

        return img, mask

    def load_orig_mask(self, i: int) -> PIL.Image.Image:
        """Load and return unmodified mask as image by index in dataset.
        Image mode of returned masks is 'L'.
        Note that the mask has
        :py:attr:`~hybrid_learning.datasets.custom.coco.mask_dataset.ConceptDataset.mask_size`
        if this is set, not the original image's size (which is only changed
        to ``img_size`` during transformation)."""
        mask_fp: str = self.mask_filepath(i)
        mask = PIL.Image.open(mask_fp)

        # Check mask size
        if self.mask_size is not None and \
                (mask.size[1], mask.size[0]) != tuple(self.mask_size):
            raise RuntimeError(
                f"Mask cached at {mask_fp} does not match mask_size "
                f"{(mask.size[1], mask.size[0])} (HxW), but was of size "
                f"{self.mask_size}. Is masks_root correct?")

        # Possibly fix mask mode:
        if mask.mode != 'L':
            mask = mask.convert(mode='L')
        return mask

    @classmethod
    def get_default_transforms(cls, img_size: Optional[Tuple[int, int]] = None,
                               mask_size: Optional[Tuple[int, int]] = None,
                               device: Union[str, torch.device] = None,
                               ) -> Callable[[PIL.Image.Image, PIL.Image.Image],
                                             Tuple[torch.Tensor, torch.Tensor]]:
        """Default transformation that pads and resizes both images and masks,
        and binarizes the masks.
        If ``mask_size`` is ``None``, the masks are scaled and
        padded to the original image, otherwise to ``mask_size``.
        """
        mask_size = mask_size or img_size
        if img_size is not None:
            size_trafos = [trafos.OnInput(trafos.PadAndResize(img_size)),
                           trafos.OnTarget(trafos.PadAndResize(mask_size))]
        elif mask_size is not None:
            size_trafos = [trafos.OnTarget(trafos.PadAndResize(mask_size))]
        else:
            size_trafos = [trafos.SameSize()]

        # noinspection PyTypeChecker
        return trafos.Compose(
            [trafos.OnBothSides(trafos.ToTensor(device=device)),
             # pad an resize both img and mask:
             *size_trafos,
             # binarize the mask:
             trafos.OnTarget(trafos.Binarize())]
        )

    def mask_exists(self, i: int) -> bool:
        """Check whether a mask for the specified index already exists."""
        # Is it a valid ID?
        if not 0 <= i < len(self):
            m_exists = False

        # Does the mask already exist?
        else:
            mask_fp = self.mask_filepath(i)
            m_exists = os.path.exists(mask_fp)
        return m_exists

    def mask_filepath(self, i: int) -> str:
        """Provide the path under which the mask for the given image ID
        should lie."""
        return os.path.join(self.masks_root, self.descriptor(i))

    ANNOTATION_TRANSFORM: Callable = kpt_proc.annotations_to_mask
    # pylint: disable=line-too-long
    """Callable used to transform a list of annotations into a mask.
    All keypoints belonging to any of the :py:attr:`body_parts` are
    marked, and keypoints belonging to a common body part are linked
    according to the person skeleton.
    For details see
    :py:func:`~hybrid_learning.datasets.custom.coco.keypoints_processing.annotations_to_mask`."""

    # pylint: enable=line-too-long

    def annotations_to_mask(self, orig_img_wh: Tuple[int, int],
                            annotations: List[dict]
                            ) -> PIL.Image.Image:
        """Create mask of all configured keypoints from ``annotations``.
        It is assumed that the original image considered in ``annotations``
        has ``orig_img_wh``.
        If :py:attr:`~hybrid_learning.datasets.custom.coco.mask_dataset.ConceptDataset.mask_size`
        is ``None``, the mask size will be ``orig_img_wh``.
        For details see :py:attr:`ANNOTATION_TRANSFORM`.
        """
        # get skeleton; see also pycocotools
        skeleton: np.ndarray = self._get_skeleton(annotations[0])
        keypoint_idxs: List[Set[int]] = \
            [set(kpt_proc.to_keypoint_idxs(body_part, self.coco))
             for body_part in self.body_parts]
        annotation_wh = orig_img_wh
        if self.mask_size is not None:
            annotations = [
                kpt_proc.pad_and_scale_annotation(
                    ann, from_size=(orig_img_wh[1], orig_img_wh[0]),
                    to_size=self.mask_size)
                for ann in annotations]
            annotation_wh = (self.mask_size[1], self.mask_size[0])
        return self.__class__.ANNOTATION_TRANSFORM(
            annotation_wh=annotation_wh,
            annotations=annotations,
            pt_radius=self.pt_radius,
            keypoint_idxs=keypoint_idxs,
            skeleton=skeleton)

    def _get_skeleton(self, annotation: Dict[str, Any] = None) -> np.ndarray:
        """Obtain the coco skeleton information.
        The skeleton is returned as a list of lists of the form
        ``[[link1_start, link2_end], [link2_start, link2_end], ...]``
        where link starting and end points are the indices of keypoints in the
        ``keypoints`` list of an annotation.

        :param annotation: a COCO annotation for the category of which the
            skeleton should be loaded; if not given, the ``'person'``
            category is used
        """
        cat_id = annotation['category_id'] if annotation is not None \
            else self.coco.getCatIds(catNms=['person'])[0]
        skeleton: np.ndarray = np.array(
            self.coco.loadCats(cat_id)[0]['skeleton']
        ) - 1
        return skeleton

    def generate_masks(self, force_rebuild: bool = False,
                       show_progress_bar: bool = True,
                       **kwargs) -> None:
        """Generate and save the masks for the images in this dataset.

        :param force_rebuild: whether to overwrite masks that already exist.
        :param show_progress_bar: whether to show the progress using
            :py:class:`tqdm.tqdm`
        :param kwargs: further arguments to the progress bar
        """
        masks_to_process = [i for i in range(len(self))
                            if force_rebuild or not self.mask_exists(i)]
        if len(masks_to_process) == 0:
            return

        if show_progress_bar:
            masks_to_process = tqdm(**{**dict(iterable=masks_to_process,
                                              unit="mask",
                                              desc="Masks newly generated: "),
                                       **kwargs})
        for i in masks_to_process:
            if self.mask_exists(i) and force_rebuild:
                mask_fp = self.mask_filepath(i)
                LOGGER.info("Overwriting file %s", mask_fp)
                os.remove(mask_fp)

            # obtain image keypoint annotations
            anns = self.raw_anns(i)

            # obtain original image's size
            img: PIL.Image.Image = self.load_orig_image(i)
            img_size = img.size

            # generate and save mask
            mask = self.annotations_to_mask(orig_img_wh=img_size,
                                            annotations=anns)
            self.save_mask(i, mask)

    def save_mask(self, i: int, mask: PIL.Image.Image) -> None:
        """Save the given mask according to the given index in the dataset.

        :param i: ID of the image the mask belongs to.
        :param mask: mask image
        """
        filepath = self.mask_filepath(i)
        os.makedirs(os.path.dirname(filepath), exist_ok=True)
        mask.save(filepath)


from future import standard_library
standard_library.install_aliases()
from builtins import str
from past.builtins import basestring
import json
import logging
import re
import sys
import urllib.parse
from datetime import datetime
from dateutil import tz
from dateutil import parser

from django.utils.formats import localize_input
from desktop.lib.parameterization import find_variables
from liboozie.oozie_api import get_oozie, DEFAULT_USER

if sys.version_info[0] > 2:
  from django.utils.translation import gettext as _
else:
  from django.utils.translation import ugettext as _

LOG = logging.getLogger()

JSON_FIELDS = ('parameters', 'job_properties', 'files', 'archives', 'prepares', 'params',
               'deletes', 'mkdirs', 'moves', 'chmods', 'touchzs')
BOOLEAN_FIELDS = ('propagate_configuration','capture_output')
NUMBER_FIELDS_OR_NULL = ('sub_workflow',)
GMT_TIME_FORMAT = "%Y-%m-%dT%H:%MGMT%z"
UTC_TIME_FORMAT = "%Y-%m-%dT%H:%MZ"
FREQUENCY_REGEX = r'^\$\{coord:(?P<frequency_unit>\w+)\((?P<frequency_number>\d+)\)\}$'

def format_field_value(field, value):
  if field in JSON_FIELDS:
    if isinstance(value, basestring):
      value = json.loads(value)
    value = [item for item in value if isinstance(item, dict) and item.get('name')]
    return json.dumps(value)
  if field in NUMBER_FIELDS_OR_NULL:
    if not isinstance(value, int) and value is not None:
      return int(value)
  if field in BOOLEAN_FIELDS:
    return str(value).lower() == 'true'
  return value

def format_dict_field_values(dictionary):
  for key in dictionary:
    dictionary[key] = format_field_value(key, dictionary[key])
  return dictionary

def model_to_dict(model):
  from django.db import models

  dictionary = {}
  for field in model._meta.fields:
    try:
      attr = getattr(model, field.name, None)
      if isinstance(attr, models.Model):
        dictionary[field.name] = attr.id
      elif isinstance(attr, datetime):
        dictionary[field.name] = str(attr)
      else:
        dictionary[field.name] = attr
    except Exception as e:
      LOG.debug(_("Could not set field %(field)s: %(exception)s") % {'field': field.name, 'exception': str(e)})
  return dictionary

def sanitize_node_dict(node_dict):
  for field in ['node_ptr', 'workflow']:
    if field in node_dict:
      del node_dict[field]
  return node_dict

def workflow_to_dict(workflow):
  workflow_dict = model_to_dict(workflow)
  node_list = [node.get_full_node() for node in workflow.node_list]
  nodes = [model_to_dict(node) for node in node_list]

  for index, node in enumerate(node_list):
    nodes[index]['child_links'] = [model_to_dict(link) for link in node.get_all_children_links()]

  workflow_dict['nodes'] = nodes

  return workflow_dict

def smart_path(path, mapping=None, is_coordinator=False):
  # Try to prepend home_dir and FS scheme if needed.
  # If path starts by a parameter try to get its value from the list of parameters submitted by the user or the coordinator.
  # This dynamic checking enable the use of <prepares> statements in a workflow scheduled manually of by a coordinator.
  # The logic is a bit complicated but Oozie is not consistent with data paths, prepare, coordinator paths and Fs action.
  if mapping is None:
    mapping = {}

  path = path.strip()
  if not path.startswith('$') and not path.startswith('/') and not urllib.parse.urlsplit(path).scheme:
    path = '/user/%(username)s/%(path)s' % {
        'username': '${coord:user()}' if is_coordinator else '${wf:user()}',
        'path': path
    }

  if path.startswith('$'):
    variables = find_variables(path)
    for var in variables:
      prefix = '${%s}' % var
      if path.startswith(prefix):
        if var in mapping:
          if not urllib.parse.urlsplit(mapping[var]).scheme and not mapping[var].startswith('$'):
            path = '%(nameNode)s%(path)s' % {'nameNode': '${nameNode}', 'path': path}
  else:
    if not urllib.parse.urlsplit(path).scheme:
      path = '%(nameNode)s%(path)s' % {'nameNode': '${nameNode}', 'path': path}

  return path

def contains_symlink(path, mapping):
  vars = find_variables(path)
  return any([var in mapping and '#' in mapping[var] for var in vars]) or '#' in path

def utc_datetime_format(utc_time):
  if utc_time and type(utc_time) is datetime:
    return utc_time.strftime(UTC_TIME_FORMAT)
  return utc_time

def oozie_to_django_datetime(dt_string):
  try:
    return localize_input(datetime.strptime(dt_string, UTC_TIME_FORMAT))
  except ValueError:
    pass

  try:
    return localize_input(datetime.strptime(dt_string, GMT_TIME_FORMAT))
  except ValueError:
    pass

  return None

class InvalidFrequency(Exception):
  pass

def oozie_to_hue_frequency(frequency_string):
  """
  Get frequency number and units from frequency, which must be of the format
  "${coord:$unit($number)}".

  frequency units and number are just different parts of the EL function.

  Returns:
    A tuple of the frequency unit and number

  Raises:
    InvalidFrequency: If the `frequency_string` does not match the frequency pattern.
  """
  matches = re.match(FREQUENCY_REGEX, frequency_string)
  if matches:
    return matches.group('frequency_unit'), matches.group('frequency_number')
  else:
    raise InvalidFrequency(_('invalid frequency: %s') % frequency_string)

def convert_to_server_timezone(date, local_tz='UTC', server_tz=None, user=DEFAULT_USER):
  api = get_oozie(user)

  if server_tz is None:
    oozie_conf = api.get_configuration()
    server_tz = oozie_conf.get('oozie.processing.timezone') or 'UTC'

  if date and date.startswith('$'):
    return date

  # To support previously created jobs
  if date.endswith('Z'):
    date = date[:-1]
    local_tz = 'UTC'

  try:
    date_local_tz = parser.parse(date)
    date_local_tz = date_local_tz.replace(tzinfo=tz.gettz(local_tz))
    date_server_tz = date_local_tz.astimezone(tz.gettz(server_tz))

    # Oozie timezone is either UTC or GMT(+/-)####
    if 'UTC' == server_tz:
      return date_server_tz.strftime('%Y-%m-%dT%H:%M') + u'Z'
    else:
      return date_server_tz.strftime('%Y-%m-%dT%H:%M') + date_server_tz.strftime('%z')
  except TypeError as ValueError:
    LOG.error("Failed to convert Oozie timestamp: %s" % date)
  return None


import statistics
import datetime
import pytz
import math
import csv

import yaml
from flask import Blueprint, render_template, request, abort
from CTFd.models import db, Challenges, Solves, Users
from CTFd.utils import get_config
from CTFd.utils.user import get_current_user, is_admin
from CTFd.utils.decorators import authed_only, admins_only
from CTFd.cache import cache

from ..models import DiscordUsers, DojoMembers
from ..utils import module_visible, module_challenges_visible, DOJOS_DIR, is_dojo_admin
from ..utils.dojo import dojo_route
from .writeups import WriteupComments, writeup_weeks, all_writeups
from .discord import discord_reputation

grades = Blueprint("grades", __name__)

def average(data):
    data = list(data)
    if not data:
        return 0.0
    return sum(data) / len(data)

def helpful_credit(reputation, max_reputation):
    if not reputation or not max_reputation:
        return 0.0
    return round(100 * (math.log2(reputation + 0.1) / math.log2(max_reputation + 0.1))) / 1000

def shared_helpful_extra_credit():
    students = yaml.safe_load(get_config("students"))
    student_ids = set(int(student["dojo_id"]) for student in students)

    all_reputation = discord_reputation()
    max_reputation = max(all_reputation.values(), default=None)

    discord_users = {
        discord_user.discord_id: discord_user.user_id
        for discord_user in DiscordUsers.query.all()
    }

    all_shared_credit = [
        helpful_credit(reputation, max_reputation)
        for discord_id, reputation in all_reputation.items()
        if discord_users.get(discord_id) not in student_ids
    ]

    return sum(all_shared_credit) / len(students)

def module_grade_report(dojo, module, user, when=None):
    m = { }

    challenges = dojo.challenges(module, user, solves_before=when)
    assigned = module.get("time_assigned", None)
    due = module.get("time_due", None)
    ec_full = module.get("time_ec_full", None)
    ec_part = module.get("time_ec_part", None)

    if assigned and due and not ec_full:
        ec_full = (assigned + (due-assigned)/2)
    if assigned and due and not ec_part:
        ec_part = (assigned + (due-assigned)/4)

    m['name'] = module['name']
    m['total_challenges'] = len(challenges)
    m['ec_threshold'] = module.get("ec_threshold", len(challenges) // 2)
    m['late_penalty'] = module.get('late_penalty', module.get('late', 0.5))
    m['time_assigned'] = assigned
    m['time_due'] = due
    m['time_ec_part'] = ec_part
    m['time_ec_full'] = ec_full

    m['solved_timely'] = 0
    m['solved_late'] = 0
    m['solved_part_ec'] = 0
    m['earned_part_ec'] = 0
    m['solved_full_ec'] = 0
    m['earned_full_ec'] = 0
    m['early_bird_ec'] = 0
    m['module_grade'] = 0

    if challenges and due and user:
        m['solved_timely'] = len([ c for c in challenges if c.solved and pytz.UTC.localize(c.solve_date) < due ])
        m['solved_late'] = len([ c for c in challenges if c.solved and pytz.UTC.localize(c.solve_date) >= due ])
        m['module_grade'] = 100 * (m['solved_timely'] + m['solved_late']*(1-m['late_penalty'])) / len(challenges)

        if ec_part:
            m['solved_part_ec'] = len([ c for c in challenges if c.solved and pytz.UTC.localize(c.solve_date) < ec_part ])
            m['earned_part_ec'] = (m['solved_part_ec'] >= m['ec_threshold'] // 2)
        if ec_full:
            m['solved_full_ec'] = len([ c for c in challenges if c.solved and pytz.UTC.localize(c.solve_date) < ec_full ])
            m['earned_full_ec'] = (m['solved_full_ec'] >= m['ec_threshold'])
        m['early_bird_ec'] = 1.0 if m['earned_full_ec'] else 0.5 if m['earned_part_ec'] else 0

    return m

def letter_grade(dojo, total_grade, module_reports=None):
    if module_reports is None:
        module_reports = []

    for dojo_grade in dojo.grades:
        if "points" in dojo_grade and total_grade >= dojo_grade["points"]:
            grade = dojo_grade["grade"]
            break
        if "modules" in dojo_grade:
            modules = dojo_grade["modules"]
            progress = len([report for report in module_reports if report["module_grade"]])
            if progress >= modules:
                grade = dojo_grade["grade"]
                break
    else:
        grade = "?"

    return grade

def overall_grade_report(dojo, user, when=None):
    discord_user = DiscordUsers.query.filter_by(user_id=user.id).first()

    reports = [ ]
    for module in dojo.modules:
        if not module_visible(dojo, module, user) or not module_challenges_visible(dojo, module, user):
            continue
        r = module_grade_report(dojo, module, user, when=when)
        if not r['total_challenges']:
            continue
        if when and r['time_assigned'] > when:
            continue
        reports.append(r)

    module_average = statistics.mean(r["module_grade"] for r in reports)

    extra_credit = {}
    extra_credit["Partial Early Bird"] = sum((0.5 if r["earned_part_ec"] and not r["earned_full_ec"] else 0) for r in reports)
    extra_credit["Full Early Bird"] = sum((1.0 if r["earned_full_ec"] else 0) for r in reports)

    for current_extra_credit in dojo.extra_credit:
        name = current_extra_credit.get("name", "???")
        id = current_extra_credit.get("id", "dojo")
        points = current_extra_credit.get("points", {})

        if isinstance(points, (int, float)):
            user_points = points
        elif id == "dojo":
            user_points = points.get(user.id, 0)
        elif id == "discord":
            user_points = points.get(discord_user.discord_id, 0) if discord_user else 0
        else:
            user_points = 0

        extra_credit[name] = user_points

    total_grade = module_average + sum(extra_credit.values())

    return dict(
        module_reports=reports,
        module_average=module_average,
        extra_credit=extra_credit,
        total_grade=total_grade,
        letter_grade=letter_grade(dojo, total_grade, reports)
    )

@grades.route("/<dojo>/grades", methods=["GET"])
@grades.route("/<dojo>/grades/<int:user_id>", methods=["GET"])
@dojo_route
@authed_only
def view_grades(dojo, user_id=None):
    if not user_id or not is_admin():
        user = get_current_user()
    else:
        user = Users.query.filter_by(id=user_id).first()

    when = request.args.get("when", None)
    if when:
        when = pytz.UTC.localize(datetime.datetime.fromtimestamp(int(when)))

    grades = overall_grade_report(dojo=dojo, user=user, when=when)

    return render_template("grades.html", grades=grades)

@grades.route("/<dojo>/grades/all", methods=["GET"])
@dojo_route
@authed_only
@cache.memoize(timeout=1800)
def dojo_grades(dojo):
    user = get_current_user()
    if not is_dojo_admin(user, dojo):
        abort(403)
    if dojo.public and not is_admin(user):
        abort(403)

    when = request.args.get("when")
    if when:
        when = datetime.datetime.fromtimestamp(int(when))

    if not dojo.public:
        grade_tokens = [ dm.grade_token for dm in DojoMembers.query.filter_by(dojo=dojo).all() ]
    else:
        grade_tokens = set()
        for csv_path in DOJOS_DIR.glob(f"{dojo.id}/*.csv"):
            with open(csv_path) as csv_file:
                grade_tokens |= set(student["Zoom Email"] for student in csv.DictReader(csv_file))

    grades = []
    unlinked_users = []
    for token in grade_tokens:
        if dojo.public:
            user = Users.query.filter_by(email=token).first()
            if not user:
                unlinked_users.append({ "id":None, "grade_token":token })
                continue
        else:
            user = DojoMembers.query.filter_by(dojo=dojo, grade_token=token).first().user

        report = overall_grade_report(dojo, user, when=when)
        grades.append({
            "id": user.id if user else None,
            "grade_token": token,
            "letter": report["letter_grade"],
            "overall": report["total_grade"],
            "extra_credit": sum(report["extra_credit"].values()),
            **{
                module["name"]: module["module_grade"]
                for module in report.get("module_reports", [])
            }
        })

    grades.sort(key=lambda k: (k["overall"] or 0.0, -(k["id"] or float("inf"))), reverse=True)

    all_module_stats = {
        name: statistics.mean(
            grade[name] for grade in grades if
            name in grade and type(grade[name]) in (float,int)
        )
        for name in grades[0]
        if name not in ["id", "grade_token", "letter"]
    } if grades else {}
    grade_statistics = [
        {
            "id": "Average",
            "grade_token": "",
            "letter": letter_grade(dojo, statistics.mean(grade["overall"] for grade in grades)),
            "overall": report["total_grade"],
            "extra_credit": statistics.mean(grade["extra_credit"] for grade in grades),
            **all_module_stats
        }
    ] if grades else []

    grades += unlinked_users

    return render_template("admin_grades.html", grades=grades, grade_statistics=grade_statistics)


from typing import Iterable

import numpy as np

from ..utils.detection_types import ImageType
from .annotation import *  # pylint: disable=W0401, W0614
from .box import crop_box_from_image, global_to_local_coords, intersection_box
from .convert import convert_b64_to_np_array, convert_np_array_to_b64, convert_pdf_bytes_to_np_array_v2

@dataclass
class Image:
    """
    The image object is the enclosing data class that is used in the core data model to manage, retrieve or store
    all information during processing. It contains metadata belonging to the image, but also the image itself
    and annotations, either given as ground truth or determined via a processing path. In addition, there are
    storage options in order not to recalculate coordinates in relation to other images.

    Data points from datasets must be mapped in this format so that the processing tools (pipeline components) can
    be called up without further adjustment.

    In the case of full pipelines, the image data model is also the highest hierarchy class in which document pages
    including their discovered features can be processed.

    An image can be provided with an image_id by providing the external_id, which can be clearly identified
    as a md5 hash string. If such an id is not given, an image_id is derived from the file_name and, if necessary,
    from the given location.

    When initializing the object, the following arguments can be specified:

    :attr:`file_name`: Should be equal to the name of a physical file representing the image. If the image is part
    of a larger document (e.g. pdf-document) the file_name should be populated as a concatenation of the document file
    and its page number.

    :attr:`location`: Full path to the document or to the physical file. Loading functions from disk use this attribute.

    :attr:`external_id`: A string or integer value for generating an image id.

    All other attributes represent containers (lists or dicts) that can be populated and managed using their own method.

    In :attr:`image`, the image may be saved as np.array. Allocation as base64 encoding string or as pdf bytes are
    possible and are converted via a :meth:`image.setter`. Other formats are rejected. As a result of the transfer,
    the width and height of the image are determined. These are accessible via :attr:`width` or :attr:`height`.
    Using :attr:`embeddings`, various bounding boxes can be saved that describe the position of the image as a
    sub-image of another image. The bounding box is accessed in relation to the embedding image via the annotation_id.
    Embeddings are often used in connection with annotations in which :attr:`image` is populated.

    All ImageAnnotations associated with the image are used in the list annotations. Other types of annotation are
    not permitted and must either be transported as  sub-category of an ImageAnnotation or placed as a summary
    annotation in the :attr:`summary`.
    """

    file_name: str
    location: str = field(default_factory=str)
    external_id: Optional[Union[str, int]] = field(default=None, init=True, repr=False)
    image_id: str = field(init=False, repr=True)
    _image_id: Optional[str] = field(default=None, init=False, repr=False)
    image: Optional[ImageType] = field(init=False, repr=False)
    _image: Optional[ImageType] = field(default=None, init=False, repr=False)
    _bbox: Optional[BoundingBox] = field(default=None, init=False, repr=False)
    _embeddings: Dict[str, BoundingBox] = field(default_factory=dict, init=False, repr=False)
    embeddings: Dict[str, BoundingBox] = field(default_factory=dict, init=False, repr=True)
    annotations: List[ImageAnnotation] = field(default_factory=list, init=False, repr=True)
    _annotations: List[ImageAnnotation] = field(default_factory=list, init=False, repr=False)
    _annotation_ids: List[str] = field(default_factory=list, init=False, repr=False)
    summary: Optional[SummaryAnnotation] = field(default=None, init=False, repr=False)

    def __post_init__(self) -> None:
        if self.external_id is not None:
            external_id = str(self.external_id)
            if is_uuid_like(external_id):
                self.image_id = external_id
            else:
                self.image_id = get_uuid(external_id)
        else:
            self.image_id = get_uuid(self.location, self.file_name)

    @property  # type: ignore
    def image_id(self) -> str:  # pylint: disable=E0102
        """
        image_id
        """
        if self._image_id is not None:
            return self._image_id
        raise ValueError("image_id not set")

    @image_id.setter
    def image_id(self, input_id: str) -> None:
        """
        image_id setter
        """
        if self._image_id is not None:
            raise AssertionError("image_id already defined and cannot be reset")
        if is_uuid_like(input_id):
            self._image_id = input_id
        elif isinstance(input_id, property):
            pass
        else:
            raise ValueError("image_id must be uuid3 string")

    @property  # type: ignore
    def image(self) -> Optional[ImageType]:  # pylint: disable=E0102
        """
        image
        """
        return self._image

    @image.setter
    def image(self, image: Optional[Union[str, ImageType, bytes]]) -> None:
        """
        Sets the image for internal storage. Will convert to numpy array before storing internally.
        Note: If the input is an np.array, ensure that the image is in BGR-format as this is the standard
        format for the whole package.
        :param image: Accepts numpy arrays, base64 encodings or bytes generated from pdf documents.
                      Everything else will be rejected.
        """

        if isinstance(image, property):
            pass
        elif isinstance(image, str):
            self._image = convert_b64_to_np_array(image)
            self.set_width_height(self._image.shape[1], self._image.shape[0])
            self._self_embedding()
        elif isinstance(image, bytes):
            self._image = convert_pdf_bytes_to_np_array_v2(image, dpi=300)
            self.set_width_height(self._image.shape[1], self._image.shape[0])
            self._self_embedding()
        else:
            assert isinstance(image, np.ndarray), f"cannot load image is of type: {type(image)}"
            self._image = image
            self.set_width_height(self._image.shape[1], self._image.shape[0])
            self._self_embedding()

    @property  # type: ignore
    def embeddings(self) -> Dict[str, BoundingBox]:  # pylint: disable=E0102
        """
        embeddings. Rather use :meth:`get_embedding`
        """
        return self._embeddings

    @embeddings.setter
    def embeddings(self, inputs: Dict[str, BoundingBox]) -> None:
        """
        embedding setter. Only defined for technical reasons. Cannot change the underlying attribute from here.
        """

    @property  # type: ignore
    def annotations(self) -> List[ImageAnnotation]:  # pylint: disable=E0102
        """
        annotations. Rather use :meth:`get_annotations` directly or with some specified filter conditions.
        """
        return self._annotations

    @annotations.setter
    def annotations(self, inputs: List[ImageAnnotation]) -> None:
        """
        annotations setter. Only defined for technical reasons. Cannot change the underlying attribute from here.
        """

    @property
    def pdf_bytes(self) -> Optional[bytes]:
        """
        pdf_bytes. This attribute will be set dynamically and is not part of the core Image data model
        """
        if hasattr(self, "_pdf_bytes"):
            return self._pdf_bytes  # type: ignore
        return None

    @pdf_bytes.setter
    def pdf_bytes(self, pdf_bytes: bytes) -> None:
        """
        pdf_bytes setter
        """
        assert isinstance(pdf_bytes, bytes)
        if not hasattr(self, "_pdf_bytes"):
            setattr(self, "_pdf_bytes", pdf_bytes)

    def clear_image(self) -> None:
        """
        Removes the :attr:`Image.image`. Useful, if the image must be a lightweight object.
        """
        self._image = None

    def get_image(self, type_id: str) -> Optional[Union[ImageType, str]]:
        """
        Get the image either in base64 string representation or as np.array.

        :param type_id: "b64" or "np"
        :return: desired image encoding representation
        """
        assert type_id in ["b64", "np"], type_id
        if type_id == "b64" and self.image is not None:
            return convert_np_array_to_b64(self.image)
        return self.image

    @property
    def width(self) -> float:  # pylint: disable=R1710
        """
        width
        """
        if self._bbox is None:
            raise ValueError("Width not available. Call set_width_height first")
        return self._bbox.width

    @property
    def height(self) -> float:  # pylint: disable=R1710
        """
        height
        """
        if self._bbox is None:
            raise ValueError("Height not available. Call set_width_height first")
        return self._bbox.height

    def set_width_height(self, width: float, height: float) -> None:
        """
        Defines bounding box of the image if not already set. Use this, if you do not want to keep the image separated
        for memory reasons.

        :param width: width of image
        :param height: height of image
        """
        if self._bbox is None:
            self._bbox = BoundingBox(ulx=0.0, uly=0.0, height=height, width=width, absolute_coords=True)
            self._self_embedding()

    def set_embedding(self, image_id: str, bounding_box: BoundingBox) -> None:
        """
        Set embedding pair. Pass an image_id and a bounding box defining the spacial position of this image with
        respect to the embedding image.

        :param image_id: A uuid of the embedding image.
        :param bounding_box: bounding box of this image in terms of the embedding image.
        """
        assert isinstance(bounding_box, BoundingBox), "bounding box must be of type BoundingBox"
        self._embeddings[image_id] = bounding_box

    def get_embedding(self, image_id: str) -> BoundingBox:
        """
        Returns the bounding box according to the image_id.

        :param image_id: uuid string of the embedding image
        :return: The bounding box of this instance in terms of the embedding image
        """

        return self._embeddings[image_id]

    def _self_embedding(self) -> None:
        if self._bbox is not None:
            assert isinstance(self.image_id, str)
            self.set_embedding(self.image_id, self._bbox)

    def dump(self, annotation: ImageAnnotation) -> None:
        """
        Dump an annotation to the Image dataclass. This is the central method for associating an annotation with
        an image. It gives the annotation an annotation_id in relation to the image_id in order to ensure uniqueness
        across all images.

        :param annotation: image annotation to store
        """
        assert isinstance(annotation, ImageAnnotation), (
            f"Annotation must be of type ImageAnnotation: "
            f"{annotation.annotation_id} but is of type {str(type(annotation))}"
        )
        if annotation._annotation_id is None:  # pylint: disable=W0212
            annotation.annotation_id = self._define_annotation_id(annotation)
        assert annotation.annotation_id not in self._annotation_ids, (
            f"Cannot dump annotation with already taken " f"id {annotation.annotation_id}"
        )
        self._annotation_ids.append(annotation.annotation_id)
        self._annotations.append(annotation)

    def get_annotation(
        self,
        category_names: Optional[Union[str, List[str]]] = None,
        annotation_ids: Optional[Union[str, List[str]]] = None,
        annotation_types: Optional[Union[str, List[str]]] = None,
    ) -> List[ImageAnnotation]:
        """
        Selection of annotations from the annotation container. Filter conditions can be defined by specifying
        the annotation_id or the category name. (Since only image annotations are currently allowed in the container,
        annotation_type is a redundant filter condition.) Only annotations that have :attr: active = 'True' are
        returned. If more than one condition is provided, only annotations will be returned that satisfy all conditions.
        If no condition is provided, it will return all active annotations.

        :param category_names: A single name or list of names
        :param annotation_ids: A single id or list of ids
        :param annotation_types: A type name or list of type names.
        :return: A (possibly empty) list of Annotations
        """

        cat_names = [category_names] if isinstance(category_names, str) else category_names
        ann_ids = [annotation_ids] if isinstance(annotation_ids, str) else annotation_ids
        ann_types = [annotation_types] if isinstance(annotation_types, str) else annotation_types

        anns = filter(lambda x: x.active, self._annotations)

        if ann_types is not None:
            for type_name in ann_types:
                anns = filter(lambda x: isinstance(x, eval(type_name)), anns)  # pylint: disable=W0123, W0640

        if cat_names is not None:
            anns = filter(lambda x: x.category_name in cat_names, anns)  # type: ignore

        if ann_ids is not None:
            anns = filter(lambda x: x.annotation_id in annotation_ids, anns)  # type: ignore

        return list(anns)

    def get_annotation_iter(
        self,
        category_names: Optional[Union[str, List[str]]] = None,
        annotation_ids: Optional[Union[str, List[str]]] = None,
        annotation_types: Optional[Union[str, List[str]]] = None,
    ) -> Iterable[ImageAnnotation]:
        """
        Get annotation as an iterator. Same as :meth:`get_annotation` but returns an iterator instead of a list.

        :param category_names: A single name or list of names
        :param annotation_ids: A single id or list of ids
        :param annotation_types: A type name or list of type names.

        :return: A (possibly empty) list of annotations
        """

        return iter(self.get_annotation(category_names, annotation_ids, annotation_types))

    def as_dict(self) -> Dict[str, Any]:
        """
        Returns the full image dataclass as dict. Uses the custom :func:`convert.as_dict` to disregard attributes
        defined by :meth:`remove_keys`.

        :return:  A custom dict.
        """

        img_dict = as_dict(self, dict_factory=dict)
        return img_dict

    @staticmethod
    def remove_keys() -> List[str]:
        """
        A list of attributes to suspend from as_dict creation.
        """

        return ["external_id", "_image_id", "_image", "_bbox", "_embeddings", "_annotations"]

    def _define_annotation_id(self, annotation: Annotation) -> str:
        attributes = annotation.get_defining_attributes()
        attributes_values = [str(getattr(annotation, attribute)) for attribute in attributes]
        assert self.image_id is not None
        return get_uuid(*attributes_values, str(self.image_id))

    def remove(self, annotation: ImageAnnotation) -> None:
        """
        Calls :meth:`List.remove`. Make sure, the element is in the list for otherwise a ValueError will be raised.

        :param annotation: The annotation to remove
        """

        self._annotations.remove(annotation)

    def image_ann_to_image(self, annotation_id: str, crop_image: bool = False) -> None:
        """
        This method is an operation that changes the state of an underlying dumped image annotation and that
        manages :attr:`ImageAnnotation.image`. An image object is generated, which is interpreted as part of the image
        by the bounding box. The image is cut out and the determinable fields such as height, width and the embeddings
        are determined. The partial image is not saved if crop_image = 'False' is set.

        :param annotation_id: An annotation id of the image annotations.
        :param crop_image: Whether to store the cropped image as np.array.
        """

        ann = self.get_annotation(annotation_ids=annotation_id)[0]
        assert isinstance(
            ann, ImageAnnotation
        ), f"Annotation must be of type ImageAnnotation: {annotation_id} but is of type {str(type(ann))}"

        new_image = Image(file_name=self.file_name, location=self.location, external_id=annotation_id)

        assert self._bbox is not None and ann.bounding_box is not None
        new_bounding_box = intersection_box(self._bbox, ann.bounding_box, self.width, self.height)
        if new_bounding_box.absolute_coords:
            width = new_bounding_box.width
            height = new_bounding_box.height
        else:
            width = new_bounding_box.width * self.width
            height = new_bounding_box.height * self.height
        new_image.set_width_height(width, height)
        new_image.set_embedding(self.image_id, bounding_box=new_bounding_box)

        if crop_image and self.image is not None:
            new_image.image = crop_box_from_image(self.image, ann.bounding_box, self.width, self.height)
        elif crop_image and self.image is None:
            raise ValueError("crop_image = 'True' requires self.image to be not None")

        ann.image = new_image

    def maybe_ann_to_sub_image(self, annotation_id: str, category_names: Union[str, List[str]]) -> None:
        """
        Provides a supplement to :meth:`image_ann_to_image` and mainly operates on the: attr:`ImageAnnotation.image` of
        the image annotation. The aim is to assign image annotations from this image one hierarchy level lower to the
        image of the image annotation. All annotations of this image are also dumped onto the image of the image
        annotation, provided that their bounding boxes are completely in the box of the annotation under consideration.

        :param annotation_id: image annotation you want to assign image annotation from this image. Note, that the
                              annotation must have a not None :attr:`image`.
        :param category_names: Filter the proposals of all image categories of this image by some given category names.
        """

        ann = self.get_annotation(annotation_ids=annotation_id)[0]
        assert ann.image is not None, "adding sub images to annotations requires ann to have an image"
        assert ann.bounding_box is not None
        box = ann.bounding_box.to_list("xyxy")
        proposals = self.get_annotation(category_names)
        points = np.array([prop.image.get_embedding(self.image_id).center for prop in proposals])  # type: ignore
        ann_ids = np.array([prop.annotation_id for prop in proposals])
        indices = np.where(
            (box[0] < points[:, 0]) & (box[1] < points[:, 1]) & (box[2] > points[:, 0]) & (box[3] > points[:, 1])
        )[0]
        selected_ids = ann_ids[indices]
        sub_images = self.get_annotation(annotation_ids=selected_ids.tolist())
        for sub_image in sub_images:
            assert sub_image.image is not None
            sub_image.image.set_embedding(
                annotation_id,
                global_to_local_coords(
                    sub_image.image.get_embedding(self.image_id), ann.image.get_embedding(self.image_id)
                ),
            )
            ann.image.dump(sub_image)

    def get_export(self) -> Dict[str, Any]:
        """
        Exporting image as dictionary. As numpy array cannot be serialized :attr:`image` values will be converted into
        base64 encodings.

        :return: Dict that e.g. can be saved to a file.
        """

        image_copy = deepcopy(self)
        image_copy._annotations = []  # pylint: disable=W0212
        image_copy.image = np.ones((1, 1, 3), dtype=np.float32)
        export_dict = image_copy.as_dict()
        export_dict["annotations"] = []
        export_dict["image"] = self.get_image(type_id="b64")
        for ann in self._annotations:
            export_dict["annotations"].append(ann.get_export())

        return export_dict


from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass
import datetime
import dateutil
from typing import List
from cjwmodule.i18n import trans, I18nMessage

import pandas as pd

def _warn_converted_to_text_because_value_not_timestamp(colname: str, value: str):
    return {
        "message": trans(
            "errors.valueNotTimestamp",
            "Column “{colname}” was converted to Text because the given value is not a Timestamp. "
            "Try entering a value that looks like “2020-01-10” or “2020-01-10T13:11”.",
            # This message doesn't take a "value" argument, but let's
            # store arguments in case we change the message in the future.
            {"colname": colname, "value": value},
        )
    }

def _warn_converted_to_text_because_value_not_number(colname: str, value: str):
    return {
        "message": trans(
            "errors.valueNotNumber",
            "Column “{colname}” was converted to Text because the given value is not a Number. "
            "Try entering a value that looks like “1234” or “12.31242”.",
            # This message doesn't take a "value" argument, but let's
            # store arguments in case we change the message in the future.
            {"colname": colname, "value": value},
        )
    }

def _warn_converted_to_text_because_types_conflict(
    colname: str, value_colnames: List[str]
):
    return {
        "message": trans(
            "errors.valueColumnsWrongType",
            "Values in column “{colname}” were converted to Text because "
            "the chosen columns have different types.",
            # This message doesn't take a "value_colnames" argument, but let's
            # store arguments in case we change the message in the future.
            {"colname": colname, "value_colnames": value_colnames},
        )
    }

def _workbench_type(series: pd.Series) -> Literal["text", "number", "timestamp"]:
    if pd.api.types.is_numeric_dtype(series):
        return "number"
    elif pd.api.types.is_datetime64_dtype(series):
        return "timestamp"
    else:
        return "text"

def _convert_to_str(series: pd.Series) -> pd.Series:
    ret = series.astype(str)
    ret[series.isna()] = None
    return ret

class FillWith(ABC):
    """Abstract class describing how to fill missing values."""

    @abstractmethod
    def run(series: pd.Series) -> Tuple[pd.Series, List]:
        """Return a new `series` with NA values filled in."""

    @classmethod
    def parse(cls, method: str, value: str, from_columns: List[Series]) -> FillWith:
        if method == "value":
            return FillValue(value)
        elif method == "pad":
            return FillPad()
        elif method == "backfill":
            return FillBackfill()
        elif method == "columns":
            return FillWithColumns(from_columns)
        else:
            raise ValueError(f"Invalid method {method}")

class FillValue(FillWith):
    """Operation that fills missing values with a provided value.

    The provided value is given as ``str``; `apply()` will attempt to convert
    it to the series type, if possible.
    """

    def __init__(self, value: str):
        self.value = value

    def run(self, series: pd.Series):
        if not series.isnull().any():
            # There are no nulls. No-op.
            return series, []

        if self.value == "" and _workbench_type(series) in {"number", "timestamp"}:
            # "" means null for timestamps and numbers
            return series, []

        value = self.value
        warnings = []

        # Try to convert `value` to series type. If we fail, convert `series`
        # to str and add a warning.
        if _workbench_type(series) == "number":
            try:
                value = float(self.value)
            except ValueError:
                warnings.append(
                    _warn_converted_to_text_because_value_not_number(series.name, value)
                )
                series = _convert_to_str(series)
        elif _workbench_type(series) == "timestamp":
            try:
                value = dateutil.parser.isoparse(value)
                # TODO test nixing timezone
                if value.tzinfo:
                    value = value.astimezone(datetime.timezone.utc).replace(tzinfo=None)
            except ValueError:
                warnings.append(
                    _warn_converted_to_text_because_value_not_timestamp(
                        series.name, value
                    )
                )
                series = _convert_to_str(series)

        # category (of text) series: value is str; make sure we can fillna() with it
        if hasattr(series, "cat") and self.value not in series.cat.categories:
            series = series.cat.add_categories([self.value])

        return series.fillna(value), warnings

class FillPad(FillWith):
    """Operation that fills missing values with previous ones in the Series."""

    def run(self, series: pd.Series):
        return series.fillna(method="pad"), []

class FillBackfill(FillWith):
    """Operation that fills missing values with next ones in the Series."""

    def run(self, series: pd.Series):
        return series.fillna(method="backfill"), []

class FillWithColumns(FillWith):
    """Operation that fills missing values using other columns' values."""

    def __init__(self, from_columns: List[pd.Series]):
        self.from_columns = from_columns

    def run(self, series: pd.Series):
        warnings = []

        output_is_categorical = hasattr(series, "cat") and all(
            hasattr(c, "cat") for c in self.from_columns
        )

        if hasattr(series, "cat"):
            series = _convert_to_str(series)

        best_type = _workbench_type(series)
        for from_column in self.from_columns:
            if _workbench_type(from_column) != best_type:
                # Unhappy path: convert everything to text
                if best_type != "text":
                    series = _convert_to_str(series)

                from_columns = [
                    col if _workbench_type(col) == "text" else _convert_to_str(col)
                    for col in self.from_columns
                ]
                warnings.append(
                    _warn_converted_to_text_because_types_conflict(
                        series.name,
                        [
                            col.name
                            for col, orig in zip(from_columns, self.from_columns)
                            if col is not orig
                        ],
                    )
                )

                break
        else:
            from_columns = self.from_columns

        ret = series.copy()
        for from_column in from_columns:
            if hasattr(from_column, "cat"):
                from_column = _convert_to_str(from_column)
            ret.fillna(from_column, inplace=True)

        if output_is_categorical:
            ret = ret.astype("category")

        return ret, warnings

def fillna(table: pd.DataFrame, colnames: List[str], fill_with: FillWith) -> None:
    warnings = []
    for colname in colnames:
        series = table[colname]
        series2, series_warnings = fill_with.run(series)
        warnings.extend(series_warnings)
        table[colname] = series2
    return warnings

def render(table, params):
    fill_with = FillWith.parse(
        params["method"], params["value"], [table[c] for c in params["from_colnames"]]
    )
    warnings = fillna(table, params["colnames"], fill_with)
    if warnings:
        return table, warnings
    else:
        return table

def _migrate_params_v0_to_v1(params):
    """
    v0: 'colnames' is comma-separated str; menus are integers
    'contenttype' (0=value, 1=show another menu) and
    'method' (0=pad, 1=backfill). value is 'fillvalue'.

    v1: 'colnames' is List[str]; menu options are value|pad|backfill.
    value is 'value'.

    (These all bring params in line with pandas.DataFrame.fillna.)
    """
    method = {
        (0, 0): "value",
        (0, 1): "value",
        (1, 0): "pad",
        (1, 1): "backfill",
    }[(params["contenttype"], params["method"])]
    return {
        "colnames": [c for c in params["colnames"].split(",") if c],
        "method": method,
        "value": params["fillvalue"],
    }

def _migrate_params_v1_to_v2(params):
    """v1: No "from_colnames" choice. v2 has one."""
    return {
        **params,
        "from_colnames": [],
    }

def migrate_params(params):
    if "contenttype" in params:
        params = _migrate_params_v0_to_v1(params)
    if "from_colnames" not in params:
        params = _migrate_params_v1_to_v2(params)
    return params


from hamcrest import assert_that, equal_to

from nodestream.pipeline.value_providers import StringFormattingValueProvider

from ...stubs import StubbedValueProvider

def test_string_format(blank_context):
    subject = StringFormattingValueProvider(
        fmt="{a}{b}", a=StubbedValueProvider(["a"]), b=StubbedValueProvider(["b"])
    )
    assert_that(subject.single_value(blank_context), equal_to("ab"))

def test_string_format_many(blank_context):
    subject = StringFormattingValueProvider(
        fmt="{a}{b}", a=StubbedValueProvider(["a"]), b=StubbedValueProvider(["b"])
    )
    assert_that(subject.many_values(blank_context), equal_to(["ab"]))


import config
import telebot
import logic
import time
from telebot import types

bot = telebot.TeleBot(config.token)
chats = {}
logic.load_sentences()

markup = types.ReplyKeyboardMarkup()

btn1 = types.KeyboardButton('Ваш сном?')
btn2 = types.KeyboardButton('Нет время сна.')
btn3 = types.KeyboardButton('У меня солидный.')

markup.add(btn1, btn2, btn3)

u_menya_solidnyi = logic.normalize_sentence("Life является странным")

@bot.message_handler(content_types=["text"])
def handle_text_message(message):

    chat_id = message.chat.id

    normalized_message_text = logic.normalize_sentence(message.text)

    if u_menya_solidnyi <= normalized_message_text:
        bot.send_message(chat_id, 'Выберите возвращение, которое отражает их слова на них:', reply_markup=markup)
        return

    if chat_id not in chats:
        chats[chat_id] = {}

    logic.add_sentence(message.text)
    answer = logic.get_random_answer(message.text)

    if "previous_text" in chats[chat_id]:
        if chats[chat_id]["previous_user"] != message.from_user.id:

            previous_text = chats[chat_id]["previous_text"]
            logic.add_answer(previous_text, message.text)

    if answer:
        bot.send_message(chat_id, answer)
        chats[chat_id]["previous_text"] = answer
        chats[chat_id]["previous_user"] = 'yarkiybot'
    else:
        chats[chat_id]["previous_text"] = message.text
        chats[chat_id]["previous_user"] = message.from_user.id

if __name__ == '__main__':
     while True:
        try:
            bot.polling(none_stop=True)
        except Exception as e:
            print(e)
            time.sleep(5)
        finally:
            logic.dump_sentences()


# Write a Python function to calculate the factorial of a number (a non- negative integer).
# The function accepts the number as argument

from functools import reduce

def Factorial(n):
    l = list(range(2, (n+1)))
    return reduce(lambda x, y: x*y, l)

print("Factorial :", Factorial(5))

BCA Programs/#98 Factorial using function.py


"""
python3 prog_2.py texts/alice.txt texts/second.txt
"""

import sys
#	NLTK modules
import nltk
from nltk import sent_tokenize as st
from nltk import word_tokenize as wt
from nltk.probability import FreqDist as fd
#	Statistical module
import LC.tok as LC
#	Statistical module
import LC.ner as NE
#	Data cleaning module
import LC.clean as cl
#	Utility module
import LC.toolbox as tool
#import bs4 to edit the output in html format
from bs4 import BeautifulSoup
import write_output as OUT

st = nltk.data.load('tokenizers/punkt/english.pickle')

def main(f_1, f_2):
	"""
	Extracts NER stats from 2 text files
	"""

	######################################
	#	Text extraction
	######################################

	t_1 = ''
	t_2 = ''

	with open(f_1, 'r') as f:
		t_1 = f.read()
	with open(f_2, 'r') as f:
		t_2 = f.read()

	######################################
	#	Text cleaning
	######################################

	t_1 = cl.clean_text(t_1)
	t_2 = cl.clean_text(t_2)

	# Initialize output
	output = OUT.init_html()
	soup = BeautifulSoup(output, 'html.parser')

	col_1 = soup.find('div', {'id': 'text-1-container'})
	col_2 = soup.find('div', {'id': 'text-2-container'})

	title_1 = soup.new_tag('p', attrs={'id' : 'title-1'})
	title_1.insert(1,'Alice in Wonderland')

	title_2 = soup.new_tag('p', attrs={'id' : 'title-2'})
	title_2.insert(1,'Second Variety')

	author_1 = soup.new_tag('p', attrs={'id' : 'author-1'})
	author_1.insert(1,'Lewis Carroll')

	author_2 = soup.new_tag('p', attrs={'id' : 'author-2'})
	author_2.insert(1,'Philip Kindred Dick')

	col_1.append(title_1)
	col_1.append(author_1)

	col_2.append(title_2)
	col_2.append(author_2)

	######################################
	#	Extracting the useful elements
	######################################

	# Sentence tokens
	sents_1 = st.tokenize(t_1)
	sents_2 = st.tokenize(t_2)

	# Word tokens (by sentence)
	s_toks_1 = []
	s_toks_2 = []

	for s_1, s_2 in zip(sents_1, sents_2):
		s_toks_1.append(wt(s_1))
		s_toks_2.append(wt(s_2))

	# Word tokens (unique list)
	toks_1 = wt(t_1)
	toks_2 = wt(t_2)

	# NER tagging

	POS_NE_1 = NE.ner_sents(sents_1)
	POS_NE_2 = NE.ner_sents(sents_2)

	PERSON_sents_1 = NE.get_ner_sents(POS_NE_1, ['PERSON'])
	PERSON_sents_2 = NE.get_ner_sents(POS_NE_2, ['PERSON'])

	# Get the entities of specific NER_tag
	ENTITIES_1 = NE.get_ner_entities(PERSON_sents_1, ['PERSON'])
	ENTITIES_2 = NE.get_ner_entities(PERSON_sents_2, ['PERSON'])

	PEOPLE_1 = LC.rank(ENTITIES_1['PERSON'])[:10]
	PEOPLE_2 = LC.rank(ENTITIES_2['PERSON'])[:10]

	# Print most frequent characters
	nc_1 = OUT.complex_container('Most frequent characters', PEOPLE_1, soup)
	nc_2 = OUT.complex_container('Most frequent characters', PEOPLE_2, soup)
	col_1.append(nc_1)
	col_2.append(nc_2)
	print('added Most frequent characters to project/index.html')

	# Extract only sentence in which appear the selected PEOPLE
	PEOPLE_1 = tool.tup_2_list(PEOPLE_1)
	PEOPLE_2 = tool.tup_2_list(PEOPLE_2)

	useful_sents_1 = NE.meaningful_sents(PERSON_sents_1, ['PERSON'], PEOPLE_1)
	useful_sents_2 = NE.meaningful_sents(PERSON_sents_2, ['PERSON'], PEOPLE_2)

	######################################
	#	Extracting informations
	######################################

	"""
	Now that we have extracted the useful sentences,
	we can begin to mine the information inside them.
	"""

	# NER_tags to look for in the sentence
	NER_tags = ['PERSON', 'LOC', 'GPE', 'DATE', 'TIME']
	POS_tags = ['NN', 'NNP', 'NNPS', 'NNS', 'VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ']

	# For every sent: find the Person and {PERSON_1 : [link1: freq, link2: freq], PERSON_2: [link1: freq, link2: freq]}
	links_1 = NE.extract_links(useful_sents_1, {'PERSON' : PEOPLE_1}, NER_tags, POS_tags)
	links_2 = NE.extract_links(useful_sents_2, {'PERSON' : PEOPLE_2}, NER_tags, POS_tags)

	# Extract the frequency of each link
	links_freqs_1 = NE.extract_link_freqs(links_1)
	links_freqs_2 = NE.extract_link_freqs(links_2)

	#with open('tests/links_freqs_1.txt', 'w') as output:
		#output.write(str(links_freqs_1))

	# Extract the most frequent items for each field
	most_freq_1 = NE.rank_link_freqs(links_freqs_1, 10)
	most_freq_2 = NE.rank_link_freqs(links_freqs_2, 10)

	# Print infos TEXT_1
	for NER in most_freq_1:
		for leaf in most_freq_1[NER]:
			for link_type in most_freq_1[NER][leaf]:
				nc_1 = OUT.complex_container(leaf + ' + ' + link_type, most_freq_1[NER][leaf][link_type], soup)
				col_1.append(nc_1)
				print('added', leaf, ' + ', link_type,'to project/index.html')

	# Print infos TEXT_2
	for NER in most_freq_2:
		for leaf in most_freq_2[NER]:
			for link_type in most_freq_2[NER][leaf]:
				nc_2 = OUT.complex_container(leaf + ' + ' + link_type, most_freq_2[NER][leaf][link_type], soup)
				col_2.append(nc_2)
				print('added', leaf, ' + ', link_type,'to project/index.html')

	# Get the max-prob sentence of lenght between 8 and 12 token

	# Select sents of lenght between 8 and 12 token containing
	selected_sents_1 = tool.sents_of_len(NE.trees_2_toks(useful_sents_1), 8, 12)
	selected_sents_2 = tool.sents_of_len(NE.trees_2_toks(useful_sents_2), 8, 12)

	sents_4_person_1 = NE.assign_sent_2_person(selected_sents_1, PEOPLE_1)
	sents_4_person_2 = NE.assign_sent_2_person(selected_sents_2, PEOPLE_2)

	# Compute the probability for each sentence for each important person in the text
	max_markow_1 = {}
	for person in sents_4_person_1:
		max_markow_1[person] = NE.get_max_markow(sents_4_person_1[person], fd(toks_1), len(t_1), False)

	max_markow_2 = {}
	for person in sents_4_person_2:
		max_markow_2[person] = NE.get_max_markow(sents_4_person_2[person], fd(toks_2), len(t_2), False)

	# Print Markov probabilities
	for person in max_markow_1:
		nc_1 = OUT.tok_sent_container('Probable sentence for ' + person, max_markow_1[person], soup)
		col_1.append(nc_1)
		print('added '+ 'Probable sentence for ' + person + 'to project/index.html')

	for person in max_markow_2:
		nc_2 = OUT.tok_sent_container('Probable sentence for ' + person, max_markow_2[person], soup)
		col_2.append(nc_2)
		print('added '+ 'Probable sentence for ' + person + 'to project/index.html')

	# Final prints
	title_1 = soup.new_tag('p', attrs={'id' : 'book-1'})
	title_1.insert(1,'Alice in Wonderland')
	soup.body.append(title_1)

	title_2 = soup.new_tag('p', attrs={'id' : 'book-2'})
	title_2.insert(1,'Second Variety')
	soup.body.append(title_2)

	# Exporting output
	with open('output/index_2.html', 'w') as h:
		h.write(str(soup))

main(sys.argv[1], sys.argv[2])


# IMPORT SELENIUM DEPENDENCIES
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.webdriver import WebDriver
from webdriver_manager.chrome import ChromeDriverManager

# CONFIGURE HEADLESS BROWSER OPTIONS
opt_headless = Options()
opt_headless.add_argument("--headless")

# CREATE HEADLESS BROWSER INSTANCE FOR BOTS
def init_browser_headless() -> WebDriver:
    browser: WebDriver = webdriver.Chrome(
        executable_path=ChromeDriverManager().install(),
        chrome_options=opt_headless,
    )
    return browser

# CREATE HEADFUL BROWSER INSTANCE FOR BOTS
def init_browser_headful() -> WebDriver:
    browser: WebDriver = webdriver.Chrome(
        executable_path=ChromeDriverManager().install()
    )
    return browser


from __future__ import absolute_import
from __future__ import print_function
from __future__ import division

import torch

from reid.data.samplers import build_train_sampler
from reid.data.transforms import build_transforms
from reid.data.datasets import init_image_dataset

class DataManager(object):
    r"""Base data manager.

    Args:
        sources (str or list): source dataset(s).
        targets (str or list, optional): target dataset(s). If not given,
            it equals to ``sources``.
        height (int, optional): target image height. Default is 256.
        width (int, optional): target image width. Default is 128.
        transforms (str or list of str, optional): transformations applied to model training.
            Default is 'random_flip'.
        norm_mean (list or None, optional): data mean. Default is None (use imagenet mean).
        norm_std (list or None, optional): data std. Default is None (use imagenet std).
        use_gpu (bool, optional): use gpu. Default is True.
    """

    def __init__(self, sources=None, targets=None, height=256, width=128, transforms='random_flip', padding=10, norm_mean=None, norm_std=None, use_gpu=False):
        self.sources = sources
        self.targets = targets
        self.height = height
        self.width = width

        if self.sources is None:
            raise ValueError('sources must not be None')

        if isinstance(self.sources, str):
            self.sources = [self.sources]

        if self.targets is None:
            self.targets = self.sources

        if isinstance(self.targets, str):
            self.targets = [self.targets]

        self.transform_tr, self.transform_te = build_transforms(
            self.height, self.width, transforms=transforms,
            norm_mean=norm_mean, norm_std=norm_std, padding=padding,
        )

        self.use_gpu = (torch.cuda.is_available() and use_gpu)

    @property
    def num_train_pids(self):
        """Returns the number of training person identities."""
        return self._num_train_pids

    @property
    def num_train_cams(self):
        """Returns the number of training cameras."""
        return self._num_train_cams

    def return_dataloaders(self):
        """Returns trainloader and testloader."""
        return self.trainloader, self.testloader

    def return_testdataset_by_name(self, name):
        """Returns query and gallery of a test dataset, each containing
        tuples of (img_path(s), pid, camid).

        Args:
            name (str): dataset name.
        """
        return self.testdataset[name]['query'], self.testdataset[name]['gallery']

class ImageDataManager(DataManager):
    r"""Image data manager.

    Args:
        root (str): root path to datasets.
        sources (str or list): source dataset(s).
        targets (str or list, optional): target dataset(s). If not given,
            it equals to ``sources``.
        height (int, optional): target image height. Default is 256.
        width (int, optional): target image width. Default is 128.
        transforms (str or list of str, optional): transformations applied to model training.
            Default is 'random_flip'.
        norm_mean (list or None, optional): data mean. Default is None (use imagenet mean).
        norm_std (list or None, optional): data std. Default is None (use imagenet std).
        use_gpu (bool, optional): use gpu. Default is True.
        split_id (int, optional): split id (*0-based*). Default is 0.
        combineall (bool, optional): combine train, query and gallery in a dataset for
            training. Default is False.
        batch_size_train (int, optional): number of images in a training batch. Default is 32.
        batch_size_test (int, optional): number of images in a test batch. Default is 32.
        workers (int, optional): number of workers. Default is 4.
        num_instances (int, optional): number of instances per identity in a batch.
            Default is 4.
        train_sampler (str, optional): sampler. Default is empty (``RandomSampler``).
        cuhk03_labeled (bool, optional): use cuhk03 labeled images.
            Default is False (defaul is to use detected images).
        cuhk03_classic_split (bool, optional): use the classic split in cuhk03.
            Default is False.
        market1501_500k (bool, optional): add 500K distractors to the gallery
            set in market1501. Default is False.

    Examples::

        datamanager = torchreid.data.ImageDataManager(
            root='path/to/reid-data',
            sources='market1501',
            height=256,
            width=128,
            batch_size_train=32,
            batch_size_test=100
        )
    """
    data_type = 'image'

    def __init__(self, root='', sources=None, targets=None, height=256, width=128, transforms='random_flip', padding=10,
                 norm_mean=None, norm_std=None, use_gpu=True, split_id=0, combineall=False,
                 batch_size_train=32, batch_size_test=32, workers=4, num_instances=4, train_sampler='',
                 cuhk03_labeled=False, cuhk03_classic_split=False, market1501_500k=False, naic2019_phase='train', naic2019_total=False):

        super(ImageDataManager, self).__init__(sources=sources, targets=targets, height=height, width=width,
                                               transforms=transforms, padding=padding, norm_mean=norm_mean, norm_std=norm_std,
                                               use_gpu=use_gpu)

        print('=> Loading train (source) dataset')
        trainset = []
        for name in self.sources:
            trainset_ = init_image_dataset(
                name,
                transform=self.transform_tr,
                mode='train',
                combineall=combineall,
                root=root,
                split_id=split_id,
                cuhk03_labeled=cuhk03_labeled,
                cuhk03_classic_split=cuhk03_classic_split,
                market1501_500k=market1501_500k,
                naic2019_phase=naic2019_phase,
                naic2019_total=naic2019_total,
            )
            trainset.append(trainset_)
        trainset = sum(trainset)

        self._num_train_pids = trainset.num_train_pids
        self._num_train_cams = trainset.num_train_cams

        train_sampler = build_train_sampler(
            trainset.train, train_sampler,
            batch_size=batch_size_train,
            num_instances=num_instances
        )

        self.trainloader = torch.utils.data.DataLoader(
            trainset,
            sampler=train_sampler,
            batch_size=batch_size_train,
            shuffle=False,
            num_workers=workers,
            pin_memory=self.use_gpu,
            drop_last=True
        )

        print('=> Loading test (target) dataset')
        self.testloader = {name: {'query': None, 'gallery': None}
                           for name in self.targets}
        self.testdataset = {name: {'query': None, 'gallery': None}
                            for name in self.targets}

        for name in self.targets:
            # build query loader
            queryset = init_image_dataset(
                name,
                transform=self.transform_te,
                mode='query',
                combineall=combineall,
                root=root,
                split_id=split_id,
                cuhk03_labeled=cuhk03_labeled,
                cuhk03_classic_split=cuhk03_classic_split,
                market1501_500k=market1501_500k,
                naic2019_phase=naic2019_phase,
                naic2019_total=naic2019_total,
            )
            self.testloader[name]['query'] = torch.utils.data.DataLoader(
                queryset,
                batch_size=batch_size_test,
                shuffle=False,
                num_workers=workers,
                pin_memory=self.use_gpu,
                drop_last=False
            )

            # build gallery loader
            galleryset = init_image_dataset(
                name,
                transform=self.transform_te,
                mode='gallery',
                combineall=combineall,
                verbose=False,
                root=root,
                split_id=split_id,
                cuhk03_labeled=cuhk03_labeled,
                cuhk03_classic_split=cuhk03_classic_split,
                market1501_500k=market1501_500k,
                naic2019_phase=naic2019_phase,
                naic2019_total=naic2019_total,
            )
            self.testloader[name]['gallery'] = torch.utils.data.DataLoader(
                galleryset,
                batch_size=batch_size_test,
                shuffle=False,
                num_workers=workers,
                pin_memory=self.use_gpu,
                drop_last=False
            )

            self.testdataset[name]['query'] = queryset.query
            self.testdataset[name]['gallery'] = galleryset.gallery

        print('\n')
        print('  **************** Summary ****************')
        print('  train            : {}'.format(self.sources))
        print('  # train datasets : {}'.format(len(self.sources)))
        print('  # train ids      : {}'.format(self.num_train_pids))
        print('  # train images   : {}'.format(len(trainset)))
        print('  # train cameras  : {}'.format(self.num_train_cams))
        print('  test             : {}'.format(self.targets))
        print('  *****************************************')
        print('\n')


import operator
import inspect

import pytest

from ftrack_api.event.expression import (
    Expression, All, Any, Not, Condition, Parser
)
from ftrack_api.exception import ParseError

@pytest.fixture()
def candidate():
    '''Return common candidate to test expressions against.'''
    return {
        'id': 10,
        'name': 'value',
        'change': {
            'name': 'value',
            'new_value': 10
        }
    }

@pytest.mark.parametrize('expression, expected', [
    pytest.mark.xfail(('', Expression())),
    ('invalid', ParseError),
    ('key=value nor other=value', ParseError),
    ('key=value', Condition('key', operator.eq, 'value')),
    ('key="value"', Condition('key', operator.eq, 'value')),
    (
        'a=b and ((c=d or e!=f) and not g.h > 10)',
        All([
            Condition('a', operator.eq, 'b'),
            All([
                Any([
                    Condition('c', operator.eq, 'd'),
                    Condition('e', operator.ne, 'f')
                ]),
                Not(
                    Condition('g.h', operator.gt, 10)
                )
            ])
        ])
    )
], ids=[
    'empty expression',
    'invalid expression',
    'invalid conjunction',
    'basic condition',
    'basic quoted condition',
    'complex condition'
])
def test_parser_parse(expression, expected):
    '''Parse expression into Expression instances.'''
    parser = Parser()

    if inspect.isclass(expected)and issubclass(expected, Exception):
        with pytest.raises(expected):
            parser.parse(expression)
    else:
        assert str(parser.parse(expression)) == str(expected)

@pytest.mark.parametrize('expression, expected', [
    (Expression(), '<Expression>'),
    (All([Expression(), Expression()]), '<All [<Expression> <Expression>]>'),
    (Any([Expression(), Expression()]), '<Any [<Expression> <Expression>]>'),
    (Not(Expression()), '<Not <Expression>>'),
    (Condition('key', '=', 'value'), '<Condition key=value>')
], ids=[
    'Expression',
    'All',
    'Any',
    'Not',
    'Condition'
])
def test_string_representation(expression, expected):
    '''String representation of expression.'''
    assert str(expression) == expected

@pytest.mark.parametrize('expression, expected', [
    # Expression
    (Expression(), True),

    # All
    (All(), True),
    (All([Expression(), Expression()]), True),
    (All([Expression(), Condition('test', operator.eq, 'value')]), False),

    # Any
    (Any(), False),
    (Any([Expression(), Condition('test', operator.eq, 'value')]), True),
    (Any([
        Condition('test', operator.eq, 'value'),
        Condition('other', operator.eq, 'value')
    ]), False),

    # Not
    (Not(Expression()), False),
    (Not(Not(Expression())), True)
], ids=[
    'Expression-always matches',

    'All-no expressions always matches',
    'All-all match',
    'All-not all match',

    'Any-no expressions never matches',
    'Any-some match',
    'Any-none match',

    'Not-invert positive match',
    'Not-double negative is positive match'
])
def test_match(expression, candidate, expected):
    '''Determine if candidate matches expression.'''
    assert expression.match(candidate) is expected

def parametrize_test_condition_match(metafunc):
    '''Parametrize condition_match tests.'''
    identifiers = []
    data = []

    matrix = {
        # Operator, match, no match
        operator.eq: {
            'match': 10, 'no-match': 20,
            'wildcard-match': 'valu*', 'wildcard-no-match': 'values*'
        },
        operator.ne: {'match': 20, 'no-match': 10},
        operator.ge: {'match': 10, 'no-match': 20},
        operator.le: {'match': 10, 'no-match': 0},
        operator.gt: {'match': 0, 'no-match': 10},
        operator.lt: {'match': 20, 'no-match': 10}
    }

    for operator_function, values in matrix.items():
        for value_label, value in values.items():
            if value_label.startswith('wildcard'):
                key_options = {
                    'plain': 'name',
                    'nested': 'change.name'
                }
            else:
                key_options = {
                    'plain': 'id',
                    'nested': 'change.new_value'
                }

            for key_label, key in key_options.items():
                identifiers.append('{} operator {} key {}'.format(
                    operator_function.__name__, key_label, value_label
                ))

                data.append((
                    key, operator_function, value,
                    'no-match' not in value_label
                ))

    metafunc.parametrize(
        'key, operator, value, expected', data, ids=identifiers
    )

def test_condition_match(key, operator, value, candidate, expected):
    '''Determine if candidate matches condition expression.'''
    condition = Condition(key, operator, value)
    assert condition.match(candidate) is expected