import os
import json
import math
import tempfile
import numpy as np
from decimal import Decimal
from colormath.color_objects import RGBColor, LabColor

from django.db import models, transaction

from common.models import EmptyModelBase, ResultBase, PaperCitation
from photos.models import Photo

from common.utils import get_content_tuple, get_opensurfaces_storage

STORAGE = get_opensurfaces_storage()

class IntrinsicSyntheticDecomposition(EmptyModelBase):
    """
    This holds the different layers of a synthetic image.  The final rendered
    result is stored as an LDR JPG in the ``photo`` attribute, and the layers are
    stored in ``multilayer_exr``.  LDR preview thumbnails of each layer are
    stored in ``*_thumb``.
    """

    #: final rendered result as an LDR JPG.
    photo = models.OneToOneField(Photo, related_name='intrinsic_synthetic')

    #: name of the artist that modeled the scene
    scene_artist = models.CharField(max_length=255, blank=True)

    #: URL where the scene can be found online
    scene_url = models.URLField(max_length=255, blank=True)

    #: options for ``FileField`` or ``ImageField`` instance
    FILE_OPTS = {
        'null': True,
        'blank': True,
        'max_length': 255,
        'storage': STORAGE,
    }

    #: decomposition stored as a multi-layer EXR file
    multilayer_exr = models.FileField(upload_to='intrinsic_syn_multilayer_exr', **FILE_OPTS)

    #: md5 hash of the multilayer exr file
    md5 = models.CharField(max_length=32, unique=True)

    # decomposition separated out into thumbnail images for web visualization

    #: small LDR thumbnail of the environment lighting layer
    env_thumb = models.ImageField(upload_to='intrinsic_syn_env', **FILE_OPTS)

    #: small LDR thumbnail of the emitter layer
    emit_thumb = models.ImageField(upload_to='intrinsic_syn_emit', **FILE_OPTS)

    #: small LDR thumbnail of the depth layer
    depth_thumb = models.ImageField(upload_to='intrinsic_syn_depth', **FILE_OPTS)

    #: small LDR thumbnail of the normal layer
    normal_thumb = models.ImageField(upload_to='intrinsic_syn_normal', **FILE_OPTS)

    #: small LDR thumbnail of the diffuse color (albedo) layer
    diff_col_thumb = models.ImageField(upload_to='intrinsic_syn_diff_col', **FILE_OPTS)

    #: small LDR thumbnail of the diffuse indirect light layer
    diff_ind_thumb = models.ImageField(upload_to='intrinsic_syn_diff_ind', **FILE_OPTS)

    #: small LDR thumbnail of the diffuse direct light layer
    diff_dir_thumb = models.ImageField(upload_to='intrinsic_syn_diff_dir', **FILE_OPTS)

    #: small LDR thumbnail of the gloss color (albedo) layer
    gloss_col_thumb = models.ImageField(upload_to='intrinsic_syn_gloss_col', **FILE_OPTS)

    #: small LDR thumbnail of the gloss indirect light layer
    gloss_ind_thumb = models.ImageField(upload_to='intrinsic_syn_gloss_ind', **FILE_OPTS)

    #: small LDR thumbnail of the gloss direct light layer
    gloss_dir_thumb = models.ImageField(upload_to='intrinsic_syn_gloss_dir', **FILE_OPTS)

    #: small LDR thumbnail of the transparent color layer
    trans_col_thumb = models.ImageField(upload_to='intrinsic_syn_trans_col', **FILE_OPTS)

    #: small LDR thumbnail of the transparent indirect light layer
    trans_ind_thumb = models.ImageField(upload_to='intrinsic_syn_trans_ind', **FILE_OPTS)

    #: small LDR thumbnail of the transparent direct light layer
    trans_dir_thumb = models.ImageField(upload_to='intrinsic_syn_trans_dir', **FILE_OPTS)

    def _download_multilayer_exr(self, callback):
        """ Download the Multilayer EXR file, call a function, then delete it. """

        temp = tempfile.NamedTemporaryFile(delete=False, suffix=".exr")
        try:
            self.multilayer_exr.seek(0)
            temp.file.write(self.multilayer_exr.read())
            temp.file.close()
            return callback(temp.name)
        finally:
            os.remove(temp.name)

    def open_multilayer_exr(self, **kwargs):
        """ Open all layers from the decomposition as a dictionary
        See ``intrinsic.synthetic.open_multilayer_exr`` for list of arguments """

        # unfortunately the OpenEXR file insists on being givin a filename, so
        # we have to download the file to a temp directory
        from intrinsic.synthetic import open_multilayer_exr
        return self._download_multilayer_exr(
            lambda filename: open_multilayer_exr(filename, **kwargs))

    def open_multilayer_exr_layers(self, layers):
        """ Open a set of layers from the decomposition, returned as a numpy
        float array (linear space) """

        # unfortunately the OpenEXR file insists on being givin a filename, so
        # we have to download the file to a temp directory
        from intrinsic.synthetic import open_multilayer_exr_layers
        return self._download_multilayer_exr(
            lambda filename: open_multilayer_exr_layers(filename, layers))

class IntrinsicPoint(EmptyModelBase):

    """ A point where we want to say something about relative shading,
    reflectance, or both, with respect to another point. """

    #: photo of interest
    photo = models.ForeignKey(Photo, related_name='intrinsic_points')

    #: x and y are a fraction of width and height
    x = models.FloatField()
    #: x and y are a fraction of width and height
    y = models.FloatField()

    #: color at this point: "RRGGBB" in sRGB hex
    sRGB = models.CharField(max_length=6)

    #: synthetic ground truth diffuse albedo intensity (mean of reflectance
    #: color channels in the ``diff_col`` layer) at this point, if known.
    synthetic_diff_intensity = models.FloatField(blank=True, null=True)

    #: synthetic ground truth ``mean(diffuse contribution) / mean(combined pixel
    #: contribution)``, if known.  So, for fully diffuse points, this is 1.0, and
    #: for fully specular points, this is 0.0.
    synthetic_diff_fraction = models.FloatField(blank=True, null=True)

    #: synthetic ground truth ``std(diffuse contribution) / mean(diffuse
    #: contribution)``, if known, and if the mean is nonzero.
    synthetic_diff_cv = models.FloatField(blank=True, null=True)

    #: at what density was this point sampled (use decimals so we can compare
    #: these with exact equality).  This is converted to a distance in pixels
    #: according to ::
    #:
    #:     max(1, int(np.sqrt(cols ** 2 + rows ** 2) * min_separation))
    min_separation = models.DecimalField(
        default=Decimal('0.07'), decimal_places=5, max_digits=7)

    #: if True: enough users voted this to be ``opaque`` (not a mirror and not
    #: transparent).
    #: if None: not yet known.
    opaque = models.NullBooleanField()

    #: CUBAM score for the ``opaque`` field.
    #: higher scores indicate higher probability of being opaque.
    #: scores are thresholded at 0.
    opaque_score = models.FloatField(blank=True, null=True, db_index=True)

    #: method used to set the opaque field (admin, CUBAM, or majority vote)
    opaque_method = models.CharField(
        max_length=1, choices=ResultBase.QUALITY_METHODS, blank=True, null=True)

    def get_rgb(self):
        """ return the color as a list """
        if not hasattr(self, '_rgb'):
            c = RGBColor()
            c.set_from_rgb_hex(self.sRGB)
            self._rgb = (c.rgb_r / 255.0, c.rgb_g / 255.0, c.rgb_b / 255.0)
        return self._rgb

    def get_intensity(self):
        """ returns mean(rgb) """
        return np.mean(self.get_rgb())

    def get_lab(self):
        """ returns the current point in L*a*b* """
        if not hasattr(self, '_lab'):
            c = RGBColor()
            c.set_from_rgb_hex(self.sRGB)
            self._lab = c.convert_to('lab').get_value_tuple()
        return self._lab

    def get_chroma(self):
        """ returns rgb / sum(rgb).  pure black has the same chromaticity as pure white. """
        rgb = self.get_rgb()
        s = np.sum(rgb)
        if s > 0:
            return np.asarray(rgb) / s
        else:
            return np.array([1.0 / 3.0, 1.0 / 3.0, 1.0 / 3.0])

    def save(self, *args, **kwargs):
        if not self.sRGB:
            rgb = self.photo.get_pixel(self.x, self.y, width='300')
            self.sRGB = '%02x%02x%02x' % rgb

        super(IntrinsicPoint, self).save(*args, **kwargs)

    def update_opacity(self, save=True):
        """ Update the opaque field by aggregating user responses and selecting
        the majority vote.  Note that the opaque field is updated periodically
        with the output of CUBAM, which is a better predictor than majority
        vote.  """
        opacities = self.opacities.filter(invalid=False) \
            .values_list('opaque', flat=True)
        if (len(opacities) >= 5 and
                (not self.opaque_method or self.opaque_method == 'M')):
            # If 100% in agreement, store in advance.  Otherwise,
            # wait for CUBAM to assign the answer.
            num_opaque = sum(int(b) for b in opacities)
            if num_opaque == 0 or num_opaque == len(opacities):
                self.opaque = (num_opaque * 2 > len(opacities))
                self.opaque_score = None
                self.opaque_method = 'M'
                if save:
                    self.save()

    def x_aspect(self):
        """ Helper for templates.  Return the x pixel coordinate, scaled by
        height instead of width (i.e. x_aspect = x_original / height) """
        return self.x * self.photo.aspect_ratio

    def get_entry_dict(self, include_photo=True):
        """ Return a dictionary of this model containing just the fields needed
        for javascript rendering.  """
        ret = {
            'id': self.id,
            'x': self.x,
            'y': self.y,
            'sRGB': self.sRGB,
        }
        if include_photo:
            ret['photo'] = self.photo.get_entry_dict()
        return ret

class IntrinsicPointOpacityResponse(ResultBase):

    """ A user's opinion of a point's opacity.  Note that "opaque" additionally
    excludes pure mirrors.  In BRDF terms, opaque points have only diffuse or
    glossy reflection components, and no transmission component. """

    #: point being classified
    point = models.ForeignKey(IntrinsicPoint, related_name='opacities')

    #: if True: not a mirror and not transparent.
    #: if False: either a mirror or transparent.
    opaque = models.BooleanField(default=False)

    #: The zoom level that the user used in the UI when deciding on opaque
    #: (1: zoomed out, larger numbers indicate zoomed in further.
    zoom = models.FloatField(null=True, blank=True)

    #: List of all ForeignKey relationships that are accessed during a mturk
    #: task.  (called by mturk.utils.get_content_model_prefetch)
    MTURK_PREFETCH = ('point', 'point__photo')

    @staticmethod
    def mturk_submit(user, hit_contents, results, time_ms,
                     time_active_ms, version,
                     mturk_assignment=None, **kwargs):

        if unicode(version) != u'1.0':
            raise ValueError(
                "Unknown version: '%s' (type: %s)" % (version, type(version)))

        new_objects = {}
        for point in hit_contents:
            key = unicode(point.id)

            new_obj, created = point.opacities.get_or_create(
                user=user,
                mturk_assignment=mturk_assignment,
                time_ms=time_ms[key],
                time_active_ms=time_active_ms[key],
                opaque=results[key][u'opaque'],
                zoom=results[key][u'zoom'],
            )

            if created:
                new_objects[get_content_tuple(point)] = [new_obj]

        # atomically update comparison objects
        with transaction.atomic():
            qset = IntrinsicPoint.objects \
                .select_for_update() \
                .filter(id__in=[c.id for c in hit_contents])
            for point in qset:
                point.update_opacity()

        return new_objects

    @staticmethod
    def get_thumb_template():
        return 'intrinsic/point_opacity_thumb.html'

    def expanded_bbox(self):
        if not hasattr(self, '_expanded_bbox'):
            x1, y1 = self.point1.x * self.photo.aspect_ratio, self.point1.y
            x2, y2 = self.point2.x * self.photo.aspect_ratio, self.point2.y
            if x2 < x1:
                x1, x2 = x2, x1
            if y2 < y1:
                y1, y2 = y2, y1
            self._expanded_bbox = [x1 - 0.1, x2 + 0.1, y1 - 0.1, y2 + 0.1]
        return self._expanded_bbox

    def view_box_svg(self):
        x = self.point.x_aspect()
        y = self.point.y
        return "%s %s 0.1 0.1" % (x - 0.05, y - 0.05)

class IntrinsicPointComparison(EmptyModelBase):

    """ Pair of points where we want to say something about their relative
    reflectance.  The user must explain which point has a darker reflectance.
    To maintain unique ordering of points, point1 and point2 must be ordered
    so that:

    .. code-block:: py

        point1.y < point2.y if point1.x == point2.x
        point1.x < point2.x if point1.x != point2.x

    """

    #: photo of interest
    photo = models.ForeignKey(
        Photo, related_name='intrinsic_comparisons')

    #: first point in the comparison.
    #: note: point1 and point2 must be from the same photo
    point1 = models.ForeignKey(
        IntrinsicPoint, related_name='comparison_point1')

    #: second point in the comparison.
    #: note: point1 and point2 must be from the same photo
    point2 = models.ForeignKey(
        IntrinsicPoint, related_name='comparison_point2')

    #: ``True`` if the luminance of the RGB pixel value at ``point1`` is darker
    #: than that of ``point2``
    point1_image_darker = models.NullBooleanField()

    #: darker: which point is darker
    DARKER_CHOICES = (('1', '1 < 2'), ('2', '1 > 1'), ('E', '1 = 2'))
    darker_to_str = dict((k, v) for (k, v) in DARKER_CHOICES)
    darker = models.CharField(
        max_length=1, choices=DARKER_CHOICES,
        blank=True, null=True)

    #: if synthetic, the ratio (point1/point2) of the intensity of the
    #: ``diff_col`` channel (diffuse albedo) of the multilayer exr rendering.
    #: If the ground truth intensity is 0, then this is not computed.
    synthetic_diff_intensity_ratio = models.FloatField(blank=True, null=True)

    #: CUBAM score for final decision
    darker_score = models.FloatField(blank=True, null=True)

    #: method used to set the ``darker`` field (admin, CUBAM, or majority vote)
    darker_method = models.CharField(
        max_length=1, choices=ResultBase.QUALITY_METHODS, blank=True, null=True)

    #: This is equal to ``darker == "E"``.  This is used for aggregating votes
    #: with CUBAM.
    reflectance_eq = models.NullBooleanField()

    #: CUBAM score for ``reflectance_eq``
    reflectance_eq_score = models.FloatField(blank=True, null=True)

    #: boolean version of the answer: if ``True``, the darker pixel
    #: has a darker reflectance
    reflectance_dd = models.NullBooleanField()

    #: CUBAM score for ``reflectance_dd``
    reflectance_dd_score = models.FloatField(blank=True, null=True)

    #: List of all ForeignKey relationships that are accessed during a mturk
    #: task.  (called by mturk.utils.get_content_model_prefetch)
    MTURK_PREFETCH = ('photo', 'point1', 'point2')

    class Meta:
        ordering = ['photo', '-darker_score']
        unique_together = (("point1", "point2"))

    def __unicode__(self):
        if self.darker:
            return self.darker_to_str[self.darker]
        else:
            return 'Unknown'

    def expanded_bbox(self):
        if not hasattr(self, '_expanded_bbox'):
            x1, y1 = self.point1.x * self.photo.aspect_ratio, self.point1.y
            x2, y2 = self.point2.x * self.photo.aspect_ratio, self.point2.y
            if x2 < x1:
                x1, x2 = x2, x1
            if y2 < y1:
                y1, y2 = y2, y1
            self._expanded_bbox = [x1 - 0.1, x2 + 0.1, y1 - 0.1, y2 + 0.1]
        return self._expanded_bbox

    def view_box_svg(self):
        bb = self.expanded_bbox()
        return "%s %s %s %s" % (bb[0], bb[2], bb[1] - bb[0], bb[3] - bb[2])

    def view_box_aspect(self):
        bb = self.expanded_bbox()
        return (bb[1] - bb[0]) / (bb[3] - bb[2])

    def save(self, *args, **kwargs):
        if (self.point1.photo_id != self.photo_id or
                self.point2.photo_id != self.photo_id):
            raise ValueError("Photo mismatch: %s %s %s" % (
                self.photo_id, self.point1.photo_id, self.point2.photo_id))
        if self.point1_image_darker is None:
            self.point1_image_darker = (
                self.point1.get_lab()[0] < self.point2.get_lab()[0])
        super(IntrinsicPointComparison, self).save(*args, **kwargs)

    def get_entry_dict(self):
        """ Return a dictionary of this model containing just the fields needed
        for javascript rendering.  """
        return {
            'id': self.id,
            'photo': self.photo.get_entry_dict(),
            'point1': self.point1.get_entry_dict(include_photo=False),
            'point2': self.point2.get_entry_dict(include_photo=False),
        }

    def svg_path(self):
        """ Return the svg path attribute """
        radius = self.photo.intrinsic_circle_size()
        p1 = np.array([self.point1.x * self.photo.aspect_ratio, self.point1.y])
        p2 = np.array([self.point2.x * self.photo.aspect_ratio, self.point2.y])
        if self.darker == '1':
            p1, p2 = p2, p1
        v = p2 - p1
        v *= radius / np.linalg.norm(v)
        p = np.array([-v[1], v[0]])
        if not self.darker:
            return 'M %s %s L %s %s' % (
                p1[0] + v[0], p1[1] + v[1],
                p2[0] - v[0], p2[1] - v[1],
            )
        if self.darker == 'E':
            p /= 4
            return 'M %s %s L %s %s L %s %s L %s %s z' % (
                p1[0] - p[0] + v[0], p1[1] - p[1] + v[1],
                p1[0] + p[0] + v[0], p1[1] + p[1] + v[1],
                p2[0] + p[0] - v[0], p2[1] + p[1] - v[1],
                p2[0] - p[0] - v[0], p2[1] - p[1] - v[1],
            )
        elif self.darker in ['1', '2']:
            angle = math.degrees(math.atan2(v[1], v[0]))
            return 'M %s %s A %s %s %s %s %s %s %s L %s %s z' % (
                p1[0] - p[0], p1[1] - p[1],
                radius, radius, angle, 1, 1,
                p1[0] + p[0], p1[1] + p[1],
                p2[0] - v[0], p2[1] - v[1],
            )
        else:
            return ''

    def svg_style(self):
        """ Return the svg style attribute """
        if self.darker:
            if self.darker_method == 'A':
                return 'fill: #49D36A'
            else:
                t = np.clip(abs(self.darker_score), 0, 1)
                lab0 = np.array([40, 30, -70])
                lab1 = np.array([70, 30, 70])
                lab = t * lab0 + (1 - t) * lab1
                rgb = np.clip(LabColor(*lab).convert_to('rgb').get_value_tuple(), 0, 255)
                return 'fill: #%02x%02x%02x' % tuple(rgb.astype(int))
        else:
            return ""

    def synthetic_darker(self, thresh=0.05):
        """ Return the expected/true judgement for a given threshold """

        # synthetic_diff_intensity_ratio: point1/point2
        if self.synthetic_diff_intensity_ratio is None:
            return None
        ratio = max(self.synthetic_diff_intensity_ratio, 1e-10)
        if ratio > 1.0 + thresh:
            return "2"
        elif (1.0 / ratio) > 1.0 + thresh:
            return "1"
        else:
            return "E"

    def darker_score_perc(self):
        """ Return the darker score as an int percentage """
        return int(self.darker_score * 100)

    @staticmethod
    def get_thumb_template():
        """ Return the template path for rendering thumbnails """
        return 'intrinsic/comparison_thumb.html'

    def comparison(self):
        """ Return ``self`` (helper for templates) """
        return self

class IntrinsicPointComparisonResponse(ResultBase):

    """ A user's answer to the comparison """

    comparison = models.ForeignKey(
        IntrinsicPointComparison, related_name='responses')

    #: '1': 1 is darker
    #: '2': 2 is darker
    #: 'E': the points are equal
    darker = models.CharField(
        max_length=1, choices=IntrinsicPointComparison.DARKER_CHOICES)

    #: 0: guessing, 1: probably, 2: definitely
    confidence = models.IntegerField()

    #: boolean version of the answer: stores whether ``darker == "E"``.  This
    #: is used for aggregating votes with CUBAM.
    reflectance_eq = models.BooleanField(default=False)

    #: boolean version of the answer: if ``True``, the darker pixel
    #: has a darker reflectance
    reflectance_dd = models.NullBooleanField()

    def __unicode__(self):
        if self.darker:
            return IntrinsicPointComparison.darker_to_str[self.darker]
        else:
            return 'Unknown'

    @staticmethod
    def mturk_grade_test(test_content_wrappers, test_contents, results):
        responses = [
            results[unicode(comparison.id)]['darker']
            for comparison in test_contents
        ]
        # Note that if darker_method != 'A' (set by admin), then CUBAM can
        # change it, so we just give it to them for free
        responses_correct = [
            (not comparison.darker or comparison.darker_method != 'A' or
             bool(unicode(responses[i]) == unicode(comparison.darker)))
            for i, comparison in enumerate(test_contents)
        ]
        return responses, responses_correct

    @staticmethod
    def mturk_submit(user, hit_contents, results, time_ms, time_active_ms, version,
                     mturk_assignment=None, **kwargs):

        if unicode(version) != u'1.0':
            raise ValueError(
                "Unknown version: '%s' (type: %s)" % (version, type(version)))

        new_objects = {}
        for comparison in hit_contents:
            key = unicode(comparison.id)
            darker = results[key][u'darker']
            confidence = results[key][u'confidence']

            reflectance_dd = None
            if darker == "E":
                reflectance_eq = True
            else:
                reflectance_eq = False
                if comparison.point1_image_darker is not None:
                    if darker == "1":
                        reflectance_dd = comparison.point1_image_darker
                    elif darker == "2":
                        reflectance_dd = not comparison.point1_image_darker

            new_obj, created = comparison.responses.get_or_create(
                user=user,
                mturk_assignment=mturk_assignment,
                time_ms=time_ms[key],
                time_active_ms=time_active_ms[key],
                darker=darker,
                confidence=confidence,
                reflectance_eq=reflectance_eq,
                reflectance_dd=reflectance_dd,
            )

            if created:
                new_objects[get_content_tuple(comparison)] = [new_obj]

        # clear any existing aggregation
        IntrinsicPointComparison.objects \
            .filter(id__in=[c.id for c in hit_contents]) \
            .update(darker=None, darker_score=None,
                    reflectance_dd=None, reflectance_dd_score=None,
                    reflectance_eq=None, reflectance_eq_score=None)

        return new_objects

    @staticmethod
    def get_thumb_template():
        return 'intrinsic/comparison_response_thumb.html'

class IntrinsicImagesAlgorithm(EmptyModelBase):
    """ An algorithm together with the parameters used """

    #: url-friendly name of the algorithm
    slug = models.CharField(max_length=128)

    #: citation for this algorithm
    citation = models.ForeignKey(PaperCitation, blank=True, null=True)

    #: json-encoded dictionary of the parameters used
    parameters = models.TextField(blank=True)

    #: if ``True``, this algorithm is a simple non-algorithm
    baseline = models.BooleanField(default=False)

    #: if ``True``, this is part of the current set of comparisons
    active = models.BooleanField(default=True)

    #: counter to keep track of stale jobs; this is only used during scheduling
    #: of decompositions.  Increment this number to invalidate current tasks.
    task_version = models.IntegerField(default=0)

    #: mean error across all IIW ("Intrinsic Images in the Wild" paper) images
    iiw_mean_error = models.FloatField(blank=True, null=True)

    #: mean runtime across all IIW ("Intrinsic Images in the Wild" paper) images
    iiw_mean_runtime = models.FloatField(blank=True, null=True)

    #: is this the best parameter setting for this slug?
    iiw_best = models.BooleanField(default=False)

    def iiw_mean_error_pretty(self):
        return '%.1f%%' % (100.0 * self.iiw_mean_error)

    def parameters_pretty_html(self):
        """ Render the parameters as HTML """

        params = json.loads(self.parameters)
        if not params:
            return "None"
        ret = []
        for p, v in params.iteritems():
            #if p == "remap_gamma_2_2":
                #continue
            if isinstance(v, (float, int, long)):
                for exp in xrange(2, 20):
                    if abs(v - 10 ** exp) < 1e-6:
                        v = "10<sup>%s</sup>" % exp
                        break
                else:
                    if isinstance(v, float):
                        v = round(v, max(0, 2 + int(-math.log10(v))))
            ret.append("%s: %s" % (p.replace('_', ' '), v))
        return "<ul><li>%s</ul>" % "<li>".join(sorted(ret))

    def save(self, *args, **kwargs):
        # make sure that json-formatted string is sorted by keys
        if self.parameters:
            self.parameters = json.dumps(
                json.loads(self.parameters), sort_keys=True)
        super(IntrinsicImagesAlgorithm, self).save(*args, **kwargs)

    class Meta:
        unique_together = (('slug', 'parameters'))

class IntrinsicImagesDecomposition(EmptyModelBase):
    """ Result of running an intrinisc images algorithm """

    #: input photograph
    photo = models.ForeignKey(
        Photo, related_name='intrinsic_images_decompositions')

    #: algorithm used
    algorithm = models.ForeignKey(
        IntrinsicImagesAlgorithm, related_name='intrinsic_images_decompositions')

    #: time taken to run in seconds
    runtime = models.FloatField(blank=True, null=True)

    #: algorithm output
    reflectance_image = models.ImageField(
        upload_to='intrinsic_reflectance',
        null=True, blank=True, max_length=255,
        storage=STORAGE,
    )

    #: recomputed shading image from the algorithm output
    shading_image = models.ImageField(
        upload_to='intrinsic_shading',
        null=True, blank=True, max_length=255,
        storage=STORAGE,
    )

    def reset_error(self, save=True):
        for attr in IntrinsicImagesDecomposition.ERROR_ATTRS:
            setattr(self, attr, None)
        self.error_comparison_thresh = None
        if save:
            self.save()

    def mean_error_pretty(self):
        return '%.1f%%' % (100.0 * self.mean_error)

    #: attributes on this instance corresponding to an error measure
    ERROR_ATTRS = [
        'mean_dense_error', 'mean_sparse_error', 'mean_error',
        'mean_eq_error', 'mean_neq_error', 'mean_sum_error',
    ]

    #: error of this across all comparisons
    mean_error = models.FloatField(blank=True, null=True)

    #: error of this across all dense comparisons
    mean_dense_error = models.FloatField(blank=True, null=True)

    #: error of this across all sparse comparisons
    mean_sparse_error = models.FloatField(blank=True, null=True)

    #: error of this decomposition from equality
    mean_eq_error = models.FloatField(blank=True, null=True)

    #: error of this decomposition from inequality
    mean_neq_error = models.FloatField(blank=True, null=True)

    #: sum of error types
    mean_sum_error = models.FloatField(blank=True, null=True)

    #: number of comparisons used in evaluating all errors
    num = models.IntegerField(blank=True, null=True)

    #: number of comparisons used in evaluating the dense error
    num_dense = models.IntegerField(blank=True, null=True)

    #: number of comparisons used in evaluating the dense error
    num_sparse = models.IntegerField(blank=True, null=True)

    #: number of comparisons used in evaluating the equality error
    num_eq = models.IntegerField(blank=True, null=True)

    #: number of comparisons used in evaluating the inequality error
    num_neq = models.IntegerField(blank=True, null=True)

    #: threshold used in the comparison error metric
    error_comparison_thresh = models.FloatField(blank=True, null=True)

    class Meta:
        unique_together = (('photo', 'algorithm'))
        ordering = ['mean_error', '-id']
        #ordering = ['-id']


# ----------------------------------------------------------------------------
# cocos2d
# Copyright (c) 2008 Daniel Moisset, Ricardo Quesada, Rayentray Tappa, Lucio Torre
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#   * Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer.
#   * Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in
#     the documentation and/or other materials provided with the
#     distribution.
#   * Neither the name of cocos2d nor the names of its
#     contributors may be used to endorse or promote products
#     derived from this software without specific prior written
#     permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
# ----------------------------------------------------------------------------
'''Particle system engine'''

import random
import pyglet
from pyglet.gl import *
import math
import copy
import numpy
import ctypes

from cocosnode import CocosNode
from euclid import Point2

rand = lambda: random.random() * 2 - 1

# PointerToNumpy by Gary Herron
# from pyglet's user list
def PointerToNumpy(a, ptype=ctypes.c_float):
    a = numpy.ascontiguousarray(a)           # Probably a NO-OP, but perhaps not
    return a.ctypes.data_as(ctypes.POINTER(ptype)) # Ugly and undocumented!

class Color( object ):
    def __init__( self, r,g,b,a ):
        self.r = r
        self.g = g
        self.b = b
        self.a = a

    def to_array(self):
        return self.r, self.g, self.b, self.a

class ParticleSystem( CocosNode ):
    #: is the particle system active ?
    active = True

    #: duration in seconds of the system. -1 is infinity
    duration = 0

    #: time elapsed since the start of the system (in seconds)
    elapsed = 0

    #: Gravity of the particles
    gravity = Point2(0.0, 0.0)

    #: position is from "superclass" CocosNode
    #: Position variance
    pos_var = Point2(0.0, 0.0)

    #: The angle (direction) of the particles measured in degrees
    angle = 0.0
    #: Angle variance measured in degrees;
    angle_var = 0.0

    #: The speed the particles will have.
    speed = 0.0
    #: The speed variance
    speed_var = 0.0

    #: Tangential acceleration
    tangential_accel = 0.0
    #: Tangential acceleration variance
    tangential_accel_var = 0.0

    #: Radial acceleration
    radial_accel = 0.0
    #: Radial acceleration variance
    radial_accel_var = 0.0

    #: Size of the particles
    size = 0.0
    #: Size variance
    size_var = 0.0

    #: How many seconds will the particle live
    life = 0
    #: Life variance
    life_var = 0

    #: Start color of the particles
    start_color = Color(0.0,0.0,0.0,0.0)
    #: Start color variance
    start_color_var = Color(0.0,0.0,0.0,0.0)
    #: End color of the particles
    end_color = Color(0.0,0.0,0.0,0.0)
    #: End color variance
    end_color_var = Color(0.0,0.0,0.0,0.0)

    #: Maximum particles
    total_particles = 0

    #:texture of the particles
    texture = pyglet.resource.image('fire.png').texture

    #:blend additive
    blend_additive = False

    #:color modulate
    color_modulate = True

    def __init__(self):
        super(ParticleSystem,self).__init__()

        # particles
        # position x 2
        self.particle_pos = numpy.zeros( (self.total_particles, 2), numpy.float32 )
        # direction x 2
        self.particle_dir = numpy.zeros( (self.total_particles, 2), numpy.float32 )
        # rad accel x 1
        self.particle_rad = numpy.zeros( (self.total_particles, 1), numpy.float32 )
        # tan accel x 1
        self.particle_tan = numpy.zeros( (self.total_particles, 1), numpy.float32 )
        # gravity x 2
        self.particle_grav = numpy.zeros( (self.total_particles, 2), numpy.float32 )
        # colors x 4
        self.particle_color = numpy.zeros( (self.total_particles, 4), numpy.float32 )
        # delta colors x 4
        self.particle_delta_color = numpy.zeros( (self.total_particles, 4), numpy.float32 )
        # life x 1
        self.particle_life = numpy.zeros( (self.total_particles, 1), numpy.float32 )
        self.particle_life.fill(-1.0)
        # size x 1
        self.particle_size = numpy.zeros( (self.total_particles, 1), numpy.float32 )

        #: How many particles can be emitted per second
        self.emit_counter = 0

        #: Count of particles
        self.particle_count = 0

        self.schedule( self.step )

    def on_enter( self ):
        super( ParticleSystem, self).on_enter()
        self.add_particle()

    def draw( self ):
        glPushMatrix()
        self.transform()

        glPointSize( self.size )

        glEnable(GL_TEXTURE_2D)
        glBindTexture(GL_TEXTURE_2D, self.texture.id )

        glEnable(GL_POINT_SPRITE)
        glTexEnvi( GL_POINT_SPRITE, GL_COORD_REPLACE, GL_TRUE )

        glEnableClientState(GL_VERTEX_ARRAY)
        vertex_ptr = PointerToNumpy( self.particle_pos )
        glVertexPointer(2,GL_FLOAT,0,vertex_ptr);

        glEnableClientState(GL_COLOR_ARRAY)
        color_ptr = PointerToNumpy( self.particle_color)
        glColorPointer(4,GL_FLOAT,0,color_ptr);

        glPushAttrib(GL_COLOR_BUFFER_BIT)
        glEnable(GL_BLEND)
        if self.blend_additive:
            glBlendFunc(GL_SRC_ALPHA, GL_ONE);
        else:
            glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);

#        mode = GLint()
#        glTexEnviv( GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, mode )
#
#        if self.color_modulate:
#            glTexEnvi( GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_MODULATE )
#        else:
#            glTexEnvi( GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_REPLACE )

        glDrawArrays(GL_POINTS, 0, self.total_particles);

        # un -blend
        glPopAttrib()

#        # restore env mode
#        glTexEnvi( GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, mode)

        # disable states
        glDisableClientState(GL_COLOR_ARRAY);
        glDisableClientState(GL_VERTEX_ARRAY);
        glDisable(GL_POINT_SPRITE);
        glDisable(GL_TEXTURE_2D);

        glPopMatrix()

    def step( self, delta ):
        if self.active:
            rate = 1.0 / self.emission_rate
            self.emit_counter += delta

#            if random.random() < 0.01:
#                delta += 0.5

            self.particle_count = sum( self.particle_life >= 0 )

            while self.particle_count < self.total_particles and self.emit_counter > rate:
                self.add_particle()
                self.emit_counter -= rate

            self.elapsed += delta

            if self.duration != -1 and self.duration < self.elapsed:
                self.stop_system()

        self.update_particles( delta )

    def add_particle( self ):
        self.init_particle()
        self.particle_count += 1

    def stop_system( self ):
        self.active = False
        self.elapsed= self.duration
        self.emit_counter = 0

    def reset_system( self ):
        self.elapsed= self.duration
        self.emit_counter = 0

    def update_particles( self, delta ):
        # radial: posx + posy
        norm = numpy.sqrt( self.particle_pos[:,0] ** 2 + self.particle_pos[:,1] ** 2 )
        # XXX prevent div by 0
        norm = numpy.select( [norm==0], [0.0000001], default=norm )
        posx = self.particle_pos[:,0] / norm
        posy = self.particle_pos[:,1] / norm

        radial = numpy.array( [posx, posy] )
        tangential = numpy.array( [-posy, posx] )

        # update dir
        radial = numpy.swapaxes(radial,0,1)
        radial *= self.particle_rad
        tangential = numpy.swapaxes(tangential,0,1)
        tangential *= self.particle_tan

        self.particle_dir +=  (tangential + radial + self.particle_grav) * delta

        # update pos with updated dir
        self.particle_pos += self.particle_dir * delta

        # life
        self.particle_life -= delta

        # color
        self.particle_color += self.particle_delta_color * delta

        # if life < 0, set alpha in 0
        self.particle_color[:,3] = numpy.select( [self.particle_life[:,0] < 0], [0], default=self.particle_color[:,3] )

#        print self.particles[0]
#        print self.pas[0,0:4]

    def init_particle( self ):
        # position
#        p=self.particles[idx]

        a = self.particle_life < 0
        idxs = a.nonzero()

        idx = -1

        if len(idxs[0]) > 0:
            idx = idxs[0][0]
        else:
            raise Exception("No empty particle")

        # position
        self.particle_pos[idx][0] = self.pos_var.x * rand()
        self.particle_pos[idx][1] = self.pos_var.y * rand()

        a = math.radians( self.angle + self.angle_var * rand() )
        v = Point2( math.cos( a ), math.sin( a ) )
        s = self.speed + self.speed_var * rand()

        dir = v * s

        # direction
        self.particle_dir[idx][0] = dir.x
        self.particle_dir[idx][1] = dir.y

        # radial accel
        self.particle_rad[idx] = self.radial_accel + self.radial_accel_var * rand()

        # tangential accel
        self.particle_tan[idx] = self.tangential_accel + self.tangential_accel_var * rand()

        # life
        life = self.particle_life[idx] = self.life + self.life_var * rand()

        # Color
        # start
        sr = self.start_color.r + self.start_color_var.r * rand()
        sg = self.start_color.g + self.start_color_var.g * rand()
        sb = self.start_color.b + self.start_color_var.b * rand()
        sa = self.start_color.a + self.start_color_var.a * rand()

        self.particle_color[idx][0] = sr
        self.particle_color[idx][1] = sg
        self.particle_color[idx][2] = sb
        self.particle_color[idx][3] = sa

        # end
        er = self.end_color.r + self.end_color_var.r * rand()
        eg = self.end_color.g + self.end_color_var.g * rand()
        eb = self.end_color.b + self.end_color_var.b * rand()
        ea = self.end_color.a + self.end_color_var.a * rand()

        delta_color_r = (er - sr) / life
        delta_color_g = (eg - sg) / life
        delta_color_b = (eb - sb) / life
        delta_color_a = (ea - sa) / life

        self.particle_delta_color[idx][0] = delta_color_r
        self.particle_delta_color[idx][1] = delta_color_g
        self.particle_delta_color[idx][2] = delta_color_b
        self.particle_delta_color[idx][3] = delta_color_a

        # size
        self.particle_size[idx] = self.size + self.size_var * rand()

        # gravity
        self.particle_grav[idx][0] = self.gravity.x
        self.particle_grav[idx][1] = self.gravity.y


""" This generates the SB at the end! """

import numpy as np

from aleph.consts import *
from reamber.algorithms.generate.sv.generators.svOsuMeasureLineMD import svOsuMeasureLineMD, SvOsuMeasureLineEvent

from reamber.osu.OsuMap import OsuMap
from math import pi

RED = "red.jpg"
BLUE = "blue.jpg"
GREEN = "green.jpg"

START = 375444
END = 383124 - 100
DURATION = END - START
EVENT_DURATION = 25

np.random.seed(0)

def EVENT(img, start, duration, opacity, positionX, positionY, scale):
    end = start + duration

    return f"Sprite,Overlay,Centre,\"{img}\",320,240\n"\
           f" V,0,{int(start)},{int(end)},{3.5},{scale}\n"\
           f" F,0,{int(start)},{int(end)},{opacity},{opacity}\n"\
           f" M,0,{int(start)},{int(end)},{positionX},{positionY}\n"

def sb(osbPath: str):
    with open(osbPath, "w") as f:
        f.write("[Events]\n"
                "//Background and Video events\n"
                "//Storyboard Layer 0 (Background)\n"
                "//Storyboard Layer 1 (Fail)\n"
                "//Storyboard Layer 2 (Pass)\n"
                "//Storyboard Layer 3 (Foreground)\n"
                "//Storyboard Layer 4 (Overlay)\n"
                ""
                "Sprite,Overlay,Centre,\"black.jpg\",320,240\n"
                " S,0,0,5000,3.158545\n"
                " F,1,1000,5000,1,0\n"
                "Sprite,Overlay,Centre,\"black.jpg\",320,240\n"
                " M,0,375444,,300.8,235.3454\n"
                " S,0,375444,,3.158545\n"
                " F,0,375444,383124,0,1\n"
                " F,0,383124,392000,1\n"
                "Sprite,Overlay,Centre,\"creditTexture.jpg\",320,240\n"
                " M,1,383124,392000,320,230,310,270\n"
                " F,0,383124,392000,0.8,0.6\n"
                " S,0,385049,,0.2087273\n"
                "Sprite,Overlay,Centre,\"leaf.png\",320,240\n"
                " S,0,383124,,0.1\n"
                " M,0,383124,392000,320,205\n"
                "Sprite,Overlay,Centre,\"eve.png\",320,240\n"
                " F,0,383124,,1\n"
                " S,0,383124,,0.18\n"
                " M,0,383124,392000,320,275\n"
                "Sprite,Overlay,Centre,\"credits.png\",320,240\n"
                " F,0,383124,,0.7\n"
                " S,0,383124,,0.4\n"
                " M,0,383124,392000,321,330\n")

        for r in np.sin(np.random.rand(50) * pi / 2) ** 0.25:
            start = DURATION * r + START
            duration = EVENT_DURATION / r

            f.write(EVENT(np.random.choice([RED, GREEN, BLUE]), start, duration,
                          np.random.rand() ** 2 * 1 / r,
                          2 * (np.random.rand() ** 0.2 - 0.5) * 500,
                          2 * (np.random.rand() - 0.5) * 500,
                          np.random.rand() ** 2 * 4
                          ))

        f.write(
                "//Storyboard Sound Samples\n")

# Redacted Path
osbPath = f"LeaF - Aleph-0 (extended ver)/LeaF - Aleph-0 (extended ver.) (Evening).osb"

sb(osbPath)


"""
    worldview.py
"""
from __future__ import absolute_import, division, print_function, unicode_literals
from collections import deque
import logging
import math
import time
import itertools

from OpenGL import GL
from PySide import QtCore, QtGui
from PySide.QtCore import Qt

from PySide.QtOpenGL import QGLWidget
import numpy

from mcedit2.rendering import worldscene, loadablechunks, sky, compass
from mcedit2.rendering.chunknode import ChunkNode
from mcedit2.rendering.frustum import Frustum
from mcedit2.rendering.geometrycache import GeometryCache
from mcedit2.rendering.layers import Layer
from mcedit2.rendering.scenegraph.matrix import MatrixState, Ortho
from mcedit2.rendering.scenegraph.misc import ClearNode
from mcedit2.rendering.scenegraph.scenenode import Node
from mcedit2.rendering.scenegraph.vertex_array import VertexNode
from mcedit2.rendering.textureatlas import TextureAtlas
from mcedit2.rendering.vertexarraybuffer import VertexArrayBuffer
from mcedit2.rendering.scenegraph import scenenode, rendernode
from mcedit2.util import profiler, raycast
from mcedit2.util.mimeformats import MimeFormats
from mcedit2.util.qglcontext import validateWidgetQGLContext
from mcedit2.util.settings import Settings
from mcedit2.widgets.infopanel import InfoPanel
from mceditlib import faces
from mceditlib.geometry import Vector, Ray
from mceditlib.exceptions import LevelFormatError, ChunkNotPresent
from mceditlib.util import displayName

log = logging.getLogger(__name__)

def worldMeshVertexSize(worldMesh):
    """
    xxx what is this for??

    :type worldMesh: WorldScene
    """

    def bufferSizes():
        for cm in worldMesh.chunkGroupNode.children:
            assert isinstance(cm, ChunkNode)
            for bm in cm.getChunkVertexNodes():
                assert isinstance(bm, VertexNode)
                for va in bm.vertexArrays:
                    assert isinstance(va, VertexArrayBuffer)
                    yield va.buffer.nbytes

    return sum(bufferSizes())

def anglesToVector(yaw, pitch):
    def nanzero(x):
        if math.isnan(x):
            return 0
        else:
            return x

    dx = -math.sin(math.radians(yaw)) * math.cos(math.radians(pitch))
    dy = -math.sin(math.radians(pitch))
    dz = math.cos(math.radians(yaw)) * math.cos(math.radians(pitch))
    return Vector(*map(nanzero, [dx, dy, dz]))

THREADED_BUFFER_SWAP = False

class BufferSwapper(QtCore.QObject):
    def __init__(self, view):
        super(BufferSwapper, self).__init__()
        self.view = view
        self.swapDone.connect(view.swapDone)

    swapDone = QtCore.Signal()

    def swap(self):
        self.view.makeCurrent()
        self.view.swapBuffers()
        self.view.doneCurrent()
        self.swapDone.emit()

class WorldView(QGLWidget):
    """
    Superclass for the following views:

    IsoWorldView: Display the world using an isometric viewing angle, without perspective.
    Click and drag to pan the viewing area. Use the mouse wheel or a UI control to zoom.

    CameraWorldView: Display the world using a first-person viewing angle with perspective.
    Click and drag to pan the camera. Use WASD or click and drag to move the camera.

    CutawayWorldView: Display a single slice of the world. Click and drag to move sideways.
    Use the mouse wheel or a UI widget to move forward or backward. Use a UI widget to zoom.

    FourUpWorldView: Display up to four other world views at once. Default to three cutaways
    and one isometric view.

    """
    viewportMoved = QtCore.Signal(QtGui.QWidget)
    cursorMoved = QtCore.Signal(QtGui.QMouseEvent)

    urlsDropped = QtCore.Signal(QtCore.QMimeData, Vector, faces.Face)
    mapItemDropped = QtCore.Signal(QtCore.QMimeData, Vector, faces.Face)

    mouseBlockPos = Vector(0, 0, 0)
    mouseBlockFace = faces.FaceYIncreasing

    doSwapBuffers = QtCore.Signal()

    def __init__(self, dimension, textureAtlas=None, geometryCache=None, sharedGLWidget=None):
        """

        :param dimension:
        :type dimension: WorldEditorDimension
        :param textureAtlas:
        :type textureAtlas: TextureAtlas
        :param geometryCache:
        :type geometryCache: GeometryCache
        :param sharedGLWidget:
        :type sharedGLWidget: QGLWidget
        :return:
        :rtype:
        """
        QGLWidget.__init__(self, shareWidget=sharedGLWidget)
        self.dimension = None
        self.worldScene = None
        self.loadableChunksNode = None
        self.textureAtlas = None

        validateWidgetQGLContext(self)

        self.bufferSwapDone = True

        if THREADED_BUFFER_SWAP:
            self.setAutoBufferSwap(False)
            self.bufferSwapThread = QtCore.QThread()
            self.bufferSwapper = BufferSwapper(self)
            self.bufferSwapper.moveToThread(self.bufferSwapThread)
            self.doSwapBuffers.connect(self.bufferSwapper.swap)
            self.bufferSwapThread.start()

        self.setAcceptDrops(True)
        self.setSizePolicy(QtGui.QSizePolicy.Policy.Expanding, QtGui.QSizePolicy.Policy.Expanding)
        self.setFocusPolicy(Qt.ClickFocus)

        self.layerToggleGroup = LayerToggleGroup()
        self.layerToggleGroup.layerToggled.connect(self.setLayerVisible)

        self.mouseRay = Ray(Vector(0, 1, 0), Vector(0, -1, 0))

        self.setMouseTracking(True)

        self.lastAutoUpdate = time.time()
        self.autoUpdateInterval = 0.5  # frequency of screen redraws in response to loaded chunks

        self.compassNode = self.createCompass()
        self.compassOrtho = Ortho((1, float(self.height()) / self.width()))
        self.compassNode.addState(self.compassOrtho)

        self.viewActions = []
        self.pressedKeys = set()

        self.setTextureAtlas(textureAtlas)

        if geometryCache is None and sharedGLWidget is not None:
            geometryCache = sharedGLWidget.geometryCache
        if geometryCache is None:
            geometryCache = GeometryCache()
        self.geometryCache = geometryCache

        self.worldNode = None
        self.skyNode = None
        self.overlayNode = scenenode.Node("WorldView Overlay")

        self.sceneGraph = None
        self.renderGraph = None

        self.frameSamples = deque(maxlen=500)
        self.frameSamples.append(time.time())

        self.cursorNode = None

        self.setDimension(dimension)

    def waitForSwapThread(self):
        while not self.bufferSwapDone:
            QtGui.QApplication.processEvents(QtCore.QEventLoop.ExcludeUserInputEvents)

    def dealloc(self):
        log.info("Deallocating GL resources for worldview %s", self)
        if THREADED_BUFFER_SWAP:
            self.waitForSwapThread()
            self.bufferSwapThread.quit()
        self.makeCurrent()
        self.renderGraph.dealloc()

    def __str__(self):
        try:
            if self.dimension:
                dimName = displayName(self.dimension.worldEditor.filename) + ": " + self.dimension.dimName
            else:
                dimName = "None"
        except Exception as e:
            return "%s trying to get node name" % e
        return "%s(%r)" % (self.__class__.__name__, dimName)

    # --- Displayed world ---

    def setDimension(self, dimension):
        """

        :param dimension:
        :type dimension: WorldEditorDimension
        :return:
        :rtype:
        """
        log.info("Changing %s to dimension %s", self, dimension)
        self.dimension = dimension
        self.waitForSwapThread()
        self.makeCurrent()
        if self.renderGraph:
            self.renderGraph.dealloc()
        self.sceneGraph = self.createSceneGraph()
        self.renderGraph = rendernode.createRenderNode(self.sceneGraph)
        self.resetLoadOrder()
        self.update()

    def setTextureAtlas(self, textureAtlas):
        self.textureAtlas = textureAtlas
        if self.worldScene:
            self.worldScene.setTextureAtlas(textureAtlas)

        if textureAtlas is not None:
            self.waitForSwapThread()
            self.makeCurrent()
            textureAtlas.load()
            self.resetLoadOrder()

    # --- Graph construction ---

    def createCompass(self):
        return compass.CompassNode()

    def createWorldScene(self):
        return worldscene.WorldScene(self.dimension, self.textureAtlas, self.geometryCache)

    def createSceneGraph(self):
        sceneGraph = scenenode.Node("WorldView SceneGraph")
        self.worldScene = self.createWorldScene()
        self.worldScene.setVisibleLayers(self.layerToggleGroup.getVisibleLayers())

        clearNode = ClearNode()
        self.skyNode = sky.SkyNode()
        self.loadableChunksNode = loadablechunks.LoadableChunksNode(self.dimension)

        self.worldNode = Node("World Container")
        self.matrixState = MatrixState()
        self.worldNode.addState(self.matrixState)
        self._updateMatrices()

        self.worldNode.addChild(self.loadableChunksNode)
        self.worldNode.addChild(self.worldScene)
        self.worldNode.addChild(self.overlayNode)

        sceneGraph.addChild(clearNode)
        sceneGraph.addChild(self.skyNode)
        sceneGraph.addChild(self.worldNode)
        sceneGraph.addChild(self.compassNode)
        if self.cursorNode:
            self.worldNode.addChild(self.cursorNode)

        return sceneGraph

    # --- Tool support ---

    def setToolCursor(self, cursorNode):
        if self.cursorNode:
            self.worldNode.removeChild(self.cursorNode)
        self.cursorNode = cursorNode
        if cursorNode:
            self.worldNode.addChild(cursorNode)

    def setToolOverlays(self, overlayNodes):
        self.overlayNode.clear()
        for node in overlayNodes:
            self.overlayNode.addChild(node)

    # --- View settings ---

    def setLayerVisible(self, layerName, visible):
        self.worldScene.setLayerVisible(layerName, visible)
        self.resetLoadOrder()

    def setDayTime(self, value):
        if self.skyNode:
            self.skyNode.setDayTime(value)

    def _updateMatrices(self):
        self.updateMatrices()
        self.updateFrustum()
        #min = self.unprojectPoint(0, 0)[0]
        #max = self.unprojectPoint(self.width(), self.height())[0]
        #self.visibleBox = BoundingBox(min, (0, 0, 0)).union(BoundingBox(max, (0, 0, 0)))

    def updateMatrices(self):
        """
        Subclasses must implement updateMatrices to set the projection and modelview matrices.

        Should set self.worldNode.projection and self.worldNode.modelview
        """
        raise NotImplementedError

    def updateFrustum(self):
        matrix = self.matrixState.projection * self.matrixState.modelview
        self.frustum = Frustum.fromViewingMatrix(numpy.array(matrix.data()))

    def getViewCorners(self):
        """
        Returns corners:
            bottom left, near
            bottom left, far
            top left, near
            top left, far
            bottom right, near
            bottom right, far
            top right, near
            top right, far

        :return:
        :rtype: list[QVector4D]
        """
        corners = [QtGui.QVector4D(x, y, z, 1.)
                   for x, y, z in itertools.product((-1., 1.), (-1., 1.), (0., 1.))]

        matrix = self.matrixState.projection * self.matrixState.modelview
        matrix, inverted = matrix.inverted()
        worldCorners = [matrix.map(corner) for corner in corners]
        worldCorners = [Vector(*((corner / corner.w()).toTuple()[:3])) for corner in worldCorners]
        return worldCorners

    def getViewBounds(self):
        """
        Get the corners of the viewing area, intersected with the world's bounds.
        xxx raycast to intersect with terrain height too

        :return:
        :rtype:
        """
        corners = self.getViewCorners()
        # Convert the 4 corners into rays extending from the near point, then interpolate each ray at the
        # current dimension's height limits
        pairs = []
        for i in range(0, 8, 2):
            pairs.append(corners[i:i+2])

        rays = [Ray.fromPoints(p1, p2) for p1, p2 in pairs]
        bounds = self.dimension.bounds
        pointPairs = [(r.atHeight(bounds.maxy), r.atHeight(bounds.miny)) for r in rays]

        return sum(pointPairs, ())

    scaleChanged = QtCore.Signal(float)

    _scale = 1. / 16

    @property
    def scale(self):
        return self._scale

    @scale.setter
    def scale(self, value):
        self._scale = value
        self._updateMatrices()
        log.debug("update(): scale %s %s", self, value)
        self.update()
        self.scaleChanged.emit(value)
        self.viewportMoved.emit(self)

    _centerPoint = Vector(0, 0, 0)

    @property
    def centerPoint(self):
        return self._centerPoint

    @centerPoint.setter
    def centerPoint(self, value):
        value = Vector(*value)
        if value != self._centerPoint:
            self._centerPoint = value
            self._updateMatrices()
            log.debug("update(): centerPoint %s %s", self, value)
            self.update()
            self.resetLoadOrder()
            self.viewportMoved.emit(self)

    def centerOnPoint(self, pos, distance=None):
        """Center the view on the given position"""
        # delta = self.viewCenter() - self.centerPoint
        # self.centerPoint = pos - delta
        self.centerPoint = pos
        self.update()

    def viewCenter(self):
        """Return the world position at the center of the view."""
        #return self.unprojectAtHeight(self.width() / 2, self.height() / 2, 64.)
        # ray = self.rayAtPosition(self.width() / 2, self.height() / 2)
        # try:
        #     point, face = raycast.rayCastInBounds(ray, self.dimension, 600)
        # except (raycast.MaxDistanceError, ValueError):
        #     point = ray.atHeight(0)
        # return point or ray.point
        return self.centerPoint

    # --- Events ---

    def resizeEvent(self, event):
        center = self.viewCenter()
        self.compassOrtho.size = (1, float(self.height()) / self.width())
        super(WorldView, self).resizeEvent(event)
        # log.info("WorldView: resized. moving to %s", center)
        # self.centerOnPoint(center)

    acceptableMimeTypes = [
        MimeFormats.MapItem,
    ]

    def dragEnterEvent(self, event):
        # xxx show drop preview as scene node
        for mimeType in self.acceptableMimeTypes:
            if event.mimeData().hasFormat(mimeType):
                event.acceptProposedAction()
                return
        if event.mimeData().hasUrls():
            event.acceptProposedAction()

    def dropEvent(self, event):
        mimeData = event.mimeData()
        x = event.pos().x()
        y = event.pos().y()
        ray = self.rayAtPosition(x, y)
        dropPosition, face = self.rayCastInView(ray)

        if mimeData.hasFormat(MimeFormats.MapItem):
            self.mapItemDropped.emit(mimeData, dropPosition, face)
        elif mimeData.hasUrls:
            self.urlsDropped.emit(mimeData, dropPosition, face)

    def keyPressEvent(self, event):
        self.augmentKeyEvent(event)
        self.pressedKeys.add(event.key())
        for action in self.viewActions:
            if action.matchKeyEvent(event):
                action.keyPressEvent(event)

    def keyReleaseEvent(self, event):
        self.augmentKeyEvent(event)
        self.pressedKeys.discard(event.key())
        for action in self.viewActions:
            if action.matchKeyEvent(event):
                action.keyReleaseEvent(event)

    def mousePressEvent(self, event):
        self.augmentMouseEvent(event)
        for action in self.viewActions:
            if action.button & event.button():
                if action.matchModifiers(event):
                    action.mousePressEvent(event)

    def mouseMoveEvent(self, event):
        self.augmentMouseEvent(event)
        for action in self.viewActions:
            if not action.button or action.button == event.buttons() or action.button & event.buttons():
                # Important! mouseMove checks event.buttons(), press and release check event.button()
                if action.matchModifiers(event):
                    if not action.key or action.key in self.pressedKeys:
                        action.mouseMoveEvent(event)
        self.cursorMoved.emit(event)
        self.update()

    def mouseReleaseEvent(self, event):
        # Ignore modifiers on mouse release event and send mouse release to any
        # actions that are set to the given button. This handles this series of inputs,
        # for example:  Control Key down, Mouse1 down, Control Key up, Mouse1 up
        self.augmentMouseEvent(event)
        for action in self.viewActions:
            if action.button & event.button():
                action.mouseReleaseEvent(event)

    wheelPos = 0

    def wheelEvent(self, event):
        self.augmentMouseEvent(event)
        for action in self.viewActions:
            if action.acceptsMouseWheel and ((action.modifiers & event.modifiers()) or action.modifiers == event.modifiers()):
                self.wheelPos += event.delta()
                # event.delta reports eighths of a degree. a standard wheel tick is 15 degrees, or 120 eighths.
                # keep count of wheel position and emit an event for each 15 degrees turned.
                # xxx will we ever need sub-click precision for wheel events?
                clicks = 0
                while self.wheelPos >= 120:
                    self.wheelPos -= 120
                    clicks += 1
                while self.wheelPos <= -120:
                    self.wheelPos += 120
                    clicks -= 1

                if action.button == action.WHEEL_UP and clicks > 0:
                    for i in range(abs(clicks)):
                        action.keyPressEvent(event)

                if action.button == action.WHEEL_DOWN and clicks < 0:
                    for i in range(abs(clicks)):
                        action.keyPressEvent(event)

    def augmentMouseEvent(self, event):
        x, y = event.x(), event.y()
        return self.augmentEvent(x, y, event)

    def augmentKeyEvent(self, event):
        globalPos = QtGui.QCursor.pos()
        mousePos = self.mapFromGlobal(globalPos)
        x = mousePos.x()
        y = mousePos.y()

        # xxx fake position of mouse event -- need to use mcedit internal event object already
        event.x = lambda: x
        event.y = lambda: y

        return self.augmentEvent(x, y, event)

    @profiler.function
    def augmentEvent(self, x, y, event):
        ray = self.rayAtPosition(x, y)

        event.ray = ray
        event.view = self

        position, face = self.rayCastInView(ray)

        self.mouseBlockPos = event.blockPosition = position
        self.mouseBlockFace = event.blockFace = face
        self.mouseRay = ray

    # --- OpenGL support ---

    def initializeGL(self, *args, **kwargs):
        GL.glEnableClientState(GL.GL_VERTEX_ARRAY)
        GL.glAlphaFunc(GL.GL_NOTEQUAL, 0)
        GL.glBlendFunc(GL.GL_SRC_ALPHA, GL.GL_ONE_MINUS_SRC_ALPHA)
        GL.glEnable(GL.GL_DEPTH_TEST)

    def resizeGL(self, width, height):
        GL.glViewport(0, 0, width, height)

        self._updateMatrices()

    maxFPS = 45

    @profiler.function
    def glDraw(self, *args, **kwargs):
        if not self.bufferSwapDone:
            return
        frameInterval = 1.0 / self.maxFPS
        if time.time() - self.frameSamples[-1] < frameInterval:
            return
        super(WorldView, self).glDraw(*args, **kwargs)

    shouldRender = True

    def paintGL(self):
        if not self.shouldRender:
            return
        try:
            with profiler.context("paintGL: %s" % self):
                self.frameSamples.append(time.time())
                if self.textureAtlas:
                    self.textureAtlas.update()

                with profiler.context("renderScene"):
                    rendernode.renderScene(self.renderGraph)

            if THREADED_BUFFER_SWAP:
                self.doneCurrent()
                self.bufferSwapDone = False
                self.doSwapBuffers.emit()
        except:
            self.shouldRender = False
            raise

    def swapDone(self):
        self.bufferSwapDone = True

    @property
    def fps(self):
        samples = 3
        if len(self.frameSamples) <= samples:
            return 0.0

        return (samples - 1) / (self.frameSamples[-1] - self.frameSamples[-samples])

    # --- Screen<->world space conversion ---

    def rayCastInView(self, ray):
        try:
            result = raycast.rayCastInBounds(ray, self.dimension, maxDistance=200)
            position, face = result

        except (raycast.MaxDistanceError, ValueError):
            # GL.glReadBuffer(GL.GL_BACK)
            # pixel = GL.glReadPixels(x, self.height() - y, 1, 1, GL.GL_DEPTH_COMPONENT, GL.GL_FLOAT)
            # depth = -1 + 2 * pixel[0, 0]
            # p = self.pointsAtPositions((x, y, depth))[0]
            #
            # face = faces.FaceYIncreasing
            # position = p.intfloor()
            defaultDistance = 200
            position = (ray.point + ray.vector * defaultDistance).intfloor()
            face = faces.FaceUp

        return position, face

    def rayAtPosition(self, x, y):
        """
        Given coordinates in screen space, return a ray in 3D space.

        Parameters:
            x and y are coordinates local to this QWidget

        :rtype: Ray
        """

        p0, p1 = self.pointsAtPositions((x, y, 0.0), (x, y, 1.0))
        return Ray(p0, (p1 - p0).normalize())

    def rayAtCenter(self):
        return self.rayAtPosition(self.width()/2, self.height()/2)

    def pointsAtPositions(self, *screenPoints):
        w = float(self.width())
        h = float(self.height())
        matrix = self.matrixState.projection * self.matrixState.modelview
        inverse, ok = matrix.inverted()

        if not ok or 0 in (w, h):
            return [Vector(0, 0, 0) for i in screenPoints]

        def _unproject():
            for x, y, z in screenPoints:
                x = float(x)
                y = float(y)
                y = h - y
                x = 2 * x / w - 1
                y = 2 * y / h - 1

                def v(p):
                    return Vector(p.x(), p.y(), p.z())

                yield v(inverse.map(QtGui.QVector3D(x, y, z)))

        return list(_unproject())

    def unprojectAtHeight(self, x, y, h):
        """
        Given coordinates in screen space, find the corresponding point in 3D space.

        Like rayAtPosition, but the third parameter is a height value in 3D space.
        """
        ray = self.rayAtPosition(x, y)
        return ray.atHeight(h)

    # --- Chunk loading ---

    _chunkIter = None

    def resetLoadOrder(self):
        self._chunkIter = None

    def makeChunkIter(self):
        x, y, z = self.viewCenter()
        return iterateChunks(x, z, 1 + max(self.width() * self.scale, self.height() * self.scale) // 32)

    def requestChunk(self):
        if self._chunkIter is None:
            self._chunkIter = self.makeChunkIter()
        try:
            for c in self._chunkIter:
                if self.worldScene.wantsChunk(c):
                    return c
        except StopIteration:
            pass

    def wantsChunk(self, c):
        if not self.worldScene.wantsChunk(c):
            return False

        if hasattr(self, 'frustum'):
            point = [
                c[0] * 16 + 8,
                self.dimension.bounds.miny + self.dimension.bounds.height / 2,
                c[1] * 16 + 8,
                1.0
            ]
            return self.frustum.visible1(point=point, radius=self.dimension.bounds.height / 2)

        return True

    def chunkNotPresent(self, cPos):
        self.worldScene.chunkNotPresent(cPos)

    def recieveChunk(self, chunk):
        t = time.time()
        if self.lastAutoUpdate + self.autoUpdateInterval < t:
            self.lastAutoUpdate = t
            log.debug("update(): receivedChunk %s %s", self, chunk)
            self.update()

        with profiler.getProfiler().context("preloadCulling"):
            if hasattr(self, 'frustum'):
                cx, cz = chunk.chunkPosition
                points = [(cx * 16 + 8, h + 8, cz * 16 + 8, 1.0)
                          for h in chunk.sectionPositions()]
                points = numpy.array(points)

                visibleSections = self.frustum.visible(points, radius=8 * math.sqrt(2))
            else:
                visibleSections = None

        return self.worldScene.workOnChunk(chunk, visibleSections)

    def chunkInvalid(self, (cx, cz), deleted):
        self.worldScene.invalidateChunk(cx, cz)
        if deleted:
            self.loadableChunksNode.dirty = True

        self.resetLoadOrder()

def iterateChunks(x, z, radius):
    """
    Yields a list of chunk positions starting from the center and going outward in a square spiral pattern.

    :param x: center block position
    :param z: center block position
    :param radius: radius, in chunks
    :type radius: int
    :return:
    :rtype:
    """
    cx = int(math.floor(x)) >> 4
    cz = int(math.floor(z)) >> 4

    yield (cx, cz)

    step = dir = 1

    while True:
        for i in range(step):
            cx += dir
            yield (cx, cz)

        for i in range(step):
            cz += dir
            yield (cx, cz)

        step += 1
        if step > radius * 2:
            raise StopIteration

        dir = -dir

class WorldViewInfo(InfoPanel):
    def __init__(self, **kwargs):
        InfoPanel.__init__(self,
                           ['centerPoint', 'mouseBlockPos', 'mouseRay', 'worldMeshVertexSize',
                            'worldMeshChunks'],
                           ['viewportMoved', 'cursorMoved'],
                           **kwargs)

    @property
    def worldMeshVertexSize(self):
        # size = worldMeshVertexSize(self.object.worldMesh)
        # return "%0.2f MB" % (size / 1000000)
        return "-1"

    @property
    def worldMeshChunks(self):
        try:
            return "%s" % len(self.object.worldScene.chunkRenderInfo)
        except AttributeError as e:
            return "%s" % e

class WorldCursorInfo(InfoPanel):
    def __init__(self, **kwargs):
        super(WorldCursorInfo, self).__init__(['blockPos', 'mouseBlockFace', 'describeBlock', 'describeAdjacent'],
                                              ['cursorMoved'], **kwargs)

        #InfoPanel.__init__(self,
        #                   ['mouseBlockPos', 'mouseBlockFace', 'describeBlock', 'describeAdjacent'],
        #                   ['cursorMoved'], **kwargs)

    @property
    def blockPos(self):
        pos = self.object.mouseBlockPos
        if pos is not None:
            x, y, z = pos
            cx = x >> 4
            cy = y >> 4
            cz = z >> 4
            return "%s (cx=%d, cy=%d, cz=%d)" % (pos, cx, cy, cz)
        else:
            return "(None)"

    @property
    def describeBlock(self):
        pos = self.object.mouseBlockPos
        if pos is not None:
            return "%s: %s" % (pos, self.describe(pos))
        else:
            return "(None)"

    dirs = dict(faces.faceDirections)

    @property
    def describeAdjacent(self):
        pos = self.object.mouseBlockPos
        if pos is not None:
            face = self.object.mouseBlockFace
            pos += self.dirs[face]
            return "%s: %s" % (pos, self.describe(pos))
        else:
            return "(None)"

    def describe(self, pos):
        """

        :type pos: Vector or tuple
        :return:
        :rtype:
        """
        dim = self.object.dimension
        try:
            result = dim.getBlocks(pos[0], pos[1], pos[2],
                                   return_Data=True,
                                   return_BlockLight=True,
                                   return_SkyLight=True)
            if result.Data is not None:
                data = result.Data[0]
            else:
                data = 0
            if result.BlockLight is not None:
                bl = result.BlockLight[0]
            else:
                bl = "N/A"
            if result.SkyLight is not None:
                sl = result.SkyLight[0]
            else:
                sl = "N/A"

            desc = "\tID: %s\tData: %s\tLight: %s\tSkyLight: %s" % (
                result.Blocks[0],
                data, bl, sl,
            )

            cPos = pos.chunkPos()
            chunk = dim.getChunk(cPos[0], cPos[2])
            if chunk.HeightMap is not None:
                ix, iz = pos[0] & 0xf, pos[2] & 0xf
                #log.info("HeightMap (%s:%s): \n%s", cPos, (ix, iz), chunk.HeightMap)
                desc += "\tHeightMap(%s:%s): %d" % (cPos, (ix, iz), chunk.HeightMap[iz, ix])

            desc += "\tName: %s" % dim.blocktypes[result.Blocks[0], result.Data[0]].displayName
            return desc
        except ChunkNotPresent:
            return "Chunk not present."
        except (EnvironmentError, LevelFormatError) as e:
            return "Error describing block: %r" % e
        except Exception as e:
            log.exception("Error describing block: %r", e)
            return "Error describing block: %r" % e

LayerToggleOptions = Settings().getNamespace("layertoggleoptions")

class LayerToggleGroup(QtCore.QObject):
    def __init__(self, *args, **kwargs):
        super(LayerToggleGroup, self).__init__(*args, **kwargs)
        self.actions = {}
        self.actionGroup = QtGui.QActionGroup(self)
        self.actionGroup.setExclusive(False)
        self.options = {}
        for layer in Layer.AllLayers:
            option = LayerToggleOptions.getOption("%s_visible" % layer, bool, layer in Layer.DefaultVisibleLayers)
            self.options[layer] = option

            action = QtGui.QAction(layer, self)
            action.setCheckable(True)
            action.setChecked(option.value())
            log.info("LAYER %s VISIBLE %s", layer, option.value())
            action.layerName = layer
            self.actions[layer] = action
            self.actionGroup.addAction(action)

        self.actionGroup.triggered.connect(self.actionTriggered)

        self.menu = QtGui.QMenu()

        for layer in Layer.AllLayers:
            self.menu.addAction(self.actions[layer])

    def actionTriggered(self, action):
        checked = action.isChecked()
        log.info("Set layer %s to %s", action.layerName, checked)
        self.options[action.layerName].setValue(checked)
        self.layerToggled.emit(action.layerName, checked)

    layerToggled = QtCore.Signal(str, bool)

    def getVisibleLayers(self):
        return [layer for layer in self.actions if self.actions[layer].isChecked()]

    def setVisibleLayers(self, layers):
        for layer in self.actions:
            self.actions[layer].setChecked(layer in layers)


# COP version 0.0
# pylint: disable=no-self-argument, no-member

from core_legacy import struct, some, no, ni

from numpy import array, average, median, std
from numpy.random import rand, randint, seed

from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score, KFold, RepeatedKFold, train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score
from sklearn.pipeline import Pipeline

import datetime as dt

def load_data(file, seps='\t,;'):
    n_seps = len(seps)
    data = []
    try:
        raw = open(file, 'r', encoding='utf8')
        lines = raw.readlines()
    except:
        raw = open(file, 'r')
        lines = raw.readlines()
    for line in lines:
        tokens = []
        for n in range(n_seps):
            test = line.split(seps[n])
            if len(test)>len(tokens): tokens = test
        if tokens[-1] == '\n': tokens.pop(-1)
        data.append(tokens)
    return data

class translation(struct):
    zero = 0.0
    _c2d = str.maketrans(',','.')
    def __init__(tclass, _from, at={'tags':1, 'types':2}):
        if some(_from) and some(at):
            data = load_data(_from)
            tags = data[at['tags']]
            types = data[at['types']]
            if len(tags) != len(types): raise ValueError('Tag Type mismatch:', len(types)-len(tags))
            super().__init__(tags=[t.strip() for t in tags], types=[t.strip() for t in types], items={})
    def translate(_, line, zero=None):
        if no(zero): zero = _.zero
        def _translate(this, value):
            value, tag = value.strip(), _.tags[this]
            if no(value): return zero
            _type, items = _.types[this], _.items
            if _type == 'n':
                try: return float(value.translate(_._c2d))
                except: return zero
            elif _type.startswith('d'):
                sep = _type[1]
                rawdate = value.split(sep)
                ref = dt.date.today()
                args = {}
                for n,in_type in enumerate(_type[2:]):
                    if in_type == 'd': args['day'] = int(rawdate[n])
                    elif in_type == 'm': args['month'] = int(rawdate[n])
                    else: args['year'] = int(rawdate[n])
                date = dt.date(**args)
                return float((ref-date).days)
            elif _type == 'l':
                if tag not in items: items[tag] = {}
                if value not in items[tag]: items[tag].update({value: len(items[tag])})
                return float(items[tag][value])
            elif _type.startswith('-'): return float(value.strip(_type[1:]))
            elif _type.endswith('-'):
                if value.strip() == _type.strip('- '): return 1.
                return zero
            else:
                if value.strip() == _type.strip(): return zero
                return 1.
        if len(line) == len(_.tags): return {tag: _translate(n,line[n]) for n,tag in ni(_.tags)}
        return {}

class patient(struct):
    DEF = None
    states = []
    prev, next = None, None
    zero = 'prev'
    ID = None
    @staticmethod
    def reset(): patient.states = []
    def __init__(pclass, _from=None, line=1, args=None):
        if some(_from):
            args = load_data(_from)[line]
            patient.DEF = {arg.strip():0 for arg in args}
            patient._lines_ = len(patient.DEF)
        elif some(pclass.DEF):
            super().__init__(**pclass.DEF)
            pclass.states.append(pclass)
            pclass.set(**args)
    @property
    def len(_):
        if no(_.DEF): raise Warning('Definition not set')
        return len(_.DEF)
    def link(this, screening):
        this.next = screening
        screening.prev = this
    def _count_id(_, n=0):
        if some(_.ID): return _.ID, n
        return _.prev._count_id(n+1)
    @property
    def id(_): return _._count_id()
    def load(data, file, translator, id_from='hn', prev_at=0, _from=1):
        last, lines = None, load_data(file)[_from:]
        for line in lines:
            raw, id = translator.translate(line), id_from
            new = patient(args=raw)
            if raw[id] == prev_at and some(last):
                reset = translator.translate(line, data.zero)
                for field in raw:
                    if reset[field] == data.zero: reset[field] = last.get(field)
                new.set(**reset)
                last.link(new)
            else: new.ID = raw[id]
            last = new
    def select(_, x_by='day', fx=lambda x:True, exclude=['hn'], y_by='outcome', groups={(0,1,2):0, (3,4,5):1}, prediction=0):
        def search_outcome_of(state, _from, this):
            check = lambda x,y:x>y
            if _from<0: check = lambda x,y:x<y
            search = state
            while search.prev:
                out = search.get(y_by)
                if check(out, this): this = out
                search = search.prev
            search = state
            while search.next:
                out = search.get(y_by)
                if check(out, this): this = out
                search = search.next
            return this
        X, Y, off = [], [], exclude + [y_by]
        for state in _.states:
            if fx(state.get(x_by)):
                out, y = state.get(y_by), None
                if prediction: out = search_outcome_of(state, prediction, out)
                for this in groups:
                    if out in this: y = groups[this]
                if no(y): continue
                x = [state.get(var) for var in state.sets if var not in off]
                X.append(x)
                Y.append(y)
        return X,Y
    @property
    def ids(_): return len([state for state in _.states if state.ID is not None])
    @property
    def long_ids(_): return len([state for state in _.states if state.ID and state.next])

def idsets_of(db):
    _set = [db.states[0]]
    for state in db.states[1:]:
        if state.ID:
            yield _set
            _set = [state]
        else: _set.append(state)
    yield _set

class model(struct):
    k, err = 10, .05
    var_mul = 4
    ideal_y = .5
    classifier=RandomForestClassifier
    scaler=StandardScaler
    random_state = 31
    cargs = {}
    auto_exclude = ['hn', 'start']
    stack, unstacked = False, []
    _ref, from_end = 0, False
    train_on_head = False
    err_step = .05
    max_splits = 10
    class TELL:
        error = True
        train = True
    tell = TELL()
    @staticmethod
    def score_of(x,y, margs={'n_estimators':100}):
        _model, scaler = model.classifier, model.scaler
        random_state = model.random_state
        if no(random_state): random_state = randint(0xFFFFFFFF)
        C = Pipeline([('scaler', scaler()), ('model', _model(**margs))])
        cv = KFold(model.k, shuffle=True, random_state=random_state)
        return cross_val_score(C, x, y, cv=cv)
    def __init__(M, database, groups={range(0,2):0, range(2,5):1}, by='outcome', prediction=1, exclude=[], **opts):
        super().__init__(**opts)
        exclude = list(set(exclude + M.auto_exclude))
        features = [feat for feat in database.states[0].sets if feat not in exclude and feat != by]
        M.set(_db=database, _selargs={'x_by':'day', 'groups':groups, 'y_by':by, 'exclude':exclude, 'prediction':prediction}, features=features, groups=groups, outcome=by, best=[])
    def train(_, on=.5, max_ids=0, err=.15, **opts):
        _.set(**opts)
        if _.random_state: seed(_.random_state)
        if max_ids == 0: max_ids = _._db.ids
        times, nest = _.k, _._db.len*_.var_mul
        DB, states = _._db, _._db.states
        all_states = states
        if _.train_on_head: states = [state for state in states if state.ID]
        sort, models = int(on*len(states)), []
        miny, maxy = _.ideal_y-err, _.ideal_y+err
        scores, splits = [], 0
        while times:
            sel = set(randint(0, len(states), sort))
            A,B = [state for n,state in ni(states) if n in sel], [state for n,state in ni(states) if n not in sel]
            DB.states = A
            if not _.train_on_head and DB.ids > max_ids: continue
            x,y = DB.select(**_._selargs)
            ay = average(y)
            if splits == _.max_splits:
                err += _.err_step
                miny, maxy, splits = _.ideal_y-err, _.ideal_y+err, 0
                if _.tell.error: print('error split threshold set to {:.2f}'.format(err))
            if not _.train_on_head and ay<miny or ay>maxy: splits+=1; continue
            X,Y = array(x), array(y)
            S = _.scaler()
            Xs = S.fit_transform(X)
            M = _.classifier(n_estimators=nest, **_.cargs)
            M.fit(Xs,Y)
            DB.states = B
            x,y = DB.select(**_._selargs)
            Xt,Yt = array(x), array(y)
            Xts = S.transform(Xt)
            Yp = M.predict(Xts)
            score = accuracy_score(Yt, Yp)
            if _.tell.train: print('{:.3f}'.format(score), end='\t')
            models.append((M,S)); scores.append(score)
            times -= 1
        if _.tell.train: print('| {:.3f}'.format(average(scores)))
        ms = zip(models, scores)
        s_ms = sorted(ms, key=lambda x:x[1], reverse=True)
        DB.states = all_states
        _.best += s_ms
    def _calculate(_, on, by, targets, outcome_from, model, models, good, bad, threshold, _to=0):
        start = _to
        DB, _._states = _._db, _._db.states
        train, test, p = [], [], [int(r>=on) for r in rand(_._db.ids)]
        for n, idset in ni(idsets_of(DB)):
            if p[n]: test.append(idset)
            else: train += idset
        DB.states = train
        _.train(on=model, err=_.err)
        if no(models): models = len(_.best)
        for idset in test:
            DB.states = idset
            (x,y), ym = DB.select(**_._selargs), []
            for ((M,S),s) in _.best[:models]:
                try:
                    X = array(x)
                    Xs = S.transform(X)
                    Yp = M.predict(Xs)
                    ym.append(Yp)
                except:
                    pass
            if len(ym) == 0: continue
            Y = array(ym).T
            p = average(Y)
            if not y[0]: p = 1-p
            if y[0]==good[0]: _.stats[good[1]].append(p)
            else: _.stats[bad[1]].append(p)
            if not _.from_end: _.assessed.append(idset[0])
            if len(idset)==1 or y[0]==_._ref or len(Y)!=len(idset): continue
            ref, target_map = None, {tag:None for tag in targets+['before']}
            #assert(len(Y)==len(idset))
            for n,state in ni(idset):
                yn = average(Y[n])
                if y[0]==0: yn = 1-yn
                if no(state.next) or yn>=threshold:
                    ref = state.get(by)
                    break
            if _.from_end:
                for state in idset:
                    if no(state.next):
                        anticipation = state.get(by)-ref
                        if anticipation:
                            target_map['before'] = anticipation
                            _.dismissed.append(idset[0])
                        else: _.in_room.append(idset[0])
            else:
                for state in idset:
                    if no(target_map['before']) and outcome_from<=state.get(_.outcome):
                        target_map['before'] = state.get(by)-ref
                        break
            for target in targets:
                for state in idset:
                    if no(target_map[target]) and state.get(target):
                        target_map[target] = state.get(by)-ref
                        break
            for tag in target_map:
                if target_map[tag] is not None: _.stats[tag].append(target_map[tag])
            _to+=1
        print('from {} to {}...'.format(start, _to))
        for tag in _.stats:
            print(tag+'\t{:.3f} ({:.3f})'.format(
                average(_.stats[tag]) if len(_.stats[tag])>0 else 0,
                median(_.stats[tag]) if len(_.stats[tag])>0 else 0)
                )
        DB.states = _._states
        return _to
    def calculate(stats, on=.5, by='day', targets=['bio'], outcome_from=4, model=.9, virtual=None, models=None, good=(0,'good'), bad=(1,'bad'), skip=0, threshold=.75, **opts):
        stats.set(**opts)
        stats._ref = skip
        stats.set(stats = {tag:[] for tag in [good[1], bad[1], 'before']+targets})
        if stats.from_end: stats.set(in_room=[], dismissed=[])
        else: stats.set(assessed=[])
        n=stats._calculate(on, by, targets, outcome_from, model, models, good, bad, threshold)
        while some(virtual) and n<virtual:
            n=stats._calculate(on, by, targets, outcome_from, model, models, good, bad, threshold, n)
            if not stats.stack: stats.unstacked += stats.best; stats.best = []
    def reset(_):
        _._db.states = _._states
        _.best = []
    def sorted(_, model=0):
        (M,S), score = _.best[model]
        fi = zip(_.features, list(M.feature_importances_))
        s_fi = sorted(fi, key=lambda x:x[1], reverse=True)
        return s_fi, score
    def swap_stack(_):
        if len(_.best)==0: _.best = _.unstacked
        else: _.best = []
    @property
    def vars_of(model, start=0, stop=None):
        vari, s = model.sorted(start)
        index = {var:[i] for var,i in vari}
        if no(stop): stop = len(model.best)
        for n in range(start+1,stop):
            vari, s = model.sorted(n)
            for var,i in vari: index[var].append(i)
        s_vari = sorted(index.items(), key=lambda x:average(x[1]), reverse=True)
        return s_vari
    def show(_):
        for var,i in _.vars_of:
            print('{}\t{:.3f}'.format(var,average(i)))
    @property
    def DB(_): return _._db