import datetime
from werkzeug.exceptions import BadRequest
import pytz
import uuid
from threading import Lock
from dateutil.relativedelta import relativedelta
from RepetitionInterval import RepetitionIntervalClass

class jobFactoryClass():
  def loadFromDB(self, jobFromDBTuple, appObj):
    jobFromDB = jobFromDBTuple[0]
    repetitionInterval = jobFromDB["repetitionInterval"]

    fieldsToDefaultToNone = ["AfterSuccessJobGUID", "AfterFailJobGUID", "AfterUnknownJobGUID"]
    for fieldToDefault in fieldsToDefaultToNone:
      if fieldToDefault not in jobFromDB:
        jobFromDB[fieldToDefault] = None

    return jobClass(
      appObj = appObj,
      name = jobFromDB["name"],
      command = jobFromDB["command"],
      enabled = jobFromDB["enabled"],
      repetitionInterval = repetitionInterval,
      pinned = jobFromDB["pinned"],
      overrideMinutesBeforeMostRecentCompletionStatusBecomesUnknown = jobFromDB["overrideMinutesBeforeMostRecentCompletionStatusBecomesUnknown"],
      StateChangeSuccessJobGUID = jobFromDB["StateChangeSuccessJobGUID"],
      StateChangeFailJobGUID = jobFromDB["StateChangeFailJobGUID"],
      StateChangeUnknownJobGUID = jobFromDB["StateChangeUnknownJobGUID"],
      AfterSuccessJobGUID = jobFromDB["AfterSuccessJobGUID"],
      AfterFailJobGUID = jobFromDB["AfterFailJobGUID"],
      AfterUnknownJobGUID = jobFromDB["AfterUnknownJobGUID"],
      guid = jobFromDB["guid"],
      loadingObjectVersion = jobFromDBTuple[1]
  )

jobFactory = jobFactoryClass()

#Class to represent a job
class jobClass():
  guid = None
  name = None
  command = None
  enabled = None
  repetitionInterval = None
  creationDate = None
  lastUpdateDate = None
  lastRunDate = None
  nextScheduledRun = None
  lastRunReturnCode = None
  lastRunExecutionGUID = None
  pinned = False
  overrideMinutesBeforeMostRecentCompletionStatusBecomesUnknown = None
  mostRecentCompletionStatus = 'Unknown'
  resetCompletionStatusToUnknownTime = None
  StateChangeSuccessJobGUID = None
  StateChangeFailJobGUID = None
  StateChangeUnknownJobGUID = None
  AfterSuccessJobGUID =None
  AfterFailJobGUID = None
  AfterUnknownJobGUID = None

  CompletionstatusLock = None
  objectVersion = None

  def __repr__(self):
    ret = 'jobClass('
    ret += 'guid:' + self.guid + ' '
    ret += 'name:' + self.name + ' '
    ret += 'command:' + self.command + ' '
    ret += 'enabled:' + str(self.enabled) + ' '
    ret += 'repetitionInterval:' + self.repetitionInterval + ' '
    ret += 'creationDate:' + str(self.creationDate) + ' '
    ret += 'lastRunReturnCode:' + str(self.lastRunReturnCode) + ' '
    ret += 'lastRunExecutionGUID:' + str(self.lastRunExecutionGUID) + ' '
    ret += ')'
    return ret

  def assertValidName(name):
    if (len(name)<2):
      raise BadRequest('Job name must be more than 2 characters')
  def assertValidRepetitionInterval(ri, enabled):
    if ri is None:
      ri = ''
    if ri == '':
      if enabled:
        raise BadRequest('Repetition interval not set but enabled is true')
      return None
    try:
      return RepetitionIntervalClass(ri)
    except:
      raise BadRequest('Invalid Repetition Interval')

  def setNewRepetitionInterval(self, newRepetitionInterval):
    self.repetitionInterval = newRepetitionInterval
    if (self.repetitionInterval != None):
      if (self.repetitionInterval != ''):
        ri = RepetitionIntervalClass(self.repetitionInterval)
        self.repetitionInterval = ri.__str__()

  def verifyJobGUID(self, appObj, jobGUID, callingJobGUID):
    if jobGUID is None:
      return None
    if jobGUID == '':
      return None
    if callingJobGUID == jobGUID:
      raise BadRequest('A follow on action can not be set to the same job')
    unused = appObj.appData['jobsData'].getJob(jobGUID) #Will throw bad request exception if job dosen't exist
    return jobGUID

  def __init__(
      self,
      appObj,
      name,
      command,
      enabled,
      repetitionInterval,
      pinned,
      overrideMinutesBeforeMostRecentCompletionStatusBecomesUnknown,
      StateChangeSuccessJobGUID,
      StateChangeFailJobGUID,
      StateChangeUnknownJobGUID,
      AfterSuccessJobGUID,
      AfterFailJobGUID,
      AfterUnknownJobGUID,
      guid=None, #used when loading from DB
      verifyDependentJobGuids=True, #False when testing
      loadingObjectVersion=None #used when loading from DB
  ):
    jobClass.assertValidName(name)
    jobClass.assertValidRepetitionInterval(repetitionInterval, enabled)
    curTime = datetime.datetime.now(pytz.timezone("UTC"))
    if guid is None:
      self.guid = str(uuid.uuid4())
    else:
      self.guid = guid
    self.name = name
    self.command = command
    self.enabled = enabled
    self.setNewRepetitionInterval(repetitionInterval)
    self.creationDate = curTime.isoformat()
    self.lastUpdateDate = curTime.isoformat()
    self.lastRunDate = None
    self.lastRunExecutionGUID = ''
    self.lastRunReturnCode = None
    self.nextScheduledRun = None
    self.setNextScheduledRun(datetime.datetime.now(pytz.timezone("UTC")))
    self.pinned = pinned
    if overrideMinutesBeforeMostRecentCompletionStatusBecomesUnknown == 0:
      overrideMinutesBeforeMostRecentCompletionStatusBecomesUnknown = None
    self.overrideMinutesBeforeMostRecentCompletionStatusBecomesUnknown = overrideMinutesBeforeMostRecentCompletionStatusBecomesUnknown
    self.mostRecentCompletionStatus = 'Unknown'
    if verifyDependentJobGuids:
      self.StateChangeSuccessJobGUID = self.verifyJobGUID(appObj, StateChangeSuccessJobGUID, self.guid)
      self.StateChangeFailJobGUID = self.verifyJobGUID(appObj, StateChangeFailJobGUID, self.guid)
      self.StateChangeUnknownJobGUID = self.verifyJobGUID(appObj, StateChangeUnknownJobGUID, self.guid)
      self.AfterSuccessJobGUID = self.verifyJobGUID(appObj, AfterSuccessJobGUID, self.guid)
      self.AfterFailJobGUID = self.verifyJobGUID(appObj, AfterFailJobGUID, self.guid)
      self.AfterUnknownJobGUID = self.verifyJobGUID(appObj, AfterUnknownJobGUID, self.guid)
    else:
      self.StateChangeSuccessJobGUID = StateChangeSuccessJobGUID
      self.StateChangeFailJobGUID = StateChangeFailJobGUID
      self.StateChangeUnknownJobGUID = StateChangeUnknownJobGUID
      self.AfterSuccessJobGUID = AfterSuccessJobGUID
      self.AfterFailJobGUID = AfterFailJobGUID
      self.AfterUnknownJobGUID = AfterUnknownJobGUID

    #fields excluded from JSON output
    self.resetCompletionStatusToUnknownTime = None
    self.CompletionstatusLock = Lock()

    self.objectVersion = loadingObjectVersion

  def _getMinutesBeforeMostRecentCompletionStatusBecomesUnknown(self, appObj):
    if self.overrideMinutesBeforeMostRecentCompletionStatusBecomesUnknown == None:
      return appObj.minutesBeforeMostRecentCompletionStatusBecomesUnknown
    return self.overrideMinutesBeforeMostRecentCompletionStatusBecomesUnknown

  def _getCaculatedValueForModeRecentCompletionStatus(self, appObj, lastRunDate, lastRunReturnCode):
    if lastRunDate is None:
      return "Unknown"
    earlyTime = appObj.getCurDateTime() - relativedelta(minutes=self._getMinutesBeforeMostRecentCompletionStatusBecomesUnknown(appObj))
    if lastRunDate < earlyTime:
      return "Unknown"
    if lastRunReturnCode==0:
      return "Success"
    else:
      return "Fail"

  def _caculatedDictWithoutAppObjDependancy(self):
    ret = dict(self.__dict__)
    del ret['CompletionstatusLock']
    del ret['resetCompletionStatusToUnknownTime']
    if self.lastRunDate is not None:
      ret['lastRunDate'] = self.lastRunDate.isoformat()

    # If there is no state change job set the value returned should be null
    ret['StateChangeSuccessJobNAME'] = None
    ret['StateChangeFailJobNAME'] = None
    ret['StateChangeUnknownJobNAME'] = None
    ret['AfterSuccessJobNAME'] = None
    ret['AfterFailJobNAME'] = None
    ret['AfterUnknownJobNAME'] = None

    return ret

  # Needed when we use extra caculated values in the dict
  def _caculatedDict(self, appObj):
    ret = self._caculatedDictWithoutAppObjDependancy()

    if self.StateChangeSuccessJobGUID is not None:
      ret['StateChangeSuccessJobNAME'] = appObj.appData['jobsData'].getJob(self.StateChangeSuccessJobGUID).name
    if self.StateChangeFailJobGUID is not None:
      ret['StateChangeFailJobNAME'] = appObj.appData['jobsData'].getJob(self.StateChangeFailJobGUID).name
    if self.StateChangeUnknownJobGUID is not None:
      ret['StateChangeUnknownJobNAME'] = appObj.appData['jobsData'].getJob(self.StateChangeUnknownJobGUID).name
    if self.AfterSuccessJobGUID is not None:
      ret['AfterSuccessJobNAME'] = appObj.appData['jobsData'].getJob(self.AfterSuccessJobGUID).name
    if self.AfterFailJobGUID is not None:
      ret['AfterFailJobNAME'] = appObj.appData['jobsData'].getJob(self.AfterFailJobGUID).name
    if self.AfterUnknownJobGUID is not None:
      ret['AfterUnknownJobNAME'] = appObj.appData['jobsData'].getJob(self.AfterUnknownJobGUID).name

    return ret

  def setNewValues(
    self,
    appObj,
    name,
    command,
    enabled,
    repetitionInterval,
    pinned,
    overrideMinutesBeforeMostRecentCompletionStatusBecomesUnknown,
    StateChangeSuccessJobGUID,
    StateChangeFailJobGUID,
    StateChangeUnknownJobGUID,
    AfterSuccessJobGUID,
    AfterFailJobGUID,
    AfterUnknownJobGUID
  ):
    self.name = name
    self.command = command
    self.enabled = enabled
    self.setNewRepetitionInterval(repetitionInterval)
    self.setNextScheduledRun(datetime.datetime.now(pytz.timezone("UTC")))
    self.pinned = pinned
    if overrideMinutesBeforeMostRecentCompletionStatusBecomesUnknown==0:
      overrideMinutesBeforeMostRecentCompletionStatusBecomesUnknown = None
    self.overrideMinutesBeforeMostRecentCompletionStatusBecomesUnknown = overrideMinutesBeforeMostRecentCompletionStatusBecomesUnknown
    self.StateChangeSuccessJobGUID = self.verifyJobGUID(appObj, StateChangeSuccessJobGUID, self.guid)
    self.StateChangeFailJobGUID = self.verifyJobGUID(appObj, StateChangeFailJobGUID, self.guid)
    self.StateChangeUnknownJobGUID = self.verifyJobGUID(appObj, StateChangeUnknownJobGUID, self.guid)
    self.AfterSuccessJobGUID = self.verifyJobGUID(appObj, AfterSuccessJobGUID, self.guid)
    self.AfterFailJobGUID = self.verifyJobGUID(appObj, AfterFailJobGUID, self.guid)
    self.AfterUnknownJobGUID = self.verifyJobGUID(appObj, AfterUnknownJobGUID, self.guid)

  def removeRemoveRelianceOnOtherJob(self, guid):
    # return True if a change is made
    changed = False
    if self.StateChangeSuccessJobGUID == guid:
      self.StateChangeSuccessJobGUID = None
      changed = True
    if self.StateChangeFailJobGUID == guid:
      self.StateChangeFailJobGUID = None
      changed = True
    if self.StateChangeUnknownJobGUID == guid:
      self.StateChangeUnknownJobGUID = None
      changed = True
    if self.AfterSuccessJobGUID == guid:
      self.AfterSuccessJobGUID = None
      changed = True
    if self.AfterFailJobGUID == guid:
      self.AfterFailJobGUID = None
      changed = True
    if self.AfterUnknownJobGUID == guid:
      self.AfterUnknownJobGUID = None
      changed = True

    return changed

  def setNextScheduledRun(self, curTime):
    ri = None
    if self.enabled == False:
      self.nextScheduledRun = None
    else:
      if (self.repetitionInterval != None):
        if (self.repetitionInterval != ''):
          ri = RepetitionIntervalClass(self.repetitionInterval)
          self.nextScheduledRun = ri.getNextOccuranceDatetime(curTime).isoformat()

  def uniqueJobNameStatic(name):
    return name.strip().upper()

  def uniqueName(self):
    return jobClass.uniqueJobNameStatic(self.name)

  def _startPostCompletionJob(self, jobGuid, eventName, triggerExecutionObj, appObj):
    #Send a post completion event
    appObj.jobExecutor.submitJobForExecution(
      jobGuid,
      executionName='Event - ' + eventName,
      manual=False,
      triggerJobObj=self,
      triggerEvent=eventName,
      callerHasJobExecutionLock=True,
      triggerExecutionObj=triggerExecutionObj
    )

  #Called from job execution thread and request processing threads
  def _setNewCompletionStatus(self, appObj, newStatus, triggerExecutionObj):
    if self.mostRecentCompletionStatus == newStatus:
      if newStatus == 'Success':
        if self.AfterSuccessJobGUID is not None:
          self._startPostCompletionJob(self.AfterSuccessJobGUID, "AfterSuccess", triggerExecutionObj, appObj)
      elif newStatus == 'Fail':
        if self.AfterFailJobGUID is not None:
          self._startPostCompletionJob(self.AfterFailJobGUID, "AfterFail", triggerExecutionObj, appObj)
      else:
        if self.AfterUnknownJobGUID is not None:
          self._startPostCompletionJob(self.AfterUnknownJobGUID, "AfterUnknown", triggerExecutionObj, appObj)
      return
    self.CompletionstatusLock.acquire()
    self.mostRecentCompletionStatus = newStatus
    self.CompletionstatusLock.release()
    if newStatus=='Success':
      if self.StateChangeSuccessJobGUID is not None:
        self._startPostCompletionJob(self.StateChangeSuccessJobGUID, "StateChangeToSuccess", triggerExecutionObj, appObj)
      else:
        if self.AfterSuccessJobGUID is not None:
          self._startPostCompletionJob(self.AfterSuccessJobGUID, "AfterSuccess", triggerExecutionObj, appObj)
    elif newStatus=='Fail':
      if self.StateChangeFailJobGUID is not None:
        self._startPostCompletionJob(self.StateChangeFailJobGUID, "StateChangeToFail", triggerExecutionObj, appObj)
      else:
        if self.AfterFailJobGUID is not None:
          self._startPostCompletionJob(self.AfterFailJobGUID, "AfterFail", triggerExecutionObj, appObj)
    else:
      if self.StateChangeUnknownJobGUID is not None:
        self._startPostCompletionJob(self.StateChangeUnknownJobGUID, "StateChangeToUnknown", triggerExecutionObj, appObj)
      else:
        if self.AfterUnknownJobGUID is not None:
          self._startPostCompletionJob(self.AfterUnknownJobGUID, "AfterUnknown", triggerExecutionObj, appObj)

  def registerRunDetails(self, appObj, newLastRunDate, newLastRunReturnCode, triggerExecutionObj):
    #print('registerRunDetails for job ' + self.name + ' - lastrundate=' + newLastRunDate.isoformat())
    self.lastRunDate = newLastRunDate
    self.lastRunReturnCode = newLastRunReturnCode
    self.lastRunExecutionGUID = triggerExecutionObj.guid
    self.resetCompletionStatusToUnknownTime = newLastRunDate + relativedelta(minutes=self._getMinutesBeforeMostRecentCompletionStatusBecomesUnknown(appObj))

    newCompletionStatus = self._getCaculatedValueForModeRecentCompletionStatus(appObj, lastRunDate=newLastRunDate, lastRunReturnCode=newLastRunReturnCode)
    self._setNewCompletionStatus(
      appObj=appObj,
      newStatus=newCompletionStatus,
      triggerExecutionObj=triggerExecutionObj
    )

  #In the loop each job needs to check if its status needs to become unknown
  # this is required because the job may need to emit an event as a result
  # This is called from the job execution thread
  def loopIteration(self, appObj, curTime):
    if self.mostRecentCompletionStatus == 'Unknown':
      return
    if curTime > self.resetCompletionStatusToUnknownTime:
      self._setNewCompletionStatus(
        appObj=appObj,
        newStatus='Unknown',
        triggerExecutionObj=None
      )


import os
from pathlib import Path
from textwrap import dedent
import re

from jsonschema import validate as json_validate
from jsonschema.exceptions import ValidationError
from tornado.escape import json_decode
from tornado.httpclient import AsyncHTTPClient

class EntrypointValidationError(Exception):
    """Exception raised if entrypoint data validation fails.

    If either basic or extended validation of entrypoint data fails, this
    exception is to be raised. No detailed explanatory text, especially
    incorporating user input, should be returned to the user if validation
    fails.

    """

    pass

class EntrypointType:
    """Base entrypoint type class.

    Subclass this and override the following methods:

    - spawner_args      default is to return entrypoint data verbatim
    - get_type_name     optional classmethod, default is based on class name
    - get_display_name  optional classmethod, default is get_type_name()
    - get_description   optional classmethod, default is empty string
    - get_options       optional, coroutine
    - validation_hook   optional, coroutine

    An `EntrypointType` has the following responsibilities:

    - Converting entrypoint data into spawner arguments
    - Providing contents for a form used to manage a type of entrypoint
    - Validating input entrypoint data in an extensible way

    Validation: Basic validation of submitted entrypoint data via JSON Schema
    plus arbitrary type-specific validation. JSON Schema validation is to
    ensure that properties have the right types and that entrypoint data has
    the exact right number of properties. We assume that if a property is
    defined in the schema, it is required.

    Subclasses of `EntrypointType` can extend the JSON Schema easily to expand
    property type and count validation. They can also provide additional
    type-specific validation to ensure that entrypoint data is correct. For
    instance, a subclass that needs to verify that a "path" property looks like
    a proper Unix path can do so by extending the `validation_hook` method.

    """

    def __init__(self, **kwargs):
        self.schema = {
            "type": "object",
            "properties": {
                "entrypoint_name": {"type": "string"},
            },
            "required": [
                "entrypoint_name"
            ]
        }
        self.executable = kwargs.get("executable", "jupyter-labhub")
        self.username = kwargs.get("username")

    def extend_schema(self, properties):
        """Extend the base schema used for validating user entrypoint data.

        For example, to extend the schema to verify that a "path" field is
        included and has the right type, call this method this way:

            self.extend_schema([{"path": {"type": "string"}}])

        The specified properties will be added to the list of base entrypoint
        data properties, included as required properties, and the min/max
        property constraint will be updated to take into account the additional
        properties.

        Args:
            properties (list): List of JSON Schema property specifications

        """

        for prop in properties:
            self.schema["properties"].update(prop)
            self.schema["required"].append(list(prop.keys())[0])
        self.schema["minProperties"] = len(self.schema["properties"])
        self.schema["maxProperties"] = self.schema["minProperties"]

    @property
    def type_name(self):
        """str: Rendering of entrypoint type suitable for use as a dict key"""
        return self.get_type_name()

    @property
    def display_name(self):
        """str: Human-readable rendering of entrypoint type"""
        return self.get_display_name()

    @property
    def description(self):
        """str: Description of entrypoint type"""
        return self.get_description()

    async def form(self, entrypoint_data=None):
        """Return entire HTML form for managing this type of entrypoint.

        If the form is being used to update an entrypoint, then the old version
        of the entrypoint data can be passed to populate the form.

        Args:
            entrypoint_data (dict): Old entrypoint data to update.

        Returns:
            str: HTML form for managing entrypoint

        """

        content = ""
        autofocus = True
        for name in self.schema["required"]:
            content += await self.form_group(name, entrypoint_data, autofocus)
            autofocus = False
        return content

    async def get_options(self, name, prop):
        """Dynamically determine options for select-type form inputs

        Sometimes the list of options available for a select form input may
        need to be determined dynamically, for instance, by querying an
        external registry of container images. This method can be overridden
        to provide that kind of logic.

        This is a coroutine because validating entrypoint data may require
        non-blocking interaction with external services.

        Args:
            name (string): Property name.
            prop (dict): Property schema.

        Returns:
            list: List of options, or None if no options are appropriate

        """

        return prop.get("enum")

    async def form_group(self, name, entrypoint_data, autofocus):
        """Render a form group element

        Form elements are either selects or inputs and that's it for now.

        Args:
            name (str): Property name.
            entrypoint_data (dict): Old entrypoint data for update.
            autofocus (bool): Whether the element should have autofocus.

        Returns:
            str: HTML form group element

        """

        prop = self.schema["properties"][name]
        options = await self.get_options(name, prop)
        if options:
            return self.form_select(name, options, entrypoint_data, autofocus)
        else:
            return self.form_input(name, entrypoint_data, autofocus)

    def form_select(self, name, options, entrypoint_data, autofocus):
        """Render a form select element

        Args:
            name (str): Property name.
            options (list): Options to present.
            entrypoint_data (dict): Current entrypoint data or None.
            autofocus (bool): Whether the element should have autofocus.

        Returns:
            str: HTML form select element

        """

        value = ""
        if entrypoint_data:
            value = entrypoint_data[name]

        content = dedent(f"""\

          {name}:
          <select
            class="form-control"
            name="{name}"
            required
            {"autofocus" if autofocus else ""}
          >
        """)
        if value:
            if value not in options:
                content += dedent(f"""\
                  <option disabled selected value>{value}</option>
                """)
        else:
            content += dedent(f"""\
              <option disabled selected value>-- choose --</option>
            """)
        for option in options:
            if value and value == option:
                content += f"  <option selected>{option}</option>"
            else:
                content += f"  <option>{option}</option>"
        content += dedent(f"""\
          </select>

        """)
        return content

    def form_input(self, name, entrypoint_data, autofocus):
        """Render a form input element

        Args:
            name (str): Property name.
            entrypoint_data (dict): Current entrypoint data or None.
            autofocus (bool): Whether the element should have autofocus.

        Returns:
            str: HTML form input element

        """

        value = ""
        if entrypoint_data:
            value = entrypoint_data[name]

        return dedent(f"""\

          {name}:
          <input
            class="form-control"
            name="{name}"
            required
            {"autofocus" if autofocus else ""}
            autocomplete="off"
            value="{value}"
          >

        """)

    async def validate(self, entrypoint_data):
        """Validate entrypoint data, both basic and extended by hook."""
        self.validate_schema(entrypoint_data)
        await self.validation_hook(entrypoint_data)

    def validate_schema(self, entrypoint_data):
        """Perform basic entrypoint data validation using JSON Schema.

        Raises:
            EntrypointValidationError: If schema-based validation fails

        """

        try:
            json_validate(entrypoint_data, self.schema)
        except ValidationError:
            raise EntrypointValidationError

    def spawner_args(self, entrypoint_data, **kwargs):
        """Convert entrypoint data into spawner arguments.

        This method returns entrypoint data verbatim by default, but can be
        used to convert entrypoint data into appropriate spawner arguments
        for use by the Hub.

        Args:
            entrypoint_data (dict): Entrypoint data
            kwargs (dict): Query filters

        Returns:
            dict: Arguments like `cmd` used during Hub `start()`

        """

        return entrypoint_data

    @classmethod
    def get_type_name(cls):
        """Render the entrypoint type for use as a dict key.

        By default, an entrypoint type's class name is converted to a type name
        by removing "EntrypointType" from the end of the class name and then
        returning the remainder in lower case. Subclasses may override this
        behavior to map the entrypoint type to a string suitable for use as a
        dictionary key.

        This method is used by the `type_name` property, which is the way that
        forms should render an entrypoint type name when templating.

        Returns:
            str: Entrypoint type name in a form usable as a dict key

        """

        return re.sub("EntrypointType$", "", cls.__name__).lower()

    @classmethod
    def get_display_name(cls):
        """Render the entrypoint type in a human-friendly string.

        By default this just uses `get_type_name()` assuming that the type name
        constructed that way is basically human-friendly enough, in particular,
        during template rendering. This may not be desirable, so this method
        can be overridden by subclasses to use an alternate scheme.

        This is used by the `display_name` property, which is how forms should
        render an entrypoint types's human-friendly name when templating.

        Returns:
            str: Entrypoint type name in a human-friendly strong format.

        """

        return cls.get_type_name()

    @classmethod
    def get_description(cls):
        """Render a description of the entrypoint type.

        By default this just returns the empty string, but can be used to
        explain what the entrypoint type is for and how to use it during
        template rendering.

        This is used by the `description` property during templating.

        Returns:
            str: Description of entrypoint type

        """

        return ""

    async def validation_hook(self, entrypoint_data):
        """Perform extended validation on user input entrypoint data.

        The base `EntrypointType` class provides basic JSON Schema validation
        of entrypoint data, but most subclasses will need to perform additional
        arbitrary validation to ensure that the inputs are sane and secure.

        For instance, if an entrypoint type uses a file path, it is wise to
        ensure that the proposed file path (a) contains no suspicious
        formatting like HTML tags, (b) actually looks like a valid POSIX path,
        and (c) actually exists and has suitable permissions.

        Another possibility is that an entrypoint type could be configured to
        allow only a limited set of entrypoints, governed by configuration and
        managed by the operator/administrator. In this case, validation should
        ensure that users are selecting valid choices from that set, and not
        other choices that don't exist. This way we mitigate user input through
        UI as a vector.

        This is a coroutine because validating entrypoint data may require
        non-blocking interaction with external services.  For instance,
        interacting with a service that can validate that a path exists on the
        file-system.

        Raises:
            EntrypointValidationError: If extended validation fails, subclasses
            should raise EntrypointValidationError with no/minimal additional
            details about how validation failed, and **especially** should not
            respond with **any** of the user's input **at all, never, ever**.

        """

        pass

class TrustedScriptEntrypointType(EntrypointType):
    """Entrypoint type based on scripts managed by an administrator.

    This entrypoint type is configured with a list of absolute script paths
    that users may choose from, to use as entrypoints. These scripts should be
    managed by the administrator or trusted designee.

    This is a usable reference implementation of `EntrypointType`. It shows how
    the entrypoint command is formatted from entrypoint data, how basic
    validation via JSON schema is extended, and how type and display names are
    customized.

    An entrypoint script in this case might look like:

        #!/bin/bash
        export SOME_VARIABLE=123
        module load vasp
        exec "$@"

    """

    def __init__(self, *args, **kwargs):
        """Initialize the trusted script entrypoint type.

        Args:
            *args (list): List of trusted entrypoint scripts

        """

        super().__init__(**kwargs)
        self.extend_schema([{"script": {"type": "string", "enum": list(args)}}])

    def spawner_args(self, entrypoint_data, **kwargs):
        """Convert entrypoint data into spawner arguments.

        Args:
            entrypoint_data (dict): Entrypoint data
            kwargs (dict): Query filters

        Returns:
            dict: Arguments prefixed with the trusted entrypoint script

        """

        batchspawner = kwargs.get("batchspawner", False)

        doc = dict()
        script = entrypoint_data["script"]
        if batchspawner:
            doc["cmd"] = [self.executable]
            doc["batchspawner_singleuser_cmd"] = (
                f"{script} batchspawner-singleuser"
            )
        else:
            doc["cmd"] = [script, self.executable]
        return doc

    @classmethod
    def get_type_name(cls):
        """Override default type name behavior."""
        return "trusted_script"

    @classmethod
    def get_display_name(cls):
        """Override default display name behavior."""
        return "trusted script"

    @classmethod
    def get_description(cls):
        """Override default description behavior."""

        return dedent("""
            Start a Jupyter notebook server using a staff-managed, pre-defined
            configuration implemented in a wrapper script.
        """)

class TrustedPathEntrypointType(EntrypointType):
    """Entrypoint type based on a path managed by an administrator.

    This entrypoint type is configured with a list of directory paths that
    users may choose from, to use as entrypoints. These paths should be managed
    by the administrator or trusted designee.

    This is a usable reference implementation of `EntrypointType`. It shows how
    the entrypoint command is formatted from entrypoint data, how basic
    validation via JSON schema is extended, and how type and display names are
    customized.

    Example: An entrypoint path could be the full absolute path to a bin
    directory of a conda environment.

    """

    def __init__(self, *args, **kwargs):
        """Initialize the trusted path entrypoint type.

        Args:
            *args (list): List of trusted directory paths

        """

        super().__init__(**kwargs)
        self.extend_schema([{"path": {"type": "string", "enum": list(args)}}])

    def spawner_args(self, entrypoint_data, **kwargs):
        """Convert entrypoint data into spawner arguments.

        Args:
            entrypoint_data (dict): Entrypoint data
            kwargs (dict): Query filters

        Returns:
            dict: Arguments prefixed with the trusted path

        """

        batchspawner = kwargs.get("batchspawner", False)

        path = Path(entrypoint_data["path"])
        doc = dict(cmd=[str(path / self.executable)])
        if batchspawner:
            doc["batchspawner_singleuser_cmd"] = str(
                path / "batchspawner-singleuser"
            )
        return doc

    @classmethod
    def get_type_name(cls):
        """Override default type name behavior."""
        return "trusted_path"

    @classmethod
    def get_display_name(cls):
        """Override default display name behavior."""
        return "trusted path"

    @classmethod
    def get_description(cls):
        """Override default description behavior."""

        return dedent("""
            Start a Jupyter notebook server using a staff-managed, pre-defined
            absolute path to the "jupyter" executable.
        """)

class ShifterEntrypointType(EntrypointType):
    """Entrypoint type for Shifter, a Docker container runtime.

    This entrypoint type leverages an external services that manages a list of
    Shifter images available for running Jupyter, that users may select from to
    use as entrypoints. These images should be managed by the administrator or
    trusted designee.

    A usable reference implementation of `EntrypointType`, this class shows all
    the same kinds of customizations as `TrustedScriptEntrypointType`, but it
    includes a non-blocking call to an external service during both form
    building and validation hook execution.

    """

    def __init__(
        self,
        shifter_api_url,
        shifter_api_token=None,
        image_filter=None,
        **kwargs
    ):
        """Initialize the Shifter entrypoint type.

        Args:
            shifter_api_url (str): URL for Shifter image service
            shifter_api_token (str): API token for Shifter image service
            image_filter (function): Image filter, default is no filtering

        """

        super().__init__(**kwargs)
        self.shifter_api_url = shifter_api_url
        self.shifter_api_token = (
            shifter_api_token or os.environ["SHIFTER_API_TOKEN"]
        )
        self.extend_schema([{"image": {"type": "string"}}])
        self.image_filter = image_filter or (lambda image: True)

    def spawner_args(self, entrypoint_data, **kwargs):
        """Convert entrypoint data into spawner arguments.

        Args:
            entrypoint_data (dict): Entrypoint data
            kwargs (dict): Query filters

        Returns:
            dict: Arguments prefixed with Shifter command and image

        """

        batchspawner = kwargs.get("batchspawner", False)

        doc = dict()
        image = entrypoint_data["image"]
        if batchspawner:
            doc["cmd"] = [self.executable]
            doc["batchspawner_singleuser_cmd"] = (
                f"shifter --image={image} batchspawner-singleuser"
            )
        else:
            doc["cmd"] = ["shifter", f"--image={image}", self.executable]
        return doc

    @classmethod
    def get_description(cls):
        """Override default description behavior."""

        return dedent("""
            Start a Jupyter notebook server using a Shifter image loaded on the
            target system.
        """)

    async def get_options(self, name, prop):
        """Get images from the Shifter image service to present as options

        Args:
            name (string): Property name.
            prop (dict): Property schema.

        Returns:
            list: List of images.

        """

        if name == "image":
            return await self.get_images()

    async def validation_hook(self, entrypoint_data):
        """Validate that the chosen image is known to Shifter.

        Raises:
            EntrypointValidationError: If the image is unknown.

        """

        try:
            images = await self.get_images()
        except:
            # FIXME log message would be good here...
            raise EntrypointValidationError
        if entrypoint_data["image"] not in images:
            raise EntrypointValidationError

    async def get_images(self):
        """Gets images from the Shifter image service."""

        client = AsyncHTTPClient()
        response = await client.fetch(
            f"{self.shifter_api_url}list/{self.username}",
            headers={"Authorization": self.shifter_api_token}
        )
        result = json_decode(response.body)
        images = result["images"]
        return [
            image["tag"][0] for image in images if self.image_filter(image)
        ]


__author__ = 'jiaodongdong'
#encoding=utf-8
from datetime import datetime
import time
def date2unix(attr):
    dalist = attr.split()

    zone = dalist[4]
    datef = dalist[0]+" "+dalist[1]+" "+dalist[2]+" "+dalist[3]+" "+dalist[5]

    dtmp = datetime.strptime(datef, "%a %b %d %X %Y")
    stampt = int(time.mktime(dtmp.timetuple()))
    if zone is "+0800":
        return stampt
    else:
        hou = int(zone[1: 3])
        mint = int(zone[3: 5])
        if zone[0] is '-':
            return (hou+8)*3600+mint*60+stampt
        else:
            return stampt - ((hou-8)*3600+mint*60)


from functools import cmp_to_key
from src.world.environments.homogeneous_environment import HomogeneousEnvironment
from src.world.person import Person

class Household(HomogeneousEnvironment):
    __slots__ = ('_city', '_city_env')
    name = "household"

    def __init__(self, city, contact_prob_between_each_two_people):
        super(Household, self).__init__(contact_prob_between_each_two_people)
        self._city = city
        self._city_env = None

    def set_city_env(self, city_env):
        self._city_env = city_env

    @classmethod
    def house_comperator_ASCENDING(cls, a,b):
        """
        Compare households by their youngest person
        """
        #Get the persons we whish to sort
        a_people = a.get_people()
        b_people = b.get_people()
        #Sort the persons by their age in ascending way
        a_people = sorted(a_people,key = cmp_to_key(Person.person_comperator_ASCENDING))
        b_people = sorted(b_people,key = cmp_to_key(Person.person_comperator_ASCENDING))
        #compare the youngest persons of each house
        return a_people[0].get_age() - b_people[0].get_age()

    @classmethod
    def house_comperator_DESCENDING(cls,a,b):
        """
        Compare households by their youngest person
        """
        #Get the persons we whish to sort
        a_people = a.get_people()
        b_people = b.get_people()
        #Sort the persons by their age in ascending way
        a_people = sorted(a_people,key = cmp_to_key(Person.person_comperator_DESCENDING))
        b_people = sorted(b_people,key = cmp_to_key(Person.person_comperator_DESCENDING))
        #compare the youngest persons of each house
        return b_people[0].get_age() - a_people[0].get_age()


import csv
import json
from PIL import Image, ImageDraw, ImageFont

# there are 5 types of  certificates (brief description)

# 1: "Big kid monthly certificate"
# the kind that's made for KET, PET, FCE and CAE classes after each month.

# 2: "Small kid monthly certificate (normal)"
# the kind that's made for Starters, Movers and Flyers classes after each month.

# 3: "Small kid monthly certificate (TOR)"
# the kind that's like type 2, but a TOR.

# 4: "Award certificate (normal)"
# the kind that's made for every class. It's with a title in the middle (no marks).

# 5: "Award certificate (Academic Excellence)"
# the kind that's like type 4, but for academic excellence.

# ** type 4 and 5 are combined to class type4

# setting up the fonts
fonts1 = {"font0":ImageFont.truetype("fonts/LibreBaskerville-Regular.ttf", size=55),
         "font1": ImageFont.truetype('fonts/LibreBaskerville-Regular.ttf',size=80),
         "font2":ImageFont.truetype("fonts/LibreBaskerville-Regular.ttf", size=60),
         "font3":ImageFont.truetype("fonts/LibreBaskerville-Bold.ttf", size=60),
         "font4":ImageFont.truetype("fonts/LibreBaskerville-Regular.ttf", size=50),
         "font5":ImageFont.truetype("fonts/LibreBaskerville-Regular.ttf", size=45)}

fonts4 = {"font0":ImageFont.truetype("fonts/LibreBaskerville-Regular.ttf", size=45),
         "font1": ImageFont.truetype('fonts/LoveloBlack.otf',size=40),
         "font2":ImageFont.truetype("fonts/LibreBaskerville-Regular.ttf", size=20),
         "font3":ImageFont.truetype("fonts/LibreBaskerville-Bold.ttf", size=20)}

fonts2 = {"font0":ImageFont.truetype("fonts/LibreBaskerville-Regular.ttf", size=100),
         "font1": ImageFont.truetype('fonts/LoveloBlack.otf',size=90),
         "font2":ImageFont.truetype("fonts/Lato-Regular.ttf", size=50),
         "font3":ImageFont.truetype("fonts/LibreBaskerville-Bold.ttf", size=45)}

fonts3 = {"font0":ImageFont.truetype("fonts/LibreBaskerville-Regular.ttf", size=55),
         "font1": ImageFont.truetype('fonts/LoveloBlack.otf',size=90),
         "font2":ImageFont.truetype("fonts/Lato-Regular.ttf", size=50),
         "font3":ImageFont.truetype("fonts/LibreBaskerville-Bold.ttf", size=45)}

fonts6 = {"font0":ImageFont.truetype("fonts/LibreBaskerville-Regular.ttf", size=65),
          "font1":ImageFont.truetype("fonts/LibreBaskerville-Bold.ttf", size=65),
          "font2":ImageFont.truetype("fonts/LibreBaskerville-Regular.ttf", size=75)
          }

# the colors
colors = {"black": "rgb(0, 0, 0)", "name_blue": "rgb(25, 27, 109)", "red": "rgb(210,19,26)"}

# title mapper
his_or_her = {"male":"his", "female":"her"}

# gender list
gender_lst = ["male","female"]

def input_dic_creator(jsonf):
    return json.loads(open(jsonf,"r").read())

class Type1:

    exam_info_dict = {"KET": {"real total": 150,
                              "txt1": "International Entry (Key)*",
                              "txt2": "Key English Test",
                              "txt3": " Level A2",
                              "mrk_dic":{"Grade A":(140,151),"Grade B":(133,140),"Grade C":(120,133),"Level A1":(100,120)}
                              },

                      "PET": {"real total": 170,
                              "txt1": "International Entry (Preliminary)*",
                              "txt2": "Preliminary English Test",
                              "txt3": " Level B1",
                              "mrk_dic":{"Grade A":(160,171),"Grade B":(150,160),"Grade C":(140,150),"Level A1":(120,140)}},

                      "FCE": {"real total": 190,
                              "txt1": "International Entry (First)*",
                              "txt2": "First Certificate in English",
                              "txt3": " Level B2",
                              "mrk_dic":{"Grade A":(180,191),"Grade B":(173,180),"Grade C":(160,173),"Level A1":(140,160)}},

                      "CAE": {"real total": 210,
                              "txt1": "International Entry (Advanced)*",
                              "txt2": "Certificate in Advanced English",
                              "txt3": " Level C1",
                              "mrk_dic": {"Grade A": (200, 211), "Grade B": (193, 200), "Grade C": (180, 193), "Level A1": (160, 180)}}
                      }

    def __init__(self, std_name: str, std_l: float, std_r: float,  std_w:float,std_s: float,
                 level: str, cls_tr: str, t_l: float, t_r: float,t_w:float, t_s: float,minus_rule=False):

        self.std_name = std_name
        self.std_r = std_r
        self.std_l = std_l
        self.std_w = std_w
        self.std_s = std_s
        self.level = level
        self.cls_tr = cls_tr
        self.t_r = t_r
        self.t_l = t_l
        self.t_w = t_w
        self.t_s = t_s
        self.minus_rule = minus_rule

        self.cert = Image.open(f"templates\\type1temp.png")
        self.sign = Image.open(f"signs2\\{cls_tr}.png")
        self.current = self.exam_info_dict[self.level]

        self.state = True
        for i in [self.std_r, self.std_l, self.std_s,self.std_w, self.t_r, self.t_l, self.t_s,self.t_w]:
            try:
                float(i)
            except TypeError or ValueError:
                self.state = False

    def image_size(self):
        return self.cert.size

    def mark_calc(self):
        std_lst = [self.std_l,self.std_r,self.std_w,self.std_s]
        t_lst = [self.t_l,self.t_r,self.t_w,self.t_s]

        rt_lst = []
        if self.minus_rule == False:
            for i in range(4):
                rt_lst.append((self.current['real total'] / t_lst[i]) * std_lst[i])

        else:
            for i in range(4):
                rt_lst.append(self.current['real total'] - int(( float(t_lst[i]) - float(std_lst[i]))))

        return_lst = []
        return_lst.append(abs(sum(rt_lst)/4))
        grade = ""
        print(self.current['mrk_dic'].keys())
        for j in self.current['mrk_dic'].keys():

            if int(abs(sum(rt_lst)/4)) in range(int(self.current['mrk_dic'][j][0]),int(self.current['mrk_dic'][j][1])):
                grade = j
                break

            else:

                return f"{self.std_name}: Marks too low to receive a certificate."

        return_lst = return_lst + rt_lst
        rt_lst = []
        title_lst = ['Overall Score',"Listening","Reading","Writing","Speaking"]
        for i in range(5):
            rt_lst.append(title_lst[i]+"   "+str(int(abs(return_lst[i]))))

        return {"grade":grade,"lst":rt_lst}

    def place_txt(self, txt: str, font: str, y_pos: int, color: str, x_pos=None):

        txt_size = fonts1[font].getsize(txt)
        if x_pos is None:
            x_pos = ((self.image_size()[0] - txt_size[0]) / 2) + (txt_size[0] / 2)

        draw = ImageDraw.Draw(self.cert)
        draw.text((x_pos, y_pos), txt, fill=colors[color], font=fonts1[font], anchor="ma")

    def place_sign(self):
        self.cert.paste(self.sign, (1870, 2870))
        txt = "Teacher " + self.cls_tr[0].upper() + self.cls_tr[1:-1] + self.cls_tr[-1]
        self.place_txt(txt, "font5", 3210, "black", 2000)

    def create(self):

        if self.state == True:
            try:
                self.place_txt("Cambridge English Entry Level Certificate in ESOL","font0",1000,"black")
                self.place_txt(f"{self.current['txt1']}", "font0", 1085, "black")

                self.place_txt(f"{self.std_name}","font1",1350,"name_blue")

                self.place_txt(f"{self.current['txt2']}","font3",1825,"black")

                txt="Council of Europe"
                txt_size = fonts1["font2"].getsize(txt)
                self.place_txt(f"Council of Europe", "font2", 1940, "black",((self.image_size()[0] - txt_size[0]) / 2)+50)
                self.place_txt(f"{self.current['txt3']}", "font2", 1940, "red",
                               ((self.image_size()[0] - txt_size[0]) / 2) + (txt_size[0] / 2) +200)

                if type(self.mark_calc()) != type({1:1}):
                    return self.mark_calc()

                self.place_txt(f"{self.mark_calc()['grade']}", "font3", 1625, "black")
                st_ypos = 2200
                y_inerval = 125
                self.place_txt(self.mark_calc()['lst'][0], "font3", st_ypos, "black")
                st_ypos += y_inerval

                for i in self.mark_calc()["lst"][1:len(self.mark_calc()["lst"])]:
                    self.place_txt(i,"font2",st_ypos,"black")
                    st_ypos+=y_inerval

                self.place_sign()

                self.cert.save(f"certificates\\t1CERT{self.std_name}.png")
                return f"Certificate successfully created for {self.std_name}"

            except IndexError:
                return f"Error in {self.std_name}"
        else:
            return f"Error in {self.std_name}"

    @staticmethod
    def create_dic(csvfile, input_dic):
        rt_dic = {}
        with open(csvfile, "r") as f:
            gen_obj = csv.reader(f)
            lst = []

            # append everything in the generator object into a list, so that it can be iterated more than once.
            for line in gen_obj:
                lst.append(line)

            c = 0
            for line in lst:
                str_c = str(c)
                while len(str(str_c)) < 3:
                    str_c = "0" + str_c
                rt_dic[f'{line[0]}{str_c}'] = (line[0], line[1], line[2], line[3],line[4], input_dic['level'],
                                               input_dic['class_tr'], input_dic['t_l'],
                                               input_dic['t_r'],input_dic['t_w'], input_dic['t_s'],input_dic['minus_rule'])
                c += 1

        return rt_dic

class Type4:

    def __init__(self,awd_title:str,des:str,std_name:str,giv_date:str,cls_name:str,gender:str,cls_tr:str):

        self.awd_title = awd_title
        self.des = des
        self.std_name = std_name
        self.giv_date = giv_date
        self.cls_name = cls_name
        self.gender = gender
        self.cls_tr = cls_tr

        self.cert = Image.open(f"templates\\type4temp.png")
        self.sign = Image.open(f"signs4\\{cls_tr}.png")

    def date_str(self):
        try:
            temp = [f"Given this on {self.giv_date},",f"for {his_or_her[self.gender]} excellence in {self.cls_name}"]
        except KeyError:
            temp = []
        return temp

    def image_size(self):
        return self.cert.size

    def place_txt(self,txt:str,font:str,y_pos:int,color:str,x_pos=None):

        txt_size = fonts4[font].getsize(txt)
        if x_pos is None:
            x_pos = ((self.image_size()[0] - txt_size[0]) / 2) + (txt_size[0]/2)

        draw = ImageDraw.Draw(self.cert)
        draw.text((x_pos,y_pos),txt,fill=colors[color],font=fonts4[font],anchor="ma")

    def place_sign(self):
        self.cert.paste(self.sign,(720,550))
        txt = "Teacher "+self.cls_tr[0].upper() + self.cls_tr[1:-1] + self.cls_tr[-1]
        self.place_txt(txt,"font3",695,"black",783)

    def create(self):
        try:
            if self.des == "":
                self.place_txt(self.awd_title, "font0", 250, "red")
            else:
                self.place_txt(self.awd_title, "font0", 225, "red")
                self.place_txt(self.des, "font2", 285, "red")

            self.place_txt(self.std_name, "font1", 390, "name_blue")
            self.place_txt(self.date_str()[0], "font2", 440, "black")
            self.place_txt(self.date_str()[1], "font2", 462, "black")

            self.place_sign()

            save_name = ""
            for i in self.std_name.split():
                save_name += i
            self.cert.save(f"certificates\\t4CERT{save_name}.png")
            return f"Certificate successfully created for {self.std_name}."
        except IndexError:
            return f"Error in {self.std_name}."

    def __repr__(self):
        return f"Type4('{self.awd_title}','{self.des}','{self.std_name}'," \
               f"'{self.giv_date}','{self.cls_name}','{self.gender}','{self.cls_tr}')"

    @staticmethod
    def create_dic(csvfile,input_dic):
        rt_dic = {}
        with open(csvfile,"r") as f:
            gen_obj = csv.reader(f)
            lst = []

            # append everything in the generator object into a list, so that it can be iterated more than once.
            for line in gen_obj:
                lst.append(line)

            # check if there are faulty data
            # in the case of Type4, only gender needs to be checked
            for line in lst:
                line[3] = line[3].lower()
                if line[3] not in gender_lst:
                    line[3] = "ERROR"

            c = 0
            for line in lst:
                str_c = str(c)
                while len(str(str_c)) <3:
                    str_c = "0" + str_c
                rt_dic[f'{line[2]}{str_c}'] = (line[0],line[1],line[2],input_dic['date'],
                                               input_dic['class_name'],line[3],input_dic['class_tr'])
                c+=1

        return rt_dic

class Type2:

    marking_sys = {"Starters":[25,10,15],
                   "Movers":[40,25,15],
                   "Flyers":[40,25,15]}

    def __init__(self,std_name:str,std_rw:float,std_l:float,std_s:float,
                 level:str,date:str,cls_tr:str,t_rw:float,t_l:float,t_s:float):

        self.std_name = std_name
        self.std_rw = std_rw
        self.std_l = std_l
        self.std_s = std_s
        self.level = level
        self.date = date
        self.cls_tr = cls_tr
        self.t_rw = t_rw
        self.t_l = t_l
        self.t_s = t_s

        self.cert = Image.open(f"templates\\type2temp.png")
        self.sign = Image.open(f"signs2\\{cls_tr}.png")
        self.shield = Image.open(f"shield.png")
        self.shadow = Image.open(f"shadow.png")

        self.state = True
        for i in [self.std_rw,self.std_l,self.std_s,self.t_rw,self.t_l,self.t_s]:
            try:
                float(i)
            except TypeError:
                self.state = False

    def image_size(self):
        return self.cert.size

    def date_str(self):
        return [f"took YLE {self.level} at Teacher Su Language Center", f"in {self.date}"]

    def place_txt(self,txt:str,font:str,y_pos:int,color:str,x_pos=None):

        txt_size = fonts2[font].getsize(txt)
        if x_pos is None:
            x_pos = ((self.image_size()[0] - txt_size[0]) / 2)+ (txt_size[0]/2)

        draw = ImageDraw.Draw(self.cert)
        draw.text((x_pos,y_pos),txt,fill=colors[color],font=fonts2[font],anchor="ma")

    def place_sign(self):
        self.cert.paste(self.sign,(1870,2870))
        txt = "Teacher "+self.cls_tr[0].upper() + self.cls_tr[1:-1] + self.cls_tr[-1]
        self.place_txt(txt,"font3",3210,"black",2000)

    def shield_calc(self):
        t_lst = [self.t_rw,self.t_l,self.t_s]
        std_lst = [self.std_rw,self.std_l,self.std_s]

        result_lst = []
        for i in range(3):
            result_lst.append((self.marking_sys[self.level][i]/float(t_lst[i]))*float(std_lst[i]))

        rt_lst = []
        for i in range(3):
            denominator = self.marking_sys[self.level][i]/5
            if result_lst[i]%denominator != 0:
                rt_lst.append((result_lst[i]//denominator)+1)
            elif result_lst[i]%denominator == 0:
                rt_lst.append(result_lst[i] // denominator)
        return rt_lst

    def place_sheield(self):

        starting_pos_x = 800
        starting_pos_y = 2000

        x_interval = 225
        y_interval = 250

        for i in self.shield_calc():
            for sh in range(int(i)):
                self.cert.paste(self.shield,(starting_pos_x,starting_pos_y))
                starting_pos_x += x_interval
            for sh in range(int(5-i)):
                self.cert.paste(self.shadow,(starting_pos_x,starting_pos_y))
                starting_pos_x += x_interval
            starting_pos_x = 800
            starting_pos_y += y_interval

        starting_pos_y = 2080
        self.place_txt("Reading & Writing","font3",starting_pos_y,"black",550)
        starting_pos_y += y_interval
        self.place_txt("Listening", "font3", starting_pos_y, "black", 550+(fonts2['font3'].getsize("Reading & Writing")[0]/4))
        starting_pos_y += y_interval
        self.place_txt("Speaking", "font3", starting_pos_y, "black", 550+(fonts2['font3'].getsize("Reading & Writing")[0]/4))

    def create(self):

        if self.state == True:
            try:
                self.place_txt(self.level, "font0", 1000, "red")
                self.place_txt(self.std_name, "font1", 1250, "name_blue")
                self.place_sign()
                self.place_txt(self.date_str()[0], "font2", 1450, "black")
                self.place_txt(self.date_str()[1], "font2", 1530, "black")
                self.place_sheield()

                self.cert.save(f"certificates\\t2CERT{self.std_name}.png")
                return f"Certificate successfully created for {self.std_name}"

            except IndexError:
                return f"Error in {self.std_name}"
        else:
            return f"Error in {self.std_name}"

    @staticmethod
    def create_dic(csvfile, input_dic):
        rt_dic = {}
        with open(csvfile, "r") as f:
            gen_obj = csv.reader(f)
            lst = []

            # append everything in the generator object into a list, so that it can be iterated more than once.
            for line in gen_obj:
                lst.append(line)

            c = 0
            for line in lst:
                str_c = str(c)
                while len(str(str_c)) < 3:
                    str_c = "0" + str_c
                rt_dic[f'{line[0]}{str_c}'] = (line[0], line[1], line[2], line[3],input_dic['level'],
                                               input_dic['date'],input_dic['class_tr'],
                                               input_dic['t_rw'],input_dic['t_l'],input_dic['t_s'])
                c += 1

        return rt_dic

class Type3:

    def __init__(self, std_name: str, std_rw: float, std_l: float, std_s: float,
                 level: str, date: str, cls_tr: str, t_rw: float, t_l: float, t_s: float):

        self.std_name = std_name
        self.std_rw = std_rw
        self.std_l = std_l
        self.std_s = std_s
        self.level = level
        self.date = date
        self.cls_tr = cls_tr
        self.t_rw = t_rw
        self.t_l = t_l
        self.t_s = t_s

        self.cert = Image.open(f"templates\\type3temp.png")
        self.sign = Image.open(f"signs2\\{cls_tr}.png")

        self.state = True
        for i in [self.std_rw, self.std_l, self.std_s, self.t_rw, self.t_l, self.t_s]:
            try:
                i = float(i)
            except TypeError:
                self.state = False

    def image_size(self):
        return self.cert.size

    def place_txt(self, txt: str, font: str, y_pos: int, color: str, x_pos=None):

        txt_size = fonts3[font].getsize(txt)
        if x_pos is None:
            x_pos = ((self.image_size()[0] - txt_size[0]) / 2)

        draw = ImageDraw.Draw(self.cert)
        draw.text((x_pos, y_pos), txt, fill=colors[color], font=fonts3[font], anchor="la")

    def place_sign(self):
        self.cert.paste(self.sign, (1870, 2770))
        txt = "Teacher " + self.cls_tr[0].upper() + self.cls_tr[1:-1] + self.cls_tr[-1]
        txt_size = fonts3["font3"].getsize(txt)
        self.place_txt(txt, "font3", 3100, "black", ((self.image_size()[0] - txt_size[0] +1500) / 2))

    def create(self):

        if self.state == True:
            try:
                self.place_txt(f"{self.level} Test Report Form","font3",900,"black")
                txt = "Candidate's Name:"
                self.place_txt(txt, "font0",1200, "red", 350)
                self.place_txt(f"{self.std_name}", "font0", 1200, "black", 370 + fonts3['font0'].getsize(txt)[0] )

                starting_y_pos = 1650
                y_interval = 230
                gap = 20

                txt = "Reading&Writing:"
                self.place_txt(txt, "font0", starting_y_pos, "red", 350)
                self.place_txt(f"{self.std_rw}\\{self.t_rw}", "font0", starting_y_pos, "black",
                               370 + fonts3['font0'].getsize(txt)[0] +gap)
                starting_y_pos+=y_interval

                txt = "Listening:"
                self.place_txt(txt, "font0", starting_y_pos, "red", 867-fonts3['font0'].getsize(txt)[0])
                self.place_txt(f"{self.std_l}\\{self.t_l}", "font0", starting_y_pos, "black",
                               867 - fonts3['font0'].getsize(txt)[0] + fonts3['font0'].getsize(txt)[0] + gap)
                starting_y_pos += y_interval

                txt = "Speaking:"
                self.place_txt(txt, "font0", starting_y_pos, "red", 867 - fonts3['font0'].getsize(txt)[0])
                self.place_txt(f"{self.std_s}\\{self.t_s}", "font0", starting_y_pos, "black",
                               867 - fonts3['font0'].getsize(txt)[0] + fonts3['font0'].getsize(txt)[0] + gap)
                starting_y_pos += y_interval

                txt = "Overall score:"
                self.place_txt(txt, "font0", starting_y_pos, "red", 867 - fonts3['font0'].getsize(txt)[0])
                self.place_txt(f"{int(abs(sum([float(i) for i in [self.std_l,self.std_s,self.std_rw]])))}"
                               f"\\{int(abs(sum([float(i) for i in [self.t_l,self.t_s,self.t_rw]])))}",
                               "font0", starting_y_pos, "black",
                               867 - fonts3['font0'].getsize(txt)[0] + fonts3['font0'].getsize(txt)[0] +gap)
                starting_y_pos += y_interval

                txt = "Date:"
                self.place_txt(txt, "font0", starting_y_pos, "black", 867 - fonts3['font0'].getsize(txt)[0])
                self.place_txt(f"{self.date}",
                               "font0", starting_y_pos, "black",
                               867 - fonts3['font0'].getsize(txt)[0] + fonts3['font0'].getsize(txt)[0] + gap)
                starting_y_pos += y_interval

                self.place_sign()

                self.cert.save(f"certificates\\t3CERT{self.std_name}.png")
                return f"Certificate successfully created for {self.std_name}"

            except IndexError:
                return f"Error in {self.std_name}"
        else:
            return f"Error in {self.std_name}"

    @staticmethod
    def create_dic(csvfile, input_dic):
        rt_dic = {}
        with open(csvfile, "r") as f:
            gen_obj = csv.reader(f)
            lst = []

            # append everything in the generator object into a list, so that it can be iterated more than once.
            for line in gen_obj:
                lst.append(line)

            c = 0
            for line in lst:
                str_c = str(c)
                while len(str(str_c)) < 3:
                    str_c = "0" + str_c
                rt_dic[f'{line[0]}{str_c}'] = (line[0], line[1], line[2], line[3], input_dic['level'],
                                               input_dic['date'], input_dic['class_tr'],
                                               input_dic['t_rw'], input_dic['t_l'], input_dic['t_s'])
                c += 1

        return rt_dic

class Type6:

    def __init__(self, std_name: str, std_l: float, std_r: float, std_w: float, std_s:float, std_o:float,
                 date:str):

        self.std_name = std_name
        self.std_l = std_l
        self.std_r = std_r
        self.std_w = std_w
        self.std_s = std_s
        self.std_o = std_o
        self.date = date

        self.cert = Image.open(f"templates\\type6temp.png")

        self.state = True

        for i in [self.std_l, self.std_r, self.std_w, self.std_s]:
            try:
                i = float(i)
            except TypeError:
                self.state = False

    def image_size(self):
        return self.cert.size

    def place_txt(self, txt1: str, txt2:str, font: str, y_pos: int, color: str, x_pos):

        draw = ImageDraw.Draw(self.cert)
        draw.text((x_pos - 100, y_pos), txt1, fill=colors[color], font=fonts6[font], anchor="rm")
        draw.text((x_pos - 30, y_pos), " - ", fill=colors[color], font=fonts6[font], anchor="mm")
        draw.text((x_pos +30, y_pos), txt2, fill=colors[color], font=fonts6[font], anchor="lm")

    def create(self):

        if self.state == True:
            try:

                self.place_txt("Candidate's Name",self.std_name,"font2",1200,"black",1000)

                st_ypos = 1700
                y_inter = 200

                title_lst = ["Listening", "Reading","Writing","Speaking"]
                band_lst = [self.std_l,self.std_r,self.std_w,self.std_s]

                for i in range(4):
                    self.place_txt(title_lst[i],str(band_lst[i]),"font0",st_ypos,'black',900)
                    st_ypos += y_inter

                self.place_txt("Overall Band Score",str(self.std_o),"font1",st_ypos,"black",900)
                self.place_txt("Date",self.date,"font0",st_ypos+270,"black",450)

                self.cert.save(f"certificates\\t6CERT{self.std_name}.png")
                return f"Certificate successfully created for {self.std_name}"

            except IndexError:
                return f"Error in {self.std_name}"
        else:
            return f"Error in {self.std_name}"

    @staticmethod
    def create_dic(csvfile, input_dic):
        rt_dic = {}
        with open(csvfile, "r") as f:
            gen_obj = csv.reader(f)
            lst = []

            # append everything in the generator object into a list, so that it can be iterated more than once.
            for line in gen_obj:
                lst.append(line)

            c = 0
            for line in lst:
                str_c = str(c)
                while len(str(str_c)) < 3:
                    str_c = "0" + str_c
                rt_dic[f'{line[0]}{str_c}'] = (line[0], line[1], line[2], line[3], line[4],line[5],input_dic['date'])
                c += 1

        return rt_dic

type_dic = {"Type4":Type4, "Type2":Type2, "Type3":Type3, "Type1":Type1, "Type6":Type6}

#x = Type6("Thiha Swan Htet",9.0,8.5,7.0,6.5,8.0, "August 2020").create()

def main(jsonfile,csvfile):
    indic = input_dic_creator(jsonfile)
    dic = type_dic[indic['type']].create_dic(csvfile, indic)
    rt_lst = []
    for i in dic.values():
        rt_lst.append(eval(f"{indic['type']}{i}").create())
    return rt_lst


import asyncio
import time
import typing
from ...objects.package import Package
from ...utils import gen_random

class Messages:
    """
    Методы для работы с сообщениями
    """

    def __init__(self, api):
        self.__api = api
        self.__task_list = {}

    async def send(self, package: Package, message: typing.Optional[str]=None,
            attachments: typing.Optional[str]=None,
            delete_last:bool = False, **kwargs):
        """
        Упрощённая форма отправки ответа
        """

        kwargs['peer_id'] = package.peer_id
        kwargs['message'] = message
        kwargs['attachments'] = attachments

        if  'random_id' not in kwargs:
            kwargs['random_id'] = gen_random()

        if delete_last:
            await self.delete(package)

        return await self.__api.messages.send(**kwargs)

    async def delete(self, package:Package):
        """
        Удалить сообщение
        """

        return await self.__api.messages.delete(
            conversation_message_ids = package.conversation_message_id,
            peer_id = package.peer_id, delete_for_all = 1)

    def check_for_waiting_reply(self, package):
        """
        Специальная функция для получения новых оповещений с беседы.
        """

        for message_task in self.__task_list.values():
            if message_task.peer_id == package.peer_id:
                if message_task.from_id == package.from_id:
                    message_task.response = package
                    return True

    async def get_reply(self, package):
        """
        Специальная функция для получения новых оповещений с беседы.
        """

        message_task = ReplyTask(package)
        self.__task_list[str(message_task)] = message_task

        while not message_task.response:
            await asyncio.sleep(0.01)

        self.__task_list.pop(str(message_task), None)

        return message_task.response

class ReplyTask:
    """
    Объект задачи для ожидания ответа
    """

    def __init__(self, package):
        self.timestamp = time.time()
        self.peer_id = package.peer_id
        self.from_id = package.from_id
        self.response = None

    def __repr__(self) -> str:
        return "<vkbotkit.framework.toolkit.replies.task>"

    def __str__(self):
        return f"${self.timestamp}_{self.peer_id}_{self.from_id}"


#!/usr/bin/python

import paho.mqtt.client as mqtt
import json
import logging

logging.basicConfig(filename='voltlog.log', level=logging.INFO, format='%(asctime)s %(message)s', datefmt='%Y-%m-%d %H:%M:%S')

def on_connect(client, userdata, flags, rc):
	client.subscribe("tele/sonoffpow/ENERGY")

def on_message(client, userdata, msg):
	carga = json.loads(msg.payload)
	volts = carga['Voltage']
	print(str(volts)+" volts")
	logging.info(volts)

client = mqtt.Client()
client.on_connect = on_connect
client.on_message = on_message

client.connect("10.0.0.254", 1883, 60)

try:
	client.loop_forever()

except KeyboardInterrupt:
	print("Exiting")


#! python3
# add bullet in front of list copied from web or somewhere else

import pyperclip

text = pyperclip.paste()

lines = text.split('\n')
""" to split the copied string by new line character """

for i in range(len(lines)):
    lines[i] = '* ' + lines[i]

text = '\n'.join(lines)

pyperclip.copy(text)

# this file has bat file


from django import forms
from .models import Produto

class Form_Produto(forms.ModelForm):
    class Meta:
        model = Produto
        fields = "__all__"
        labels = {
                'nome_produto': 'Nome do Produto',
                'codigo_produto': 'Codigo do Produto',
                'quatidade': 'Quantidade',
                'quantidade_minima': 'Quantidade Mínima',
                'categoria': 'Categoria',
                'preco_custo': 'Preço do Produto',
                'preco_venda' : 'Preço de Venda'
        }


import json

from project.tests.base import BaseTestCase

class TestEncryptService(BaseTestCase):
    def test_encrypt_returns_a_value(self):
        """Ensure the api can call the encryptor properly"""
        with self.client:
            plaintext = "Hello, I'm a string to encrypt."
            response = self.client.get("/encrypt/{plaintext}")
            data = json.loads(response.data.decode())
            assert response.status_code == 200
            assert data['data']['ciphertext'] != plaintext

    # def test_ping(self):
    #     with self.client:
    #         response = self.client.get('/encrypt/ping')
    #         assert response.status_code == 200


from django.db import models

class User(models.Model):
    username = models.CharField(max_length=100, unique=True, primary_key=True)
    email = models.EmailField()
    profile_image = models.ImageField(upload_to='images', default='Screenshot_from_2023-01-07_20-45-18.png')

    def __str__(self):
        return self.username

    def save(
        self, force_insert=False, force_update=False, using=None, update_fields=None
    ):
        if '.ac.ir' not in self.email:
            return
        else:
            self.username = self.username.lower()
            super().save()

    def delete(self, using=None, keep_parents=False):
        if self.username == 'han':
            raise Exception('You can delete this user')
        else:
            super().delete()

    class Meta:
        ordering = ['username']


# cell_inverter.py <filename.xlsx> - inverts the row and column positions of all cells in the spreadsheet

import openpyxl
from openpyxl.utils import get_column_letter, column_index_from_string
import sys

if len(sys.argv) == 2:
	work_book = openpyxl.load_workbook(sys.argv[1])
else:
	print('\ncell_inverter.py <filename.xlsx> - inverts the row and column positions of all cells in the spreadsheet')

sheet = work_book.active
new_sheet = {}

# store every value from every cell in the new_sheet dict with the key being the inverted position
for i in range(1, sheet.max_row+1):
	for cell in sheet[i]:
		new_col = str(get_column_letter(cell.row))
		new_row = str(column_index_from_string(cell.column))
		new_sheet[new_col+new_row] = cell.value

# write the cell values to their inverted positions
for position in new_sheet.keys():
	sheet[position].value = new_sheet[position]

# always save to a new file name
work_book.save('inverted.xlsx')


from datetime import datetime

import dash_bootstrap_components as dbc
import pandas as pd
import plotly.express as px
import plotly.graph_objects as go

from dash import callback, dcc, dash_table, html
from dash.dependencies import Input, Output, State

from external import datasets

class CierreTable():
    def __init__(
        self,
        signal_update,
        id=None,
        cierre_id=None,
        offset_store_id=None,
        sort_action=None,
    ):
        self.signal_update = signal_update
        self.id = id
        self.cierre_id = cierre_id
        self.offset_store_id = offset_store_id
        self.sort_action = sort_action

    def display(self):
        table_args = {
            'style_table': {'overflow': 'auto'},
        }
        if (self.id):
            table_args['id'] = self.id
        if (self.sort_action):
            table_args['sort_action'] = self.sort_action

        self.layout = dash_table.DataTable(**table_args)
        self.register_callbacks()

        return self.layout

    def register_callbacks(self):
        if (self.cierre_id):
            @callback(
                Output(self.layout, 'columns'),
                Output(self.layout, 'data'),
                Input(self.signal_update, 'data'),
                Input(self.cierre_id, 'value'),
                prevent_initial_call=False,
            )
            def update_table(_, cierre_date):
                df = datasets.stats_cierre(dataviz=True)

                cierre_date = datetime.strptime(cierre_date, '%Y-%m-%d').date()
                df = df.query('FECHA_CIERRE == @cierre_date')

                if (df.shape[0] == 0):
                    return ([], [])

                columns = [{'name': i, 'id': i} for i in df.columns]
                data = df.to_dict('records')

                return (columns, data)
        elif (self.offset_store_id):
            @callback(
                Output(self.layout, 'columns'),
                Output(self.layout, 'data'),
                Input(self.signal_update, 'data'),
                Input(self.offset_store_id, 'data'),
                prevent_initial_call=False,
            )
            def update_table(_, offset_data):
                df = datasets.stats_cierre(dataviz=True)
                df = datasets.filter_dataframe(
                    df, 'FECHA_CIERRE', offset_data)
                columns = [{'name': i, 'id': i, } for i in df.columns]
                data = df.to_dict('records')

                return (columns, data)


from __future__ import division

import base64
import errno
import gzip
import json
import os
import shutil
import struct
import tempfile
import time
from collections import namedtuple
from io import open
from logging import getLogger

import OpenSSL
import boto3
import botocore.exceptions
from Crypto.Cipher import AES
from Crypto.Hash import SHA256
from boto3.exceptions import RetriesExceededError, S3UploadFailedError
from boto3.s3.transfer import TransferConfig
from botocore.client import Config

from .compat import (PKCS5_PAD, PKCS5_UNPAD, PKCS5_OFFSET, TO_UNICODE)
from .constants import (AMZ_MATDESC, AMZ_KEY, AMZ_IV,
                        SFC_DIGEST, SHA256_DIGEST, UTF8)

RESULT_STATUS_ERROR = u'ERROR'
RESULT_STATUS_UPLOADED = u'UPLOADED'
RESULT_STATUS_DOWNLOADED = u'DOWNLOADED'
RESULT_STATUS_COLLISION = u'COLLISION'
RESULT_STATUS_SKIPPED = u'SKIPPED'
RESULT_STATUS_RENEW_TOKEN = u'RENEW_TOKEN'
RESULT_STATUS_NOT_FOUND_FILE = u'NOT_FOUND_FILE'

DEFAULT_CONCURRENCY = 1
DEFAULT_MAX_RETRY = 5
ERRORNO_WSAECONNABORTED = 10053  # network connection was aborted

"""
S3 Location: S3 bucket name + path
"""
S3Location = namedtuple(
    "S3Location", [
        "bucket_name",  # S3 bucket name
        "s3path"  # S3 path name

    ])

"""
Material Description
"""
MaterialDescriptor = namedtuple(
    "MaterialDescriptor", [
        "smk_id",  # SMK id
        "query_id",  # query id
        "key_size"  # key size, 128 or 256
    ]
)

"""
Encryption Material
"""
SnowflakeS3FileEncryptionMaterial = namedtuple(
    "SnowflakeS3FileEncryptionMaterial", [
        "query_stage_master_key",  # query stage master key
        "query_id",  # query id
        "smk_id"  # SMK id
    ]
)

def matdesc_to_unicode(matdesc):
    """
    Convert Material Descriptor to Unicode String
    """
    return TO_UNICODE(
        json.dumps({
            u'queryId': matdesc.query_id,
            u'smkId': str(matdesc.smk_id),
            u'keySize': str(matdesc.key_size)
        },
            separators=(',', ':')))

class SnowflakeS3Util(object):
    """
    S3 Utility class
    """
    # magic number, given from the AWS error message.
    DATA_SIZE_THRESHOLD = 5242880

    @staticmethod
    def create_s3_client(stage_credentials, use_accelerate_endpoint=False):
        """
        Creates a s3client object with a stage credential
        :param stage_credentials: a stage credential
        :param use_accelerate_endpoint: is accelerate endpoint?
        :return: s3client
        """
        logger = getLogger(__name__)
        security_token = stage_credentials[
            u'AWS_TOKEN'] if u'AWS_TOKEN' in stage_credentials else None
        logger.debug(u"AWS_ID: %s", stage_credentials[u'AWS_ID'])

        config = Config(
            signature_version=u's3v4',
            s3={
                'use_accelerate_endpoint': use_accelerate_endpoint,
            })
        s3client = boto3.resource(
            u's3',
            region_name=stage_credentials['region'],
            aws_access_key_id=stage_credentials[u'AWS_ID'],
            aws_secret_access_key=stage_credentials[u'AWS_KEY'],
            aws_session_token=security_token,
            config=config,
        )
        return s3client

    @staticmethod
    def upload_one_file_to_s3(meta):
        """
        Uploads a file to S3
        :param meta: a file meta
        """
        logger = getLogger(__name__)
        s3location = SnowflakeS3Util.extract_bucket_name_and_path(
            meta[u'stage_location'])
        s3path = s3location.s3path + meta[u'dst_file_name']
        s3_metadata = {
            u'Content-Type': u'application/octet-stream',
            SFC_DIGEST: meta[SHA256_DIGEST],
        }
        if u'encryption_material' in meta:
            data_file = SnowflakeS3Util.encrypt_file(
                s3_metadata, meta[u'encryption_material'],
                meta[u'real_src_file_name'], tmp_dir=meta[u'tmp_dir'])
            logger.debug(
                u'encrypted data file=%s, size=%s', data_file,
                os.path.getsize(data_file))
        else:
            logger.debug(u'not encrypted data file')
            data_file = meta[u'real_src_file_name']

        akey = SnowflakeS3Util.get_s3_file_object(meta, meta[u'dst_file_name'])
        if meta[u'result_status'] == RESULT_STATUS_RENEW_TOKEN:
            # need renew token
            return
        elif akey and meta[u'result_status'] == RESULT_STATUS_UPLOADED and \
                not meta.get(u'overwrite'):
            logger.info(
                u'file already exists, checking digest: file=%s', s3path)
            try:
                sfc_digest = akey.metadata[SFC_DIGEST]
                if sfc_digest == meta[SHA256_DIGEST]:
                    logger.info(u'file digest matched: digest=%s',
                                sfc_digest)
                    meta[u'dst_file_size'] = 0
                    meta[u'error_details'] = \
                        (u'File with the same destination name '
                         u'and checksum already exists')
                    meta[u'result_status'] = RESULT_STATUS_SKIPPED
                    return
                else:
                    logger.info(
                        (u"file digest didn't match: "
                         u"digest_s3=%s, digest_local=%s"),
                        sfc_digest, meta[SHA256_DIGEST])
            except botocore.exceptions.ClientError as e:
                # this could happen in the last minute
                if e.response[u'Error'][u'Code'] != '404':
                    raise e
                logger.debug(u'ignored. file not found: %s, %s',
                             s3location.bucket_name, s3path)

        logger.debug(u'putting a file: %s', s3path)
        put_callback = meta[u'put_callback']
        put_callback_output_stream = meta[u'put_callback_output_stream']

        max_concurrency = meta[u'parallel']
        last_err = None
        max_retry = DEFAULT_MAX_RETRY
        for retry in range(max_retry):
            try:
                akey.upload_file(
                    data_file,
                    Callback=put_callback(
                        data_file,
                        os.path.getsize(data_file),
                        output_stream=put_callback_output_stream) if \
                        put_callback else None,
                    ExtraArgs={
                        u'Metadata': s3_metadata,
                        u'ContentEncoding': u'gzip',
                    },
                    Config=TransferConfig(
                        multipart_threshold=SnowflakeS3Util.DATA_SIZE_THRESHOLD,
                        max_concurrency=max_concurrency,
                        num_download_attempts=10,
                    )
                )
                break
            except botocore.exceptions.ClientError as err:
                if err.response[u'Error'][u'Code'] == u'ExpiredToken':
                    logger.debug(u"AWS Token expired. Renew and retry")
                    meta[u'result_status'] = RESULT_STATUS_RENEW_TOKEN
                    return
                logger.exception(
                    u"Failed to upload a file: %s, err: %s",
                    data_file, err)
                raise err
            except OpenSSL.SSL.SysCallError as err:
                last_err = err
                if err.args[0] not in (
                        ERRORNO_WSAECONNABORTED,
                        errno.ECONNRESET,
                        errno.ETIMEDOUT,
                        errno.EPIPE,
                        -1):
                    raise err
                if err.args[0] == ERRORNO_WSAECONNABORTED:
                    # connection was disconnected by S3
                    # because of too many connections. retry with
                    # less concurrency to mitigate it
                    max_concurrency = meta[u'parallel'] - int(
                        retry * meta[u'parallel'] / max_retry)
                    max_concurrency = max(DEFAULT_CONCURRENCY, max_concurrency)
                    meta['last_max_concurrency'] = max_concurrency
                logger.info(
                    'Failed to upload a file: %s, err: %s. Retrying with '
                    'max concurrency: %s',
                    data_file, err, max_concurrency)
                if 'no_sleeping_time' not in meta:
                    sleeping_time = min(2 ** retry, 16)
                    logger.debug(u"sleeping: %s", sleeping_time)
                    time.sleep(sleeping_time)
            except S3UploadFailedError as err:
                last_err = err
                logger.info(
                    'Failed to upload a file: %s, err: %s. Retrying',
                    data_file, err)
                if 'no_sleeping_time' not in meta:
                    sleeping_time = min(2 ** retry, 16)
                    logger.debug(u"sleeping: %s", sleeping_time)
                    time.sleep(sleeping_time)

        else:
            if last_err:
                raise last_err
            else:
                raise Exception(
                    "Unknown Error in uploading a file: %s",
                    data_file)

        logger.debug(u'DONE putting a file')
        meta[u'dst_file_size'] = meta[u'upload_size']
        meta[u'result_status'] = RESULT_STATUS_UPLOADED

    @staticmethod
    def download_one_file_from_s3(meta):
        """
        Downloads a file from S3
        :param meta: file meta
        """
        logger = getLogger(__name__)
        full_dst_file_name = os.path.join(
            meta[u'local_location'],
            os.path.basename(meta[u'dst_file_name']))
        full_dst_file_name = os.path.realpath(full_dst_file_name)
        # TODO: validate full_dst_file_name is under the writable directory
        base_dir = os.path.dirname(full_dst_file_name)
        if not os.path.exists(base_dir):
            os.makedirs(base_dir)

        get_callback = meta[u'get_callback']
        get_callback_output_stream = meta[u'get_callback_output_stream']

        akey = SnowflakeS3Util.get_s3_file_object(meta, meta[u'src_file_name'])
        meta[u'src_file_size'] = akey.content_length

        max_concurrency = meta[u'parallel']
        last_err = None
        max_retry = DEFAULT_MAX_RETRY
        for retry in range(max_retry):
            try:
                akey.download_file(
                    full_dst_file_name,
                    Callback=get_callback(
                        meta[u'src_file_name'],
                        meta[u'src_file_size'],
                        output_stream=get_callback_output_stream) if \
                        get_callback else None,
                    Config=TransferConfig(
                        multipart_threshold=SnowflakeS3Util.DATA_SIZE_THRESHOLD,
                        max_concurrency=max_concurrency,
                        num_download_attempts=10,
                    )
                )
                break
            except botocore.exceptions.ClientError as err:
                if err.response[u'Error'][u'Code'] == u'ExpiredToken':
                    logger.debug(u"AWS Token expired. Renew and retry")
                    meta[u'result_status'] = RESULT_STATUS_RENEW_TOKEN
                    return
                logger.exception(
                    u"Failed to download a file: %s, err: %s",
                    full_dst_file_name, err)
                raise err
            except OpenSSL.SSL.SysCallError as err:
                last_err = err
                if err.args[0] not in (
                        ERRORNO_WSAECONNABORTED,
                        errno.ECONNRESET,
                        errno.ETIMEDOUT,
                        errno.EPIPE,
                        -1):
                    raise err
                if err.args[0] == ERRORNO_WSAECONNABORTED:
                    # connection was disconnected by S3
                    # because of too many connections. retry with
                    # less concurrency to mitigate it
                    max_concurrency = meta[u'parallel'] - int(
                        retry * meta[u'parallel'] / max_retry)
                    max_concurrency = max(DEFAULT_CONCURRENCY, max_concurrency)
                    meta['last_max_concurrency'] = max_concurrency
                logger.info(
                    'Failed to download a file: %s, err: %s. Retrying with '
                    'max concurrency: %s',
                    full_dst_file_name, err, max_concurrency)
                if 'no_sleeping_time' not in meta:
                    sleeping_time = min(2 ** retry, 16)
                    logger.debug(u"sleeping: %s", sleeping_time)
                    time.sleep(sleeping_time)
            except RetriesExceededError as err:
                last_err = err
                logger.info(
                    'Failed to download a file: %s, err: %s. Retrying',
                    full_dst_file_name, err)
                if 'no_sleeping_time' not in meta:
                    sleeping_time = min(2 ** retry, 16)
                    logger.debug(u"sleeping: %s", sleeping_time)
                    time.sleep(sleeping_time)
        else:
            if last_err:
                raise last_err
            else:
                raise Exception(
                    "Unknown Error in downloading a file: %s",
                    full_dst_file_name)

        if u'encryption_material' in meta:
            logger.debug(
                u'encrypted data file=%s', full_dst_file_name)
            s3_metadata = {
                AMZ_KEY: akey.metadata[AMZ_KEY],
                AMZ_IV: akey.metadata[AMZ_IV],
            }
            tmp_dst_file_name = SnowflakeS3Util.decrypt_file(
                s3_metadata, meta[u'encryption_material'],
                full_dst_file_name, tmp_dir=meta[u'tmp_dir'])
            shutil.copyfile(tmp_dst_file_name, full_dst_file_name)
            os.unlink(tmp_dst_file_name)
        else:
            logger.debug(u'not encrypted data file=%s',
                         full_dst_file_name)

        statinfo = os.stat(full_dst_file_name)
        meta[u'dst_file_size'] = statinfo.st_size

    @staticmethod
    def extract_bucket_name_and_path(stage_location):
        bucket_name = stage_location
        s3path = u''

        # split stage location as bucket name and path
        if u'/' in stage_location:
            bucket_name = stage_location[0:stage_location.index(u'/')]
            s3path = stage_location[stage_location.index(u'/') + 1:]
            if s3path and not s3path.endswith(u'/'):
                s3path += u'/'

        return S3Location(
            bucket_name=bucket_name,
            s3path=s3path)

    @staticmethod
    def get_secure_random(byte_length):
        return os.urandom(byte_length)

    @staticmethod
    def encrypt_file(s3_metadata, encryption_material, in_filename,
                     chunk_size=AES.block_size * 4 * 1024, tmp_dir=None):
        """
        Encrypts a file
        :param s3_metadata: S3 metadata output
        :param encryption_material: encryption material
        :param in_filename: input file name
        :param chunk_size: read chunk size
        :param tmp_dir: temporary directory, optional
        :return: a encrypted file
        """
        logger = getLogger(__name__)
        decoded_key = base64.standard_b64decode(
            encryption_material.query_stage_master_key)
        key_size = len(decoded_key)
        logger.debug(u'key_size = %s', key_size)

        # Generate key for data encryption
        iv_data = SnowflakeS3Util.get_secure_random(AES.block_size)
        file_key = SnowflakeS3Util.get_secure_random(key_size)
        data_cipher = AES.new(key=file_key, mode=AES.MODE_CBC, IV=iv_data)

        temp_output_fd, temp_output_file = tempfile.mkstemp(
            text=False, dir=tmp_dir,
            prefix=os.path.basename(in_filename) + "#")
        padded = False
        logger.debug(u'unencrypted file: %s, temp file: %s, tmp_dir: %s',
                     in_filename, temp_output_file, tmp_dir)
        with open(in_filename, u'rb') as infile:
            with os.fdopen(temp_output_fd, u'wb') as outfile:
                while True:
                    chunk = infile.read(chunk_size)
                    if len(chunk) == 0:
                        break
                    elif len(chunk) % AES.block_size != 0:
                        chunk = PKCS5_PAD(chunk, AES.block_size)
                        padded = True
                    outfile.write(data_cipher.encrypt(chunk))
                if not padded:
                    outfile.write(data_cipher.encrypt(
                        AES.block_size * chr(AES.block_size).encode(UTF8)))

        # encrypt key with QRMK
        key_cipher = AES.new(key=decoded_key, mode=AES.MODE_ECB)
        enc_kek = key_cipher.encrypt(PKCS5_PAD(file_key, AES.block_size))

        mat_desc = MaterialDescriptor(
            smk_id=encryption_material.smk_id,
            query_id=encryption_material.query_id,
            key_size=key_size * 8)
        s3_metadata[AMZ_MATDESC] = matdesc_to_unicode(mat_desc)
        s3_metadata[AMZ_KEY] = base64.b64encode(enc_kek).decode('utf-8')
        s3_metadata[AMZ_IV] = base64.b64encode(iv_data).decode('utf-8')
        return temp_output_file

    @staticmethod
    def decrypt_file(s3_metadata, encryption_material, in_filename,
                     chunk_size=AES.block_size * 4 * 1024, tmp_dir=None):
        """
        Decrypts a file and stores the output in the temporary directory
        :param s3_metadata: S3 metadata input
        :param encryption_material: encryption material
        :param in_filename: input file name
        :param chunk_size: read chunk size
        :param tmp_dir: temporary directory, optional
        :return: a decrypted file name
        """
        logger = getLogger(__name__)
        key_base64 = s3_metadata[AMZ_KEY]
        iv_base64 = s3_metadata[AMZ_IV]

        decoded_key = base64.standard_b64decode(
            encryption_material.query_stage_master_key)
        key_bytes = base64.standard_b64decode(key_base64)
        iv_bytes = base64.standard_b64decode(iv_base64)

        key_cipher = AES.new(key=decoded_key, mode=AES.MODE_ECB)
        file_key = PKCS5_UNPAD(key_cipher.decrypt(key_bytes))

        data_cipher = AES.new(key=file_key, mode=AES.MODE_CBC, IV=iv_bytes)

        temp_output_fd, temp_output_file = tempfile.mkstemp(
            text=False, dir=tmp_dir,
            prefix=os.path.basename(in_filename) + "#")
        total_file_size = 0
        prev_chunk = None
        logger.info(u'encrypted file: %s, tmp file: %s',
                    in_filename, temp_output_file)
        with open(in_filename, u'rb') as infile:
            with os.fdopen(temp_output_fd, u'wb') as outfile:
                while True:
                    chunk = infile.read(chunk_size)
                    if len(chunk) == 0:
                        break
                    total_file_size += len(chunk)
                    d = data_cipher.decrypt(chunk)
                    outfile.write(d)
                    prev_chunk = d
                if prev_chunk is not None:
                    total_file_size -= PKCS5_OFFSET(prev_chunk)
                outfile.truncate(total_file_size)
        return temp_output_file

    @staticmethod
    def compress_file_with_gzip(file_name, tmp_dir):
        """
        Compresses a file by GZIP
        :param file_name: a file name
        :param tmp_dir: temprary directory where an GZIP file will be created
        :return: a paif of gzip file name and size
        """
        logger = getLogger(__name__)
        base_name = os.path.basename(file_name)
        gzip_file_name = os.path.join(tmp_dir, base_name + u'_c.gz')
        logger.debug(u'gzip file: %s, original file: %s', gzip_file_name,
                     file_name)
        fr = open(file_name, u'rb')
        fw = gzip.GzipFile(gzip_file_name, u'wb')
        shutil.copyfileobj(fr, fw)
        fw.close()
        fr.close()
        SnowflakeS3Util.normalize_gzip_header(gzip_file_name)

        statinfo = os.stat(gzip_file_name)
        return gzip_file_name, statinfo.st_size

    @staticmethod
    def normalize_gzip_header(gzip_file_name):
        """
        Normalize GZIP file header. For consistent file digest, this removes
        creation timestamp from the header.
        :param gzip_file_name: gzip file name
        """
        logger = getLogger(__name__)
        with open(gzip_file_name, u'r+b') as f:
            # reset the timestamp in gzip header
            f.seek(4, 0)
            f.write(struct.pack('<L', 0))
            # reset the file name in gzip header
            f.seek(10, 0)
            byte = f.read(1)
            while byte:
                value = struct.unpack('B', byte)[0]
                # logger.debug(u'ch=%s, byte=%s', value, byte)
                if value == 0:
                    break
                f.seek(-1, 1)  # current_pos - 1
                f.write(struct.pack('B', 0x20))  # replace with a space
                byte = f.read(1)

    @staticmethod
    def get_digest_and_size_for_file(file_name):
        """
        Gets file digest and size
        :param file_name: a file name
        :return:
        """
        CHUNK_SIZE = 16 * 4 * 1024
        f = open(file_name, 'rb')
        m = SHA256.new()
        while True:
            chunk = f.read(CHUNK_SIZE)
            if chunk == b'':
                break
            m.update(chunk)

        statinfo = os.stat(file_name)
        file_size = statinfo.st_size
        digest = base64.standard_b64encode(m.digest()).decode(UTF8)
        logger = getLogger(__name__)
        logger.debug(u'getting digest and size: %s, %s, file=%s', digest,
                     file_size, file_name)
        return digest, file_size

    @staticmethod
    def get_s3_file_object(meta, filename):
        """
        Gets S3 file object
        :param meta: file meta object
        :return: S3 object if no error, otherwise None. Check meta[
        u'result_status'] for status.
        """
        import logging
        logger = getLogger(__name__)
        s3client = meta[u's3client']
        s3location = SnowflakeS3Util.extract_bucket_name_and_path(
            meta[u'stage_location'])
        s3path = s3location.s3path + filename.lstrip('/')

        if logger.getEffectiveLevel() == logging.DEBUG:
            tmp_meta = {}
            for k, v in meta.items():
                if k != 'stage_credentials':
                    tmp_meta[k] = v
            logger.debug(
                u"s3location.bucket_name: %s, "
                u"s3location.s3path: %s, "
                u"s3fullpath: %s, "
                u'meta: %s',
                s3location.bucket_name,
                s3location.s3path,
                s3path, tmp_meta)

        try:
            # HTTP HEAD request
            akey = s3client.Object(s3location.bucket_name, s3path)
            akey.load()
        except botocore.exceptions.ClientError as e:
            if e.response[u'Error'][u'Code'] == u'ExpiredToken':
                logger.debug(u"AWS Token expired. Renew and retry")
                meta[u'result_status'] = RESULT_STATUS_RENEW_TOKEN
                return None
            elif e.response[u'Error'][u'Code'] == u'404':
                logger.debug(u'not found. bucket: %s, path: %s',
                             s3location.bucket_name, s3path)
                meta[u'result_status'] = RESULT_STATUS_NOT_FOUND_FILE
                return akey
            logger.debug(
                u"Failed to get metadata for %s, %s: %s",
                s3location.bucket_name, s3path, e)
            meta[u'result_status'] = RESULT_STATUS_ERROR
            return None

        meta[u'result_status'] = RESULT_STATUS_UPLOADED
        return akey

    @staticmethod
    def upload_one_file_to_s3_with_retry(meta):
        """
        Uploads one file to S3 with retry
        :param meta: a file meta
        """
        logger = getLogger(__name__)
        for _ in range(10):
            # retry
            SnowflakeS3Util.upload_one_file_to_s3(meta)
            if meta[u'result_status'] == RESULT_STATUS_UPLOADED:
                for _ in range(10):
                    _ = SnowflakeS3Util.get_s3_file_object(
                        meta, meta[u'dst_file_name'])
                    if meta[u'result_status'] == RESULT_STATUS_NOT_FOUND_FILE:
                        time.sleep(1)  # wait 1 second
                        logger.debug('not found. double checking...')
                        continue
                    break
                else:
                    # not found. retry with the outer loop
                    logger.debug('not found. gave up. reuploading...')
                    continue
            break
        else:
            # could not upload a file even after retry
            meta[u'result_status'] = RESULT_STATUS_ERROR