import collections
import contextlib
import functools
import hashlib
import itertools
import json
import operator
import os
import sys

# -----------------------------------------------------------------------------
def coroutine(fun):
    """
    Return a decorator function that primes communications with a coroutine.

    """
    def primed(*args, **kwargs):
        """
        Wrapper for coroutines that 'primes' it by sending the first message.

        """
        coro = fun(*args, **kwargs)
        coro.send(None)
        return coro
    return primed

# -----------------------------------------------------------------------------
def string_types():
    """
    Return the string types for the current Python version.

    """
    is_python_2 = str is bytes
    if is_python_2:
        return (str, unicode)  # pylint: disable=E0602
    else:
        return (str, bytes)

# -----------------------------------------------------------------------------
def is_string(obj):
    """
    Return true if obj is a string. Works in Python 2 and Python 3.

    """
    return isinstance(obj, string_types())

# -----------------------------------------------------------------------------
@coroutine
def index_builder_coro(index = None):
    """
    Return a coroutine that builds an index (map) from supplied tuples.

    The last element in each tuple is treated as
    the payload value that is to be indexed, and
    the preceeding elements as a hierarchy of
    indices by which it is to be indexed.

    The index itself is a nested default-dictionary
    with len(tup) levels of nesting, containing a
    list at the innermost level to hold the payload.

    >>> builder = index_builder_coro()
    >>> builder.send(None)
    >>> index   = builder.send(('a', 'b', 1))
    >>> index   = builder.send(('a', 'c', 2))
    >>> print(json.dumps(index, sort_keys=True))
    {"a": {"b": [1], "c": [2]}}

    """
    _dedict   = collections.defaultdict
    _repeat   = itertools.repeat
    _reduce   = functools.reduce
    _partial  = functools.partial
    _getitem  = operator.getitem
    tupl      = yield None
    if index is None:
        index = _reduce(lambda ctor, _: _partial(_dedict, ctor),
                        _repeat(None, len(tupl) - 1), list)()
    while True:
        tmp = index
        _reduce(_getitem, tupl[:-1], tmp).append(tupl[-1])
        tupl = yield index

# -----------------------------------------------------------------------------
def build_index(itable_tup):
    """
    Return an index for an iterable container of tuples, all the same length.

    The last element in each tuple is treated as
    the payload value that is to be indexed, and
    the preceeding elements as a hierarchy of
    indices by which it is to be indexed.

    The index itself is a nested default-dictionary
    with len(tup) levels of nesting, containing a
    list at the innermost level to hold the payload.

    >>> idx = build_index((('a','b', 1), ('a','c', 2)), 3)
    >>> import json
    >>> print(json.dumps(idx, sort_keys=True))
    {"a": {"b": [1], "c": [2]}}

    This function is a bit of an exercise in making
    use of the functional programming capabilities
    that come with Python, so the implementation is
    a little more terse and opaque than it might
    otherwise be...

    For each tuple in iter_tup, we construct a slot in
    the index using the lazy initialisation capability
    that is provided by the collections.defaultdict
    class.

    All but the last element in the tuple are used
    for indexing. The last element is the payload
    which gets added to the inner list after the
    structure has been initialised. The basic
    capability is provided by the nested defaultdict
    data structure returned from the idx_ctor
    function, which in turn is built with the aid
    of the higher order lambda function
    (_partial(_dedict, ctor)).

    All this is plugged together using calls to
    functools.reduce, which is used to construct
    the custom index data structure; to add each
    tuple to the index, as well as to iterate over
    the input.

    """
    _dedict   = collections.defaultdict
    _chain    = itertools.chain
    _repeat   = itertools.repeat
    _reduce   = functools.reduce
    _partial  = functools.partial
    _getitem  = operator.getitem

    # Peek ahead to determine the tuple length (& reconstitute the iterator).
    iter_tup  = iter(itable_tup)
    try:
        first_tup = iter_tup.__next__()
    except StopIteration:
        return {}
    idx_depth = len(first_tup) - 1
    iter_all  = _chain([first_tup], iter_tup)

    # Create a constructor for a custom index data structure.
    idx_ctor = _reduce(lambda ctor, _: _partial(_dedict, ctor),
                       _repeat(None, idx_depth), list)

    # This function adds a sigle tuple to the custom index data structure.
    def _add_tup_to_idx(idx, tup):
        """
        Add a sigle tuple to the custom index data structure.

        """
        _reduce(_getitem, tup[:-1], idx).append(tup[-1])
        return idx

    # This iterates over all tuples adding each to the index in turn.
    return _reduce(_add_tup_to_idx, iter_all, idx_ctor())

# -----------------------------------------------------------------------------
# TODO: Refactor this so the parameters go in a named tuple and re-enable the
#       pylint warning.
def walkobj(obj,                                  # pylint: disable=R0912,R0913
            gen_leaf    = False,
            gen_nonleaf = False,
            gen_path    = False,
            gen_obj     = False,
            path        = (),
            memo        = None):
    """
    Generate a walk over a treelike structure of mappings and other iterables.

    Adapted from:
    http:/code.activestate.com/recipes/577982-recursively-walk-python-objects/

    This function performs a (depth-first left to
    right recursive) traversal over the provided
    data structure, which we assume to consist of
    a finite; treelike arrangement of nested
    collections.Mapping and collections.Iterable
    types.

    This function can be configured to yield
    information about nodes in the tree: leaf
    nodes; non-leaf (internal) nodes or both.

    The information that is yielded may also be
    configured: the path to the node can be
    delivered as can the object at the node
    itself.

    """
    # If the object is elemental, it cannot be
    # decomposed, so we must bottom out the
    # recursion and yield the object and its'
    # path before returning control back up
    # the stack.
    #
    is_itable    = isinstance(obj, collections.Iterable)
    is_leaf      = (not is_itable) or is_string(obj)

    if is_leaf:
        if gen_leaf:
            if gen_path and gen_obj:
                yield (path, obj)
            elif gen_path:
                yield path
            elif gen_obj:
                yield obj
        return

    # Since this is a recursive function, we need
    # to be on our guard against any references
    # to objects back up the call stack (closer
    # to the root of the tree). Any such
    # references would be circular, leading to
    # an infinite tree, and causing us to blow
    # our stack in a fit of unbounded recursion.
    #
    # If we detect that we've already visited
    # this object (using identity not equality),
    # then the safe thing to do is to halt the
    # recursive descent and return control back
    # up the stack.
    #
    _id = id(obj)
    if memo is None:
        memo = set()
    if _id in memo:
        return
    memo.add(_id)

    # If the object is not elemental (i.e. it is
    # an Iterable), then it may be decomposed, so
    # we should recurse down into each component,
    # yielding the results as we go. Of course,
    # we need different iteration functions for
    # mappings vs. other iterables.
    #
    def mapiter(mapping):
        """
        Return an iterator over the specified mapping or other iterable.

        This function selects the appropriate
        iteration function to use.

        """
        return getattr(mapping, 'iteritems', mapping.items)()
    itfcn = mapiter if isinstance(obj, collections.Mapping) else enumerate

    for pathpart, component in itfcn(obj):

        childpath = path + (pathpart,)
        if gen_nonleaf:
            if gen_path and gen_obj:
                yield (childpath, component)
            elif gen_path:
                yield childpath
            elif gen_obj:
                yield component

        for result in walkobj(obj         = component,
                              gen_leaf    = gen_leaf,
                              gen_nonleaf = gen_nonleaf,
                              gen_path    = gen_path,
                              gen_obj     = gen_obj,
                              path        = childpath,
                              memo        = memo):
            yield result

    # We only need to guard against infinite
    # recursion within a branch of the call-tree.
    # There is no danger in visiting the same item
    # instance in sibling branches, so we can
    # forget about objects once we are done
    # with them and about to pop the stack.
    #
    memo.remove(_id)
    return

# -----------------------------------------------------------------------------
def flatten_ragged(raggedlist, memo = None):
    """
    Flatten a 'ragged' list (a list-of-lists).

    """
    # If the object is elemental, it cannot be
    # decomposed, so we must bottom out the
    # recursion and yield the object and its'
    # path before returning control back up the
    # stack.
    #
    is_itable    = isinstance(raggedlist, collections.Iterable)

    is_leaf      = (not is_itable) or is_string(raggedlist)

    if is_leaf:
        yield raggedlist
        return

    # Since this is a recursive function, we need
    # to be on our guard against any references
    # to objects back up the call stack (closer
    # to the root of the tree). Any such
    # references would be circular, leading to
    # an infinite tree, and causing us to blow
    # our stack in a fit of unbounded recursion.
    #
    # If we detect that we've already visited this
    # object (using identity not equality), then
    # the safe thing to do is to halt the
    # recursive descent and return control back
    # up the stack.
    #
    _id = id(raggedlist)
    if memo is None:
        memo = set()
    if _id in memo:
        return
    memo.add(_id)

    for element in flatten_ragged(raggedlist, memo):
        yield element

    # We only need to guard against infinite
    # recursion within a branch of the call-tree.
    # There is no danger in visiting the same
    # item instance in sibling branches, so we
    # can forget about objects once we are done
    # with them and about to pop the stack.
    #
    memo.remove(_id)
    return

# -----------------------------------------------------------------------------
def decompose_map(srcmap):
    """
    Yield a single entry dict for each key-value pair in the specified mapping.

    """
    for key, value in sorted(srcmap.items()):
        yield {key: value}

# -----------------------------------------------------------------------------
def write_jseq(filepath, iterable):
    """
    Write a sequence of JSON objects to the specified file.

    """
    with open(filepath, 'wt') as file:
        for item in iterable:
            file.write('{line}\n'.format(
                                    line = json.dumps(item, sort_keys = True)))

# -----------------------------------------------------------------------------
def load(filepath):
    """
    Load the specified file with the format determined by the file extension.

    """
    if filepath.endswith('.yaml'):
        # The yaml module is not part of the standard
        # library. It is part of the configuration
        # controlled runtime environment in a0_env.
        # We import it in function scope because the
        # util module is imported before a0_env is
        # enabled in the bootstrap process.
        #
        import yaml
        with open(filepath, 'rt') as file:
            return yaml.safe_load(file)

    if filepath.endswith('.json'):
        with open(filepath, 'rt') as file:
            return json.load(file)

    else:
        raise RuntimeError('File format not supported: %s', filepath)

# -----------------------------------------------------------------------------
def save(filepath, data):
    """
    Save the specified file with the format determined by the file extension.

    """
    if filepath.endswith('.yaml'):
        # The yaml module is not part of the standard
        # library. It is part of the configuration
        # controlled runtime environment in a0_env.
        # We import it in function scope because the
        # util module is imported before a0_env is
        # enabled in the bootstrap process.
        #
        import yaml
        with open(filepath, 'wt') as file:
            yaml.safe_dump(data, file, default_flow_style=True)

    if filepath.endswith('.json'):
        with open(filepath, 'wt') as file:
            json.dump(data, file, sort_keys=True, indent=4)

    else:
        raise RuntimeError('File format not supported: %s', filepath)

# -----------------------------------------------------------------------------
def ensure_dir_exists(path):
    """
    Ensure that a specified directory exists, creating it if necessary.

    """
    try:
        os.makedirs(path, exist_ok = True)
    except OSError:
        if not os.path.isdir(path):
            raise

# -----------------------------------------------------------------------------
def ensure_file_exists(filepath):
    """
    Ensure that a specified file exists, creating it if necessary.

    """
    dirpath = os.path.dirname(filepath)
    ensure_dir_exists(dirpath)
    if not os.path.isfile(filepath):
        with open(filepath, 'w') as _:
            pass

# -----------------------------------------------------------------------------
def merge_dicts(first, second):
    """
    Merge two dictionaries. second takes priority.

    """
    return dict(_merge_dicts(first, second))

# -----------------------------------------------------------------------------
def _merge_dicts(first, second):
    """
    Merge two dictionaries (recursive function). second takes priority.

    """
    for key in set(first.keys()).union(second.keys()):
        _in_first  = key in first
        _in_second = key in second
        if _in_first and _in_second:
            _isdict_first  = isinstance(first[key], dict)
            _isdict_second = isinstance(second[key], dict)
            if _isdict_first and _isdict_second:
                yield (key, dict(_merge_dicts(first[key], second[key])))
            else:
                # second overwrites first if both are present.
                yield(key, second[key])
        elif _in_first:
            yield (key, first[key])
        else:
            yield (key, second[key])

# -----------------------------------------------------------------------------
def iter_yaml_docs(text):
    """
    Yield text sections corresponding to YAML documents in the specified text.

    """
    iter_lines = iter(text.splitlines())
    is_inside  = False
    doc_lines  = []

    for line in iter_lines:

        # State transition: outside to inside
        if line.strip() == '---':
            is_inside = True
            continue

        # State transition: inside to outside
        elif line.strip() == '...':
            if doc_lines:
                yield '\n'.join(doc_lines)
                doc_lines = []
            is_inside = False
            continue

        # state is inside -> accumulate lines
        # state is outside -> do nothing
        if is_inside:
            doc_lines.append(line)
        else:
            continue

    # At the end, if we have anything to return, yield it.
    if doc_lines:
        yield '\n'.join(doc_lines)

# -----------------------------------------------------------------------------
def iter_dirs(dirpath_root):
    """
    Yield directory names & paths found immediately under dirpath_root.

    """
    for name in os.listdir(dirpath_root):
        path = os.path.join(dirpath_root, name)
        if not os.path.isdir(path):
            continue
        yield (name, path)

# -----------------------------------------------------------------------------
def iter_files(dirpath_root):
    """
    Yield file names & paths found immediately under dirpath_root.

    """
    for name in os.listdir(dirpath_root):
        path = os.path.join(dirpath_root, name)
        if not os.path.isfile(path):
            continue
        yield (name, path)

# -----------------------------------------------------------------------------
def sha256(filepath,
           blocksize = 65536):
    """
    Return the sha256 digest of the specified file.

    """
    hasher = hashlib.sha256()
    with open(filepath, 'rb') as file:
        block = file.read(blocksize)
        while len(block) > 0:
            hasher.update(block)
            block = file.read(blocksize)
    return hasher.hexdigest()

# -----------------------------------------------------------------------------
@contextlib.contextmanager
def sys_path_context(path):
    """
    Context manager that extends sys.path temporarily.

    """
    sys.path.insert(0, path)
    (yield)
    popped = sys.path.pop(0)
    assert popped == path

# -----------------------------------------------------------------------------
@contextlib.contextmanager
def sys_argv_context(new_argv):
    """
    Context manager that replaces sys.argv temporarily.

    """
    old_argv = sys.argv
    sys.argv = new_argv
    (yield)
    sys.argv = old_argv


import os

from django.conf import settings

from rest_framework import pagination

class AutoDetectedPageNumberPagination(pagination.PageNumberPagination):
    template = os.path.join(os.path.dirname(__file__), 'templates/browsable_api/numbers.html')
    page_size = getattr(settings, 'REST_API_PAGE_SIZE', 20)
    page_size_query_param = getattr(settings,
                                    'REST_API_PAGE_SIZE_QUERY_PARAM',
                                    'page_size')
    max_page_size = getattr(settings,
                            'REST_API_MAX_PAGE_SIZE',
                            100)

    def get_page_size(self, request):
        if self.page_size_query_param:
            try:
                if int(request.query_params[self.page_size_query_param]) <= 0:
                    return None
                else:
                    return pagination._positive_int(
                        request.query_params[self.page_size_query_param],
                        strict=True,
                        cutoff=self.max_page_size
                    )
            except (KeyError, ValueError):
                pass

        return self.page_size


#!/usr/bin/python
# -*- coding: utf-8 -*-

import queue

import multiprocessing

import magic
import logging
import dbApi
import signal

import traceback
import os.path

import random
random.seed()

import tqdm

import scanner.runState
import UniversalArchiveInterface as uar
import scanner.hashFile as hasher

IMAGE_EXTS = ("bmp", "eps", "gif", "im", "jpeg", "jpg", "msp", "pcx", "png", "ppm", "spider", "tiff", "webp", "xbm")
ARCH_EXTS = ("zip", "rar", "cbz", "cbr", "7z", "cb7")

class HashEngine(object):

	def __init__(self, inputQueue, outputQueue, threads=2, pHash=True, integrity=True):
		self.log           = logging.getLogger("Main.HashEngine")
		self.tlog          = logging.getLogger("Main.HashEngineThread")
		self.hashWorkers   = threads
		self.inQ           = inputQueue
		self.outQ          = outputQueue
		self.archIntegrity = integrity

		self.runStateMgr   = multiprocessing.Manager()
		self.manNamespace  = self.runStateMgr.Namespace()
		self.dbApi         = self.getDbConnection()

	def getDbConnection(self):
		'''
		Intended to be overridden in unit-tests
		'''
		return dbApi.DbApi()

	def runThreads(self):
		self.manNamespace.stopOnEmpty = False
		self.manNamespace.run = True
		args = (self.inQ,
			self.outQ,
			self.manNamespace,
			self.archIntegrity)

		self.pool = multiprocessing.pool.Pool(processes=self.hashWorkers, initializer=createHashThread, initargs=args)

	def close(self):
		self.log.info("Closing threadpool")
		self.manNamespace.run = False
		self.pool.terminate()

	def haltEarly(self):
		self.manNamespace.run = False

	def gracefulShutdown(self):
		self.manNamespace.stopOnEmpty = True

		self.pool.close()
		self.pool.join()

	def cleanPathCache(self, fqPathBase):

		self.log.info("Querying for all files on specified path.")

		itemsCursor = self.dbApi.getUniqueOnBasePath(fqPathBase)
		items = []
		retItems = 0
		for item in tqdm.tqdm(itemsCursor):
			retItems += 1
			items.append(item[0])
			if not scanner.runState.run:
				print("Breaking due to exit flag")
				return

		self.log.info("Looking for files in the DB that are not on disk anymore.")

		self.log.info("Recieved items = %d", retItems)
		self.log.info("total unique items = %s", len(items))

		for itemPath in tqdm.tqdm(items, desc="Exist check"):
			if not os.path.exists(itemPath):
				self.log.info("Item %s does not exist. Should delete from DB", itemPath)
				self.dbApi.deleteBasePath(itemPath)

			try:
				if not scanner.runState.run:
					print("Breaking due to exit flag")
					return
			except BrokenPipeError:
				self.log.error("Runstate thread exited? Halting")
				return

			self.outQ.put("clean")

def createHashThread(inQueue, outQueue, runMgr, integrity):
	# Make all the thread-pool threads ignore SIGINT, so they won't freak out on CTRL+C
	signal.signal(signal.SIGINT, signal.SIG_IGN)

	runner = HashThread(inQueue, outQueue, runMgr, integrity)
	runner.run()

class HashThread(object):

	loggerPath = "Main.HashEngine"

	def __init__(self, inputQueue, outputQueue, runMgr, integrity=True):

		# If we're running as a multiprocessing thread, inject that into
		# the logger path
		threadName = multiprocessing.current_process().name
		if threadName:
			self.tlog      = logging.getLogger("%s.%s" % (self.loggerPath, threadName))
		else:
			self.tlog      = logging.getLogger(self.loggerPath)

		self.runMgr        = runMgr
		self.inQ           = inputQueue
		self.outQ          = outputQueue
		self.archIntegrity = integrity

		self.dbApi         = self.getDbConnection()

	def getDbConnection(self):
		'''
		Intended to be overridden in unit-tests
		'''
		return dbApi.DbApi()

	def putProgQueue(self, value):
		if self.outQ:
			self.outQ.put(value)

	def run(self):
		try:
			while self.runMgr.run:
				try:
					filePath, fileName = self.inQ.get(timeout=0.5)
					# self.tlog.info("Scan task! %s", filePath)
					self.processFile(filePath)
				except queue.Empty:
					if self.runMgr.stopOnEmpty:
						self.tlog.info("Hashing thread out of tasks. Exiting.")
						break
				# self.tlog.info("HashThread loopin! stopOnEmpty = %s, run = %s", self.runMgr.stopOnEmpty, self.runMgr.run)
		except FileNotFoundError:
			print("Multiprocessing manager shut down?")

		except BrokenPipeError:
			print("Multiprocessing manager shut down?")

		self.tlog.info("Scanner exiting.")

		self.inQ.close()
		self.outQ.close()

		self.inQ.join_thread()
		self.outQ.join_thread()

	def scanArchive(self, archPath, archData):
		# print("Scanning archive", archPath)

		added_items = 0

		archIterator = uar.ArchiveReader(archPath, fileContents=archData)

		fnames = [item[0] for item in archIterator]
		fset = set(fnames)
		if len(fnames) != len(fset):
			print(fnames)
			print(fset)
			raise ValueError("Wat?")

		self.dbApi.begin()

		try:
			for fName, fp in archIterator:

				fCont = fp.read()

				fName, hexHash, pHash, imX, imY = hasher.hashFile(archPath, fName, fCont)

				insertArgs = {
							"fsPath"       :archPath,
							"internalPath" :fName,
							"itemHash"     :hexHash,
							"pHash"        :pHash,
							"imgX"         :imX,
							"imgY"         :imY
						}

				self.dbApi.insertIntoDb(**insertArgs)
				self.putProgQueue("processed")

				added_items += 1

				if not scanner.runState.run:
					break
		except:
			print(archPath)
			self.dbApi.rollback()
			raise
		finally:
			archIterator.close()

		self.tlog.info("Added %s hashes to DB from archive", added_items)
		self.dbApi.commit()

	def processImageFile(self, wholePath, dbFilePath):

		scan = True
		have = self.dbApi.getItem(fspath=dbFilePath, wantCols = ['phash', 'imgx', 'imgy'])
		if have and all(have):
			scan = False

		# print("Have hashes - ", dummy_itemHash, pHash, dHash)
		if scan:
			with open(wholePath, "rb") as fp:
				fCont = fp.read()
				try:

					fName, hexHash, pHash, imX, imY = hasher.hashFile(wholePath, "", fCont)

					insertArgs = {
								"fsPath"       :wholePath,
								"internalPath" :fName,     # fname == '' in this case
								"itemHash"     :hexHash,
								"pHash"        :pHash,
								"imgX"         :imX,
								"imgY"         :imY
							}

					# insert or update data row
					self.dbApi.upsert(**insertArgs)

					self.outQ.put("processed")

				except (IndexError, UnboundLocalError):
					self.tlog.error("Error while processing fileN")
					self.tlog.error("%s", wholePath)
					self.tlog.error("%s", traceback.format_exc())

				# self.log.info("Scanned bare image %s, %s, %s", fileN, pHash, dHash)

		else:
			self.outQ.put("skipped")

	def hashBareFile(self, wholePath, dbPath, doPhash=True):
		with open(wholePath, "rb") as fp:
			fCont = fp.read()

		fName, hexHash, pHash, imX, imY = hasher.hashFile(wholePath, "", fCont)

		insertArgs = {
					"fsPath"       :wholePath,
					"internalPath" :fName,     # fname == '' in this case
					"itemHash"     :hexHash,
					"pHash"        :pHash,
					"imgX"         :imX,
					"imgY"         :imY
				}

		self.dbApi.insertIntoDb(**insertArgs)

		self.putProgQueue("processed")

	def getFileMd5(self, wholePath):

		with open(wholePath, "rb") as fp:
			fCont = fp.read()
		hexHash = hasher.getMd5Hash(fCont)
		return hexHash, fCont

	def processArchive(self, wholePath):

		fType = "none"
		fCont = None
		archRow = self.dbApi.getItemsOnBasePathInternalPath(wholePath, "")

		contHashes = self.dbApi.getItemsOnBasePath(wholePath)

		haveImInfo = [(bool(item['imgx']) and bool(item['imgy'])) for item in contHashes if item['pHash']]

		if not all(haveImInfo):
			self.tlog.info("Missing image size information for archive %s. Rescanning.", wholePath)
			self.dbApi.deleteBasePath(wholePath)

		elif len(archRow) > 1:
			# print("archRow", archRow)
			raise ValueError("Multiple hashes for a single file? Wat?")
		elif archRow and archRow[0]['itemhash']:

			if not self.archIntegrity:
				self.putProgQueue("skipped")
				return
			item = archRow.pop()
			curHash, fCont = self.getFileMd5(wholePath)

			if curHash == item['itemhash']:
				# print("Skipped", wholePath)
				self.putProgQueue("hash_match")
				return
			else:
				self.tlog.warn("Archive %s has changed! Rehashing!", wholePath)
				self.dbApi.deleteBasePath(wholePath)

		else:
			if archRow:
				self.tlog.info("Missing whole archive hash! Rescanning!")
			self.dbApi.deleteBasePath(wholePath)
			curHash, fCont = self.getFileMd5(wholePath)

			insertArgs = {
						"fsPath"       :wholePath,
						"internalPath" :'',
						"itemHash"     :curHash
					}
			self.dbApi.insertIntoDb(**insertArgs)
			self.putProgQueue("processed")

		# TODO: Use `fCont` to prevent having to read each file twice.

		try:
			fType = magic.from_file(wholePath, mime=True)
			if not isinstance(fType, str):
				fType = fType.decode("ascii")

			# So some versions of libmagic return application/CDFV2-corrupt for some
			# thumbs.db, while some return application/CDFV2 for the /same/ file.
			# In any event, I've never seen a application/CDFV2 file that wasn't
			# one of the garbage application/CDFV2 file, so just pretend
			# the corrupt ones aren't corrupt, since we don't wany any of them
			# anyways.
			if fType == "application/CDFV2-corrupt":
				fType = 'application/CDFV2'

		except magic.MagicException:
			self.tlog.error("REALLY Corrupt Archive! ")
			self.tlog.error("%s", wholePath)
			self.tlog.error("%s", traceback.format_exc())
			fType = "none"
		except IOError:
			self.tlog.error("Something happened to the file before processing (did it get moved?)! ")
			self.tlog.error("%s", wholePath)
			self.tlog.error("%s", traceback.format_exc())
			fType = "none"

		if 	fType == 'application/zip' or \
			fType == 'application/x-rar' or \
			fType == 'application/x-7z-compressed':

			# self.tlog.info("Scanning into archive - %s - %s", fileN, wholePath)

			try:
				self.scanArchive(wholePath, fCont)

			except KeyboardInterrupt:
				raise
			except:
				self.tlog.error("Archive is damaged, corrupt, or not actually an archive: %s", wholePath)
				self.tlog.error("Error Traceback:")
				self.tlog.error(traceback.format_exc())
				# print("wat?")

			# print("Archive scan complete")
			return

	def processFile(self, wholePath):
		if wholePath.startswith("/content"):
			raise ValueError("Wat?")

		# print("path", wholePath)
		if wholePath.lower().endswith(ARCH_EXTS):
			self.processArchive(wholePath)
		else:

			# Get list of all hashes for items on wholePath
			extantItems = self.dbApi.getItemsOnBasePath(wholePath)
			haveFileHashList = [item['itemhash'] != "" for item in extantItems]

			haveAllStatList = [(bool(item['imgx']) and bool(item['imgy'])) for item in extantItems if item['pHash']]

			# Only rescan if we don't have hashes for all the items in the archive (no idea how that would happen),
			# or we have no items for the archive

			if all(haveFileHashList) and len(extantItems) and all(haveAllStatList):

				self.putProgQueue("skipped")
				return

			elif wholePath.lower().endswith(IMAGE_EXTS):  # It looks like an image.

				self.processImageFile(wholePath, wholePath)
				# self.tlog.info("Skipping Image = %s", wholePath)

			# Rehash the overall archive if we don't have a hash-value for the archive with no internalpath.
			elif not any([item['itemhash'] and not item['internalPath'] for item in extantItems]):
				# print("File", wholePath)
				try:
					self.hashBareFile(wholePath, wholePath)

				except IOError:

					self.tlog.error("Something happened to the file before processing (did it get moved?)! ")
					self.tlog.error("%s", wholePath)
					self.tlog.error("%s", traceback.format_exc())
				except (IndexError, UnboundLocalError):
					self.tlog.error("Error while processing wholePath")
					self.tlog.error("%s", wholePath)
					self.tlog.error("%s", traceback.format_exc())

			else:
				self.putProgQueue("skipped")
				# self.tlog.info("Skipping file = %s", wholePath)


import json

class c2s():

    def __init__(self, pair_file_dev, pair_file_test, pair_file_train,
                 data_file_dev, data_file_test, data_file_train,
                 pair , out_true, out_false, json_file):

        self.pair_file = [pair_file_dev,pair_file_test,pair_file_train]
        self.data_file =[data_file_dev,data_file_test,data_file_train]
        self.pair = pair
        self.out_ture = out_true
        self.out_false = out_false
        self.json_file = json_file

    def get_pair(self):
        for file in self.pair_file:
            with open(file, encoding='utf-8') as f:
                self.pair.update(json.load(f))
        return self.pair

    def get_data(self):
        for file in self.data_file:
            with open(file, encoding='utf-8') as f:
                data = json.load(f)
                for line in data:
                    if line.get('label') == 'True' and line.get( "scenario") == 'causal':
                        self.out_ture[str(line.get('id'))] = line.get( "sent")
                    if line.get('label') == 'False' and line.get( "scenario") == 'causal':
                        self.out_false[str(line.get('id'))] = line.get("sent")

        return  self.out_ture, self.out_false

    def out_put(self):
        write = []
        i= 0
        for id, sent in self.out_ture.items():
            i = i+1
            write_sub = {'id' : i, 'True':sent, 'False':self.out_false[self.pair[id]]}
            write.append(write_sub)
        with open(self.json_file, 'w', encoding='utf-8', newline='\n') as fs:
            json.dump(write, fs, ensure_ascii=False,indent=1)
        return write

pair = {}
out_ture = {}
out_false = {}
a = c2s('cs2_convert/data/pair_id_dev.json','cs2_convert/data/pair_id_test.json','cs2_convert/data/pair_id_train.json',
        'cs2_convert/data/dev.json','cs2_convert/data/test.json','cs2_convert/data/train.json',
        pair, out_ture, out_false,'cs2_convert/cs2_convert.json')

pair = a.get_pair()
out = a.get_data()
result = a.out_put()


#creating image border
#with the help of cv2.copyMakeBorder() function.
#it take parameters like (img,border_width(4 sides),bordertype,val_border)
#border width = top,bottom,right,left
import numpy as np
import cv2

img1 = cv2.imread("C:\\Users\\MSI\\Desktop\\1\\lion.jpg")
brdr = cv2.copyMakeBorder(img1,10,10,5,5,cv2.BORDER_CONSTANT,value = [255,0,125])

cv2.imshow("lion",img1)
cv2.imshow("border waala ",brdr)
cv2.imwrite("C:\\Users\\MSI\\Desktop\\1\\lion_border_waala.jpg",brdr)

cv2.waitKey(0)
cv2.destroyAllWindows()


import pickle
import glob
import numpy as np
import ipdb

mode = ['strong', 'weak']
fnames = sorted(glob.glob('./rate_of_convergence/' + mode[0] + '*'))

table = np.zeros([3, 5, 2])

ipdb.set_trace()

for i in range(5):
    for j in range(3):
        f = open(fnames[3*i + j], 'rb')
        tmp = pickle.load(f)
        f.close()
        table[j, i, :] = tmp['params']

f = open('rate_of_conv_table.txt', 'wb')
pickle.dump(table, f)
f.close()


# !/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2018/4/14 16:47
# @Author  : Yunhao Cao
# @File    : tool.py
import codecs
import os

import cv2
import numpy as np

import config

__author__ = 'Yunhao Cao'

level_list = config.LV_LIST
input_shape = config.TRAIN_SHAPE

compare_path = os.path.join

train_set_dir = config.TRAINSET_DIR
validation_set_dir = config.VALIDATION_DIR
sync_name = 'sync.txt'

def get_max_index(l):
    max_index = 0
    max_item = 0
    for i, item in enumerate(l):
        if item > max_item:
            max_item = item
            max_index = i

    return max_index

def get_v_range(level):
    start = level_list[level - 1] if level > 0 else 0
    if (level + 1) < len(level_list):
        end = level_list[level]
    else:
        end = None
    return start, end

def TRAIN_SET_DIR_i(i):
    return compare_path(train_set_dir, 'set_' + i)

def train_set_images(i):
    return compare_path(TRAIN_SET_DIR_i(i), 'images')

def get_all(dir_: str) -> list:
    return list(sorted([os.path.join(dir_, name) for name in os.listdir(dir_) if not name.startswith('.')]))

def read_text(filename):
    with codecs.open(filename, "r", 'utf8') as f:
        return [_.split('\n')[0] for _ in f.readlines() if _]

def write_text(filename, lines, end='\n'):
    with codecs.open(filename, 'w', 'utf8') as f:
        f.write(end.join([str(_) for _ in lines]))

def read_image(filename):
    print('[read_image]', filename)
    return cv2.imread(filename)

def image_cut(data, shape):
    return cv2.resize(data, shape, interpolation=cv2.INTER_CUBIC)

def image_save(filename, data):
    cv2.imwrite(filename, data)

def Array(i, dtype=None):
    return np.array(i, dtype=dtype)

def ArraySplit(array, rate):
    n = int(array.shape[0] * rate)
    return array[:n], array[n:]

def ArrayCut(array, out_shape, mode=5):
    """
    1 2 3
    4 5 6
    7 8 9
    :param array: np.ndarray
    :param out_shape: (height, width)
    :param mode: 1~9
    :return: cut array
    """
    in_shape = array.shape[:2]
    y_diff = in_shape[0] - out_shape[0]
    x_diff = in_shape[1] - out_shape[1]
    y_start = x_start = 0
    y_end = out_shape[0]
    x_end = out_shape[1]
    if mode in (4, 5, 6):
        y_start = y_diff // 2
        y_end = in_shape[0] - y_start - y_diff % 2
    if mode in (2, 5, 8):
        x_start = x_diff // 2
        x_end = in_shape[1] - x_start - x_diff % 2
    if mode in (7, 8, 9):
        y_start = y_diff
        y_end = None
    if mode in (3, 6, 9):
        x_start = x_diff
        x_end = None

    return array[y_start:y_end, x_start: x_end]

if __name__ == '__main__':
    pass


#!/usr/bin/env python3
import itertools
import string

import click

import utils

class Challenge(utils.BaseChallenge):
    def solve(self, _input, debug=False):
        """
        >>> Challenge().default_solve()
        220480
        """
        cave = Cave.from_cave_text(_input)
        cave.tick_many(debug=debug)
        return cave.get_outcome()

    def play(self):
        cave = Cave.from_cave_text(self.input)
        snapshots = [cave.show(show_health=True, add_stats=True)]
        while not cave.has_finished():
            cave.tick()
            snapshots.append(cave.show(show_health=True, add_stats=True))

        index = 0
        while True:
            print(snapshots[index])
            click.echo(
                f"At step {index}/{len(snapshots) - 1}: left for previous, "
                f"right for next, q to quit")
            user_input = click.getchar()
            while user_input not in ('q', '\x1b[D', '\x1b[C'):
                click.echo(
                    "Invalid command: left for previous, right for next, q to "
                    "quit")
                user_input = click.getchar()
            if user_input == 'q':
                break
            if user_input == '\x1b[D':
                index = max(0, index - 1)
            else:
                index = min(len(snapshots) - 1, index + 1)

class Cave:
    @classmethod
    def from_cave_text(cls, cave_text, elf_attack=3):
        """
        >>> Cave.from_cave_text(
        ...     "#####\\n"
        ...     "#.GE#\\n"
        ...     "#####\\n"
        ... )
        Cave(spaces=((False, False, False, False, False),
            (False, True, True, True, False),
                (False, False, False, False, False)),
            units=(Goblin(position=(2, 1), attack=3, health=200),
                Elf(position=(3, 1), attack=3, health=200)))
        """
        lines = list(filter(None, cave_text.splitlines()))
        spaces = tuple(
            tuple(
                spot in ('.', 'G', 'E')
                for spot in line
            )
            for line in lines
        )
        units = tuple(sorted(
            Goblin((x, y))
            if spot == 'G' else
            Elf((x, y), attack=elf_attack)
            for y, line in enumerate(lines)
            for x, spot in enumerate(line)
            if spot in ('G', 'E')
        ))

        return cls(spaces, units)

    @classmethod
    def parse_distances(cls, distances_text):
        """
        >>> dict(sorted(Cave.parse_distances(
        ...     "#########\\n"
        ...     "#.G1012G#\\n"
        ...     "#.32123.#\\n"
        ...     "#..323..#\\n"
        ...     "#G..3..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..G..G#\\n"
        ...     "#########\\n"
        ... ).items()))
        {(2, 2): 3, (3, 1): 1, (3, 2): 2, (3, 3): 3, (4, 1): 0, (4, 2): 1, \
(4, 3): 2, (4, 4): 3, (5, 1): 1, (5, 2): 2, (5, 3): 3, (6, 1): 2, (6, 2): 3}
        """
        lines = list(filter(None, distances_text.splitlines()))
        distance_digits = set(string.hexdigits) - set(string.ascii_uppercase)
        return {
            (x, y): int(spot)
            for y, line in enumerate(lines)
            for x, spot in enumerate(line)
            if spot in distance_digits
        }

    def __init__(self, spaces, units, step_count=0):
        self.spaces = spaces
        self.units_at_start = set(units)
        self.units = units
        self.step_count = step_count

    def __repr__(self):
        return f"{type(self).__name__}" \
               f"(spaces={repr(self.spaces)}, " \
               f"units={repr(self.units)})"

    @classmethod
    def sort_key_health_reading_order(cls, unit):
        return unit.health, cls.sort_key_reading_order_position(unit.position)

    @classmethod
    def sort_key_reading_order(cls, unit):
        return cls.sort_key_reading_order_position(unit.position)

    @classmethod
    def sort_key_reading_order_position(cls, position):
        x, y = position
        return y, x

    def get_outcome(self, allow_provisional=False):
        """
        >>> Cave.from_cave_text(
        ...     "#########\\n"
        ...     "#G..G..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G.....G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..G..G#\\n"
        ...     "#########\\n"
        ... ).get_outcome()
        0
        >>> Cave.from_cave_text(
        ...     "#######\\n"
        ...     "#.G...#\\n"
        ...     "#...EG#\\n"
        ...     "#.#.#G#\\n"
        ...     "#..G#E#\\n"
        ...     "#.....#\\n"
        ...     "#######\\n"
        ... ).tick_many().get_outcome()
        27730
        >>> Cave.from_cave_text(
        ...     "#######\\n"
        ...     "#G..#E#\\n"
        ...     "#E#E.E#\\n"
        ...     "#G.##.#\\n"
        ...     "#...#E#\\n"
        ...     "#...E.#\\n"
        ...     "#######\\n"
        ... ).tick_many().get_outcome()
        36334
        >>> Cave.from_cave_text(
        ...     "#######\\n"
        ...     "#E..EG#\\n"
        ...     "#.#G.E#\\n"
        ...     "#E.##E#\\n"
        ...     "#G..#.#\\n"
        ...     "#..E#.#\\n"
        ...     "#######\\n"
        ... ).tick_many().get_outcome()
        39514
        >>> Cave.from_cave_text(
        ...     "#######\\n"
        ...     "#E.G#.#\\n"
        ...     "#.#G..#\\n"
        ...     "#G.#.G#\\n"
        ...     "#G..#.#\\n"
        ...     "#...E.#\\n"
        ...     "#######\\n"
        ... ).tick_many().get_outcome()
        27755
        >>> Cave.from_cave_text(
        ...     "#######\\n"
        ...     "#.E...#\\n"
        ...     "#.#..G#\\n"
        ...     "#.###.#\\n"
        ...     "#E#G#G#\\n"
        ...     "#...#G#\\n"
        ...     "#######\\n"
        ... ).tick_many().get_outcome()
        28944
        >>> Cave.from_cave_text(
        ...     "#########\\n"
        ...     "#G......#\\n"
        ...     "#.E.#...#\\n"
        ...     "#..##..G#\\n"
        ...     "#...##..#\\n"
        ...     "#...#...#\\n"
        ...     "#.G...G.#\\n"
        ...     "#.....G.#\\n"
        ...     "#########\\n"
        ... ).tick_many().get_outcome()
        18740
        """
        if not allow_provisional and not self.has_finished():
            raise Exception("Cave has not finished yet")
        return self.get_total_health() * self.step_count

    def get_total_health(self):
        """
        >>> Cave.from_cave_text(
        ...     "#########\\n"
        ...     "#G..G..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..E..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..G..G#\\n"
        ...     "#########\\n"
        ... ).get_total_health()
        1800
        >>> Cave.from_cave_text(
        ...     "#######\\n"
        ...     "#G..#E#\\n"
        ...     "#E#E.E#\\n"
        ...     "#G.##.#\\n"
        ...     "#...#E#\\n"
        ...     "#...E.#\\n"
        ...     "#######\\n"
        ... ).tick_many().get_total_health()
        982
        >>> Cave.from_cave_text(
        ...     "#######\\n"
        ...     "#E..EG#\\n"
        ...     "#.#G.E#\\n"
        ...     "#E.##E#\\n"
        ...     "#G..#.#\\n"
        ...     "#..E#.#\\n"
        ...     "#######\\n"
        ... ).tick_many().get_total_health()
        859
        >>> Cave.from_cave_text(
        ...     "#######\\n"
        ...     "#E.G#.#\\n"
        ...     "#.#G..#\\n"
        ...     "#G.#.G#\\n"
        ...     "#G..#.#\\n"
        ...     "#...E.#\\n"
        ...     "#######\\n"
        ... ).tick_many().get_total_health()
        793
        >>> Cave.from_cave_text(
        ...     "#######\\n"
        ...     "#.E...#\\n"
        ...     "#.#..G#\\n"
        ...     "#.###.#\\n"
        ...     "#E#G#G#\\n"
        ...     "#...#G#\\n"
        ...     "#######\\n"
        ... ).tick_many().get_total_health()
        536
        >>> Cave.from_cave_text(
        ...     "#########\\n"
        ...     "#G......#\\n"
        ...     "#.E.#...#\\n"
        ...     "#..##..G#\\n"
        ...     "#...##..#\\n"
        ...     "#...#...#\\n"
        ...     "#.G...G.#\\n"
        ...     "#.....G.#\\n"
        ...     "#########\\n"
        ... ).tick_many().get_total_health()
        937
        """
        return sum(
            unit.health
            for unit in self.units
        )

    def tick_many(self, count=None, debug=False):
        """
        >>> print(Cave.from_cave_text(
        ...     "#########\\n"
        ...     "#G..G..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..E..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..G..G#\\n"
        ...     "#########\\n"
        ... ).tick_many(3).show())
        #########
        #.......#
        #..GGG..#
        #..GEG..#
        #G..G...#
        #......G#
        #.......#
        #.......#
        #########
        >>> print(Cave.from_cave_text(
        ...     "#########\\n"
        ...     "#G..G..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..E..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..G..G#\\n"
        ...     "#########\\n"
        ... ).tick_many().show())
        #########
        #.......#
        #..GGG..#
        #..G.G..#
        #G..G...#
        #......G#
        #.......#
        #.......#
        #########
        >>> cave_a = Cave.from_cave_text(
        ...     "#######\\n"
        ...     "#.G...#\\n"
        ...     "#...EG#\\n"
        ...     "#.#.#G#\\n"
        ...     "#..G#E#\\n"
        ...     "#.....#\\n"
        ...     "#######\\n"
        ... )
        >>> print(cave_a.tick_many(1).show(show_health=True))
        #######
        #..G..#   G(200)
        #...EG#   E(197), G(197)
        #.#G#G#   G(200), G(197)
        #...#E#   E(197)
        #.....#
        #######
        >>> print(cave_a.tick_many(1).show(show_health=True))
        #######
        #...G.#   G(200)
        #..GEG#   G(200), E(188), G(194)
        #.#.#G#   G(194)
        #...#E#   E(194)
        #.....#
        #######
        >>> print(cave_a.tick_many(1).show(show_health=True))
        #######
        #...G.#   G(200)
        #..GEG#   G(200), E(179), G(191)
        #.#.#G#   G(191)
        #...#E#   E(191)
        #.....#
        #######
        >>> print(cave_a.tick_many(20).show())
        #######
        #...G.#
        #..G.G#
        #.#.#G#
        #...#E#
        #.....#
        #######
        >>> print(cave_a.tick_many().show())
        #######
        #G....#
        #.G...#
        #.#.#G#
        #...#.#
        #....G#
        #######
        >>> print(Cave.from_cave_text(
        ...     "#######\\n"
        ...     "#G..#E#\\n"
        ...     "#E#E.E#\\n"
        ...     "#G.##.#\\n"
        ...     "#...#E#\\n"
        ...     "#...E.#\\n"
        ...     "#######\\n"
        ... ).tick_many().show(show_health=True))
        #######
        #...#E#   E(200)
        #E#...#   E(197)
        #.E##.#   E(185)
        #E..#E#   E(200), E(200)
        #.....#
        #######
        >>> print(Cave.from_cave_text(
        ...     "#######\\n"
        ...     "#E..EG#\\n"
        ...     "#.#G.E#\\n"
        ...     "#E.##E#\\n"
        ...     "#G..#.#\\n"
        ...     "#..E#.#\\n"
        ...     "#######\\n"
        ... ).tick_many().show(show_health=True))
        #######
        #.E.E.#   E(164), E(197)
        #.#E..#   E(200)
        #E.##.#   E(98)
        #.E.#.#   E(200)
        #...#.#
        #######
        >>> print(Cave.from_cave_text(
        ...     "#######\\n"
        ...     "#E.G#.#\\n"
        ...     "#.#G..#\\n"
        ...     "#G.#.G#\\n"
        ...     "#G..#.#\\n"
        ...     "#...E.#\\n"
        ...     "#######\\n"
        ... ).tick_many().show(show_health=True))
        #######
        #G.G#.#   G(200), G(98)
        #.#G..#   G(200)
        #..#..#
        #...#G#   G(95)
        #...G.#   G(200)
        #######
        >>> print(Cave.from_cave_text(
        ...     "#######\\n"
        ...     "#.E...#\\n"
        ...     "#.#..G#\\n"
        ...     "#.###.#\\n"
        ...     "#E#G#G#\\n"
        ...     "#...#G#\\n"
        ...     "#######\\n"
        ... ).tick_many().show(show_health=True))
        #######
        #.....#
        #.#G..#   G(200)
        #.###.#
        #.#.#.#
        #G.G#G#   G(98), G(38), G(200)
        #######
        >>> print(Cave.from_cave_text(
        ...     "#########\\n"
        ...     "#G......#\\n"
        ...     "#.E.#...#\\n"
        ...     "#..##..G#\\n"
        ...     "#...##..#\\n"
        ...     "#...#...#\\n"
        ...     "#.G...G.#\\n"
        ...     "#.....G.#\\n"
        ...     "#########\\n"
        ... ).tick_many().show(show_health=True))
        #########
        #.G.....#   G(137)
        #G.G#...#   G(200), G(200)
        #.G##...#   G(200)
        #...##..#
        #.G.#...#   G(200)
        #.......#
        #.......#
        #########
        """
        if count is None:
            steps = itertools.count()
        else:
            steps = range(count)
        for _ in steps:
            if self.has_finished():
                break
            self.tick(debug=debug)

        return self

    def tick(self, debug=False):
        """
        >>> cave_a = Cave.from_cave_text(
        ...     "#########\\n"
        ...     "#G..G..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..E..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..G..G#\\n"
        ...     "#########\\n"
        ... )
        >>> move_order_a = cave_a.get_move_order()
        >>> units_by_position_a = cave_a.get_units_by_position()
        >>> move_order_a[0].tick(cave_a.spaces, units_by_position_a)
        False
        >>> print(cave_a.show())
        #########
        #.G.G..G#
        #.......#
        #.......#
        #G..E..G#
        #.......#
        #.......#
        #G..G..G#
        #########
        >>> move_order_a[1].tick(cave_a.spaces, units_by_position_a)
        False
        >>> print(cave_a.show())
        #########
        #.G....G#
        #...G...#
        #.......#
        #G..E..G#
        #.......#
        #.......#
        #G..G..G#
        #########
        >>> print(Cave.from_cave_text(
        ...     "#########\\n"
        ...     "#G..G..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..E..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..G..G#\\n"
        ...     "#########\\n"
        ... ).tick().show())
        #########
        #.G...G.#
        #...G...#
        #...E..G#
        #.G.....#
        #.......#
        #G..G..G#
        #.......#
        #########
        """
        if self.has_finished():
            return self

        units_by_position = self.get_units_by_position()
        move_order = self.get_move_order()
        finished = False
        if debug:
            print(self.step_count, len(move_order), ":", end="", flush=True)
        for unit in move_order:
            if debug:
                print(f"{move_order.index(unit)}, ", end="", flush=True)
            if unit.is_dead():
                continue
            finished = unit.tick(
                self.spaces, units_by_position, debug=False)
            if finished:
                break
        else:
            self.step_count += 1
        if debug:
            print("done")

        self.units = [unit for unit in self.units if unit.is_alive()]
        if finished and not self.has_finished():
            raise Exception(
                "A unit reported that the game has finished, but there are "
                "many unit types")

        return self

    def has_finished(self):
        """
        >>> Cave((), []).has_finished()
        True
        >>> Cave.from_cave_text(
        ...     "#########\\n"
        ...     "#G..G..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..E..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..G..G#\\n"
        ...     "#########\\n"
        ... ).has_finished()
        False
        >>> Cave.from_cave_text(
        ...     "#########\\n"
        ...     "#G..G..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G.....G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..G..G#\\n"
        ...     "#########\\n"
        ... ).has_finished()
        True
        """
        return len(set(map(type, self.units))) <= 1

    def get_units_by_position(self):
        return {
            unit.position: unit
            for unit in self.units
        }

    def get_move_order(self):
        """
        >>> [unit.position for unit in Cave.from_cave_text(
        ...     "#######\\n"
        ...     "#.G.E.#\\n"
        ...     "#E.G.E#\\n"
        ...     "#.G.E.#\\n"
        ...     "#######\\n"
        ... ).get_move_order()]
        [(2, 1), (4, 1), (1, 2), (3, 2), (5, 2), (2, 3), (4, 3)]
        >>> [unit.position for unit in Cave.from_cave_text(
        ...     "#########\\n"
        ...     "#G..G..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..E..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..G..G#\\n"
        ...     "#########\\n"
        ... ).get_move_order()]
        [(1, 1), (4, 1), (7, 1), (1, 4), (4, 4), (7, 4), (1, 7), (4, 7), (7, 7)]
        """
        return sorted(self.units, key=self.unit_sort_key_reading_order)

    def unit_sort_key_reading_order(self, unit):
        return self.position_sort_key_reading_order(unit.position)

    def position_sort_key_reading_order(self, position):
        x, y = position
        return y, x

    def show(self, distances=None, show_health=False, add_stats=False):
        """
        >>> print(Cave.from_cave_text(
        ...     "#######\\n"
        ...     "#.G.E.#\\n"
        ...     "#E.G.E#\\n"
        ...     "#.G.E.#\\n"
        ...     "#######\\n"
        ... ).show())
        #######
        #.G.E.#
        #E.G.E#
        #.G.E.#
        #######
        >>> print(Cave.from_cave_text(
        ...     "#######\\n"
        ...     "#.G.E.#\\n"
        ...     "#E.G.E#\\n"
        ...     "#.G.E.#\\n"
        ...     "#######\\n"
        ... ).show(show_health=True, add_stats=True))
        #######
        #.G.E.#   G(200), E(200)
        #E.G.E#   E(200), G(200), E(200)
        #.G.E.#   G(200), E(200)
        #######
        Step: 0, health: 1400, outcome: 0
        """
        units_by_position = self.get_units_by_position()
        shown = "\n".join(
            self.show_line(y, line, units_by_position, distances, show_health)
            for y, line in enumerate(self.spaces)
        )
        if add_stats:
            shown += (
                f"\nStep: {self.step_count}, "
                f"health: {self.get_total_health()}, "
                f"outcome: {self.get_outcome(allow_provisional=True)}"
            )

        return shown

    def show_line(self, y, line, units_by_position, distances, show_health):
        line_str = "".join(
            str(distances[(x, y)])
            if distances and (x, y) in distances else
            "#"
            if not space else
            "."
            if (x, y) not in units_by_position else
            units_by_position[(x, y)].show()
            for x, space in enumerate(line)
        )
        if show_health:
            line_str += "   {}".format(", ".join(
                units_by_position[(x, y)].show(show_health=True)
                for x in range(len(line))
                if (x, y) in units_by_position
            )).rstrip()

        return line_str

class Unit:
    show_symbol = NotImplemented

    def __init__(self, position, attack=3, health=200):
        self.position = position
        self.attack = attack
        self.health = health
        self.cached_distances = None
        self.cached_distances_key = None

    def __repr__(self):
        return f"{type(self).__name__}" \
               f"(position={self.position}, " \
               f"attack={self.attack}, " \
               f"health={self.health})"

    def __lt__(self, other):
        return self.position < other.position

    def is_alive(self):
        """
        >>> Elf((0, 0)).is_alive()
        True
        >>> Elf((0, 0), health=1).is_alive()
        True
        >>> Elf((0, 0), health=0).is_alive()
        False
        >>> Elf((0, 0), health=-1).is_alive()
        False
        """
        return self.health > 0

    def is_dead(self):
        """
        >>> Elf((0, 0)).is_dead()
        False
        >>> Elf((0, 0), health=1).is_dead()
        False
        >>> Elf((0, 0), health=0).is_dead()
        True
        >>> Elf((0, 0), health=-1).is_dead()
        True
        """
        return not self.is_alive()

    def tick(self, spaces, units_by_position, debug=False):
        """
        >>> cave_a = Cave.from_cave_text(
        ...     "#########\\n"
        ...     "#G..G..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..E..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..G..G#\\n"
        ...     "#########\\n"
        ... )
        >>> units_by_position_a = cave_a.get_units_by_position()
        >>> cave_a.units[0].tick(cave_a.spaces, units_by_position_a)
        False
        >>> print(cave_a.show())
        #########
        #.G.G..G#
        #.......#
        #.......#
        #G..E..G#
        #.......#
        #.......#
        #G..G..G#
        #########
        >>> cave_b = Cave.from_cave_text(
        ...     "#########\\n"
        ...     "#.G.G..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..E..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..G..G#\\n"
        ...     "#########\\n"
        ... )
        >>> units_by_position_b = cave_b.get_units_by_position()
        >>> units_by_position_b[(4, 1)].tick(cave_b.spaces, units_by_position_b)
        False
        >>> print(cave_b.show())
        #########
        #.G....G#
        #...G...#
        #.......#
        #G..E..G#
        #.......#
        #.......#
        #G..G..G#
        #########
        >>> cave_c = Cave.from_cave_text(
        ...     "G.E\\n"
        ... )
        >>> units_by_position_c = cave_c.get_units_by_position()
        >>> unit_c = units_by_position_c[(0, 0)]
        >>> unit_c.get_enemies(units_by_position_c)
        [Elf(position=(2, 0), attack=3, health=200)]
        >>> unit_c.tick(cave_c.spaces, units_by_position_c)
        False
        >>> print(cave_c.show(show_health=True))
        .GE   G(200), E(197)
        """
        enemies = self.get_enemies(units_by_position)
        if not enemies:
            return True
        next_position, closest_enemy = self.get_next_step_to_closest_enemy(
            spaces, enemies, units_by_position, debug=debug)
        if not next_position:
            return False
        if next_position != self.position:
            self.move(next_position, units_by_position, spaces=spaces)
            next_position, closest_enemy = self.get_next_step_to_closest_enemy(
                spaces, enemies, units_by_position, debug=debug)
        if next_position == self.position:
            self.attack_enemy(closest_enemy, units_by_position)

        return False

    def get_next_step_to_closest_enemy(self, spaces, enemies,
                                       units_by_position, debug=False):
        """
        >>> cave_a = Cave.from_cave_text(
        ...     "#########\\n"
        ...     "#G..G..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..E..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..G..G#\\n"
        ...     "#########\\n"
        ... )
        >>> units_by_position_a = cave_a.get_units_by_position()
        >>> enemies_a = [unit for unit in cave_a.units if isinstance(unit, Elf)]
        >>> cave_a.units[0].get_next_step_to_closest_enemy(
        ...     cave_a.spaces, enemies_a, units_by_position_a)
        ((2, 1), Elf(position=(4, 4), attack=3, health=200))
        >>> cave_b = Cave.from_cave_text(
        ...     "#########\\n"
        ...     "#GE.G..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..E..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..G..G#\\n"
        ...     "#########\\n"
        ... )
        >>> units_by_position_b = cave_b.get_units_by_position()
        >>> enemies_b = [unit for unit in cave_b.units if isinstance(unit, Elf)]
        >>> cave_b.units[0].get_next_step_to_closest_enemy(
        ...     cave_b.spaces, enemies_b, units_by_position_b)
        ((1, 1), Elf(position=(2, 1), attack=3, health=200))
        """
        distances = self.get_distances(spaces, enemies, units_by_position)
        closest_enemy = self.get_closest_enemy(enemies, distances)
        if not closest_enemy:
            return None, None
        next_step = self.get_next_step(
            distances, closest_enemy.position, debug=debug)

        return next_step, closest_enemy

    def get_next_step(self, distances, target, debug=False):
        """
        >>> Goblin((1, 1)).get_next_step(Cave.parse_distances(
        ...     "#########\\n"
        ...     "#012G6.G#\\n"
        ...     "#123456.#\\n"
        ...     "#23456..#\\n"
        ...     "#G456..G#\\n"
        ...     "#656....#\\n"
        ...     "#.6.....#\\n"
        ...     "#G..G..G#\\n"
        ...     "#########\\n"
        ... ), (4, 4))
        (2, 1)
        >>> Goblin((1, 1)).get_next_step(Cave.parse_distances(
        ...     "#########\\n"
        ...     "#01.G..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..E..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..G..G#\\n"
        ...     "#########\\n"
        ... ), (2, 1))
        (1, 1)
        >>> Goblin((4, 1)).get_next_step(Cave.parse_distances(
        ...     "#########\\n"
        ...     "#.G1012G#\\n"
        ...     "#.32123.#\\n"
        ...     "#..323..#\\n"
        ...     "#G..3..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..G..G#\\n"
        ...     "#########\\n"
        ... ), (4, 4))
        (4, 2)
        >>> Goblin((4, 1)).get_next_step(Cave.parse_distances(
        ...     "#########\\n"
        ...     "#G21012G#\\n"
        ...     "#.32123.#\\n"
        ...     "#..323..#\\n"
        ...     "#G..3..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..G..G#\\n"
        ...     "#########\\n"
        ... ), (4, 4))
        (4, 2)
        """
        if distances[target] == 0:
            raise Exception("Tried to get next steps for same position")
        elif distances[target] == 1:
            return self.position

        stack = [target]
        next_steps = set()
        if debug:
            print("Get next step: ", end="", flush=True)
        while stack:
            position = stack.pop(0)
            next_distance, next_positions = self.get_next_step_iteration(
                distances, position)
            if debug:
                print(next_distance, ", ", end="", flush=True)
            if next_distance == 1:
                next_steps.update(next_positions)
                continue
            for next_position in next_positions:
                if next_position not in stack:
                    stack.append(next_position)
        if debug:
            print(f"done: {len(next_steps)}")

        if not next_steps:
            raise Exception(
                f"No next steps found from {self.position} to {target}")

        return min(next_steps, key=Cave.sort_key_reading_order_position)

    def get_next_step_iteration(self, distances, position):
        """
        >>> distances_a = Cave.parse_distances(
        ...     "#########\\n"
        ...     "#.G1012G#\\n"
        ...     "#.32123.#\\n"
        ...     "#..323..#\\n"
        ...     "#G..3..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..G..G#\\n"
        ...     "#########\\n"
        ... )
        >>> Goblin((4, 1)).get_next_step_iteration(distances_a, (4, 4))
        (2, [(4, 3)])
        """
        distance = distances[position]
        next_distance = distance - 1
        next_positions = [
            next_position
            for next_position, next_position_distance in distances.items()
            if next_position_distance == next_distance
            and self.are_neighbours(next_position, position)
        ]
        return next_distance, next_positions

    def are_neighbours(self, lhs, rhs):
        """
        >>> Goblin((0, 0)).are_neighbours((0, 0), (0, 1))
        True
        >>> Goblin((0, 0)).are_neighbours((0, 0), (-1, 0))
        True
        >>> Goblin((0, 0)).are_neighbours((0, 0), (0, 0))
        False
        >>> Goblin((0, 0)).are_neighbours((0, 0), (1, 1))
        False
        >>> Goblin((0, 0)).are_neighbours((0, 0), (-5, 4))
        False
        """
        lhs_x, lhs_y = lhs
        rhs_x, rhs_y = rhs
        return abs(lhs_x - rhs_x) + abs(lhs_y - rhs_y) == 1

    def get_closest_enemy(self, enemies, distances):
        """
        >>> cave_a = Cave.from_cave_text(
        ...     "#########\\n"
        ...     "#G..G..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..E..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..G..G#\\n"
        ...     "#########\\n"
        ... )
        >>> units_by_position_a = cave_a.get_units_by_position()
        >>> unit_a = units_by_position_a[(1, 1)]
        >>> enemies_a = [
        ...     unit for unit in cave_a.units
        ...     if not isinstance(unit, type(unit_a))
        ... ]
        >>> distances_a = unit_a.get_distances(
        ...     cave_a.spaces, enemies_a, units_by_position_a)
        >>> unit_a.get_closest_enemy(enemies_a, distances_a)
        Elf(position=(4, 4), attack=3, health=200)
        >>> cave_b = Cave.from_cave_text(
        ...     "#########\\n"
        ...     "#.G.G..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..E..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..G..G#\\n"
        ...     "#########\\n"
        ... )
        >>> units_by_position_b = cave_b.get_units_by_position()
        >>> unit_b = units_by_position_b[(4, 1)]
        >>> enemies_b = [
        ...     unit for unit in cave_b.units
        ...     if not isinstance(unit, type(unit_b))
        ... ]
        >>> distances_b = unit_b.get_distances(
        ...     cave_b.spaces, enemies_b, units_by_position_b)
        >>> unit_b.get_closest_enemy(enemies_b, distances_b)
        Elf(position=(4, 4), attack=3, health=200)
        >>> cave_c = Cave.from_cave_text(
        ...     "####\\n"
        ...     "#GE#\\n"
        ...     "#E.#\\n"
        ...     "####\\n"
        ... )
        >>> units_by_position_c = cave_c.get_units_by_position()
        >>> unit_c = units_by_position_c[(1, 1)]
        >>> enemies_c = [
        ...     unit for unit in cave_c.units
        ...     if not isinstance(unit, type(unit_c))
        ... ]
        >>> distances_c = unit_c.get_distances(
        ...     cave_c.spaces, enemies_c, units_by_position_c)
        >>> unit_c.get_closest_enemy(enemies_c, distances_c)
        Elf(position=(2, 1), attack=3, health=200)
        >>> units_by_position_c[(1, 2)].health = 100
        >>> unit_c.get_closest_enemy(enemies_c, distances_c)
        Elf(position=(1, 2), attack=3, health=100)
        >>> cave_c = Cave.from_cave_text(
        ...     "#####\\n"
        ...     "#G.E#\\n"
        ...     "#...#\\n"
        ...     "#E..#\\n"
        ...     "#####\\n"
        ... )
        >>> units_by_position_c = cave_c.get_units_by_position()
        >>> unit_c = units_by_position_c[(1, 1)]
        >>> enemies_c = [
        ...     unit for unit in cave_c.units
        ...     if not isinstance(unit, type(unit_c))
        ... ]
        >>> distances_c = unit_c.get_distances(
        ...     cave_c.spaces, enemies_c, units_by_position_c)
        >>> unit_c.get_closest_enemy(enemies_c, distances_c)
        Elf(position=(3, 1), attack=3, health=200)
        >>> units_by_position_c[(1, 3)].health = 100
        >>> unit_c.get_closest_enemy(enemies_c, distances_c)
        Elf(position=(3, 1), attack=3, health=200)
        """
        enemies_distances = {
            enemy: distances[enemy.position]
            for enemy in enemies
            if enemy.position in distances
        }
        if not enemies_distances:
            return None
        closest_distance = min(enemies_distances.values())
        closest_enemies = [
            enemy
            for enemy, distance in enemies_distances.items()
            if distance == closest_distance
        ]
        if closest_distance == 1:
            closest_enemy = min(
                closest_enemies, key=Cave.sort_key_health_reading_order)
        else:
            closest_enemy = min(
                closest_enemies, key=Cave.sort_key_reading_order)

        return closest_enemy

    def get_distances(self, spaces, enemies, units_by_position):
        """
        >>> cave_a = Cave.from_cave_text(
        ...     "#########\\n"
        ...     "#G..G..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..E..G#\\n"
        ...     "#.......#\\n"
        ...     "#.......#\\n"
        ...     "#G..G..G#\\n"
        ...     "#########\\n"
        ... )
        >>> units_by_position_a = cave_a.get_units_by_position()
        >>> unit_a = units_by_position_a[(1, 1)]
        >>> enemies_a = [
        ...     unit for unit in cave_a.units
        ...     if not isinstance(unit, type(unit_a))
        ... ]
        >>> distances_a = unit_a.get_distances(
        ...     cave_a.spaces, enemies_a, units_by_position_a)
        >>> print(cave_a.show(distances_a))
        #########
        #012G6.G#
        #123456.#
        #23456..#
        #G456..G#
        #656....#
        #.6.....#
        #G..G..G#
        #########
        >>> unit_b = units_by_position_a[(4, 4)]
        >>> enemies_b = [
        ...     unit for unit in cave_a.units
        ...     if not isinstance(unit, type(unit_b))
        ... ]
        >>> distances_b = unit_b.get_distances(
        ...     cave_a.spaces, enemies_b, units_by_position_a)
        >>> print(cave_a.show(distances_b))
        #########
        #G..3..G#
        #..323..#
        #.32123.#
        #3210123#
        #.32123.#
        #..323..#
        #G..3..G#
        #########
        >>> unit_c = units_by_position_a[(4, 1)]
        >>> enemies_c = [
        ...     unit for unit in cave_a.units
        ...     if not isinstance(unit, type(unit_c))
        ... ]
        >>> distances_c = unit_c.get_distances(
        ...     cave_a.spaces, enemies_c, units_by_position_a)
        >>> print(cave_a.show(distances_c))
        #########
        #G21012G#
        #.32123.#
        #..323..#
        #G..3..G#
        #.......#
        #.......#
        #G..G..G#
        #########
        """
        cached_distances_key = \
            hash((spaces, tuple(sorted(units_by_position.items()))))
        if cached_distances_key == self.cached_distances_key:
            return self.cached_distances
        enemy_positions = {enemy.position for enemy in enemies}
        distances = {self.position: 0}
        stack = [self.position]
        closest_distance = None
        while stack:
            position = stack.pop(0)
            distance = distances[position]
            next_distance = distance + 1
            if closest_distance is not None \
                    and next_distance > closest_distance:
                continue
            for neighbour in self.get_neighbours(position, spaces):
                if neighbour in distances:
                    continue
                neighbour_x, neighbour_y = neighbour
                if not spaces[neighbour_y][neighbour_x]:
                    continue
                if neighbour in enemy_positions:
                    distances[neighbour] = next_distance
                    if closest_distance is None:
                        closest_distance = next_distance
                    elif next_distance < closest_distance:
                        closest_distance = next_distance
                elif neighbour not in units_by_position:
                    distances[neighbour] = next_distance
                    stack.append(neighbour)

        self.cached_distances_key = cached_distances_key
        self.cached_distances = distances

        return distances

    NEIGHBOUR_OFFSETS = [
        (-1, 0),
        (1, 0),
        (0, -1),
        (0, 1),
    ]

    def get_neighbours(self, position, spaces):
        """
        >>> sorted(Elf((0, 0)).get_neighbours((0, 0), ((True,),)))
        []
        >>> sorted(Elf((0, 0)).get_neighbours((0, 0), ((True,) * 3,) * 3))
        [(0, 1), (1, 0)]
        >>> sorted(Elf((0, 0)).get_neighbours((1, 1), ((True,) * 3,) * 3))
        [(0, 1), (1, 0), (1, 2), (2, 1)]
        >>> sorted(Elf((0, 0)).get_neighbours((2, 2), ((True,) * 3,) * 3))
        [(1, 2), (2, 1)]
        """
        x, y = position
        return (
            neighbour
            for neighbour in (
                (x + d_x, y + d_y)
                for d_x, d_y in self.NEIGHBOUR_OFFSETS
            )
            if self.is_position_space(neighbour, spaces)
        )

    def is_position_space(self, position, spaces):
        x, y = position
        return (
            0 <= x < len(spaces[0])
            and 0 <= y < len(spaces)
        )

    def move(self, next_position, units_by_position, spaces):
        """
        >>> elf, goblin = Elf((0, 0)), Goblin((1, 0))
        >>> units_by_position_a = {(0, 0): elf, (1, 0): goblin}
        >>> elf.move((0, 1), units_by_position_a, ((True, True), (True, True)))
        >>> units_by_position_a
        {(1, 0): Goblin(position=(1, 0), attack=3, health=200),
            (0, 1): Elf(position=(0, 1), attack=3, health=200)}
        """
        if self.is_dead():
            raise Exception(f"Unit {self} tried to move but it was dead")
        if units_by_position.get(self.position) != self:
            raise Exception(
                f"Unit {self} tried to move from {self.position} but it wasn't "
                f"there: instead {units_by_position.get(self.position)} was")
        if next_position in units_by_position:
            raise Exception(
                f"Unit {self} tried to move to {next_position} but "
                f"{units_by_position[next_position]} was there")
        manhattan_distance = utils.Point2D(*self.position)\
            .manhattan_distance(utils.Point2D(*next_position))
        if manhattan_distance != 1:
            raise Exception(
                f"Unit {self} at {self.position} tried to move to "
                f"{next_position} but the distance was {manhattan_distance}")
        if not self.is_position_space(next_position, spaces):
            raise Exception(
                f"Unit {self} tried to move to {next_position} but it was not "
                f"a space")
        del units_by_position[self.position]
        self.position = next_position
        units_by_position[self.position] = self

    def attack_enemy(self, enemy, units_by_position):
        """
        >>> elf, goblin = Elf((0, 0)), Goblin((1, 0))
        >>> units_by_position_a = {(0, 0): elf, (1, 0): goblin}
        >>> elf.attack_enemy(goblin, units_by_position_a)
        >>> units_by_position_a
        {(0, 0): Elf(position=(0, 0), attack=3, health=200),
            (1, 0): Goblin(position=(1, 0), attack=3, health=197)}
        >>> goblin.health = 3
        >>> elf.attack_enemy(goblin, units_by_position_a)
        >>> units_by_position_a
        {(0, 0): Elf(position=(0, 0), attack=3, health=200)}
        >>> goblin.health = 1
        >>> units_by_position_a = {(0, 0): elf, (1, 0): goblin}
        >>> elf.attack_enemy(goblin, units_by_position_a)
        >>> units_by_position_a
        {(0, 0): Elf(position=(0, 0), attack=3, health=200)}
        """
        if self.is_dead():
            raise Exception(f"Unit {self} tried to attack but it was dead")
        if enemy.is_dead():
            raise Exception(f"Tried to attack {enemy} but it was dead")
        if units_by_position.get(self.position) != self:
            raise Exception(
                f"Unit {self} tried to attack from {self.position} but it "
                f"wasn't there: instead {units_by_position.get(self.position)} "
                f"was")
        if units_by_position.get(enemy.position) != enemy:
            raise Exception(
                f"Tried to attack {enemy} at {enemy.position} but it "
                f"wasn't there: instead "
                f"{units_by_position.get(enemy.position)} was")
        manhattan_distance = utils.Point2D(*self.position)\
            .manhattan_distance(utils.Point2D(*enemy.position))
        if manhattan_distance != 1:
            raise Exception(
                f"Unit {self} at {self.position} tried to attack {enemy} at "
                f"{enemy.position} but their distance was {manhattan_distance}")
        enemy.health -= self.attack
        if enemy.is_dead():
            del units_by_position[enemy.position]

    def get_enemies(self, units_by_position):
        """
        >>> Goblin((1, 1)).get_enemies({
        ...     (1, 1): Goblin(position=(1, 1), attack=3, health=200),
        ...     (1, 4): Goblin(position=(1, 4), attack=3, health=200),
        ...     (1, 7): Goblin(position=(1, 7), attack=3, health=200),
        ...     (4, 1): Goblin(position=(4, 1), attack=3, health=200),
        ...     (4, 4): Elf(position=(4, 4), attack=3, health=200),
        ...     (4, 7): Goblin(position=(4, 7), attack=3, health=200),
        ...     (7, 1): Goblin(position=(7, 1), attack=3, health=200),
        ...     (7, 4): Goblin(position=(7, 4), attack=3, health=200),
        ...     (7, 7): Goblin(position=(7, 7), attack=3, health=200),
        ... })
        [Elf(position=(4, 4), attack=3, health=200)]
        >>> [type(unit).__name__ for unit in Elf((4, 4)).get_enemies({
        ...     (1, 1): Goblin(position=(1, 1), attack=3, health=200),
        ...     (1, 4): Goblin(position=(1, 4), attack=3, health=200),
        ...     (1, 7): Goblin(position=(1, 7), attack=3, health=200),
        ...     (4, 1): Goblin(position=(4, 1), attack=3, health=200),
        ...     (4, 4): Elf(position=(4, 4), attack=3, health=200),
        ...     (4, 7): Goblin(position=(4, 7), attack=3, health=200),
        ...     (7, 1): Goblin(position=(7, 1), attack=3, health=200),
        ...     (7, 4): Goblin(position=(7, 4), attack=3, health=200),
        ...     (7, 7): Goblin(position=(7, 7), attack=3, health=200),
        ... })]
        ['Goblin', 'Goblin', 'Goblin', 'Goblin', 'Goblin', 'Goblin', 'Goblin',
            'Goblin']
        """
        cls = type(self)
        return [
            unit
            for unit in units_by_position.values()
            if not isinstance(unit, cls)
        ]

    def show(self, show_health=False):
        """
        >>> Elf((4, 4)).show()
        'E'
        >>> Elf((4, 4)).show(True)
        'E(200)'
        >>> Goblin((4, 4)).show()
        'G'
        >>> Goblin((4, 4), health=123).show()
        'G'
        >>> Goblin((4, 4), health=123).show(True)
        'G(123)'
        """
        if show_health:
            return f"{self.show_symbol}({self.health})"
        else:
            return self.show_symbol

class Goblin(Unit):
    show_symbol = "G"

class Elf(Unit):
    show_symbol = "E"

Challenge.main()
challenge = Challenge()


from flask import Flask, request, jsonify

app = Flask(__name__)
@app.route('/')
def home():
    return f'Hello there from Bonnie Flask API Endpoint Server'

@app.route('/hello', methods=['GET'])
def hello_get():
    name = request.args.get('name', 'World')
    middlename = request.args.get('middlename', 'no middle name provided')
    nameCapital = name.upper()
    middlenameCapital = middlename.upper()
    return f'Hello {nameCapital} {middlenameCapital}!'

@app.route('/hello', methods=['POST'])
def hello_post():
    data = request.get_json()
    if data is None:
        return jsonify({'error': 'Invalid JSON'}), 400

    name = data.get('name', 'World')
    return jsonify({'message': f'Hello {name}!'})

if __name__ == '__main__':
    app.run(debug=True)


import itertools
import warnings
import numpy as np

import bloopy.analysis.analysis_utils as anutil
from bloopy.individual import individual
import bloopy.utils as utils

def classify_points(bsize, bound_list, nidxs_dict, method='bounded'):
    if method not in ["circular", "bounded", "Hamming"]:
        raise Exception("Unknown neighbour generator given!")
    var_ranges = anutil.get_variable_ranges(bound_list)

    count_saddles = 0
    count_minima = 0
    count_maxima = 0
    regular_points = 0
    total = 0

    #A point is classified as type 0,1,2,3
    # 0 is regular, 1 is local minimum,
    # 2 is local maximum, and 3 is saddlepoint
    # we save points as tuple (type, fit, node_index) for convenience
    point_class = dict()
    for x in itertools.product(*var_ranges):
        nr_dims_is_minimal = 0
        isRegular = False
        xidx, xfit = nidxs_dict[x]
        if method == "circular":
            nbours = anutil.generate_circular_neighbours_perdim(x, var_ranges)
        elif method == "Hamming":
            nbours = anutil.generate_Hamming_neighbours_perdim(x, var_ranges)
        else:
            nbours = anutil.generate_bounded_neighbours_perdim(x, var_ranges)
        for dim in range(len(nbours)):
            nbours_larger = 0
            if len(nbours[dim]) == 0:
                raise Exception("Should already have been removed when processing space.")
            elif len(nbours[dim]) == 1:# Only one neighbour
                n1 = nbours[dim][0]
                n1_idxs, n1fit = nidxs_dict[n1]
                if n1fit == xfit:#This is for the failfit
                    if xfit > 10000:
                        isRegular = True
                        break
                    else:
                        print(xfit, "1")
                        warnings.warn("Equal fitness found")
                        #raise Exception("Equal fitness found")
                if n1fit >= xfit:
                    nbours_larger += 1
                    nr_dims_is_minimal += 1
            elif len(nbours[dim]) == 2:# Only one neighbour
                n1, n2 = nbours[dim]
                n1_idxs, n1fit = nidxs_dict[n1]
                n2_idxs, n2fit = nidxs_dict[n2]
                if n1fit == xfit or n2fit == xfit:#This is for the failfit
                    if xfit > 10000:
                        isRegular = True
                        break
                    else:
                        print(xfit, "2")
                        warnings.warn("Equal fitness found")
                        #raise Exception("Equal fitness found")
                if n1fit >= xfit:
                    nbours_larger += 1
                if n2fit >= xfit:
                    nbours_larger += 1
                if nbours_larger == 1:#Its not a critical point as it has
                    # a non-zero gradient in this dimension
                    isRegular = True
                    break
                elif nbours_larger == 2:
                    nr_dims_is_minimal += 1
            else:# More neighbours
                nbours_larger = 0
                for ni in nbours[dim]:
                    ni_idxs, nifit = nidxs_dict[ni]
                    if nifit == xfit:#This is for the failfit
                        if xfit > 10000:
                            isRegular = True
                            break
                        else:
                            warnings.warn("Equal fitness found")
                            print(xfit, "N")
                            #raise Exception("Equal fitness found")
                    if nifit >= xfit:
                        nbours_larger += 1
                if nbours_larger == len(nbours[dim]):#minimal in this dim
                    nr_dims_is_minimal += 1
                elif nbours_larger == 0:#maximal in this dim
                    pass
                else:#Its not a critical point as it has
                    # a non-zero gradient in this dimension
                    isRegular = True
                    break

        total += 1
        if x in point_class.keys():
            raise Exception("SOMETHING WRONG")

        if isRegular:
            regular_points += 1
            point_class[x] = [0, xfit, xidx]
            continue

        if nr_dims_is_minimal == 0:
            count_maxima += 1
            point_class[x] = [2, xfit, xidx]
        elif nr_dims_is_minimal == len(bound_list):
            count_minima += 1
            point_class[x] = [1, xfit, xidx]
        else:
            count_saddles += 1
            point_class[x] = [3, xfit, xidx]
    return total, count_minima, count_maxima, count_saddles, regular_points, point_class

def strong_local_minima(p, globfit, space_dict):
    loc_min_idxs = []
    for pt, value in space_dict.items():
        ptype, pfit, pidx = value
        if ptype != 1:
            continue
        if pfit <= (1+p)*globfit:
            loc_min_idxs.append(pidx)
    return loc_min_idxs

def strong_critical_points(p, globfit, space_dict):
    loc_min_idxs = []
    for pt, value in space_dict.items():
        ptype, pfit, pidx = value
        if ptype == 0:
            continue
        if pfit <= (1+p)*globfit:
            loc_min_idxs.append(pidx)
    return loc_min_idxs

def sizes_minima(sspace, bsize, bound_list, fitfunc, method="bounded"):
    if method not in ["circular", "bounded", "Hamming"]:
        raise Exception("Unknown neighbour generator given!")
    indiv = individual(bsize, boundary_list=bound_list)
    var_ranges = anutil.get_variable_ranges(bound_list)
    count_minima = 0
    total = 0
    hole_depths = []
    for x in itertools.product(*var_ranges):
        utils.set_bitstring(indiv, list(x))
        xfit = fitfunc(indiv.bitstring)

        if method == "circular":
            nbours = anutil.generate_circular_neighbours(x, var_ranges)
        elif method == "Hamming":
            nbours = anutil.generate_Hamming_neighbours(x, var_ranges)
        else:
            nbours = anutil.generate_bounded_neighbours(x, var_ranges)

        if xfit > 10000:
            is_minimal = False
        else:
            is_minimal = True
            nfits = []
            for nbour in nbours:
                utils.set_bitstring(indiv, nbour)
                nfit = fitfunc(indiv.bitstring)
                nfits.append(nfit)
                if nfit < xfit:# Neighbour has lower fitness
                    is_minimal = False
            if is_minimal:
                hole_depth = min(nfits) - xfit
                hole_depths.append([xfit, hole_depth])
                count_minima += 1
        total += 1
    return count_minima, total, np.array(hole_depths)


import torch
import numpy as np

from prediction.model_mlp import ModelMLP

class Prediction():
    def __init__(self, data_heroes, data_skills):
        hero_size = 32
        skill_size = 48
        player_hidden = 512
        player_out = 256
        team_hidden = 512
        team_out = 256
        match_hidden = 256
        p_dropout = 0.2

        self.model = ModelMLP(hero_size, skill_size, player_hidden, player_out, team_hidden, team_out, match_hidden, p_dropout)

        checkpoint = torch.load("data/model_cpu.pt")
        self.model.load_state_dict(checkpoint['model_state_dict'])

        self.model = self.model.to(torch.device("cpu"))

        self.model.eval()

        self.lookup_skills = {s["name"]: s["index"] for s in data_skills.values()}
        self.lookup_heroes = {h["name"]: h["index"] for h in data_heroes.values()}

    def __encode_players(self, features, index, players):
        for p, player in enumerate(players):
            hero_name = player["hero_name"]
            player_skills = player["skills"]

            features[index, p, 0] = self.lookup_heroes[hero_name]

            for i, skill in enumerate(player_skills):
                features[index, p, i + 1] = self.lookup_skills[skill]

    def predict_one(self, players):
        features = np.zeros((1, 10, 5), dtype=np.int)

        self.__encode_players(features, 0, players)

        y_pred = self.model(torch.from_numpy(features), False)

        return y_pred.detach().numpy()[0, 0]

    def predict_many(self, matches):
        features = np.zeros((len(matches), 10, 5), dtype=np.int)

        for i, players in enumerate(matches):
            self.__encode_players(features, i, players)

        y_pred = self.model(torch.from_numpy(features), False)

        return y_pred.detach().numpy()[:, 0]


# -*— coding: utf-8 -*-
from aip import AipNlp
import config

client = AipNlp(config.BD_NLP_APP_ID, config.BD_NLP_API_KEY, config.BD_NLP_SECRET_KEY)

# 20000字节 词法分析
def divide_words(text):
    if len(text) == 0:
        return {
            'all': []
        }
    result = client.lexer(text)
    if 'error_code' in result.keys():
        raise Exception('\ndevide words error \n text:%s\nError code:' % text + str(
            result['error_code']) + '\nError Message:' + result['error_msg'])
    else:
        words = {
            'all': []
        }
        items = result['items']
        for item in items:
            content = item['item']
            words['all'].append(content)
            if not item['ne'] == '':
                if 'ne' in words.keys():
                    words['ne'].append(content)
                else:
                    words['ne'] = [content]
            if not item['pos'] == '':
                temp = item['pos']
                if temp in words.keys():
                    words[temp].append(content)
                else:
                    words[temp] = [content]
    return words

# 256字节 句法分析
def parser(text):
    if len(text) == 0:
        return {
            'all': []
        }
    options = {'mode': 1}
    result = client.depParser(text, options=options)
    if 'error_code' in result.keys():
        raise Exception('\n parse text error \n text:%s\nError code:' % text + str(
            result['error_code']) + '\nError Message:' + result['error_msg'])
    else:
        words = {
            'all': []
        }
        items = result['items']
        for item in items:
            content = item['word']
            words['all'].append(content)
            postag, deprel = item['postag'], item['deprel']
            if postag in words.keys():
                words[postag].append(content)
            else:
                words[postag] = [content]
            if deprel in words.keys():
                words[deprel].append(content)
            else:
                words[deprel] = [content]
        return words

# 512字节 文本流畅度
def smooth_of_text(text):
    if len(text) == 0:
        return 0
    result = client.dnnlm(text)
    if 'error_code' in result.keys():
        raise Exception('\nsmooth of text error\ntext:%s\nError code:' % text + str(
            result['error_code']) + '\nError Message:' + result['error_msg'])
    else:
        return result['ppl']

def similar_of_words(word1, word2):
    if len(word1) == 0 or len(word2) == 0:
        return 0
    result = client.wordSimEmbedding(word1, word2)
    if 'error_code' in result.keys():
        raise Exception('\nsimilar of words error \n word1:%s,word2:%s\nError code:' % (word1, word2) + str(
            result['error_code']) + '\nError Message:' + result['error_msg'])
    else:
        return result['score']

# 512字节 文本相似度
def similar_of_texts(text1, text2):
    if len(text1) == 0 or len(text2) == 0:
        return 0
    result = client.simnet(text1, text2)
    if 'error_code' in result.keys():
        if result['error_code'] == 282301 or result['error_code'] == 282302 or result['error_code'] == 28303:
            return 0
        else:
            raise Exception('\nsimilar of texts error \n text1:%s,text2:%s\nError code:' % (text1, text2) +
                            str(result['error_code']) + '\nError Message:' + result['error_msg'])
    else:
        return result['score']

# 2048字节 情感倾向
def text_classify(text):
    if len(text) == 0:
        return {
            'positive': 0,
            'negative': 0,
            'sentiment': 'neutral'
        }
    result = client.sentimentClassify(text)
    if 'error_code' in result.keys():
        raise Exception('\ntext classify error \n text:%s\nError code:' % text + str(
            result['error_code']) + '\nError Message:' + result['error_msg'])
    else:
        sentiment = result['items'][0]
        return {
            'positive': sentiment['positive_prob'],
            'negative': sentiment['negative_prob'],
            'sentiment': ['negative', 'neutral', 'positive'][sentiment['sentiment']]
        }

# 65535字节 文本关键词
def label_of_article(content, title='0'):
    if len(content) == 0:
        return ''
    result = client.keyword(title, content)
    if 'error_code' in result.keys():
        raise Exception('\nlabel of article error \n title:%s,content:%s\nError code:' % (content, title) + str(
            result['error_code']) + '\nError Message:' + result['error_msg'])
    else:
        return result['items']

# 511字节 文本错误数量，基本不可用
def wrongs_of_text(text):
    if len(text) == 0:
        return 0
    result = client.ecnet(text)
    if 'error_code' in result.keys():
        raise Exception('\nerror of article \n text: %s\nError code:' % text + str(
            result['error_code']) + '\nError Message:' + result['error_msg'])
    else:
        return len(result['item']['vec_fragment'])

# 512字节 文本对话情绪
def emotion(text):
    if len(text) == 0:
        return {
            'pessimistic': 0,
            'neutral': 0
        }
    result = client.emotion(text)
    if 'error_code' in result.keys():
        raise Exception('\nerror of article \n text: %s\nError code:' % text + str(
            result['error_code']) + '\nError Message:' + result['error_msg'])
    else:
        return {
            'pessimistic': result['items'][1]['prob'],
            'neutral': result['items'][0]['prob']
        }

if __name__ == '__main__':
    # print(divide_words(''))
    print(divide_words('我想吃饭'))
    # print(smooth_of_text('我想吃饭'))
    # print(similar_of_texts('的', '麦当劳'))
    # print(text_classify('我不想吃饭'))
    # print(label_of_article(
    #     '肯德基和麦当劳是世界快餐巨头。肯德基诞生于1952年，由创始人哈兰·山德士创建，'
    #     '在世界上115个国家和地区开设有18000多家肯德基餐厅，肯德基在1987 年进入中国，收到广大消费者的喜欢。麦当劳，1955年创立于美国芝加哥，在中国大陆早期译名是“麦克唐纳快餐”，'
    #     '直到后期才统一使用现在的名字。麦当劳遍布全球六大洲119个国家，在很多国家代表着一种美式生活方式。'
    #     '在中国，因为肯德基要比麦当劳更受欢迎，很多人会觉得肯德基是世界上最大的餐饮企业，但其实麦当劳才是。麦当劳在世界上拥有3万多家门店，而肯德基的门店数量只是拥麦当劳的三分之一。'))


import sys, os, json, re, time, math
from copy import deepcopy

from multiprocessing import Pipe
from threading import Lock

class Subscription:

    def __init__(self, sub_id: int, url: str):
        self.sub_id = deepcopy(sub_id); del sub_id
        self.url = deepcopy(url); del url

        self.DEFAULT_nextUpdate_delay = 60  # Seconds

        self._lock = Lock()

        self.data: str = None   # This needs to be printable
        self._data_custom = None    # Opaque parser data storage
        self.dataTime = 0       # POSIX's epoch
        self.nextUpdate = 0     # POSIX's epoch

    def isUpdateNeeded(self):
        if math.floor(time.time()) >= self.nextUpdate:
            return True
        else:
            return False

    def putNoUpdate(self, nextUpdate: int = None, delay: int = None):
        if nextUpdate == None:
            if delay == None:
                nextUpdate = math.floor(time.time()) + self.DEFAULT_nextUpdate_delay
            else:
                nextUpdate = math.floor(time.time()) + delay
        self.nextUpdate = deepcopy(nextUpdate)

    def putData(self,
        data: str = None,
        dataTime: int = None,
        nextUpdate: int = None
    ):
        if dataTime == None:
            dataTime = math.floor(time.time())
        if nextUpdate == None:
            nextUpdate = dataTime + self.DEFAULT_nextUpdate_delay
        self.data = deepcopy(data)
        self.dataTime = deepcopy(dataTime)
        self.nextUpdate = deepcopy(nextUpdate)

class Chat:

    def __init__(self, chat_id):
        self.chat_id = deepcopy(chat_id); del chat_id
        self.subs: dict = {} # [int, Subscription]
        self.last_sub_id: int = None
        self.lastReport = 0     # POSIX's epoch

    def createSub(self, sub_url):
        MAXIMUM_SUBS = 500
        MIN_SUB_ID = 1          # sub_id can't be 0
        if self.last_sub_id == None:
            new_sub_id = MIN_SUB_ID
        else:
            new_sub_id = (self.last_sub_id + 1)%(MAXIMUM_SUBS+MIN_SUB_ID)
        while new_sub_id != self.last_sub_id:
            if new_sub_id in self.subs:
                new_sub_id = (new_sub_id + 1)%(MAXIMUM_SUBS+MIN_SUB_ID)
                continue
            self.subs[new_sub_id] = Subscription(new_sub_id, sub_url)
            self.last_sub_id = new_sub_id
            return self.subs[new_sub_id]
        else:
            # If there is more then MAXIMUM_SUBS subs, do nothing
            return None

    def deleteSub(self,sub_id):
        if sub_id not in self.subs:
            return False
        with self.subs[sub_id]._lock:
            del self.subs[sub_id]
        return True

def updatePush(pipe: Pipe,
        chat_id: int,
        sub_id: int,
        timestamp: int = None,
):
    if timestamp == None:
        timestamp = math.floor(time.time())
    # By design, if you push (-1 * chat_id) or (-1 * sub_id), it means that corresponding object needs to be deleted or not accessible anymore.
    pipe.send((chat_id,sub_id,timestamp))

def sanitizeURL(url: str):
    import urllib.parse as ulP

    return ulP.urlunsplit(ulP.urlsplit(url))

def onlyPageURL(url: str):
    import urllib.parse as ulP

    p0 = ulP.urlsplit(url)
    p1 = p0._replace(**{'query':'','fragment':''})

    return ulP.urlunsplit(p1)