from __future__ import division

import errno
import functools
import gzip
import itertools
import io
import mmap
import os
import platform
import signal
import sys
import tempfile
import textwrap
import threading
import warnings
import weakref

from distutils.version import LooseVersion

try:
    from StringIO import StringIO
except ImportError:
    class StringIO(object):
        pass

import numpy as np

from .extern import six
from .extern.six import (iteritems, string_types, integer_types, text_type,
                         binary_type, next)
from .extern.six.moves import zip, reduce

BLOCK_SIZE = 2880  # the FITS block size

if six.PY3:
    cmp = lambda a, b: (a > b) - (a < b)
elif six.PY2:
    cmp = cmp

class PyFITSDeprecationWarning(Warning):
    """
    A warning class to indicate a deprecated feature.
    """

class NotifierMixin(object):
    """
    Mixin class that provides services by which objects can register
    listeners to changes on that object.

    All methods provided by this class are underscored, since this is intended
    for internal use to communicate between classes in a generic way, and is
    not machinery that should be exposed to users of the classes involved.

    Use the ``_add_listener`` method to register a listener on an instance of
    the notifier.  This registers the listener with a weak reference, so if
    no other references to the listener exist it is automatically dropped from
    the list and does not need to be manually removed.

    Call the ``_notify`` method on the notifier to update all listeners
    upon changes.  ``_notify('change_type', *args, **kwargs)`` results
    in calling ``listener._update_change_type(*args, **kwargs)`` on all
    listeners subscribed to that notifier.

    If a particular listener does not have the appropriate update method
    it is ignored.

    Examples
    --------

    >>> class Widget(NotifierMixin):
    ...     state = 1
    ...     def __init__(self, name):
    ...         self.name = name
    ...     def update_state(self):
    ...         self.state += 1
    ...         self._notify('widget_state_changed', self)
    ...
    >>> class WidgetListener(object):
    ...     def _update_widget_state_changed(self, widget):
    ...         print('Widget {0} changed state to {1}'.format(
    ...             widget.name, widget.state))
    ...
    >>> widget = Widget('fred')
    >>> listener = WidgetListener()
    >>> widget._add_listener(listener)
    >>> widget.update_state()
    Widget fred changed state to 2
    """

    _listeners = None

    def _add_listener(self, listener):
        """
        Add an object to the list of listeners to notify of changes to this
        object.  This adds a weakref to the list of listeners that is
        removed from the listeners list when the listener has no other
        references to it.
        """

        if self._listeners is None:
            self._listeners = weakref.WeakValueDictionary()

        self._listeners[id(listener)] = listener

    def _remove_listener(self, listener):
        """
        Removes the specified listener from the listeners list.  This relies
        on object identity (i.e. the ``is`` operator).
        """

        if self._listeners is None:
            return

        try:
            del self._listeners[id(listener)]
        except KeyError:
            pass

    def _notify(self, notification, *args, **kwargs):
        """
        Notify all listeners of some particular state change by calling their
        ``_update_<notification>`` method with the given ``*args`` and
        ``**kwargs``.

        The notification does not by default include the object that actually
        changed (``self``), but it certainly may if required.
        """

        if self._listeners is None:
            return

        method_name = '_update_{0}'.format(notification)
        for listener in self._listeners.valuerefs():
            # Use valuerefs instead of itervaluerefs; see
            # https://github.com/astropy/astropy/issues/4015
            listener = listener()  # dereference weakref
            if listener is None:
                continue

            if hasattr(listener, method_name):
                method = getattr(listener, method_name)
                if callable(method):
                    method(*args, **kwargs)

    def __getstate__(self):
        """
        Exclude listeners when saving the listener's state, since they may be
        ephemeral.
        """

        # TODO: This hasn't come up often, but if anyone needs to pickle HDU
        # objects it will be necessary when HDU objects' states are restored to
        # re-register themselves as listeners on their new column instances.
        try:
            state = super(NotifierMixin, self).__getstate__()
        except AttributeError:
            # Chances are the super object doesn't have a getstate
            state = self.__dict__.copy()

        state['_listeners'] = None
        return state

def first(iterable):
    """
    Returns the first item returned by iterating over an iterable object.

    Example:

    >>> a = [1, 2, 3]
    >>> first(a)
    1
    """

    return next(iter(iterable))

def itersubclasses(cls, _seen=None):
    """
    itersubclasses(cls)

    Generator over all subclasses of a given class, in depth first order.

    >>> list(itersubclasses(int)) == [bool]
    True
    >>> class A(object): pass
    >>> class B(A): pass
    >>> class C(A): pass
    >>> class D(B,C): pass
    >>> class E(D): pass
    >>>
    >>> for cls in itersubclasses(A):
    ...     print(cls.__name__)
    B
    D
    E
    C
    >>> # get ALL (new-style) classes currently defined
    >>> [cls.__name__ for cls in itersubclasses(object)] #doctest: +ELLIPSIS
    [...'tuple', ...'type', ...]

    From http://code.activestate.com/recipes/576949/
    """

    if not isinstance(cls, type):
        raise TypeError('itersubclasses must be called with '
                        'new-style classes, not %.100r' % cls)
    if _seen is None:
        _seen = set()
    try:
        subs = cls.__subclasses__()
    except TypeError:  # fails only when cls is type
        subs = cls.__subclasses__(cls)
    for sub in sorted(subs, key=lambda s: s.__name__):
        if sub not in _seen:
            _seen.add(sub)
            yield sub
            for sub in itersubclasses(sub, _seen):
                yield sub

class lazyproperty(object):
    """
    Works similarly to property(), but computes the value only once.

    Adapted from the recipe at
    http://code.activestate.com/recipes/363602-lazy-property-evaluation
    """

    def __init__(self, fget, fset=None, fdel=None, doc=None):
        self._fget = fget
        self._fset = fset
        self._fdel = fdel
        if doc is None:
            self.__doc__ = fget.__doc__
        else:
            self.__doc__ = doc
        self._key = self._fget.__name__

    def __get__(self, obj, owner=None):
        if obj is None:
            return self
        try:
            return obj.__dict__[self._key]
        except KeyError:
            val = self._fget(obj)
            obj.__dict__[self._key] = val
            return val

    def __set__(self, obj, val):
        obj_dict = obj.__dict__
        if self._fset:
            ret = self._fset(obj, val)
            if ret is not None and obj_dict.get(self._key) is ret:
                # By returning the value set the setter signals that it took
                # over setting the value in obj.__dict__; this mechanism allows
                # it to override the input value
                return
        obj_dict[self._key] = val

    def __delete__(self, obj):
        if self._fdel:
            self._fdel(obj)
        if self._key in obj.__dict__:
            del obj.__dict__[self._key]

    def getter(self, fget):
        return self.__ter(fget, 0)

    def setter(self, fset):
        return self.__ter(fset, 1)

    def deleter(self, fdel):
        return self.__ter(fdel, 2)

    def __ter(self, f, arg):
        args = [self._fget, self._fset, self._fdel, self.__doc__]
        args[arg] = f
        cls_ns = sys._getframe(1).f_locals
        for k, v in iteritems(cls_ns):
            if v is self:
                property_name = k
                break

        cls_ns[property_name] = lazyproperty(*args)

        return cls_ns[property_name]

# TODO: This can still be made to work for setters by implementing an
# accompanying metaclass that supports it; we just don't need that right this
# second
class classproperty(property):
    """
    Similar to `property`, but allows class-level properties.  That is,
    a property whose getter is like a `classmethod`.

    The wrapped method may explicitly use the `classmethod` decorator (which
    must become before this decorator), or the `classmethod` may be omitted
    (it is implicit through use of this decorator).

    .. note::

        classproperty only works for *read-only* properties.  It does not
        currently allow writeable/deleteable properties, due to subtleties of how
        Python descriptors work.  In order to implement such properties on a class
        a metaclass for that class must be implemented.

    Parameters
    ----------
    fget : callable
        The function that computes the value of this property (in particular,
        the function when this is used as a decorator) a la `property`.

    doc : str, optional
        The docstring for the property--by default inherited from the getter
        function.

    lazy : bool, optional
        If True, caches the value returned by the first call to the getter
        function, so that it is only called once (used for lazy evaluation
        of an attribute).  This is analogous to `lazyproperty`.  The ``lazy``
        argument can also be used when `classproperty` is used as a decorator
        (see the third example below).  When used in the decorator syntax this
        *must* be passed in as a keyword argument.

    Examples
    --------

    ::

        >>> class Foo(object):
        ...     _bar_internal = 1
        ...     @classproperty
        ...     def bar(cls):
        ...         return cls._bar_internal + 1
        ...
        >>> Foo.bar
        2
        >>> foo_instance = Foo()
        >>> foo_instance.bar
        2
        >>> foo_instance._bar_internal = 2
        >>> foo_instance.bar  # Ignores instance attributes
        2

    As previously noted, a `classproperty` is limited to implementing
    read-only attributes::

        >>> class Foo(object):
        ...     _bar_internal = 1
        ...     @classproperty
        ...     def bar(cls):
        ...         return cls._bar_internal
        ...     @bar.setter
        ...     def bar(cls, value):
        ...         cls._bar_internal = value
        ...
        Traceback (most recent call last):
        ...
        NotImplementedError: classproperty can only be read-only; use a
        metaclass to implement modifiable class-level properties

    When the ``lazy`` option is used, the getter is only called once::

        >>> class Foo(object):
        ...     @classproperty(lazy=True)
        ...     def bar(cls):
        ...         print("Performing complicated calculation")
        ...         return 1
        ...
        >>> Foo.bar
        Performing complicated calculation
        1
        >>> Foo.bar
        1

    If a subclass inherits a lazy `classproperty` the property is still
    re-evaluated for the subclass::

        >>> class FooSub(Foo):
        ...     pass
        ...
        >>> FooSub.bar
        Performing complicated calculation
        1
        >>> FooSub.bar
        1
    """

    def __new__(cls, fget=None, doc=None, lazy=False):
        if fget is None:
            # Being used as a decorator--return a wrapper that implements
            # decorator syntax
            def wrapper(func):
                return cls(func, lazy=lazy)

            return wrapper

        return super(classproperty, cls).__new__(cls)

    def __init__(self, fget, doc=None, lazy=False):
        self._lazy = lazy
        if lazy:
            self._cache = {}
        fget = self._wrap_fget(fget)

        super(classproperty, self).__init__(fget=fget, doc=doc)

        # There is a buglet in Python where self.__doc__ doesn't
        # get set properly on instances of property subclasses if
        # the doc argument was used rather than taking the docstring
        # from fget
        if doc is not None:
            self.__doc__ = doc

    def __get__(self, obj, objtype=None):
        if self._lazy and objtype in self._cache:
            return self._cache[objtype]

        if objtype is not None:
            # The base property.__get__ will just return self here;
            # instead we pass objtype through to the original wrapped
            # function (which takes the class as its sole argument)
            val = self.fget.__wrapped__(objtype)
        else:
            val = super(classproperty, self).__get__(obj, objtype=objtype)

        if self._lazy:
            if objtype is None:
                objtype = obj.__class__

            self._cache[objtype] = val

        return val

    def getter(self, fget):
        return super(classproperty, self).getter(self._wrap_fget(fget))

    def setter(self, fset):
        raise NotImplementedError(
            "classproperty can only be read-only; use a metaclass to "
            "implement modifiable class-level properties")

    def deleter(self, fdel):
        raise NotImplementedError(
            "classproperty can only be read-only; use a metaclass to "
            "implement modifiable class-level properties")

    @staticmethod
    def _wrap_fget(orig_fget):
        if isinstance(orig_fget, classmethod):
            orig_fget = orig_fget.__func__

        # Using stock functools.wraps instead of the fancier version
        # found later in this module, which is overkill for this purpose

        @functools.wraps(orig_fget)
        def fget(obj):
            return orig_fget(obj.__class__)

        # Set the __wrapped__ attribute manually for support on Python 2
        fget.__wrapped__ = orig_fget

        return fget

class PyfitsDeprecationWarning(UserWarning):
    pass

class PyfitsPendingDeprecationWarning(UserWarning):
    pass

# TODO: Provide a class deprecation marker as well.
def deprecated(since, message='', name='', alternative='', pending=False):
    """
    Used to mark a function as deprecated.

    To mark an attribute as deprecated, replace that attribute with a
    depcrecated property.

    Parameters
    ------------
    since : str
        The release at which this API became deprecated.  This is required.

    message : str, optional
        Override the default deprecation message.  The format specifier
        %(func)s may be used for the name of the function, and %(alternative)s
        may be used in the deprecation message to insert the name of an
        alternative to the deprecated function.

    name : str, optional
        The name of the deprecated function; if not provided the name is
        automatically determined from the passed in function, though this is
        useful in the case of renamed functions, where the new function is just
        assigned to the name of the deprecated function.  For example:
            def new_function():
                ...
            oldFunction = new_function

    alternative : str, optional
        An alternative function that the user may use in place of the
        deprecated function.  The deprecation warning will tell the user about
        this alternative if provided.

    pending : bool, optional
        If True, uses a PyfitsPendingDeprecationWarning instead of a
        PyfitsDeprecationWarning.

    """

    def deprecate(func, message=message, name=name, alternative=alternative,
                  pending=pending):
        if isinstance(func, classmethod):
            try:
                func = func.__func__
            except AttributeError:
                # classmethods in Python2.6 and below lack the __func__
                # attribute so we need to hack around to get it
                method = func.__get__(None, object)
                if hasattr(method, '__func__'):
                    func = method.__func__
                elif hasattr(method, 'im_func'):
                    func = method.im_func
                else:
                    # Nothing we can do really...  just return the original
                    # classmethod
                    return func
            is_classmethod = True
        else:
            is_classmethod = False

        if not name:
            name = func.__name__

        altmessage = ''
        if not message or type(message) == type(deprecate):
            if pending:
                message = ('The %(func)s function will be deprecated in a '
                           'future version.')
            else:
                message = (
                    'The %(func)s function is deprecated as of version '
                    '%(since)s and may be removed in a future version.')
            if alternative:
                altmessage = '\n\n        Use %s instead.' % alternative

        message = ((message % {'func': name, 'alternative': alternative,
                               'since': since}) + altmessage)

        @functools.wraps(func)
        def deprecated_func(*args, **kwargs):
            if pending:
                category = PyfitsPendingDeprecationWarning
            else:
                category = PyfitsDeprecationWarning

            warnings.warn(message, category, stacklevel=2)

            return func(*args, **kwargs)

        old_doc = deprecated_func.__doc__
        if not old_doc:
            old_doc = ''
        old_doc = textwrap.dedent(old_doc).strip('\n')
        altmessage = altmessage.strip()
        if not altmessage:
            altmessage = message.strip()
        new_doc = (('\n.. deprecated:: %(since)s'
                    '\n    %(message)s\n\n' %
                    {'since': since, 'message': altmessage.strip()}) + old_doc)
        if not old_doc:
            # This is to prevent a spurious 'unexected unindent' warning from
            # docutils when the original docstring was blank.
            new_doc += r'\ '

        deprecated_func.__doc__ = new_doc

        if is_classmethod:
            deprecated_func = classmethod(deprecated_func)
        return deprecated_func

    if type(message) == type(deprecate):
        return deprecate(message)

    return deprecate

def ignore_sigint(func):
    """
    This decorator registers a custom SIGINT handler to catch and ignore SIGINT
    until the wrapped function is completed.
    """

    @functools.wraps(func)
    def wrapped(*args, **kwargs):
        # Get the name of the current thread and determine if this is a single
        # threaded application
        curr_thread = threading.currentThread()
        single_thread = (threading.activeCount() == 1 and
                         curr_thread.getName() == 'MainThread')

        class SigintHandler(object):
            def __init__(self):
                self.sigint_received = False

            def __call__(self, signum, frame):
                warnings.warn('KeyboardInterrupt ignored until %s is '
                              'complete!' % func.__name__)
                self.sigint_received = True

        sigint_handler = SigintHandler()

        # Define new signal interput handler
        if single_thread:
            # Install new handler
            old_handler = signal.signal(signal.SIGINT, sigint_handler)

        try:
            func(*args, **kwargs)
        finally:
            if single_thread:
                if old_handler is not None:
                    signal.signal(signal.SIGINT, old_handler)
                else:
                    signal.signal(signal.SIGINT, signal.SIG_DFL)

                if sigint_handler.sigint_received:
                    raise KeyboardInterrupt

    return wrapped

def pairwise(iterable):
    """Return the items of an iterable paired with its next item.

    Ex: s -> (s0,s1), (s1,s2), (s2,s3), ....
    """

    a, b = itertools.tee(iterable)
    for _ in b:
        # Just a little trick to advance b without having to catch
        # StopIter if b happens to be empty
        break
    return zip(a, b)

def isiterable(obj):
    """Returns true of the given object is iterable."""

    # In Python2.6 and up this is simply a matter of checking isinstance
    # collections.Iterable, but this unavailable in Python 2.5 and below
    try:
        from collections import Iterable
        if isinstance(obj, Iterable):
            return True
    except ImportError:
        pass

    try:
        iter(obj)
        return True
    except TypeError:
        return False

def encode_ascii(s):
    """
    In Python 2 this is a no-op.  Strings are left alone.  In Python 3 this
    will be replaced with a function that actually encodes unicode strings to
    ASCII bytes.
    """

    return s

def decode_ascii(s):
    """
    In Python 2 this is a no-op.  Strings are left alone.  In Python 3 this
    will be replaced with a function that actually decodes ascii bytes to
    unicode.
    """

    return s

def isreadable(f):
    """
    Returns True if the file-like object can be read from.  This is a common-
    sense approximation of io.IOBase.readable.
    """

    if hasattr(f, 'closed') and f.closed:
        # This mimics the behavior of io.IOBase.readable
        raise ValueError('I/O operation on closed file')

    if not hasattr(f, 'read'):
        return False

    if hasattr(f, 'mode') and not any((c in f.mode for c in 'r+')):
        return False

    # Not closed, has a 'read()' method, and either has no known mode or a
    # readable mode--should be good enough to assume 'readable'
    return True

def iswritable(f):
    """
    Returns True if the file-like object can be written to.  This is a common-
    sense approximation of io.IOBase.writable.
    """

    if hasattr(f, 'closed') and f.closed:
        # This mimics the behavior of io.IOBase.writable
        raise ValueError('I/O operation on closed file')

    if not hasattr(f, 'write'):
        return False

    if hasattr(f, 'mode') and not any((c in f.mode for c in 'wa+')):
        return False

    # Note closed, has a 'write()' method, and either has no known mode or a
    # mode that supports writing--should be good enough to assume 'writable'
    return True

def isfile(f):
    """
    Returns True if the given object represents an OS-level file (that is,
    isinstance(f, file)).

    On Python 3 this also returns True if the given object is higher level
    wrapper on top of a FileIO object, such as a TextIOWrapper.
    """

    return isinstance(f, file)

def fileobj_open(filename, mode):
    """
    A wrapper around the `open()` builtin.

    This exists because in Python 3, `open()` returns an `io.BufferedReader` by
    default.  This is bad, because `io.BufferedReader` doesn't support random
    access, which we need in some cases.  In the Python 3 case (implemented in
    the py3compat module) we must call open with buffering=0 to get a raw
    random-access file reader.
    """

    return open(filename, mode)

def fileobj_name(f):
    """
    Returns the 'name' of file-like object f, if it has anything that could be
    called its name.  Otherwise f's class or type is returned.  If f is a
    string f itself is returned.
    """

    if isinstance(f, string_types):
        return f
    elif isinstance(f, gzip.GzipFile):
        # The .name attribute on GzipFiles does not always represent the name
        # of the file being read/written--it can also represent the original
        # name of the file being compressed
        # See the documentation at
        # https://docs.python.org/3/library/gzip.html#gzip.GzipFile
        # As such, for gzip files only return the name of the underlying
        # fileobj, if it exists
        return fileobj_name(f.fileobj)
    elif hasattr(f, 'name'):
        return f.name
    elif hasattr(f, 'filename'):
        return f.filename
    elif hasattr(f, '__class__'):
        return str(f.__class__)
    else:
        return str(type(f))

def fileobj_closed(f):
    """
    Returns True if the given file-like object is closed or if f is not a
    file-like object.
    """

    if hasattr(f, 'closed'):
        return f.closed
    elif hasattr(f, 'fileobj') and hasattr(f.fileobj, 'closed'):
        return f.fileobj.closed
    elif hasattr(f, 'fp') and hasattr(f.fp, 'closed'):
        return f.fp.closed
    else:
        return False

def fileobj_mode(f):
    """
    Returns the 'mode' string of a file-like object if such a thing exists.
    Otherwise returns None.
    """

    # Go from most to least specific--for example gzip objects have a 'mode'
    # attribute, but it's not analogous to the file.mode attribute
    if hasattr(f, 'fileobj') and hasattr(f.fileobj, 'mode'):
        fileobj = f.fileobj
    elif hasattr(f, 'fp') and hasattr(f.fp, 'mode'):
        fileobj = f.fp
    elif hasattr(f, 'mode'):
        fileobj = f
    else:
        return None

    return _fileobj_normalize_mode(fileobj)

def _fileobj_normalize_mode(f):
    """Takes care of some corner cases in Python where the mode string
    is either oddly formatted or does not truly represent the file mode.
    """

    # I've noticed that sometimes Python can produce modes like 'r+b' which I
    # would consider kind of a bug--mode strings should be normalized.  Let's
    # normalize it for them:
    mode = f.mode

    if isinstance(f, gzip.GzipFile):
        # GzipFiles can be either readonly or writeonly
        if mode == gzip.READ:
            return 'rb'
        elif mode == gzip.WRITE:
            return 'wb'
        else:
            # This shouldn't happen?
            return None

    if '+' in mode:
        mode = mode.replace('+', '')
        mode += '+'

    if _fileobj_is_append_mode(f) and 'a' not in mode:
        mode = mode.replace('r', 'a').replace('w', 'a')

    return mode

def _fileobj_is_append_mode(f):
    """Normally the way to tell if a file is in append mode is if it has
    'a' in the mode string.  However on Python 3 (or in particular with
    the io module) this can't be relied on.  See
    http://bugs.python.org/issue18876.
    """

    if 'a' in f.mode:
        # Take care of the obvious case first
        return True

    # We might have an io.FileIO in which case the only way to know for sure
    # if the file is in append mode is to ask the file descriptor
    if not hasattr(f, 'fileno'):
        # Who knows what this is?
        return False

    # Call platform-specific _is_append_mode
    # If this file is already closed this can result in an error
    try:
        return _is_append_mode_platform(f.fileno())
    except (ValueError, IOError):
        return False

if sys.platform.startswith('win32'):
    # This global variable is used in _is_append_mode to cache the computed
    # size of the ioinfo struct from msvcrt which may have a different size
    # depending on the version of the library and how it was compiled
    _sizeof_ioinfo = None

    def _make_is_append_mode():
        # We build the platform-specific _is_append_mode function for Windows
        # inside a function factory in order to avoid cluttering the local
        # namespace with ctypes stuff
        from ctypes import (cdll, c_size_t, c_void_p, c_int, c_char,
                            Structure, POINTER, cast)

        from ctypes.util import find_msvcrt

        def _dummy_is_append_mode(fd):
            warnings.warn(
                'Could not find appropriate MS Visual C Runtime '
                'library or library is corrupt/misconfigured; cannot '
                'determine whether your file object was opened in append '
                'mode.  Please consider using a file object opened in write '
                'mode instead.')
            return False

        msvcrt_dll = find_msvcrt()
        if msvcrt_dll is None:
            # If for some reason the C runtime can't be located then we're dead
            # in the water.  Just return a dummy function
            return _dummy_is_append_mode

        msvcrt = cdll.LoadLibrary(msvcrt_dll)

        # Constants
        IOINFO_L2E = 5
        IOINFO_ARRAY_ELTS = 1 << IOINFO_L2E
        IOINFO_ARRAYS = 64
        FAPPEND = 0x20
        _NO_CONSOLE_FILENO = -2

        # Types
        intptr_t = POINTER(c_int)

        class my_ioinfo(Structure):
            _fields_ = [('osfhnd', intptr_t),
                        ('osfile', c_char)]

        # Functions
        _msize = msvcrt._msize
        _msize.argtypes = (c_void_p,)
        _msize.restype = c_size_t

        # Variables
        # Since we don't know how large the ioinfo struct is just treat the
        # __pioinfo array as an array of byte pointers
        __pioinfo = cast(msvcrt.__pioinfo, POINTER(POINTER(c_char)))

        # Determine size of the ioinfo struct; see the comment above where
        # _sizeof_ioinfo = None is set
        global _sizeof_ioinfo
        if __pioinfo[0] is not None:
            _sizeof_ioinfo = _msize(__pioinfo[0]) // IOINFO_ARRAY_ELTS

        if not _sizeof_ioinfo:
            # This shouldn't happen, but I suppose it could if one is using a
            # broken msvcrt, or just happened to have a dll of the same name
            # lying around.
            return _dummy_is_append_mode

        def _is_append_mode(fd):
            global _sizeof_ioinfo
            if fd != _NO_CONSOLE_FILENO:
                idx1 = fd >> IOINFO_L2E # The index into the __pioinfo array
                # The n-th ioinfo pointer in __pioinfo[idx1]
                idx2 = fd & ((1 << IOINFO_L2E) - 1)
                if 0 <= idx1 < IOINFO_ARRAYS and __pioinfo[idx1] is not None:
                    # Doing pointer arithmetic in ctypes is irritating
                    pio = c_void_p(cast(__pioinfo[idx1], c_void_p).value +
                                   idx2 * _sizeof_ioinfo)
                    ioinfo = cast(pio, POINTER(my_ioinfo)).contents
                    return bool(ord(ioinfo.osfile) & FAPPEND)
            return False

        return _is_append_mode

    _is_append_mode_platform = _make_is_append_mode()
    del _make_is_append_mode
else:
    import fcntl

    def _is_append_mode_platform(fd):
        return bool(fcntl.fcntl(fd, fcntl.F_GETFL) & os.O_APPEND)

def fileobj_is_binary(f):
    """
    Returns True if the give file or file-like object has a file open in binary
    mode.  When in doubt, returns True by default.
    """

    # This is kind of a hack for this to work correctly with _File objects,
    # which, for the time being, are *always* binary
    if hasattr(f, 'binary'):
        return f.binary

    if io is not None and isinstance(f, io.TextIOBase):
        return False

    mode = fileobj_mode(f)
    if mode:
        return 'b' in mode
    else:
        return True

def translate(s, table, deletechars):
    """
    This is a version of string/unicode.translate() that can handle string or
    unicode strings the same way using a translation table made with
    string.maketrans.
    """

    if isinstance(s, str):
        return s.translate(table, deletechars)
    elif isinstance(s, text_type):
        table = dict((x, ord(table[x])) for x in range(256)
                     if ord(table[x]) != x)
        for c in deletechars:
            table[ord(c)] = None
        return s.translate(table)

def indent(s, shift=1, width=4):
    indented = '\n'.join(' ' * (width * shift) + l if l else ''
                         for l in s.splitlines())
    if s[-1] == '\n':
        indented += '\n'

    return indented

def fill(text, width, *args, **kwargs):
    """
    Like :func:`textwrap.wrap` but preserves existing paragraphs which
    :func:`textwrap.wrap` does not otherwise handle well.  Also handles section
    headers.
    """

    paragraphs = text.split('\n\n')

    def maybe_fill(t):
        if all(len(l) < width for l in t.splitlines()):
            return t
        else:
            return textwrap.fill(t, width, *args, **kwargs)

    return '\n\n'.join(maybe_fill(p) for p in paragraphs)

# On MacOS X 10.8 and earlier, there is a bug that causes numpy.fromfile to
# fail when reading over 2Gb of data. If we detect these versions of MacOS X,
# we can instead read the data in chunks. To avoid performance penalties at
# import time, we defer the setting of this global variable until the first
# time it is needed.
CHUNKED_FROMFILE = None

def _array_from_file(infile, dtype, count, sep):
    """Create a numpy array from a file or a file-like object."""

    if isfile(infile):

        global CHUNKED_FROMFILE
        if CHUNKED_FROMFILE is None:
            if sys.platform == 'darwin' and LooseVersion(platform.mac_ver()[0]) < LooseVersion('10.9'):
                CHUNKED_FROMFILE = True
            else:
                CHUNKED_FROMFILE = False

        if CHUNKED_FROMFILE:
            chunk_size = int(1024 ** 3 / dtype.itemsize)  # 1Gb to be safe
            if count < chunk_size:
                return np.fromfile(infile, dtype=dtype, count=count, sep=sep)
            else:
                array = np.empty(count, dtype=dtype)
                for beg in range(0, count, chunk_size):
                    end = min(count, beg + chunk_size)
                    array[beg:end] = np.fromfile(infile, dtype=dtype, count=end - beg, sep=sep)
                return array
        else:
            return np.fromfile(infile, dtype=dtype, count=count, sep=sep)
    else:
        # treat as file-like object with "read" method; this includes gzip file
        # objects, because numpy.fromfile just reads the compressed bytes from
        # their underlying file object, instead of the decompressed bytes
        read_size = np.dtype(dtype).itemsize * count
        s = infile.read(read_size)
        return np.fromstring(s, dtype=dtype, count=count, sep=sep)

_OSX_WRITE_LIMIT = (2 ** 32) - 1
_WIN_WRITE_LIMIT = (2 ** 31) - 1

def _array_to_file(arr, outfile):
    """
    Write a numpy array to a file or a file-like object.

    Parameters
    ----------
    arr : `~numpy.ndarray`
        The Numpy array to write.
    outfile : file-like
        A file-like object such as a Python file object, an `io.BytesIO`, or
        anything else with a ``write`` method.  The file object must support
        the buffer interface in its ``write``.

    If writing directly to an on-disk file this delegates directly to
    `ndarray.tofile`.  Otherwise a slower Python implementation is used.
    """

    if isfile(outfile):
        write = lambda a, f: a.tofile(f)
    else:
        write = _array_to_file_like

    # Implements a workaround for a bug deep in OSX's stdlib file writing
    # functions; on 64-bit OSX it is not possible to correctly write a number
    # of bytes greater than 2 ** 32 and divisible by 4096 (or possibly 8192--
    # whatever the default blocksize for the filesystem is).
    # This issue should have a workaround in Numpy too, but hasn't been
    # implemented there yet: https://github.com/astropy/astropy/issues/839
    #
    # Apparently Windows has its own fwrite bug:
    # https://github.com/numpy/numpy/issues/2256

    if (sys.platform == 'darwin' and arr.nbytes >= _OSX_WRITE_LIMIT + 1 and
            arr.nbytes % 4096 == 0):
        # chunksize is a count of elements in the array, not bytes
        chunksize = _OSX_WRITE_LIMIT // arr.itemsize
    elif sys.platform.startswith('win'):
        chunksize = _WIN_WRITE_LIMIT // arr.itemsize
    else:
        # Just pass the whole array to the write routine
        return write(arr, outfile)

    # Write one chunk at a time for systems whose fwrite chokes on large
    # writes.
    idx = 0
    arr = arr.view(np.ndarray).flatten()
    while idx < arr.nbytes:
        write(arr[idx:idx + chunksize], outfile)
        idx += chunksize

def _array_to_file_like(arr, fileobj):
    """
    Write a `~numpy.ndarray` to a file-like object (which is not supported by
    `numpy.ndarray.tofile`).
    """

    if arr.flags.contiguous:
        # It suffices to just pass the underlying buffer directly to the
        # fileobj's write (assuming it supports the buffer interface, which
        # unfortunately there's no simple way to check)
        fileobj.write(arr.data)
    elif hasattr(np, 'nditer'):
        # nditer version for non-contiguous arrays
        for item in np.nditer(arr):
            fileobj.write(item.tostring())
    else:
        # Slower version for Numpy versions without nditer;
        # The problem with flatiter is it doesn't preserve the original
        # byteorder
        byteorder = arr.dtype.byteorder
        if ((sys.byteorder == 'little' and byteorder == '>')
                or (sys.byteorder == 'big' and byteorder == '<')):
            for item in arr.flat:
                fileobj.write(item.byteswap().tostring())
        else:
            for item in arr.flat:
                fileobj.write(item.tostring())

def _write_string(f, s):
    """
    Write a string to a file, encoding to ASCII if the file is open in binary
    mode, or decoding if the file is open in text mode.
    """

    # Assume if the file object doesn't have a specific mode, that the mode is
    # binary
    binmode = fileobj_is_binary(f)

    if binmode and isinstance(s, text_type):
        s = encode_ascii(s)
    elif not binmode and not isinstance(f, text_type):
        s = decode_ascii(s)
    elif isinstance(f, StringIO) and isinstance(s, np.ndarray):
        # Workaround for StringIO/ndarray incompatibility
        s = s.data

    f.write(s)

def _convert_array(array, dtype):
    """
    Converts an array to a new dtype--if the itemsize of the new dtype is
    the same as the old dtype and both types are not numeric, a view is
    returned.  Otherwise a new array must be created.
    """

    if array.dtype == dtype:
        return array
    elif (array.dtype.itemsize == dtype.itemsize and not
            (np.issubdtype(array.dtype, np.number) and
             np.issubdtype(dtype, np.number))):
        # Includes a special case when both dtypes are at least numeric to
        # account for ticket #218: https://aeon.stsci.edu/ssb/trac/pyfits/ticket/218
        return array.view(dtype)
    else:
        return array.astype(dtype)

def _unsigned_zero(dtype):
    """
    Given a numpy dtype, finds its "zero" point, which is exactly in the
    middle of its range.
    """

    assert dtype.kind == 'u'
    return 1 << (dtype.itemsize * 8 - 1)

def _is_pseudo_unsigned(dtype):
    return dtype.kind == 'u' and dtype.itemsize >= 2

def _is_int(val):
    return isinstance(val, integer_types + (np.integer,))

def _str_to_num(val):
    """Converts a given string to either an int or a float if necessary."""

    try:
        num = int(val)
    except ValueError:
        # If this fails then an exception should be raised anyways
        num = float(val)
    return num

def _pad_length(stringlen):
    """Bytes needed to pad the input stringlen to the next FITS block."""

    return (BLOCK_SIZE - (stringlen % BLOCK_SIZE)) % BLOCK_SIZE

def _words_group(input, strlen):
    """
    Split a long string into parts where each part is no longer
    than `strlen` and no word is cut into two pieces.  But if
    there is one single word which is longer than `strlen`, then
    it will be split in the middle of the word.
    """

    words = []
    nblanks = input.count(' ')
    nmax = max(nblanks, len(input) // strlen + 1)
    arr = np.fromstring((input + ' '), dtype=(binary_type, 1))

    # locations of the blanks
    blank_loc = np.nonzero(arr == ' '.encode('latin1'))[0]
    offset = 0
    xoffset = 0
    for idx in range(nmax):
        try:
            loc = np.nonzero(blank_loc >= strlen + offset)[0][0]
            offset = blank_loc[loc - 1] + 1
            if loc == 0:
                offset = -1
        except:
            offset = len(input)

        # check for one word longer than strlen, break in the middle
        if offset <= xoffset:
            offset = xoffset + strlen

        # collect the pieces in a list
        words.append(input[xoffset:offset])
        if len(input) == offset:
            break
        xoffset = offset

    return words

def _tmp_name(input):
    """
    Create a temporary file name which should not already exist.  Use the
    directory of the input file as the base name of the mkstemp() output.
    """

    if input is not None:
        input = os.path.dirname(input)
    f, fn = tempfile.mkstemp(dir=input)
    os.close(f)
    return fn

def _get_array_mmap(array):
    """
    If the array has an mmap.mmap at base of its base chain, return the mmap
    object; otherwise return None.
    """

    if isinstance(array, mmap.mmap):
        return array

    base = array
    while hasattr(base, 'base') and base.base is not None:
        if isinstance(base.base, mmap.mmap):
            return base.base
        base = base.base


import unittest
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

url = "http://[2605:fd00:4:1001:f816:3eff:feb2:3536]/"

class TestLogout(unittest.TestCase):

    def setUp(self):
        self.driver = webdriver.Firefox()

    def test_logout(self):
        driver = self.driver
        # login first
        driver.get(url)
        # enter Username credential
        elem1 = driver.find_element_by_xpath("/html/body/div/div/div/div/div[2]/div[2]/div/div/div[2]/div/input")
        assert elem1 != None
        elem1.send_keys("zuhao")
        elem1.send_keys(Keys.ENTER)
        # enter Password credential
        elem2 = driver.find_element_by_xpath("/html/body/div/div/div/div/div[2]/div[2]/div/div/div[3]/div/input")
        assert elem2 != None
        elem2.send_keys("5201314mwxely")
        elem2.send_keys(Keys.ENTER)
        # click Log in button
        elem3 = driver.find_element_by_xpath("/html/body/div/div/div/div/div[2]/div[2]/div/div/button/span[1]")
        assert elem3 != None
        elem3.click()
        # wait for redirection to the homepage
        driver.implicitly_wait(10)
        # test Logout button
        elem4 = driver.find_element_by_xpath("/html/body/div/div/div[1]/header/div/p[5]/a")
        assert elem4 != None
        elem4 = elem4.click()
        # wait for the expected condition
        try:
            WebDriverWait(driver, 10).until(
                EC.title_is(("React App"))
            )
        except Exception as e:
            print("Wrong page: {0}".format(e))

    def tearDown(self):
        self.driver.close()

if __name__ == "__main__":
    unittest.main()


from datetime import date
from django import forms
from django.forms.widgets import Select, RadioSelect
from django.conf import settings

from baseapp.modelspackage.animals import Animal
from baseapp.modelspackage.colours import Colour
from browse.modelspackage.sites import SiteManager, Site

from itertools import chain

# This tuple is defined elsewhere but I am not comfortable
# using that tuple unless they are all unified in one place
# otherwise we have too strong a coupling between the applications
GENDER_CHOICES = (('male', 'Male'), ('female', 'Female'))

EDUCATION_LEVEL_CHOICES = ((None, 'Unknown'),
							('primary to junior high', 'Primary to Junior High'),
							('school certificate', 'School Certificate'),
							('Bachelor Degree','Bachelor Degree' ),
							('higher school certificate', 'Higher School Certificate'),
							('Diploma', 'Diploma'),
							('Graduate Certificate', 'Graduate Certificate'),
							('Advanced Diploma', 'Advanced Diploma'),
							('Masters Degree', 'Masters Degree'),
							('Vocational Graduate Certificate','Vocational Graduate Certificate'),
							('Vocational Graduate Diploma', 'Vocational Graduate Diploma'),
							('Doctoral Degree', 'Doctoral Degree'),
							('Associate Degree', 'Associate Degree'),
							('TAFE Certificate', 'TAFE Certificate'),
							('Graduate Diploma', 'Graduate Diploma')
						)

def get_years():
  current_year = date.today().year
  return ((i, i) for i in range(current_year - 100, current_year - 18))

def get_sites():
	siteManager = SiteManager(client_json=settings.PPCLIENT)
	sites =  siteManager.all()
	choices = []
	for site in sites:
		choices.append((site.label,site.name))
	return choices

class LoginForm(forms.Form):

  animal      = forms.ModelChoiceField(queryset = Animal.objects.order_by('name'), initial = 1, widget = Select(attrs={'class':'form-control'}))
  colour      = forms.ModelChoiceField(queryset = Colour.objects.all(), initial = 1, widget = Select(attrs={'class':'form-control'}))
  birth_year  = forms.ChoiceField(choices = get_years(), widget = Select(attrs={'class':'form-control'}))
  gender = forms.ChoiceField(choices = GENDER_CHOICES, widget = RadioSelect, initial = 'male')

#class Password_Reset_Form(forms.Form):
class ColourAnimalHelperForm(forms.Form):
	pwd_site = forms.ChoiceField(choices = get_sites(), widget= Select(attrs={'class':'form-control'}))
	pwd_highest_qual = forms.ChoiceField(choices = sorted(EDUCATION_LEVEL_CHOICES), widget=Select(attrs={'class':'form-control'}), required=False)
	pwd_father_highest_qual = forms.ChoiceField(choices = sorted(EDUCATION_LEVEL_CHOICES), widget=Select(attrs={'class':'form-control'}), required=False)
	pwd_mother_highest_qual = forms.ChoiceField(choices = sorted(EDUCATION_LEVEL_CHOICES), widget=Select(attrs={'class':'form-control'}), required=False)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on June 18 2018

@author: kushal

Chatzigeorgiou Group
Sars International Centre for Marine Molecular Biology

GNU GENERAL PUBLIC LICENSE Version 3, 29 June 2007
"""

from PyQt5 import QtCore, QtGui, QtWidgets
from .main_widget import ProjectBrowserWidget
from ...pyqtgraphCore.console import ConsoleWidget
from .pytemplates.mainwindow_pytemplate import Ui_MainWindow
#from spyder.widgets.variableexplorer import objecteditor
import pandas as pd
import os
from ...common import configuration, get_project_manager
from functools import partial

class ProjectBrowserWindow(QtWidgets.QMainWindow):
    def __init__(self, parent=None):
        QtWidgets.QMainWindow.__init__(self, parent)

        self.ui = Ui_MainWindow()
        self.ui.setupUi(self)

        self.project_browser = ProjectBrowserWidget(self, get_project_manager().dataframe)
        self.current_tab = 0
        self.project_browser.tab_widget.currentChanged.connect(self.set_current_tab)
        self.setCentralWidget(self.project_browser)

        self.ui.actionto_pickle.triggered.connect(partial(self.save_path_dialog, file_ext='pikl', save_root=True))
        self.ui.actionto_csv.triggered.connect(partial(self.save_path_dialog, file_ext='csv', save_root=True))
        self.ui.actionto_excel.triggered.connect(partial(self.save_path_dialog, file_ext='xlsx', save_root=True))

        self.ui.actionto_pickle_tab.triggered.connect(partial(self.save_path_dialog, file_ext='pikl', save_root=False))
        self.ui.actionto_csv_tab.triggered.connect(partial(self.save_path_dialog, file_ext='csv', save_root=False))
        self.ui.actionto_excel_tab.triggered.connect(partial(self.save_path_dialog, file_ext='xlsx', save_root=False))

        self.ui.actionDataframe_editor.triggered.connect(self.edit_root_dataframe)

        self.ui.actionViewCurrent_dataframe.triggered.connect(self.view_current_dataframe)
        self.ui.actionCurrent_tab_filter_history.triggered.connect(self.view_filters)

        self.ui.actionUpdate_current_tab.triggered.connect(self.update_tab_from_child_dataframe)
        self.ui.actionUpdate_all_tabs.triggered.connect(self.update_tabs_from_child_dataframes)

        self.ui.actionSave_to_project.triggered.connect(self._save_dataframe)

        ns = {'pd': pd,
              'project_browser': self.project_browser,
              'this': self,
              'get_dataframe': lambda: self.get_current_dataframe()[0],
              'get_root_dataframe': lambda: self.get_current_dataframe(get_root=True)[0],
              'set_root_dataframe': lambda df: get_project_manager().set_dataframe(df)
              }

        txt = "Namespaces:          \n" \
              "pandas as pd         \n" \
              "Useful callables:\n" \
              "get_dataframe() - returns dataframe of the current tab\n" \
              "get_root_dataframe() - always return dataframe of root tab (i.e. entire dataframe)\n" \
              "set_root_dataframe() - save the root dataframe\n" \
              "self as this         \n" \

        cmd_history_file = os.path.join(configuration.console_history_path, 'project_browser.pik')

        self.ui.dockConsole.setWidget(ConsoleWidget(namespace=ns, text=txt, historyFile=cmd_history_file))
        # self.resizeDocks([self.ui.dockConsole], [235], QtCore.Qt.Vertical)
        self.ui.dockConsole.hide()

        self._status_bar = None
        self.status_bar = self.statusBar()

        self.setWindowTitle('Project Browser')

    @property
    def status_bar(self) -> QtWidgets.QStatusBar:
        return self._status_bar

    @status_bar.setter
    def status_bar(self, status_bar: QtWidgets.QStatusBar):
        self._status_bar = status_bar

    def set_current_tab(self, ix: int):
        self.current_tab = ix

    def save_path_dialog(self, file_ext: str, save_root=False):
        path = QtWidgets.QFileDialog.getSaveFileName(None, 'Save DataFrame as', '', f'(*.{file_ext})')
        if path == '':
            return
        if path[0].endswith('.' + file_ext):
            path = path[0]
        else:
            path = path[0] + '.' + file_ext

        if file_ext == 'pikl':
            self.get_current_dataframe(get_root=save_root)[0].to_pickle(path, protocol=4)
        elif file_ext == 'csv':
            self.get_current_dataframe(get_root=save_root)[0].to_csv(path)
        elif file_ext == '.xlsx':
            self.get_current_dataframe(get_root=save_root)[0].to_excel(path)

    def get_current_dataframe(self, get_root=False) -> (pd.DataFrame, list):
        if self.current_tab == 0 or get_root:
            dataframe = get_project_manager().get_dataframe()
            return dataframe, []
        else:
            tab_name = self.project_browser.tab_widget.widget(self.current_tab).tab_name
            child = get_project_manager().child_dataframes[tab_name]
        return child['dataframe'], child['filter_history']

    def _save_dataframe(self):
        get_project_manager().save_dataframe()

    def edit_root_dataframe(self):
        QtWidgets.QMessageBox.warning(self, 'Proceed with caution!',
                                      'You are about to edit the project DataFrame, be careful!\n'
                                      'If you make a mistake press "Close"')

        df = objecteditor.oedit(get_project_manager().get_dataframe())
        if df is not None:
            get_project_manager().set_dataframe(df)

    def view_current_dataframe(self):
        objecteditor.oedit(self.get_current_dataframe()[0])

    def view_filters(self):
        filters = self.get_current_dataframe()[1]

        QtWidgets.QMessageBox.information(self, 'Filters for current tab', '\n'.join(filters))

    def reload_all_tabs(self):
        self.status_bar.showMessage('Please wait, loading tabs...')
        for i in range(1, self.project_browser.tab_widget.count()):
            self.project_browser.tab_widget.removeTab(i)

        # for child in configuration.project_manager.child_dataframes.keys():
        #     configuration.project_manager.child_dataframes.pop(child)

        get_project_manager().create_sub_dataframe()

        for child in get_project_manager().child_dataframes.keys():
            dataframe = get_project_manager().child_dataframes[child]['dataframe']
            filter_history = get_project_manager().child_dataframes[child]['filter_history']
            self.project_browser.add_tab(dataframe, filter_history, name=child)
        self.status_bar.showMessage('Finished loading tabs!')

    def update_tab_from_child_dataframe(self, tab_name=None):
        if tab_name is None:
            tab_name = self.project_browser.tab_widget.widget(self.current_tab).tab_name

        child = get_project_manager().child_dataframes[tab_name]
        dataframe = child['dataframe']
        filter_history = child['filter_history']
        self.project_browser.tabs[tab_name].dataframe = dataframe
        self.project_browser.tabs[tab_name].filter_history = filter_history
        self.project_browser.tabs[tab_name].populate_tab()

    def update_tabs_from_child_dataframes(self):
        for tab_name in self.project_browser.tabs.keys():
            if tab_name == 'root':
                continue
            self.update_tab_from_child_dataframe(tab_name)


# coding=UTF-8

def get_corridor_lut():
        # Console version
    CORRIDOR_LUT = {'usd_ntd' : True,
                    'usd_thb' : True,
                    'usd_khr' : True,
                    'usd_cny' : True,
                    'usd_php' : True,
                    'usd_inr' : True,
                    'usd_idr' : True,
                    'usd_jpy' : True,
                    'usd_vnd' : True,
                    'sgd_php' : True}
    return CORRIDOR_LUT

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser(description='Process arguments.')
    parser.add_argument('-l', action='store_true', dest='list', help='List all corridors')
    parser.add_argument('-s', default='usd', dest='src', help='Source currency, default:usd')
    parser.add_argument('-d', default='ntd', dest='dst', help='Destination currency, default:ntd')
    parser.add_argument('-app', default=False, type=bool, dest='app', help='Run as web app')
    args = parser.parse_args()

    corridorName = args.src + '_' + args.dst
    if args.list:
        lut = get_corridor_lut()
        for corridor in lut.keys():
            print('Corridor: {}'.format(corridor.upper()))
    elif args.app:
        from app import start_app
        start_app()
    else:
        lut = get_corridor_lut()
        if lut.get(corridorName, False):
            corridor = __import__(corridorName)
            corridor.get_current_forex_price()


from dotenv import load_dotenv, find_dotenv
from langchain.chains import TransformChain, LLMChain, SimpleSequentialChain
from langchain.llms import OpenAI
from langchain.prompts import PromptTemplate

_ = load_dotenv(find_dotenv())  # read local .env file

with open("./interstellar.txt") as f:
    state_of_the_union = f.read()

def transform_func(inputs: dict) -> dict:
    text = inputs["text"]
    shortened_text = "\n\n".join(text.split("\n\n")[:3])
    return {"output_text": shortened_text}

# ---------------------------
# Transform Chain
# ---------------------------

transform_chain = TransformChain(
    input_variables=["text"], output_variables=["output_text"], transform=transform_func
)

# ---------------------------
# LLM Chain
# ---------------------------
template = """Summarize this text:

{output_text}

Summary:"""
prompt = PromptTemplate(input_variables=["output_text"], template=template)
llm_chain = LLMChain(llm=OpenAI(), prompt=prompt)

# ---------------------------
# Simple sequence
# ---------------------------
sequential_chain = SimpleSequentialChain(chains=[transform_chain, llm_chain])

result = sequential_chain.run(state_of_the_union)

print(result)


import subprocess
import pytest

from ..helpers import BaseWFC3

@pytest.mark.skip(reason="Duplicate test.")
class TestUVIS14Single(BaseWFC3):
    """
    Test pos UVIS2 GD-71
    """

    detector = 'uvis'

    def _single_raw_calib(self, rootname):
        raw_file = '{}_raw.fits'.format(rootname)

        # Prepare input file.
        self.get_input_file(raw_file)

        # Run CALWF3
        subprocess.call(['calwf3.e', raw_file, '-vt'])

        # Compare results
        outputs = [('{}_flt.fits'.format(rootname),
                    '{}_flt_ref.fits'.format(rootname))]
        self.compare_outputs(outputs)

    # Ported from ``calwf3_uv_14``.
    @pytest.mark.parametrize(
        'rootname', ['ibbq01moq'])
        # 'rootname', ['ibbq01moq',
        #             'ibbq01mpq',
        #             'ibbq01mqq',
        #             'ibbq01mrq',
        #             'ibbq01msq',
        #             'ibbq01mtq',
        #             'ibbq01muq',
        #             'ibbq01mvq',
        #             'ibbq01mwq',
        #             'ibbq01mxq',
        #             'ibbq01myq',
        #             'ibbq01n0q',
        #             'ibbq01n1q',
        #             'ibbq01n2q',
        #             'ibbq01n3q'])
    def test_uvis_14single(self, rootname):
        self._single_raw_calib(rootname)


from data.alerts import AlertCodes

class AlertPeripheralHandler(object):

    def __init__(self, events, drivers):
        self.events = events
        self.alert_driver = drivers.alert

    def subscribe(self):
        self.events.alerts_queue.observer.subscribe(self, self.on_new_alert)
        self.events.mute_alerts.observer.subscribe(self, self.on_mute)

    def on_new_alert(self, alert):
        if alert.is_medical_condition():
            self.alert_driver.set_medical_condition_alert(
                not alert.is_medical_condition(),
                self.events.mute_alerts._alerts_muted)
        elif alert.is_system_alert():
            self.alert_driver.set_system_fault_alert(
                not alert.is_system_alert(),
                self.events.mute_alerts._alerts_muted)
        else:
            self.alert_driver.set_medical_condition_alert(
                not alert.is_medical_condition(),
                self.events.mute_alerts._alerts_muted)
            self.alert_driver.set_system_fault_alert(
                not alert.is_system_alert(),
                self.events.mute_alerts._alerts_muted)

    def on_mute(self, mute):
        self.alert_driver.set_buzzer(
            mute or
            self.events.alerts_queue.last_alert == AlertCodes.OK)


from pytorch_lightning.callbacks import Callback
import torch.nn.functional as F
import torch
from models.simsiam.knn_monitor import knn_monitor
import wandb
from tqdm import tqdm
# from copy import copy
import copy
from typing import Optional
class KnnMonitorInsertWandb(Callback):

    def __init__(self,
                 train_data_loader,
                 test_data_loader,
                 ):
        super().__init__()
        self.memory_data_loader=copy.deepcopy(train_data_loader)
        self.test_data_loader=test_data_loader
        self.memory_data_loader.dataset.transform=self.test_data_loader.dataset.transform
        every_n_train_steps=2
        every_n_val_epochs=2
        self.__init_triggers(every_n_train_steps, every_n_val_epochs)

    def on_validation_epoch_end(self,trainer, pl_module):
        net=pl_module.model.backbone
        memory_data_loader=self.memory_data_loader
        test_data_loader=self.test_data_loader
        epoch=pl_module.current_epoch
        accuracy=knn_monitor(net, memory_data_loader, test_data_loader, epoch, k=200, t=0.1, hide_progress=False)
        pl_module.log("accuracy_knn_monitor",accuracy)
        pl_module.log("epoch",epoch)

        # wandb.log({"accuracy_knn_monitor":accuracy,
        #            "epoch":epoch},
        #           )

    def __init_triggers(
        self, every_n_train_steps: Optional[int], every_n_val_epochs: Optional[int],
    ) -> None:

        # Default to running once after each validation epoch if neither
        # every_n_train_steps nor every_n_val_epochs is set
        if every_n_train_steps is None and every_n_val_epochs is None:
            self._every_n_val_epochs = 1
            self._every_n_train_steps = 0
            log.debug("Both every_n_train_steps and every_n_val_epochs are not set. Setting every_n_val_epochs=1")
        else:
            self._every_n_val_epochs = every_n_val_epochs or 0
            self._every_n_train_steps = every_n_train_steps or 0

        # period takes precedence over every_n_val_epochs for backwards compatibility

        self._period = self._every_n_val_epochs


from itertools import combinations

N, M = map(int,input().split())
data = list(map(str,input().split()))

lst = list(combinations(data,N))
vowel = "aeiou"

st = []

for i in lst:
    vow_cnt = 0
    not_vow = 0

    for j in i:
        if j in vowel:
            vow_cnt += 1
        else:
            not_vow += 1

    if vow_cnt == 0 or not_vow<2:
        continue

    i = sorted(i)
    i = ''.join(i)
    st.append(i)

st = sorted(list(set(st)))

for i in range(len(st)):
    print(st[i])


import pickle
import numpy as np
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.cross_validation import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn import linear_model
from sklearn.svm import SVR
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.manifold import TSNE
import random
from sklearn.preprocessing import label_binarize
from sklearn import metrics
from sklearn import cross_validation
import matplotlib.pyplot as plt

# Preparing data
with open('X.npy', 'rb+') as fobj:
    X = pickle.load(fobj)
with open('y.npy', 'rb+') as fobj:
    y = pickle.load(fobj)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=42)

# Accuracy
cv_error = []
models = []
for i in ['RandomForest_small', 'RandomForest_big', 'Gradientboost_small', 'Gradientboost_big', 'LogisticRegression_l1', 'LogisticRegression_l2']:
    if i == 'RandomForest_small':
        model = RandomForestClassifier()
    elif i == 'RandomForest_big':
        model = RandomForestClassifier(n_estimators = 20)
    elif i == 'Gradientboost_small':
        model = GradientBoostingClassifier()
    elif i == 'Gradientboost_big':
        model = GradientBoostingClassifier(n_estimators=200)
    elif i == 'LogisticRegression_l1':
        model = LogisticRegression(penalty='l1')
    elif i == 'LogisticRegression_l2':
        model = LogisticRegression(penalty='l2')
    model.fit(X_train, y_train)
    models.append(model)
    error.append(1 - np.mean(model.predict(X_test) == y_test))

cv_error = []
for model in models:
    cv_error.append(1 - np.mean(cross_validation.cross_val_score(model, X, y, scoring="accuracy")))

#plot error
index = np.arange(6)
fig, ax = plt.subplots()
bar_width = 0.35
opacity = 0.4
error_config = {'ecolor': '0.3'}
rects1 = plt.bar(index, error, bar_width,
                 alpha=opacity,
                 color='b',
                 error_kw=error_config,
                 label='Validation _error')

rects2 = plt.bar(index + bar_width, cv_error, bar_width,
                 alpha=opacity,
                 color='r',
                 error_kw=error_config,
                 label='Cross_validation_error')
plt.xlabel('Model')
plt.ylabel('Error')
plt.title('Validation error and cv_error of different models')
plt.xticks(index + bar_width, ('RF_10', 'RF_20', 'GB_100', 'GB_200', 'LR_l1', 'LR_l2'))
plt.legend(loc=0)
plt.savefig('error.png')


from django.db import models

# Create your models here.
class Subscriber(models.Model):
    email = models.EmailField(unique=True)
    name = models.CharField(max_length=100)
    isActive = models.BooleanField(default=True)

class Campaign(models.Model):
    subject = models.CharField(max_length=255)
    preview_text = models.TextField()
    article_url = models.URLField()
    html_content = models.TextField()
    plain_text_content = models.TextField()
    published_date = models.DateTimeField(auto_now_add=True)


from collections import namedtuple
from typing import List, Tuple
import matplotlib.pyplot as plt
import numpy as np
import torch
from environments.distance_measures import EuclidianDistancer

class Environment:
    def __init__(self, depot_location, node_locations, demands,
                 capacities, depot_tws, node_tws, durations,
                 service_windows, time_factors,
                 n_vehicle: int = 10, max_tour_len: int = 20,
                 active_vehicles_num=6, env_type: str = 'synthetic', problem_type='cvrp'):
        """
        get data from dataloader
        """
        assert env_type in ['synthetic', 'real'], 'env type not supported'
        assert problem_type in ['cvrp', 'pdp'], 'problem type not supported'

        self.depot_location = depot_location  # batch_size x 2
        self.node_locations = node_locations  # batch_size x n_nodes x 2
        self.node_locations = np.concatenate(
            (depot_location, node_locations), axis=1)
        self.demands = demands  # batch_size x n_nodes
        self.capacities = capacities  # batch_size
        self.depot_tws = depot_tws  # batch_size x 1 x 2
        self.node_tws = node_tws  # batch_size x n_nodes x 2
        self.durations = durations  # batch_size x 1
        self.service_windows = service_windows  # batch_size x 1
        self.time_factors = time_factors  # batch_size x 1

        self.n_vehicles = n_vehicle  # K
        self.act_vehicles = active_vehicles_num  # number of active vehicles

        self.max_tour_len = max_tour_len
        self.env_type = env_type

        self.n_instances = self.node_locations.shape[0]
        self.n_nodes = self.node_locations.shape[1]

        self.depo_collocations = []
        self.depo_num = self.depot_location.shape[1]

        for i in range(self.depot_location.shape[0]):
            cur_batch = []
            indxs = list(range(self.depo_num, self.n_nodes))
            prev_bord = 0
            cur_bord = len(indxs) // self.depo_num + len(indxs) % self.depo_num
            for j in range(self.depo_num):
                cur_batch.append(indxs[prev_bord : cur_bord])
                prev_bord = cur_bord
                cur_bord += len(indxs) // self.depo_num
            self.depo_collocations.append(cur_batch)

        self.distancer = EuclidianDistancer()

        self.distance_matrix = self.distancer.batched_distance_matrix(
            self.node_locations)

        self.mask = np.zeros(
            shape=(self.n_instances, self.n_vehicles, self.n_nodes), dtype=bool)
        # self.mask[:, :, 0] = 1  # init depot
        self.tour_plan = np.zeros(shape=(self.n_instances,
                                         self.n_vehicles,
                                         self.max_tour_len), dtype=int)

        self.vehicle_indices = np.arange(0, self.n_vehicles)
        # set to zeros on init (all vehicle in one place)
        self.vehicle_return_to_dep_cost = np.zeros(shape=(self.n_instances,
                                                          self.n_vehicles,
                                                          1))
        # generate initial positions [n_instances x n_vehicle x 2
        # if vehicle positions are not the same it doesn't work
        self.vehicle_positions = np.repeat(self.depot_location[:, :1],
                                           self.n_vehicles,
                                           axis=1).reshape(self.n_instances,
                                                           self.n_vehicles, 2)
        # self.vehicle_positions = np.stack([np.stack([self.depot_location] * self.n_vehicles)
        #                                    for _ in range(self.n_instances)])
        self.vehicle_current_time = np.zeros(
            shape=(self.n_instances, self.n_vehicles, 1), dtype=float)

        self.vehicle_to_node_id = np.zeros(
            shape=(self.n_instances, self.n_vehicles, 1))

        self.vehicle_remain_capacities = np.repeat(self.capacities.reshape(-1, 1),
                                                   self.n_vehicles, axis=1)

        self.vehicle_depot_deadlines = np.tile(self.depot_tws, self.n_vehicles).reshape(self.n_instances,
                                                                                self.n_vehicles,
                                                                                2)

        self.vehicle_node_tws = np.repeat(
            self.node_tws[:, np.newaxis, ...], self.n_vehicles, axis=1)

        # self.vehicle_remain_time = np.repeat(self.depot_location, self.n_instances)

        self.n_step = 0
        self.done_games = np.zeros(shape=(self.n_instances,))
        self.done_nodes = np.zeros(
            shape=(self.n_instances, self.n_nodes), dtype=bool)
        self.done_nodes[:, 0] = True

        self.missed_nodes_multiplier = 1
        self.num_act_veh_multiplier = 1

    def reset(self):
        """

        :return: state (
            s0 <
                mask[active_vehicles x n_nodes], <- visited states = [True, False, ..., False]
                S_[K x L], <- tour plan for vehicles
                v_[K x 5], <- vehicle features
                idx_[active_vehicles], <- indices of current active vehicles
            >
        )
        """
        self.act_veh_idx = np.arange(self.act_vehicles)[None, :]
        self.act_veh_idx = np.repeat(self.act_veh_idx, self.n_instances, 0)

        self.acted_veh = np.zeros(
            (self.n_instances, self.n_vehicles)).astype(bool)
        self.acted_veh[:, :self.act_vehicles] = True

        self.mask = np.ones(shape=(self.n_instances, self.n_vehicles, self.n_nodes),
                             dtype=bool)

        self.mask[:, :, :self.depo_num] = 0
        # self.mask[:, :, 0] = 1  # init depot
        self.tour_plan = np.zeros(shape=(self.n_instances,
                                         self.n_vehicles,
                                         self.max_tour_len),
                                  dtype=int)
        self.n_step = np.zeros((self.n_instances, self.n_vehicles), dtype=int)
        self.done_nodes = np.zeros(
            shape=(self.n_instances, self.n_nodes), dtype=bool)
        self.done_nodes[:, 0] = True
        return [
            np.take_along_axis(self.mask, self.act_veh_idx[..., None], 1),
            self.tour_plan,
            self.get_vehicle_features(),
            self.act_veh_idx
        ]

    def step(self, actions: np.ndarray):
        """

        :param actions: matrix_[KxN], N=1 n_samples x K x 1
        :return: state(
            mask[active_vehicles x n_nodes],
            S_[KxL],
            v_[K x 5]
        ),
        reward (

        )
        done True | False

        1. for each vehicle in actions

        """
        for i, sample in enumerate(actions):
            # update return cost
            self._update_vehicle_return_cost_and_time(i, sample)
            self._update_vehicle_position(i, sample)
            self._update_plan(i, sample)
            self._update_mask(i, sample)
            self._update_active_vehicles(i)
            self._update_done_nodes(i, sample)

        # self.n_step += 1
        np.put_along_axis(arr=self.n_step, indices=self.act_veh_idx, values=np.take_along_axis(
            self.n_step, self.act_veh_idx, 1) + 1, axis=1)

        # check done game

        return [
            np.take_along_axis(self.mask, self.act_veh_idx[..., None], 1),
            self.tour_plan,
            self.get_vehicle_features(),
            self.act_veh_idx
        ]

    def _update_done_nodes(self, n_sample, actions):
        self.done_nodes[n_sample, actions] = True

    def _update_active_vehicles(self, n_sample):
        for i in range(self.act_vehicles):
            if (self.mask[n_sample, self.act_veh_idx[n_sample][i], 1:]).all():
                # skip case of fully ended route
                if self.acted_veh[n_sample].sum() == self.n_vehicles:
                    continue
                self.act_veh_idx[n_sample, i] = self.acted_veh[n_sample].sum()
                self.acted_veh[n_sample, self.acted_veh[n_sample].sum()] = True

    def _update_vehicle_to_node_id(self, n_sample, actions):
        self.vehicle_to_node_id[n_sample] = actions

    def _update_current_time(self, n_sample, actions):
        """
        (distance * time_factors) + durations
        Args:
            n_sample:
            actions:

        Returns:

        """

        self.vehicle_current_time[n_sample] += self.distance_matrix[n_sample][actions]

    def _update_vehicle_return_cost_and_time(self, n_sample, actions):
        """
        distance to depot
        Args:
            n_sample:
            actions:

        Returns:

        """

        # previous_p = np.take_along_axis(
        #     arr=np.take_along_axis(
        #         self.tour_plan,
        #         self.act_veh_idx[..., None],
        #         axis=1
        #     ),
        #     indices=np.take_along_axis(
        #         self.n_step,
        #         self.act_veh_idx,
        #         axis=1
        #     )[..., None],
        #     axis=2
        # )

        # next_p = actions

        # prev = np.take_along_axis(self.distance_matrix, previous_p, 1)
        # np.put_along_axis(self.vehicle_return_to_dep_cost, self.act_veh_idx[..., None], axis=1, values=prev[..., 0])

        for i, (previous_p, next_p) in enumerate(zip(self.tour_plan[n_sample][self.act_veh_idx[n_sample],
                                                                              self.n_step[n_sample][self.act_veh_idx[n_sample]]],
                                                     actions.ravel())):
            self.vehicle_return_to_dep_cost[n_sample, self.act_veh_idx[n_sample, i],
                                            0] = self.distance_matrix[n_sample, previous_p, 0]
            # TODO: always false??
            if next_p == 0:
                self.vehicle_current_time[n_sample, self.act_veh_idx[n_sample, i], 0] += self.distance_matrix[n_sample, previous_p, next_p] * \
                    self.time_factors[n_sample]
                self.vehicle_depot_deadlines[n_sample, self.act_veh_idx[n_sample, i], 1] -= self.distance_matrix[n_sample, previous_p, next_p] * \
                    self.time_factors[n_sample]
            else:
                self.vehicle_current_time[n_sample, self.act_veh_idx[n_sample, i], 0] += self.distance_matrix[n_sample, previous_p, next_p] * \
                    self.time_factors[n_sample] + \
                    self.durations[n_sample, next_p - self.depo_num]
                self.vehicle_depot_deadlines[n_sample, self.act_veh_idx[n_sample, i], 1] -= self.distance_matrix[n_sample, previous_p, next_p] * \
                    self.time_factors[n_sample] + \
                    self.durations[n_sample, next_p - self.depo_num]

    def _update_vehicle_position(self, n_sample, actions):
        self.vehicle_positions[n_sample, self.act_veh_idx[n_sample]] = self.node_locations[n_sample][actions.ravel(
        )]

    def _update_plan(self, n_sample, actions):
        self.tour_plan[n_sample, self.act_veh_idx[n_sample],
                       self.n_step[n_sample, self.act_veh_idx[n_sample]]] = actions.ravel()

    def _update_mask(self, n_sample, actions):
        """
        1. visited
        2. capacities
        3. time windows
        4. service time
        5. pdp constraint

        compare with service_windows and mask unreachable
        Returns:

        """
        # mark visited
        for i, action in enumerate(actions):
            if (self.n_step[n_sample, self.act_veh_idx[n_sample, i]] == 0) and (action < self.depo_num):
                # first step, we are going to the different depots
                self.mask[n_sample, self.act_veh_idx[n_sample, i], self.depo_collocations[n_sample][action]] = 0
            elif (action < self.depo_num):
                # if next depot => end of tour for this vehicle
                self.mask[n_sample, self.act_veh_idx[n_sample, i], :] = 1
            else:
                self.mask[n_sample, :, action] = 1

        # update capacities
        demand_indices = actions.ravel() - self.depo_num
        demands_ = self.demands[n_sample][demand_indices]
        demands_[demand_indices < 0] = 0

        mask_same_nodes = actions != actions[0]
        mask_same_nodes[0] = True

        self.vehicle_remain_capacities[n_sample, self.act_veh_idx[n_sample][mask_same_nodes]
                                       ] -= self.demands[n_sample][demand_indices[mask_same_nodes]].ravel()

        for i in range(len(self.vehicle_remain_capacities[n_sample][self.act_veh_idx[n_sample]])):
            capacities_mask = np.where(self.demands[n_sample] > self.vehicle_remain_capacities[n_sample][self.act_veh_idx[n_sample][i]],
                                       True,
                                       False)
            self.mask[n_sample, self.act_veh_idx[n_sample]
                      [i], self.depo_num:][capacities_mask] = 1

        # update node tws
        for i in range(self.act_vehicles):
            current_node = self.tour_plan[n_sample, self.act_veh_idx[n_sample]
                                          [i], self.n_step[n_sample][self.act_veh_idx[n_sample][i]]]
            available_nodes_indices = np.where(
                ~self.mask[n_sample, self.act_veh_idx[n_sample][i], self.depo_num:])
            left_bounds = self.distance_matrix[n_sample][current_node][available_nodes_indices]
            left_bounds = left_bounds * \
                self.time_factors[n_sample] + \
                self.vehicle_current_time[n_sample,
                                          self.act_veh_idx[n_sample][i]]
            left_bounds += self.durations[n_sample][available_nodes_indices]

            new_left_bounds = np.maximum(
                self.vehicle_node_tws[n_sample, self.act_veh_idx[n_sample][i]][available_nodes_indices][:, 0], left_bounds)
            self.vehicle_node_tws[n_sample,
                                  self.act_veh_idx[n_sample][i]][available_nodes_indices][:, 0] = new_left_bounds
            unreachable_by_tws = np.where(
                self.vehicle_node_tws[n_sample, self.act_veh_idx[n_sample][i], :, 0] > self.vehicle_node_tws[n_sample, self.act_veh_idx[n_sample][i], :, 1], True, False)
            self.mask[n_sample, self.act_veh_idx[n_sample][i]][np.concatenate(
                (np.array([False for _ in range(self.depo_num)]), unreachable_by_tws))] = 1

            # update by service time (right bound of depot time)
            if self.vehicle_depot_deadlines[n_sample, self.act_veh_idx[n_sample][i], 1] <= 0:
                self.mask[n_sample, self.act_veh_idx[n_sample][i]][self.depo_num:] = 1

            for ind, routes in enumerate(self.depo_collocations[n_sample]):
                cur_node = self.tour_plan[n_sample, self.act_veh_idx[n_sample, i], self.n_step[n_sample, self.act_veh_idx[n_sample, i]]]
                if (cur_node != ind) and (cur_node not in routes):
                    self.mask[n_sample, self.act_veh_idx[n_sample][i], routes] = 1
        return self.mask, self.tour_plan, self.get_vehicle_features(), 1.0

    def render(self, n_sample, path=None):
        plt.figure(figsize=(10, 10))
        plt.scatter(self.node_locations[n_sample]
                    [:, 0], self.node_locations[n_sample][:, 1])
        plt.scatter(self.depot_location[n_sample, :, 0],
                    self.depot_location[n_sample, :, 1])

        colors = ['b', 'g', 'r', 'c', 'm', 'y', 'k']

        for i in range(self.n_vehicles):
            for j in range(self.depo_num):
                steps = self.tour_plan[n_sample, i, :self.n_step[n_sample, i] + 1]
                if len(steps) == 0:
                    continue

                c = colors[i % len(colors)]
                x, y = self.depot_location[n_sample, j]
                dx, dy = self.node_locations[n_sample, steps[0]]
                plt.arrow(x, y, dx - x, dy - y, shape='full', lw=0.2,
                        length_includes_head=True, head_width=.01, color=c)
                for j in range(1, len(steps)):
                    x, y = self.node_locations[n_sample, steps[j - 1]]
                    dx, dy = self.node_locations[n_sample, steps[j]]
                    plt.arrow(x, y, dx - x, dy - y, shape='full', lw=0.2,
                            length_includes_head=True, head_width=.01, color=c)
        plt.xticks(fontsize=24)
        plt.yticks(fontsize=24)
        plt.show()
        if path:
            plt.savefig(path)

    def get_vehicle_features(self):
        """
        1. vehicle index
        2. current return cost to depot
        3-4. current position of vehicle
        5. current time
        """
        return np.concatenate(
            [
                np.repeat(np.arange(self.n_vehicles)[
                          None, :, None], self.n_instances, 0),
                self.vehicle_return_to_dep_cost,
                self.vehicle_positions,
                self.vehicle_current_time
            ],
            axis=2)

    def get_log_likelihood(self, _log_p, pi):
        """ _log_p: (batch, decode_step, num_vehicles, n_nodes)
                pi: (batch, decode_step, num_vehicles, 1), predicted tour
        """
        log_p = torch.gather(input=_log_p, dim=-1, index=pi)
        return torch.sum(log_p.squeeze(-1), 1).sum(dim=-1)

    def get_costs(self, pi, pi_act_ids):
        """ self.node_locations: (batch, n_nodes, 2), Coordinates of depot + customer nodes
                pi: (batch, decode_step, num_vehicles, 1), predicted tour
                d: (batch, decode_step, num_vehicles, 2)
                Note: first element of pi is not depot, the first selected node in the path
        """
        ret_cost = torch.tensor(
            [0 for _ in range(pi.shape[0])], dtype=torch.float, device=pi.device)
        for batch in range(pi.shape[0]):
            for ind, step in enumerate(self.n_step[batch]):
                ret_cost[batch] += self.distance_matrix[batch,
                                                        0, self.tour_plan[batch, ind, 0]]
                ret_cost[batch] += self.distance_matrix[batch][self.tour_plan[batch,
                                                                              ind, :step][:-1], self.tour_plan[batch, ind, :step][1:]].sum()
                ret_cost[batch] += self.distance_matrix[batch,
                                                        self.tour_plan[batch, ind, step], 0]

        # missed nodes
        ret_cost += torch.tensor((~self.done_nodes).sum(-1),
                                 device=pi.device) * self.missed_nodes_multiplier

        # num_act_vehicles
        ret_cost += torch.tensor(self.acted_veh.sum(-1),
                                 device=pi.device) * self.num_act_veh_multiplier

        return ret_cost

        d = torch.gather(input=torch.tensor(self.node_locations[:, :, None, :], device=pi.device).repeat(1, 1, self.n_vehicles, 1), dim=1,
                         index=pi.repeat(1, 1, 1, 2))
        # d = torch.gather(input = self.xy, dim = 1, index = pi[:,:,None].expand(self.batch,pi.size(1),2))
        return (torch.sum((d[:, 1:] - d[:, :-1]).norm(p=2, dim=-1), dim=1)
                # distance from depot to first selected node
                + (d[:, 0] - torch.tensor(self.depot_location[:, None, :],
                   device=pi.device)).norm(p=2, dim=-1)
                # distance from last selected node (!=0 for graph with longest path) to depot
                + (d[:, -1] - torch.tensor(self.depot_location[:,
                   None, :], device=pi.device)).norm(p=2, dim=-1)
                ).sum(dim=-1)


from collections import defaultdict
d= defaultdict(list)
n, m = list(map(int, (input().split())))
B=[]
for i in range(0, n):
    d[input()].append(i+1)
for i in range(0, m):
    B.append(input())
for i in B:
    print(*d[B[i]] or [-1])


from flask import Flask, request
from flask_cors import CORS, cross_origin
from werkzeug.utils import secure_filename
import os
import pandas as pd
import json
from validate import validator_compute
from derive_index import derive_index_videos_based, derive_index_channel_based
from filter_videos_dataset import filter_datasets
from extract_keywords import extract_keywords
from youtube_api import find_youtube_resources
from ml_strategies import extend_videos_with_ml
from flask_compress import Compress

app = Flask(__name__)
Compress(app)
cors = CORS(app)
app.config['CORS_HEADERS'] = 'Content-Type'

def contains_all_items(expected_items, items):
    return all(item in items for item in expected_items)

videos_dataset_cols = ['role_models_and_representations', 'channel_name', 'educational_subjects', 'educational_values', 'messages', 'age_range', 'video_keywords', 'channel_id']
def is_proper_videos_dataset(columns):
    return contains_all_items(videos_dataset_cols, columns)

channels_dataset_cols = ['channel_name', 'classification', 'keywords', 'channel_id']
def is_proper_channels_dataset(columns):
    return contains_all_items(channels_dataset_cols, columns)

dataset_validation = {
    'videos': is_proper_videos_dataset,
    'channels': is_proper_channels_dataset
}

@app.route('/upload-dataset', methods=['POST'])
@cross_origin()
def upload_dataset():
    # get file
    file = request.files['file']
    dataset_type = request.form.get('datasetType')
    file_name = secure_filename(file.filename)

    # store file
    dir_name = os.path.dirname(__file__)
    file_dir = os.path.join(dir_name, f'datasets/{dataset_type}')
    destination = "/".join([file_dir, file_name])
    file.save(destination)

    # validate dataset
    df = pd.read_csv(destination)
    cols = df.columns.values
    dataset_validation_fn = dataset_validation[dataset_type]
    is_valid = dataset_validation_fn(cols)
    if not is_valid:
        os.remove(destination)

    if dataset_type == "videos":
        indexed_videos_df = derive_index_videos_based(df)
        indexed_videos_df.to_csv(destination, index=False)

    if is_valid:
        return { 'successful': True }
    else:
        return { 'successful': False, 'msg': 'Invalid csv columns' }

@app.route('/datasets', methods=['GET'])
@cross_origin()
def get_datasets():
    dir_name = os.path.dirname(__file__)
    videos_dir = os.path.join(dir_name, 'datasets/videos')
    channels_dir = os.path.join(dir_name, 'datasets/channels')

    videos = [f for f in os.listdir(videos_dir)]
    channels = [f for f in os.listdir(channels_dir)]

    return { 'successful': True, 'data': {
        'videos': videos,
        'channels': channels
    }}

@app.route('/channels-from-dataset', methods=['GET'])
@cross_origin()
def get_channels_from_dataset():
    channels_dataset_name = request.args.get('dataset')

    dir_name = os.path.dirname(__file__)
    channels_dir = os.path.join(dir_name, 'datasets/channels')
    channels_df = pd.read_csv("/".join([channels_dir, channels_dataset_name]))

    channels = json.loads(channels_df[['channel_id', 'channel_name']].to_json(orient = "records"))

    return { 'successful': True, 'data': {
        'channels': channels
    }}

@app.route('/find-similar-resources', methods=['POST'])
@cross_origin()
def find_similar_resources():
    content = request.json
    if not validator_compute.validate(content):
        return {"successful": False,  "msg": validator_compute.errors}

    filter_by = content['filter']['filterBy']

    # load datasets
    serve_filtered_datasets = content['datasets']['retrieveFiltered']
    videos_dataset_name = content['datasets']['videos']
    channels_dataset_name = content['datasets']['channels']
    dir_name = os.path.dirname(__file__)
    videos_dir = os.path.join(dir_name, 'datasets/videos')
    channels_dir = os.path.join(dir_name, 'datasets/channels')
    videos_df = pd.read_csv("/".join([videos_dir, videos_dataset_name]))
    channels_df = pd.read_csv("/".join([channels_dir, channels_dataset_name]))

    # get channel ids
    channel_ids = channels_df['channel_id'].value_counts().index.values

    # derive index
    if filter_by == "DLPACS":
        (videos_df, channels_df) = derive_index_channel_based(videos_df, channels_df)

    # filter
    channel_ids = content['filter']['channels'] if "channels" in content['filter'] else None
    diversity = content['filter']['diversity'] if "diversity" in content['filter'] else None
    learning = content['filter']['learning'] if "learning" in content['filter'] else None
    messages = content['filter']['messages'] if "messages" in content['filter'] else None
    age = content['filter']['age'] if "age" in content['filter'] else None
    classification = content['filter']['classification'] if "classification" in content['filter'] else None
    status = content['filter']['status'] if "status" in content['filter'] else None
    topics = content['filter']['topics']
    (videos_df, channels_df) = filter_datasets(videos_df, channels_df, age, diversity, learning, messages, classification, status, topics, filter_by, channel_ids)

    # extract keywords
    source = content['keywords']['source']
    algorithm = content['keywords']['algorithm']
    keywords_num = content['keywords']['keywords']['num']
    keywords_from_num = content['keywords']['keywords']['from']
    keywords_manually_appended = content['keywords']['keywords']['manually_appended']
    extracted_keywords = extract_keywords(videos_df, channels_df, source, algorithm, keywords_num, keywords_from_num)
    keywords = keywords_manually_appended + extracted_keywords if keywords_manually_appended else extracted_keywords

    # send API request
    max_results = content['youtube']['maxResults']
    resource_type = content['youtube']['resource']
    order_by = content['youtube']['order']
    resources = find_youtube_resources(keywords, max_results, resource_type, order_by, topics, channel_ids)

    if resource_type == "video":
        resources = extend_videos_with_ml(resources)

    return {"successful": True, 'data': {
        'resourceType': resource_type,
        'resources': resources,
        'keywords': extracted_keywords,
        'filteredVideosDataset': json.loads(videos_df.to_json(orient = "records")) if serve_filtered_datasets == True else None,
        'filteredChannelsDataset': json.loads(channels_df.to_json(orient = "records")) if serve_filtered_datasets == True else None
    }}

if __name__ == '__main__':
    app.run(host='0.0.0.0', port='8001')


import pandas as pd
import numpy as np
from sklearn.cross_validation import  train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score, log_loss

from sklearn.datasets import make_classification
from sklearn.metrics import confusion_matrix, precision_recall_curve, auc, roc_auc_score, roc_curve, recall_score, classification_report

data = pd.read_csv('traces_50_HW.csv', delim_whitespace=True)
data1 = pd.read_csv('model.csv')

X = data.values
y = data1.values

print(data.head)

clf = RandomForestClassifier(max_depth=20, random_state=0)
X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.3, random_state=0)

X_train = np.nan_to_num(X_train)
y_train = y_train.astype(int)
X_test= np.nan_to_num(X_test)

clf.fit(X_train, y_train)
prediction = clf.predict(X_test)

cnf_matrix = confusion_matrix(y_test, prediction)

print(cnf_matrix)

print(classification_report(y_test, prediction))

# acc = accuracy_score(y_test, prediction)
# rec = recall_score(y_test, prediction)

good = 0
for i in range(len(y_test)):
    if y_test[i] == prediction[i]:
        good += 1

print("Accuracy: {:.4%}".format(good/len(y_test)))
# print("Recall: {:.4%}".format())
# print("Accuracy of undersampled testing dataset", accuracy_score(y_test_undersample, y_pred_undersample))


from collections import defaultdict
def minAreaRect(points):
    """
    :type points: List[List[int]]
    :rtype: int
    """
    columns = defaultdict(list)
    for x, y in points:
        columns[x].append(y)

    seen, result = {}, float('inf')

    for x2 in sorted(columns):
        column = columns[x2]
        column.sort()
        for j, y2 in enumerate(column):
            for i in range(j):
                y1 = column[i]
                if (y1, y2) in seen:
                    result = min(result, (x2 - seen[y1,y2]) * (y2 - y1))
                seen[y1, y2] = x2
    return result if result != float('inf') else 0

a = input()
print(minAreaRect(eval(a)))


from __future__ import print_function

import boto3
import base64
import json
from decimal import Decimal

region = "us-east-1"
dynamoDB = boto3.resource('dynamodb', region_name=region)
my_table = dynamoDB.Table("<username>-trades")

sns = boto3.resource('sns', region_name=region)
topic = sns.Topic("arn:aws:sns:us-east-1::<username>_check_trade")

def lambda_handler(event, context):
    records = event['Records']
    for r in records:
        if r is not None:
            data = base64.b64decode(r['kinesis']['data'])
            # print(data)
            data = eval(data)
            data_to_DB = {'symbol':data['symbol'],
                            'price': Decimal(str(data['price'])),
                            'size': data['size'],
                            'trade_time': data['trade_time'],
                            'epoch_time': data['epoch_time'],
                            'id': data['id'],
                            'shard_id':r['eventID']}
            my_table.put_item(Item=data_to_DB)

            data_to_TP = {'symbol':data['symbol'],
                            'price': data['price'],
                            'size': data['size'],
                            'trade_time': data['trade_time'],
                            'epoch_time': data['epoch_time'],
                            'id': data['id'],
                            'shard_id':r['eventID']}
            topic.publish(Message=json.dumps(data_to_TP))
        else:
            print("No data to be stored")

    return "Saved " + str(len(records)) + "trades to DB"


#############################################
############ Advent of Code 2022 ############
################# Day 4 #####################
#############################################
from typing import List

def readData(path: str) -> List:
    data = []  #
    currentPair = []
    with open(path, "r") as f:
        for l in f.readlines():
            assignment = l.split(",")  # split two range assignment
            assert len(assignment) == 2
            for a in assignment:
                s = a.split("-")
                currentPair.append((int(s[0]), int(s[1])))
            data.append(currentPair)
            currentPair = []

    return data

def checkTotalOverlap(data):
    overlapCounter = 0
    for first, second in data:  # one pair of elves with their range (first, second)
        firstRange = list(range(first[0], first[1] + 1))
        secondRange = list(range(second[0], second[1] + 1))
        if set(firstRange).issubset(secondRange):
            overlapCounter += 1
        elif set(secondRange).issubset(firstRange):
            overlapCounter += 1

    return overlapCounter

def checkOverlapExists(data):
    overlapCounter = 0
    for first, second in data:  # one pair of elves with their range (first, second)
        firstRange = list(range(first[0], first[1] + 1))
        secondRange = list(range(second[0], second[1] + 1))
        if len(set(firstRange).intersection(secondRange)) > 0:
            print(f"Ranges {first} and {second} overlap!")
            overlapCounter += 1

    return overlapCounter

if __name__ == "__main__":
    path = "data/day4.txt"
    data = readData(path)
    part1 = checkTotalOverlap(data)
    part2 = checkOverlapExists(data)
    print(f"Result from first part: {part1}")
    print(f"Result from second part: {part2}")


import pandas as pd
import keras
from keras.models import Sequential
from keras.layers import Dense, Dropout
from sklearn.preprocessing import LabelEncoder
from keras.utils import np_utils

base = pd.read_csv('/home/igorbaio/Documentos/DeepLearningA-Z/Base_de_Dados/iris.csv')
previsores = base.iloc[:, 0:4].values
classe = base.iloc[:, 4].values

labelEncoder = LabelEncoder()
classe = labelEncoder.fit_transform(classe)
classe_dummy = np_utils.to_categorical(classe)

classificador = Sequential()

classificador.add(Dense(units = 8, activation = 'relu', input_dim = 4 ))
classificador.add(Dropout(0.25))
classificador.add(Dense(units = 8, activation = 'relu'))
classificador.add(Dropout(0.2))
classificador.add(Dense(units = 3, activation = 'softmax'))

classificador.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy', metrics=['accuracy'])

classificador.fit(previsores, classe, batch_size = 10, epochs = 1000)

classificador_json = classificador.to_json()
with open("classificador_breast_iris.json", "w") as json_file:
    json_file.write(classificador_json)

classificador.save_weights('classificador_breast_iris.h5')