import sys, os, signal, io, six

from zope.interface import implementer

from twisted.python.log import msg
from twisted.python.filepath import FilePath
from twisted.python.procutils import which
from twisted.python.runtime import platform
from twisted.trial.unittest import SkipTest, TestCase
from twisted.plugin import IPlugin
from twisted.internet import reactor
from twisted.internet.task import deferLater
from twisted.internet.protocol import ProcessProtocol
from twisted.internet.defer import Deferred
from twisted.internet.error import ProcessTerminated
from twisted.application.service import IService, IServiceCollection

from axiom.store import Store
from axiom.item import Item
from axiom.attributes import boolean
from axiom.scripts import axiomatic
from axiom.listversions import SystemVersion
from axiom.iaxiom import IAxiomaticCommand
from axiom.test.util import callWithStdoutRedirect
from twisted.plugins.axiom_plugins import AxiomaticStart

from axiom.test.reactorimporthelper import SomeItem

@implementer(IService)
class RecorderService(Item):
    """
    Minimal L{IService} implementation which remembers if it was ever started.
    This is used by tests to make sure services get started when they should
    be.
    """
    started = boolean(
        doc="""
        A flag which is initially false and set to true once C{startService} is
        called.
        """, default=False)

    name = "recorder"

    def setServiceParent(self, parent):
        """
        Do the standard Axiom thing to make sure this service becomes a child
        of the top-level store service.
        """
        IServiceCollection(parent).addService(self)

    def startService(self):
        """
        Remember that this method was called.
        """
        self.started = True

    def stopService(self):
        """
        Ignore this event.
        """

class StartTests(TestCase):
    """
    Test the axiomatic start sub-command.
    """
    maxDiff = None
    def setUp(self):
        """
        Work around Twisted #3178 by tricking trial into thinking something
        asynchronous is happening.
        """
        return deferLater(reactor, 0, lambda: None)

    def _getRunDir(self, dbdir):
        return dbdir.child("run")

    def _getLogDir(self, dbdir):
        return self._getRunDir(dbdir).child("logs")

    def test_getArguments(self):
        """
        L{Start.getArguments} adds a I{--pidfile} argument if one is not
        present and a I{--logfile} argument if one is not present and
        daemonization is enabled and adds a I{--dbdir} argument pointing at the
        store it is passed. It also adds I{--journal-mode} if this is passed.
        """
        dbdir = FilePath(self.mktemp())
        store = Store(dbdir, journalMode='WAL')
        run = self._getRunDir(dbdir)
        logs = self._getLogDir(dbdir)
        start = axiomatic.Start()

        logfileArg = ["--logfile", logs.child("axiomatic.log").path]

        # twistd on Windows doesn't support PID files, so on Windows,
        # getArguments should *not* add --pidfile.
        if platform.isWindows():
            pidfileArg = []
        else:
            pidfileArg = ["--pidfile", run.child("axiomatic.pid").path]
        restArg = [
            "axiomatic-start", "--dbdir", dbdir.path, "--journal-mode", b"WAL"]

        self.assertEqual(
            start.getArguments(store, []),
            logfileArg + pidfileArg + restArg)
        self.assertEqual(
            start.getArguments(store, ["--logfile=foo"]),
            ["--logfile=foo"] + pidfileArg + restArg)
        self.assertEqual(
            start.getArguments(store, ["--logfile", "foo"]),
            ["--logfile", "foo"] + pidfileArg + restArg)
        self.assertEqual(
            start.getArguments(store, ["-l", "foo"]),
            ["-l", "foo"] + pidfileArg + restArg)
        self.assertEqual(
            start.getArguments(store, ["--nodaemon"]),
            ["--nodaemon"] + pidfileArg + restArg)
        self.assertEqual(
            start.getArguments(store, ["-n"]),
            ["-n"] + pidfileArg + restArg)
        self.assertEqual(
            start.getArguments(store, ["--pidfile=foo"]),
            ["--pidfile=foo"] + logfileArg + restArg)
        self.assertEqual(
            start.getArguments(store, ["--pidfile", "foo"]),
            ["--pidfile", "foo"] + logfileArg + restArg)

    def test_logDirectoryCreated(self):
        """
        If L{Start.getArguments} adds a I{--logfile} argument, it creates the
        necessary directory.
        """
        dbdir = FilePath(self.mktemp())
        store = Store(dbdir)
        start = axiomatic.Start()
        start.getArguments(store, ["-l", "foo"])
        self.assertFalse(self._getLogDir(dbdir).exists())
        start.getArguments(store, [])
        self.assertTrue(self._getLogDir(dbdir).exists())

    def test_parseOptions(self):
        """
        L{Start.parseOptions} adds axiomatic-suitable defaults for any
        unspecified parameters and then calls L{twistd.run} with the modified
        argument list.
        """
        argv = []
        def fakeRun():
            argv.extend(sys.argv)
        options = axiomatic.Options()
        options['dbdir'] = dbdir = self.mktemp()
        start = axiomatic.Start()
        start.parent = options
        start.run = fakeRun
        original = sys.argv[:]
        try:
            start.parseOptions(["-l", "foo", "--pidfile", "bar"])
        finally:
            sys.argv[:] = original
        self.assertEqual(
            argv,
            [sys.argv[0],
             "-l", "foo", "--pidfile", "bar",
             "axiomatic-start", "--dbdir", os.path.abspath(dbdir)])

    def test_parseOptionsHelp(self):
        """
        L{Start.parseOptions} writes usage information to stdout if C{"--help"}
        is in the argument list it is passed and L{twistd.run} is not called.
        """
        start = axiomatic.Start()
        start.run = None
        result, output = callWithStdoutRedirect(self.assertRaises, SystemExit, start.parseOptions, ["--help"])

        # Some random options that should be present.  This is a bad test
        # because we don't control what C{opt_help} actually does and we don't
        # even really care as long as it's the same as what I{twistd --help}
        # does.  We could try running them both and comparing, but then we'd
        # still want to do some sanity check against one of them in case we end
        # up getting the twistd version incorrectly somehow... -exarkun
        self.assertIn("--reactor", output.getvalue())
        if not platform.isWindows():
            # This isn't an option on Windows, so it shouldn't be there.
            self.assertIn("--uid", output.getvalue())

        # Also, we don't want to see twistd plugins here.
        self.assertNotIn("axiomatic-start", output.getvalue())

    def test_checkSystemVersion(self):
        """
        The L{IService} returned by L{AxiomaticStart.makeService} calls
        L{checkSystemVersion} with its store when it is started.

        This is done for I{axiomatic start} rather than somewhere in the
        implementation of L{Store} so that it happens only once per server
        startup.  The overhead of doing it whenever a store is opened is
        non-trivial.
        """
        dbdir = self.mktemp()
        store = Store(dbdir)
        service = AxiomaticStart.makeService(
            {'dbdir': dbdir, 'debug': False, 'journal-mode': None})
        self.assertEqual(store.query(SystemVersion).count(), 0)
        service.startService()
        self.assertEqual(store.query(SystemVersion).count(), 1)
        return service.stopService()

    def test_axiomOptions(self):
        """
        L{AxiomaticStart.options} takes database location, debug, and
        journal mode setting parameters.
        """
        options = AxiomaticStart.options()
        options.parseOptions([])
        self.assertEqual(options['dbdir'], None)
        self.assertFalse(options['debug'])
        self.assertEqual(options['journal-mode'], None)
        options.parseOptions(
            ["--dbdir", "foo", "--debug", "--journal-mode", "WAL"])
        self.assertEqual(options['dbdir'], 'foo')
        self.assertTrue(options['debug'])
        self.assertEqual(options['journal-mode'], 'WAL')

    def test_makeService(self):
        """
        L{AxiomaticStart.makeService} returns the L{IService} powerup of the
        L{Store} at the directory in the options object it is passed.
        """
        dbdir = FilePath(self.mktemp())
        store = Store(dbdir)
        recorder = RecorderService(store=store)
        self.assertFalse(recorder.started)
        store.powerUp(recorder, IService)
        store.close()

        service = AxiomaticStart.makeService(
            {'dbdir': dbdir, 'debug': False, 'journal-mode': None})
        service.startService()
        service.stopService()

        store = Store(dbdir)
        self.assertTrue(store.getItemByID(recorder.storeID).started)

    def _getAxiomaticScript(self):
        here = FilePath(__file__)
        # Try to find it relative to the source of this test.
        bin = here.parent().parent().parent().child("bin")
        axiomatic = bin.child("axiomatic")
        if axiomatic.exists():
            # Great, use that one.
            axiomatic = axiomatic.path
        else:
            # Try to find it on the path, instead.
            axiomatics = which("axiomatic")
            if axiomatics:
                # Great, it was on the path.
                axiomatic = axiomatics[0]
            else:
                # Nope, not there, give up.
                raise SkipTest(
                    "Could not find axiomatic script on path or at {}".format(
                        axiomatic.path))
        return axiomatic

    def test_reactorSelection(self):
        """
        L{AxiomaticStart} optionally takes the name of a reactor in the form
        --reactor [shortName] and installs it instead of the default reactor.
        """
        # Since this process is already hopelessly distant from the state in
        # which I{axiomatic start} operates, it would make no sense to try a
        # functional test of this behavior in this process.  Since the
        # behavior being tested involves lots of subtle interactions between
        # lots of different pieces of code (the reactor might get installed
        # at the end of a ten-deep chain of imports going through as many
        # different projects), it also makes no sense to try to make this a
        # unit test.  So, start a child process and try to use the alternate
        # reactor functionality there.

        axiomatic = self._getAxiomaticScript()

        # Create a store for the child process to use and put an item in it.
        # This will force an import of the module that defines that item's
        # class when the child process starts.  The module imports the default
        # reactor at the top-level, making this the worst-case for the reactor
        # selection code.
        storePath = self.mktemp()
        store = Store(storePath)
        SomeItem(store=store)
        store.close()

        # Install select reactor because it is available on all platforms, and
        # it is still an error to try to install the select reactor even if
        # the already installed reactor was the select reactor.
        argv = [
            sys.executable,
            axiomatic, "-d", storePath,
            "start", "--reactor", "select", "-n"]
        expected = [
            "reactor class: twisted.internet.selectreactor.SelectReactor.",
            "reactor class: <class 'twisted.internet.selectreactor.SelectReactor'>"]
        proto, complete = AxiomaticStartProcessProtocol.protocolAndDeferred(expected)

        environ = os.environ.copy()
        reactor.spawnProcess(proto, sys.executable, argv, env=environ)
        return complete

    def test_reactorSelectionLongOptionStyle(self):
        """
        L{AxiomaticStart} optionally takes the name of a reactor in the form
        --reactor=[shortName] and installs it instead of the default reactor.
        """

        axiomatic = self._getAxiomaticScript()

        # Create a store for the child process to use and put an item in it.
        # This will force an import of the module that defines that item's
        # class when the child process starts.  The module imports the default
        # reactor at the top-level, making this the worst-case for the reactor
        # selection code.
        storePath = self.mktemp()
        store = Store(storePath)
        SomeItem(store=store)
        store.close()

        # Install select reactor because it is available on all platforms, and
        # it is still an error to try to install the select reactor even if
        # the already installed reactor was the select reactor.
        argv = [
            sys.executable,
            axiomatic, "-d", storePath,
            "start", "--reactor=select", "-n"]
        expected = [
            "reactor class: twisted.internet.selectreactor.SelectReactor.",
            "reactor class: <class 'twisted.internet.selectreactor.SelectReactor'>"]
        proto, complete = AxiomaticStartProcessProtocol.protocolAndDeferred(expected)

        environ = os.environ.copy()
        reactor.spawnProcess(proto, sys.executable, argv, env=environ)
        return complete

    def test_reactorSelectionShortOptionStyle(self):
        """
        L{AxiomaticStart} optionally takes the name of a reactor in the form
        -r [shortName] and installs it instead of the default reactor.
        """

        axiomatic = self._getAxiomaticScript()

        # Create a store for the child process to use and put an item in it.
        # This will force an import of the module that defines that item's
        # class when the child process starts.  The module imports the default
        # reactor at the top-level, making this the worst-case for the reactor
        # selection code.
        storePath = self.mktemp()
        store = Store(storePath)
        SomeItem(store=store)
        store.close()

        # Install select reactor because it is available on all platforms, and
        # it is still an error to try to install the select reactor even if
        # the already installed reactor was the select reactor.
        argv = [
            sys.executable,
            axiomatic, "-d", storePath,
            "start", "-r", "select", "-n"]
        expected = [
            "reactor class: twisted.internet.selectreactor.SelectReactor.",
            "reactor class: <class 'twisted.internet.selectreactor.SelectReactor'>"]
        proto, complete = AxiomaticStartProcessProtocol.protocolAndDeferred(expected)

        environ = os.environ.copy()
        reactor.spawnProcess(proto, sys.executable, argv, env=environ)
        return complete

class AxiomaticStartProcessProtocol(ProcessProtocol):
    """
    L{AxiomaticStartProcessProtocol} watches an I{axiomatic start} process
    and fires a L{Deferred} when it sees either successful reactor
    installation or process termination.

    @ivar _success: A flag which is C{False} until the expected text is found
        in the child's stdout and C{True} thereafter.

    @ivar _output: A C{str} giving all of the stdout from the child received
        thus far.
    """
    _success = False
    _output = ""

    def protocolAndDeferred(cls, expected):
        """
        Create and return an L{AxiomaticStartProcessProtocol} and a
        L{Deferred}.  The L{Deferred} will fire when the protocol receives
        the given string on standard out or when the process ends, whichever
        comes first.
        """
        proto = cls()
        proto._complete = Deferred()
        proto._expected = expected
        return proto, proto._complete
    protocolAndDeferred = classmethod(protocolAndDeferred)

    def errReceived(self, bytes):
        """
        Report the given unexpected stderr data.
        """
        msg("Received stderr from axiomatic: {!r}".format(bytes))

    def outReceived(self, bytes):
        """
        Add the given bytes to the output buffer and check to see if the
        reactor has been installed successfully, firing the completion
        L{Deferred} if so.
        """
        msg("Received stdout from axiomatic: {!r}".format(bytes))
        self._output += six.ensure_str(bytes)
        if not self._success:
            for line in self._output.splitlines():
                for expectedLine in self._expected:
                    if expectedLine in line:
                        msg("Received expected output")
                        self._success = True
                        self.transport.signalProcess("TERM")

    def processEnded(self, reason):
        """
        Check that the process exited in the way expected and that the required
        text has been found in its output and fire the result L{Deferred} with
        either a value or a failure.
        """
        self._complete, result = None, self._complete
        if self._success:
            if platform.isWindows() or (
                # Windows can't tell that we SIGTERM'd it, so sorry.
                reason.check(ProcessTerminated) and
                reason.value.signal == signal.SIGTERM):
                result.callback(None)
                return
        # Something went wrong.
        result.errback(reason)

class TestMisc(TestCase):
    """
    Test things not directly involving running axiomatic commands.
    """
    def test_axiomaticCommandProvides(self):
        """
        Test that AxiomaticCommand itself does not provide IAxiomaticCommand or
        IPlugin, but subclasses do.
        """
        self.assertFalse(IAxiomaticCommand.providedBy(axiomatic.AxiomaticCommand), 'IAxiomaticCommand provided')
        self.assertFalse(IPlugin.providedBy(axiomatic.AxiomaticCommand), 'IPlugin provided')

        class _TestSubClass(axiomatic.AxiomaticCommand):
            pass

        self.assertTrue(IAxiomaticCommand.providedBy(_TestSubClass), 'IAxiomaticCommand not provided')
        self.assertTrue(IPlugin.providedBy(_TestSubClass), 'IPlugin not provided')

class AxiomaticTests(TestCase):
    """
    Test things relating to the I{axiomatic} command itself.
    """
    def test_journalMode(self):
        """
        I{axiomatic} sets the journal mode of the store according to
        I{--journal-mode}.
        """
        options = axiomatic.Options()
        options['dbdir'] = self.mktemp()
        options['journal-mode'] = 'WAL'
        self.assertEqual(options.getStore().journalMode, u'WAL')


from setuptools import setup, find_packages
import pkg_resources
import pathlib

with pathlib.Path('requirements.txt').open() as requirements_txt:
    requirements = [str(requirement) for requirement \
                    in pkg_resources.parse_requirements(requirements_txt)]

setup(
    name='pyhac',
    description='A human action controller running on different platforms',
    license='Apache License 2.0',
    long_description=open('README.md').read(),
    long_description_content_type='text/markdown',
    install_requires=requirements,
    include_package_data=True,
    author='JAQQ',
    author_email='dabit-lucas@gmail.com, chen.jiunhan@gmail.com',
    packages=find_packages(),
    keywords='human action controller',
    url='https://github.com/dabit-lucas/hac'
)


# import libraries
import sys
import pandas as pd
from sqlalchemy import create_engine

def load_data(messages_filepath, categories_filepath):
    ''' load data

    Parameters
    ----------------------
    messages_filepath,categories_filepath: str, pathlib.Path, py._path.local.LocalPath or any \
    object with a read() method (such as a file handle or StringIO)
        The string could be a URL. Valid URL schemes include http, ftp, s3, and
        file. For file URLs, a host is expected. For instance, a local file could
        be file://localhost/path/to/table.csv

    Returns
    -----------------
    result : pd.DataFrame

    '''
    # load messages dataset
    messages = pd.read_csv(messages_filepath,index_col='id')

    # load categories dataset
    categories = pd.read_csv(categories_filepath,index_col='id')

    # merge datasets
    df = pd.merge(messages,categories,left_index=True,right_index=True)

    return df

def clean_data(df):
    '''clean data

    Parameters
    -------------------
    df:pd.DataFrame
        data that needs to be cleaned

    Returns
    -------------------
    result : pd.DataFrame
    '''

    # create a dataframe of the 36 individual category columns
    categories = df['categories'].str.split(pat=';',expand=True)

    # select the first row of the categories dataframe
    row = categories.iloc[0,:]

    # use this row to extract a list of new column names for categories.
    # one way is to apply a lambda function that takes everything
    # up to the second to last character of each string with slicing
    category_colnames = [i.split('-')[0] for i in row]

    # rename the columns of `categories`
    categories.columns = category_colnames

    for column in categories:
        # set each value to be the last character of the string
        categories[column] = categories[column].str[-1]

        # convert column from string to numeric
        categories[column] = categories[column].astype('int')

    # drop the original categories column from `df`
    df.drop('categories', axis=1, inplace=True)

    # concatenate the original dataframe with the new `categories` dataframe
    df = pd.merge(df,categories,left_index=True,right_index=True)

    # drop duplicates
    df = df.drop_duplicates()

    return df

def save_data(df, database_filename):
    '''save data

    Parameters
    --------------------
    df:pd.DataFrame
        data that needs to be saved
    database_filename: str
        the url of database

    Returns
    -------------------
    None
    '''

    engine = create_engine(database_filename)
    df.to_sql('DisasterResponse', engine, index=False, if_exists='replace')

def main():
    if len(sys.argv) == 4:

        messages_filepath, categories_filepath, database_filepath = sys.argv[1:]

        print('Loading data...\n    MESSAGES: {}\n    CATEGORIES: {}'
              .format(messages_filepath, categories_filepath))
        df = load_data(messages_filepath, categories_filepath)

        print('Cleaning data...')
        df = clean_data(df)

        print('Saving data...\n    DATABASE: {}'.format('sqlite:///'+database_filename))
        save_data(df, database_filepath)

        print('Cleaned data saved to database!')

    else:
        print('Please provide the filepaths of the messages and categories '\
              'datasets as the first and second argument respectively, as '\
              'well as the filepath of the database to save the cleaned data '\
              'to as the third argument. \n\nExample: python process_data.py '\
              'disaster_messages.csv disaster_categories.csv '\
              'DisasterResponse.db')

if __name__ == '__main__':
    main()


import emoji
from nonebot.adapters.onebot.v11 import GroupMessageEvent, MessageSegment, Message

# CONSTANT
PLAIN_TEXT = ['face', 'reply', 'at', 'text']

def _extract(string: str, emoji_list: list[dict[str,str]],
    string_only : list[str], emoji_only : list[str]) -> tuple[list[str],list[str]]:
    if len(emoji_list):
        parts = string.partition(emoji_list[0]['emoji'])
        string_only.append(parts[0])
        emoji_only.append(parts[1])
        _extract(parts[2], emoji_list[1:], string_only, emoji_only)
    else:
        string_only.append(string)
    return string_only, emoji_only

def split_emoji(string : str) -> tuple[list[str], list[str]]:
    string_only, emoji_only = [], []
    emoji_list = emoji.emoji_list(string)
    return _extract(string, emoji_list, string_only, emoji_only)

def merge_emoji(text_only : list[str], emoji_only : list[str]) -> str:
    i = 0
    string = ""
    while i < len(text_only) and i < len(emoji_only):
        string += text_only[i] + emoji_only[i]
        i += 1
    while i < len(text_only):
        string += text_only[i]
        i += 1
    while i < len(emoji_only):
        string += emoji_only[i]
        i += 1
    return string

class MessageFragments():

    def __init__(self, message : Message) -> None:
        # list[[list[text], list[emoji]]]
        self.fragments =[None] * len(message)
        self.plain_text = []
        self.message = Message()
        if message:
            self.message = message
            i = 0
            while i < len(message):
                if message[i].type == 'text':
                    text_list, emoji_list = split_emoji(
                        message[i].data['text'].replace("http://", "").replace("https://", ""))
                    self.fragments[i] = [text_list, emoji_list]
                    for text in self.fragments[i][0]:
                        self.plain_text.append(text)
                elif message[i].type not in PLAIN_TEXT:
                    del message[i]
                    i -= 1
                i += 1

    def copy(self):
        return MessageFragments([])._copy_from(self.message, self.plain_text, self.fragments)

    def update_plain_text(self, text_list : list[str]) -> None:
        count = 0
        self.plain_text = text_list
        for i in range(len(self.fragments)):
            if isinstance(self.fragments[i], list):
                for j in range(len(self.fragments[i][0])):
                    self.fragments[i][0][j] = text_list[count]
                    count += 1
                self.message[i] = MessageSegment.text(
                    merge_emoji(
                        self.fragments[i][0],
                        self.fragments[i][1]
                    )
                )

    def get_message(self) -> Message:
        return self.message

    def get_plain_text(self) -> list[str]:
        return self.plain_text

    def _copy_from(self, message: Message, plain_text : list[str],
        fragments:list):
        self.fragments = fragments.copy()
        self.message = message.copy()
        self.plain_text = plain_text.copy()
        return self


"""
This file contains primitives for multi-gpu communication.
This is useful when doing distributed training.
"""
import math
import pickle
import torch
import torch.utils.data as data
import torch.distributed as dist

from torch.utils.data.sampler import Sampler, BatchSampler

__all__ = ['get_world_size', 'get_rank', 'synchronize', 'is_main_process',
           'all_gather', 'make_data_sampler', 'make_batch_data_sampler',
           'reduce_dict', 'reduce_loss_dict']

# reference: https://github.com/facebookresearch/maskrcnn-benchmark/blob/master/maskrcnn_benchmark/utils/comm.py
def get_world_size():
    if not dist.is_available():
        return 1
    if not dist.is_initialized():
        return 1
    return dist.get_world_size()

def get_rank():
    if not dist.is_available():
        return 0
    if not dist.is_initialized():
        return 0
    return dist.get_rank()

def is_main_process():
    return get_rank() == 0

def synchronize():
    """
    Helper function to synchronize (barrier) among all processes when
    using distributed training
    """
    if not dist.is_available():
        return
    if not dist.is_initialized():
        return
    world_size = dist.get_world_size()
    if world_size == 1:
        return
    dist.barrier()

def all_gather(data):
    """
    Run all_gather on arbitrary picklable data (not necessarily tensors)
    Args:
        data: any picklable object
    Returns:
        list[data]: list of data gathered from each rank
    """
    world_size = get_world_size()
    if world_size == 1:
        return [data]

    # serialized to a Tensor
    buffer = pickle.dumps(data)
    storage = torch.ByteStorage.from_buffer(buffer)
    tensor = torch.ByteTensor(storage).to("cuda")

    # obtain Tensor size of each rank
    local_size = torch.IntTensor([tensor.numel()]).to("cuda")
    size_list = [torch.IntTensor([0]).to("cuda") for _ in range(world_size)]
    dist.all_gather(size_list, local_size)
    size_list = [int(size.item()) for size in size_list]
    max_size = max(size_list)

    # receiving Tensor from all ranks
    # we pad the tensor because torch all_gather does not support
    # gathering tensors of different shapes
    tensor_list = []
    for _ in size_list:
        tensor_list.append(torch.ByteTensor(size=(max_size,)).to("cuda"))
    if local_size != max_size:
        padding = torch.ByteTensor(size=(max_size - local_size,)).to("cuda")
        tensor = torch.cat((tensor, padding), dim=0)
    dist.all_gather(tensor_list, tensor)

    data_list = []
    for size, tensor in zip(size_list, tensor_list):
        buffer = tensor.cpu().numpy().tobytes()[:size]
        data_list.append(pickle.loads(buffer))

    return data_list

def reduce_dict(input_dict, average=True):
    """
    Args:
        input_dict (dict): all the values will be reduced
        average (bool): whether to do average or sum
    Reduce the values in the dictionary from all processes so that process with rank
    0 has the averaged results. Returns a dict with the same fields as
    input_dict, after reduction.
    """
    world_size = get_world_size()
    if world_size < 2:
        return input_dict
    with torch.no_grad():
        names = []
        values = []
        # sort the keys so that they are consistent across processes
        for k in sorted(input_dict.keys()):
            names.append(k)
            values.append(input_dict[k])
        values = torch.stack(values, dim=0)
        dist.reduce(values, dst=0)
        if dist.get_rank() == 0 and average:
            # only main process gets accumulated, so only divide by
            # world_size in this case
            values /= world_size
        reduced_dict = {k: v for k, v in zip(names, values)}
    return reduced_dict

def reduce_loss_dict(loss_dict):
    """
    Reduce the loss dictionary from all processes so that process with rank
    0 has the averaged results. Returns a dict with the same fields as
    loss_dict, after reduction.
    """
    world_size = get_world_size()
    if world_size < 2:
        return loss_dict
    with torch.no_grad():
        loss_names = []
        all_losses = []
        for k in sorted(loss_dict.keys()):
            loss_names.append(k)
            all_losses.append(loss_dict[k])
        all_losses = torch.stack(all_losses, dim=0)
        dist.reduce(all_losses, dst=0)
        if dist.get_rank() == 0:
            # only main process gets accumulated, so only divide by
            # world_size in this case
            all_losses /= world_size
        reduced_losses = {k: v for k, v in zip(loss_names, all_losses)}
    return reduced_losses

def make_data_sampler(dataset, shuffle, distributed):
    if distributed:
        return DistributedSampler(dataset, shuffle=shuffle)
    if shuffle:
        sampler = data.sampler.RandomSampler(dataset)
    else:
        sampler = data.sampler.SequentialSampler(dataset)
    return sampler

def make_batch_data_sampler(sampler, images_per_batch, num_iters=None, start_iter=0):
    batch_sampler = data.sampler.BatchSampler(sampler, images_per_batch, drop_last=True)
    if num_iters is not None:
        batch_sampler = IterationBasedBatchSampler(batch_sampler, num_iters, start_iter)
    return batch_sampler

# Code is copy-pasted from https://github.com/facebookresearch/maskrcnn-benchmark/blob/master/maskrcnn_benchmark/data/samplers/distributed.py
class DistributedSampler(Sampler):
    """Sampler that restricts data loading to a subset of the dataset.
    It is especially useful in conjunction with
    :class:`torch.nn.parallel.DistributedDataParallel`. In such case, each
    process can pass a DistributedSampler instance as a DataLoader sampler,
    and load a subset of the original dataset that is exclusive to it.
    .. note::
        Dataset is assumed to be of constant size.
    Arguments:
        dataset: Dataset used for sampling.
        num_replicas (optional): Number of processes participating in
            distributed training.
        rank (optional): Rank of the current process within num_replicas.
    """

    def __init__(self, dataset, num_replicas=None, rank=None, shuffle=True):
        if num_replicas is None:
            if not dist.is_available():
                raise RuntimeError("Requires distributed package to be available")
            num_replicas = dist.get_world_size()
        if rank is None:
            if not dist.is_available():
                raise RuntimeError("Requires distributed package to be available")
            rank = dist.get_rank()
        self.dataset = dataset
        self.num_replicas = num_replicas
        self.rank = rank
        self.epoch = 0
        self.num_samples = int(math.ceil(len(self.dataset) * 1.0 / self.num_replicas))
        self.total_size = self.num_samples * self.num_replicas
        self.shuffle = shuffle

    def __iter__(self):
        if self.shuffle:
            # deterministically shuffle based on epoch
            g = torch.Generator()
            g.manual_seed(self.epoch)
            indices = torch.randperm(len(self.dataset), generator=g).tolist()
        else:
            indices = torch.arange(len(self.dataset)).tolist()

        # add extra samples to make it evenly divisible
        indices += indices[: (self.total_size - len(indices))]
        assert len(indices) == self.total_size

        # subsample
        offset = self.num_samples * self.rank
        indices = indices[offset: offset + self.num_samples]
        assert len(indices) == self.num_samples

        return iter(indices)

    def __len__(self):
        return self.num_samples

    def set_epoch(self, epoch):
        self.epoch = epoch

class IterationBasedBatchSampler(BatchSampler):
    """
    Wraps a BatchSampler, resampling from it until
    a specified number of iterations have been sampled
    """

    def __init__(self, batch_sampler, num_iterations, start_iter=0):
        self.batch_sampler = batch_sampler
        self.num_iterations = num_iterations
        self.start_iter = start_iter

    def __iter__(self):
        iteration = self.start_iter
        while iteration <= self.num_iterations:
            # if the underlying sampler has a set_epoch method, like
            # DistributedSampler, used for making each process see
            # a different split of the dataset, then set it
            if hasattr(self.batch_sampler.sampler, "set_epoch"):
                self.batch_sampler.sampler.set_epoch(iteration)
            for batch in self.batch_sampler:
                iteration += 1
                if iteration > self.num_iterations:
                    break
                yield batch

    def __len__(self):
        return self.num_iterations

if __name__ == '__main__':
    pass


# Proje
#Tanımlamalar
"""
Oyun için gereken modülleri ekledik.
pygame oyun modülü,
sys, çıkış yapmak için kullanılıyor.
"""
import pygame
import sys
pygame.init()

#Değişkenler tanımlanıyor,

tumcizgiler=[]        #üretilen her bir ışını burada saklıyoruz ki sonra temizlemek mümkün olsun
elipsvarmi=False      #ekranda ELIPS/filtre olup olmadığını bilmek için
buttonstart_visible = True   #ilk açılışta start butonu görünsün, ve sonra gizlensin,
buttonreset_visible = False  #ilk açılışta RESET butonu gizlensin, start baslınca görünsün

#renk tanimlari yapılıyor
#renkler RGB modeline uygun olarak bir TUPLE şeklinde tanımlanır
gray=(176,176,176)
yesil=(0,255,0)
kirmizi=(255,0,0)
mavi=(0,0,255)

#ışınlara basılması durumunda filtrenin alacağı rengi tutacak olan değişkenleri belirledik.
yesildeger=0
kirmizideger=0
mavideger=0

#Ekran ayarları yapılıyor, ekran genişliği 1000, yükseklik 700, arka plan GRİ, ve başlığı IŞIK ve RENKLER
screen = pygame.display.set_mode((1000,700), pygame.SWSURFACE) #pygame penceresine başlık attık
screen.fill(gray)
pygame.display.set_caption("Işık Ve Renkler")

#projede kullanılacak görseller tanımlanıyor,
proje = pygame.image.load('projeiçinfoto.png')      #Kadın görseli
yesilfener = pygame.image.load('yesil_fener.png')   #Yeşil Fener görseli
mavifener = pygame.image.load('mavi_fener.png')     #Mavi Fener Görseli
kirmizifener = pygame.image.load('kirmizi_fener.png') #Kırmızı Fener Görseli

#KIRMIZI ışın yayan bir buton geliştirdik,
#Bu butonunun görseli, yukarıdaki görsellerden kirmizifener dir,
button_kirmizi_fener = kirmizifener
button_kirmizi_fener = pygame.transform.scale(kirmizifener, (100, 100))
button_rect_kirmizi_fener = button_kirmizi_fener.get_rect()
button_rect_kirmizi_fener.center = (900,100)

#YEŞİL ışın yayan bir buton geliştiriyoruz,
#Bu butonunun görseli, yukarıdaki görsellerden yesilfener dir,
button_yesil_fener = yesilfener
button_yesil_fener = pygame.transform.scale(yesilfener, (100, 100))
button_rect_yesil_fener = button_yesil_fener.get_rect()
button_rect_yesil_fener.center = (900,300)

#MAVİ ışın yayan bir buton geliştiriyoruz,
#Bu butonunun görseli, yukarıdaki görsellerden mavifener dir,
button_mavi_fener = mavifener
button_mavi_fener = pygame.transform.scale(mavifener, (100, 100))
button_rect_mavi_fener = button_mavi_fener.get_rect()
button_rect_mavi_fener.center = (900,500)

#Tüm fenerlerin ve kadin görselinin ekrana yerleşmesini için
screen.blit(button_kirmizi_fener, button_rect_kirmizi_fener)
screen.blit(button_yesil_fener, button_rect_yesil_fener)
screen.blit(button_mavi_fener, button_rect_mavi_fener)
screen.blit(proje,(50,90))

# Görüntüyü güncelliyoruz
pygame.display.flip()

#Filtre üretimi için gereken fonksiyondur.
#hangi renk filtre üreteceğini aldığı parametrelerdeki değerler belirlemektedir.

def filtre_uret(r,g,b):
    print("Filtre Üret Fonk Çağırıldı")
    global elipsvarmi
    elipsvarmi=True
    global mavideger
    global kirmizideger
    global yesildeger
    mavideger=b
    kirmizideger=r
    yesildeger=g
    #Tüm ışınları sil
    oklarisil()
    #eğer r değerin 0 dan büyükse kırmızı ışınları çiz
    if r>0:
        kirmiziisinlar()
    #eğer g değerin 0 dan büyükse yeşil ışınları çiz
    if g>0:
        yesilisinlar()
    #eğer b değerin 0 dan büyükse kırmızı ışınları çiz
    if b>0:
        maviisinlar()
    #bu değerlere baglı olarak elips olustur
    ellipseciz()

def ellipseciz():
    #kullanıcının belirledği R,G,B değerlerine göre ellips çizer
    print("Elips çiz fonk çağırıldı")
    global elipsvarmi
    elipsvarmi=True
    pygame.draw.ellipse(screen, (kirmizideger, yesildeger, mavideger), (550, 100, 100, 400))

#Bu fonksiyon yeşil ışın çizer
#Bunun için yesildeger isimli değişkene ihtiyaç duyar
#En fazla 10 adet yeşil ışın çizebilmektedir.
#Bu nedenle yesildeger değişkeninin 255/10 olan yaklaşık 25 arttırır
#Işın çizimi bitince gider ve elipsin yeniden oluşmasını sağlar, çünkü değişen GREEN değerini yansıtmak gerek
def yesilisinlar():
    global elipsvarmi
    if elipsvarmi:
        global yesildeger
        if yesildeger<250:
            yesildeger+=25
        print("Yeşil Işın Değeri:",yesildeger)
        for i in range(yesildeger//25):
            pygame.draw.aaline(screen,yesil,(860,300),(600,175+(25*i)))
            cizgi=(860,300,600,175+(25*i))
            tumcizgiler.append(cizgi)

        ellipseciz()
#Bu fonksiyon Kırmızı ışın çizer
#Bunun için kirmizideger isimli değişkene ihtiyaç duyar
#En fazla 10 adet kırmızı ışın çizebilmektedir.
#Bu nedenle kirmizideger değişkeninin 255/10 olan yaklaşık 25 arttırır
#Işın çizimi bitince gider ve elipsin yeniden oluşmasını sağlar, çünkü değişen RED değerini yansıtmak gerek

def kirmiziisinlar():
    global elipsvarmi
    if elipsvarmi:
        global kirmizideger
        if kirmizideger<250:
            kirmizideger+=25
        print("Kırmızı Işın Değeri:",kirmizideger)
        for i in range(kirmizideger//25):
            pygame.draw.aaline(screen,kirmizi,(860,120),(600,125+(25*i)))
            cizgi=(860,120,600,125+(25*i))
            tumcizgiler.append(cizgi)
        ellipseciz()

#Bu fonksiyon mavi ışın çizer
#Bunun için mavideger isimli değişkene ihtiyaç duyar
#En fazla 10 adet mavi ışın çizebilmektedir.
#Bu nedenle yesildeger değişkeninin 255/10 olan yaklaşık 25 arttırır
#Işın çizimi bitince gider ve elipsin yeniden oluşmasını sağlar, çünkü değişen BLUE değerini yansıtmak gerek

def maviisinlar():
    global elipsvarmi

    if elipsvarmi:
        global mavideger
        if mavideger<250:
            mavideger+=25
        print("MAvi Işın Değeri:",mavideger)
        for i in range(mavideger//25):
            pygame.draw.aaline(screen,mavi,(860,490),(600,225+(25*i)))
            cizgi=(860,490,600,225+(25*i))
            tumcizgiler.append(cizgi)
        ellipseciz()

#OYun başlaması için gereken butondur.
#Başlar başlamaz bir ellips çizer ve diğer butonların da calısmasını sağlar
def start():
    print("Start Butonuna Basıldı")
    ellipseciz()

#Okların silinmesini sağlayan koddur.
#aslında gerçek anlamda okları silmiyoruz.
#daha önce çizilen her bir okun koordinatları üzerinde GRAY yeni oklar olusturuyoruz.
#Bir bakıma üstünden GRİ ile geçiyoruz
def oklarisil():
    global tumcizgiler
    print("Oklar silindi")
    if len(tumcizgiler)>0:
        for _ in range(10):
            for x1,y1,x2,y2 in tumcizgiler:
                pygame.draw.aaline(screen,gray,(x1,y1),(x2,y2),5)
    tumcizgiler=[]

#RESET butonuna basınca, önce okları siler ve sonra yenidne r:0,g:0,b:0 bir elips olusturur
def reset():
    oklarisil()

    global elipsvarmi
    elipsvarmi=True
    filtre_uret(0,0,0)

#Oyundan çıkmak için  gereken kodlardır
def quit():
    print("Quit buttona basildi")
    pygame.quit()
    sys.exit()

#START, RESET ve QUIT butonlarının oluştuğu ve konumlandığı ve ayrıca üzerindeki METİNlerin yazıldığı alandır
start_button = pygame.Rect(50, 50, 75, 25)
reset_button = pygame.Rect(50, 100, 75, 25)
quit_button = pygame.Rect(50, 150, 75, 25)
font = pygame.font.Font(None, 30)
start_text = font.render("START", True, (255, 255, 255))
reset_text = font.render("RESET", True, (255, 255, 255))
quit_text = font.render("QUIT", True, (255, 255, 255))

#KIRMIZI, YEŞİL ve MAVİ Filtre butonlarının oluştuğu ve konumlandığı ve ayrıca üzerindeki METİNlerin yazıldığı alandır
kirmizi_filtre_button = pygame.Rect(520, 50, 50, 25)
yesil_filtre_button = pygame.Rect(570, 50, 50, 25)
mavi_filtre_button = pygame.Rect(620, 50, 50, 25)
font = pygame.font.Font(None, 30)
kirmizi_filtre_text = font.render("R", True, (255, 255, 255))
yesil_filtre_text = font.render("G", True, (255, 255, 255))
mavi_filtre_text = font.render("B", True, (255, 255, 255))

#Basılan tusları kontrol ettiğimiz alandayız
while True:
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            pygame.quit()
            sys.exit()
        if event.type == pygame.MOUSEBUTTONDOWN:
            #Start butonu basıldığında, kendini gizlesin ve
            # reset butonu aktif olsun diye değişkenlere değer atarız
            #arından start fonk cagırırız
            if start_button.collidepoint(event.pos):
                buttonstart_visible = not buttonstart_visible
                buttonreset_visible=not buttonreset_visible
                start()
            #Reset butonu basıldıgında reset fonk cagırırız
            if reset_button.collidepoint(event.pos):
                reset()
            #QUIT butonu basıldığında quit fonk cagırırız
            if quit_button.collidepoint(event.pos):
                quit()
            #Yeşil fener butonu basıldıgında yesilisinlar isimli fonk cagırırız
            if button_rect_yesil_fener.collidepoint(event.pos):
                yesilisinlar()
            #Kırmızı fener butonu basıldıgında kirmiziisinlar isimli fonk cagırırız
            if button_rect_kirmizi_fener.collidepoint(event.pos):
                kirmiziisinlar()
            #Mavi fener butonu basıldıgında maviisinlar isimli fonk cagırırız
            if button_rect_mavi_fener.collidepoint(event.pos):
                maviisinlar()

            #Kırmızı filtre butonu basıldıgında filtre_uret isimli fonk cagırırız
            #ve bu fonksiyona red : 255, greeen: 0 , blue: 0 değerlerini parametre olarak göndeririz.
            if kirmizi_filtre_button.collidepoint(event.pos):
                filtre_uret(255,0,0)
            #Mavi filtre butonu basıldıgında filtre_uret isimli fonk cagırırız
            #ve bu fonksiyona red : 0, greeen: 0 , blue: 255 değerlerini parametre olarak göndeririz.
            if mavi_filtre_button.collidepoint(event.pos):
                filtre_uret(0,0,255)
            #Yeşil filtre butonu basıldıgında filtre_uret isimli fonk cagırırız
            #ve bu fonksiyona red : 0, greeen: 255 , blue: 0 değerlerini parametre olarak göndeririz.
            if yesil_filtre_button.collidepoint(event.pos):
                filtre_uret(0,255,0)

    #Çıkışl butonunu ve textini yerleştiriyoruz
    pygame.draw.rect(screen, (255, 0, 0), quit_button)
    screen.blit(quit_text, (60, 155))

    #Filtre butonlarını konumlandırıp yerleştiriyororuz
    pygame.draw.rect(screen, (255, 0, 0), kirmizi_filtre_button)
    pygame.draw.rect(screen, (0, 255, 0), yesil_filtre_button)
    pygame.draw.rect(screen, (0, 0, 255), mavi_filtre_button)
    screen.blit(kirmizi_filtre_text, (540, 50))
    screen.blit(yesil_filtre_text, (590, 50))
    screen.blit(mavi_filtre_text, (640, 50))

    #Start butonu görünecekse, ona göre yerleşim yapıyoruz
    if buttonstart_visible:
        pygame.draw.rect(screen, (0, 255, 0), start_button)
        screen.blit(start_text, (60, 55))

    #Reset butonu görünecekse ona göre yerleşim yapıyoruz
    if buttonreset_visible:
        pygame.draw.rect(screen, (0, 0, 255), reset_button)
        screen.blit(reset_text, (60, 105))
        pygame.draw.rect(screen, gray, start_button)

    pygame.display.update()


import CNNModels.icentia11k_make_wfdb as wfdbcreator
import numpy as np
import random
import scipy.signal as siglib
import numpy as np
from ecgdetectors import Detectors

# This function is called by the data generator to create one batch of data.
# patient: number of the patient in the icentia dataset
# subset: number of the subset from the patient that is used
# batchsize: the size of the batch
def getData_1dcnn(patient, subset, batchsize=1024, multi=True, nperseg_value=64):
    number_as_string = f'{patient:05d}'
    path = '/shared_data/icentia11k/'+number_as_string+'_batched.pkl.gz'
    signals, signal_lab = icentiaDataProcessor(path)
    startindex = subset*batchsize
    endindex = (subset+1)*batchsize
    signals = signals[startindex:endindex]
    signal_lab = signal_lab[startindex:endindex]
    # ab hier kommt dein eigener code, es wird über signals iteriert und für jedes einzelen signal dann die passende vorverarbeitung durchgeführt, so wie du sie brauchst
    # zusätzlich wird für jedes label ein codierung anglegt, aber das wäre dann wieder abhängig von deiner verarbeitung
    spectro = []
    spectro_label = []
    for i in range(len(signals)):
        signal = signals[i]
        signal = siglib.resample(signal, int(len(signal)*1.2))
        signal = np.ravel(signal)
        signal = preprocessData(signal, 9000, 300, False)
        f, t, Sxx = siglib.spectrogram(signal, fs=300, nfft=512, nperseg=64)
        Sxx = 10*np.log10(Sxx+1e-12)
        Sxx = np.reshape(Sxx, (Sxx.shape[0], Sxx.shape[1], 1))

        if signal_lab[i] == 'N':  # Zuordnung zu "Normal"
            label = np.asarray([1, 0, 0, 0])

        if signal_lab[i] == 'A':  # Zuordnung zu "Vorhofflimmern"
            label = np.asarray([0, 1, 0, 0])

        if multi:  # Zuordnung zu "Noise"
            if signal_lab[i] == '~':
                label = np.asarray([0, 0, 1, 0])

            if signal_lab[i] == 'O':  # Zuordnung zu "Other"
                label = np.asarray([0, 0, 0, 1])
        spectro.append(Sxx)
        spectro_label.append(label)
        del f, t, Sxx, label
    return spectro, spectro_label

# This function is called by the data generator to create one batch of data.
# patient: number of the patient in the icentia dataset
# subset: number of the subset from the patient that is used
# batchsize: the size of the batch
def getData_spectro(patient, subset, batchsize=1024, multi=True, nperseg_value=64):
    number_as_string = f'{patient:05d}'
    path = '/shared_data/icentia11k/'+number_as_string+'_batched.pkl.gz'
    signals, signal_lab = icentiaDataProcessor(path)
    startindex = subset*batchsize
    endindex = (subset+1)*batchsize
    signals = signals[startindex:endindex]
    signal_lab = signal_lab[startindex:endindex]
    spectro = []
    spectro_label = []
    for i in range(len(signals)):
        signal = signals[i]
        signal = siglib.resample(signal, int(len(signal)*1.2))
        signal = np.ravel(signal)
        signal = preprocessData(signal, 9000, 300, False)
        f, t, Sxx = siglib.spectrogram(signal, fs=300, nfft=512, nperseg=64)
        Sxx = 10*np.log10(Sxx+1e-12)
        Sxx = np.reshape(Sxx, (Sxx.shape[0], Sxx.shape[1], 1))

        if signal_lab[i] == 'N':  # Zuordnung zu "Normal"
            label = np.asarray([1, 0, 0, 0])

        if signal_lab[i] == 'A':  # Zuordnung zu "Vorhofflimmern"
            label = np.asarray([0, 1, 0, 0])

        if multi:  # Zuordnung zu "Noise"
            if signal_lab[i] == '~':
                label = np.asarray([0, 0, 1, 0])

            if signal_lab[i] == 'O':  # Zuordnung zu "Other"
                label = np.asarray([0, 0, 0, 1])
        spectro.append(Sxx)
        spectro_label.append(label)
        del f, t, Sxx, label
    return spectro, spectro_label

# function opens corresponding file and returns the fitting signals and labels

def icentiaDataProcessor(path):
    lischte = wfdbcreator.make_wfdb(path)

    # import all data
    records = []
    labels = []
    timestamps = []
    for i in range(len(lischte)):
        records.append(lischte[i][0])
        labels.append(lischte[i][1])
        timestamps.append(lischte[i][2])
        where = np.where(labels[i] != '')
        labels[i] = labels[i][where]
        timestamps[i] = timestamps[i][where]

    # create segments
    segments = []
    segments_label = []

    for j in range(len(records)):
        for i in range(len(labels[j])-1):
            if(labels[j][i] == '(N' and labels[j][i+1] == ')'):
                segments_label.append('N')
                segments.append(
                    records[j][timestamps[j][i]:timestamps[j][i+1]])
            elif(labels[j][i] == '(AFIB' and labels[j][i+1] == ')'):
                segments_label.append('A')
                segments.append(
                    records[j][timestamps[j][i]:timestamps[j][i+1]])
            elif(labels[j][i] == '(AFL' and labels[j][i+1] == ')'):
                segments_label.append('O')
                segments.append(
                    records[j][timestamps[j][i]:timestamps[j][i+1]])
            elif(labels[j][i] == 'None' or labels[j][i] == ')'):
                pass
            else:
                pass

    # length of recordings that I want to create, similar to the length of recordings in the original trainingsset
    random_numbers = np.load('patientdata_evaluated/random_numbers.npy')
    max_duration = 60

    signals = []
    signals_label = []
    fs = 250
    for i in range(len(segments)):
        current_segment = segments[i]
        index = 0
        while(len(current_segment) > fs*max_duration):
            time = random_numbers[index]
            new_signal = current_segment[0:fs*time]
            current_segment = current_segment[fs*time:-1]
            signals.append(new_signal)
            signals_label.append(segments_label[i])
            index = index+1

    return (signals, signals_label)

# Preprocessing of the incoming data
# the signal is cut before the first and after the last detected r_peak if it is to short
# than the signal is repeated multiple times to reach the desired length
# afterwards the signal is normalized

def preprocessData(data, targetsize, samplefreq, debug=False):
    ret_signal = 0
    repeat_signal = 0
    # looking for r_peaks and repeating the signal
    if(data.size < targetsize):
        if(debug):
            print('Length of the signal ->'+str(data.size) +
                  '<- is shorter than '+str(targetsize))
        detector = Detectors(samplefreq)
        try:
            r_peaks = detector.hamilton_detector(data)
            if(debug):
                print('R-Peaks are detected!')
            repeat_signal = data[r_peaks[0]:r_peaks[-1]]
        except:
            if(debug):
                print('R-Peaks could not be detected!')
            repeat_signal = data

        if(repeat_signal.size == 0):
            if(debug):
                print('Something went wrong, just use the original signal to repeat.')
            repeat_signal = data
            if(debug and repeat_signal.size == 0):
                print('I have no idea.')
        repeats = int(np.ceil(targetsize / repeat_signal.size))
        if(debug):
            print('Signal is repeated ' + str(repeats) + ' times.')
        for i in range(repeats):
            ret_signal = np.append(ret_signal, repeat_signal)
    else:
        if(debug):
            print('Length of the signal ->'+str(data.size) +
                  '<- is bigger than '+str(targetsize))
        ret_signal = data

    # cutting the signal to the desired length
    # same ammount is cut at the start and the end of the signal
    if(ret_signal.size > targetsize):
        if(debug):
            print('Signal is cut to length: ' + str(targetsize))
        cutindex = int(((ret_signal.size-targetsize)/2))
        ret_signal = ret_signal[cutindex:-1]
        ret_signal = ret_signal[0:targetsize]
        if(debug):
            print('Size of the new signal: ' + str(ret_signal.size))

    # normalising the signal
    ret_signal = (ret_signal - np.min(ret_signal)) / \
        (np.max(ret_signal) - np.min(ret_signal))

    return ret_signal


#!/usr/bin/env python3

from xmlrpc.server import SimpleXMLRPCServer
from xmlrpc.server import SimpleXMLRPCRequestHandler
from xmlrpc.client import Fault
from socketserver import ThreadingMixIn

import requests
import concurrent.futures

S = requests.Session()
URL = "https://fi.wikipedia.org/w/api.php"

def validatePage(title):
    PARAMS = {
        "action": "query",
        "format": "json",
        "titles": title,
        "prop": "links",
        "pllimit": "max"
    }
    R = S.get(url=URL, params=PARAMS)
    DATA = R.json()
    PAGES = DATA["query"]["pages"]
    if "-1" in PAGES:
        print("Page is invalid")
        return False
    else:
        print("Page is valid...")
        return True

def deleteDuplicates(theList):

    #https://stackoverflow.com/questions/42764335/python-find-duplicates-in-the-first-col-of-a-2d-list-and-remove-one-of-them-base
    #https://stackoverflow.com/questions/7436267/python-transform-a-dictionary-into-a-list-of-lists
    output_lst = dict()
    for n, v in theList:
        if n in output_lst:
            output_lst[n] = max(output_lst[n], v)
        else:
            output_lst[n] = v    # The question is, what is the complexity of this operation
    returnList = list(map(list, output_lst.items()))
    return returnList

def getPageLinks(pageTitle, currentPath):
    allLinks = []
    PARAMS = {
        "action": "query",
        "format": "json",
        "titles": pageTitle,
        "prop": "links",
        "pllimit": "max"
    }
    R = S.get(url=URL, params=PARAMS)
    DATA = R.json()
    PAGES = DATA["query"]["pages"]
    if "-1" not in PAGES:   #basically if we found a wikipedia page or not
        for k, v in PAGES.items():
            try:
                for l in v["links"]:
                    allLinks.append([ l["title"], currentPath + " >> " + l["title"]])
                    #print([ l["title"], currentPath + " >> " + l["title"] ])
            except KeyError:
                print("Failed to get links")

    while 'continue' in DATA:   #getting additional pages with plcontinue
        PARAMS = {
            "action": "query",
            "format": "json",
            "titles": pageTitle,
            "prop": "links",
            "pllimit": "max",
            "plcontinue": DATA["continue"]["plcontinue"]
        }
        R = S.get(URL, params=PARAMS)
        DATA = R.json()
        PAGES = DATA["query"]["pages"]
        if "-1" not in PAGES:
            for k, v in PAGES.items():
                try:
                    for l in v["links"]:
                        allLinks.append([ l["title"], currentPath + " >> "+ l["title"]])
                        #print([ l["title"], currentPath + " >> " + l["title"] ])
                except KeyError:
                    print("Failed to get links")

    print("Found " + str(len(allLinks)) + " links, returning them back")
    return allLinks

def runParallel(inputArray, endTitle):
    newArray = []
    for i in range(0, len(inputArray), 10):
        with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
            try:
                args = ((inputArray[k][0], inputArray[k][1]) for k in range(i,i+9))
                for out in executor.map(lambda p: getPageLinks(*p), args):
                    newArray = newArray + out
            except:
                print("Error occurred in indexing")
            newArray = deleteDuplicates(newArray)
            print(str(len(newArray)) + " is the size now..")

        print("Finished a cycle")
        titleList = [element[0] for element in newArray]
        for i in range(len(titleList)):
            if titleList[i].upper() == endTitle.upper():
                print(newArray[i][1])
                result = newArray[i][1]
                return result, "found"
    newArray = deleteDuplicates(newArray)
    return newArray, "unfound"

def findPath(startTitle, endTitle):

    currentPath = startTitle
    startArray = getPageLinks(startTitle, currentPath)

    #if only one link is returned, get page links from the single page only
    while len(startArray) == 1:
        startArray = getPageLinks(startArray[0][0], startArray[0][1])

    # while the result has not been found
    while(True):
        # run a function in parallel that finds links of a page (10 simultaneously)
        result, didFind = runParallel(startArray, endTitle)
        if didFind == "found":
            return result
        else:
            #move on to next level
            startArray = result

class SimpleThreadedXMLRPCServer(ThreadingMixIn, SimpleXMLRPCServer):
    pass

#https://stackoverflow.com/questions/53621682/multi-threaded-xml-rpc-python3-7-1
def run_server(host="localhost", port=8000):
    server_addr = (host, port)
    server = SimpleThreadedXMLRPCServer(server_addr)
    #server.register_function(deleteDuplicates)
    #server.register_function(getPageLinks)
    #server.register_function(runParallel)
    server.register_function(validatePage)
    server.register_function(findPath)

    print("Server thread started. Testing server ...")
    print('listening on {} port {}'.format(host, port))

    server.serve_forever()

if __name__ == "__main__":
    run_server()


from typing import List, Iterable, cast
from xml.etree.ElementTree import Element, indent

from utils import make_tokenstream, parser_tester, dump

if __name__ == "__main__":
    sample_let: list[str] = [
        "let s = \"string constant\";",
        "let z = null;",
        "let a[1] = a[2];",
        "let game = squaregame.new();",
        "let y = y + 2;",
        "let i = i * (-j);",
        "let i = i | j;",
    ]

    let_tokenstreams: List[List[Element]] = [make_tokenstream(x) for x in sample_let]

    for ts in let_tokenstreams:
        working_on: str = ' '.join([cast(str, x.text) for x in cast(Iterable[Element], ts)])
        print(f"working on: {working_on}")
        res: Element = parser_tester(ts, "_subroutine_let")
        indent(res)
        dump(res)
        assert res
        print()

    print("finished")


"""
Plan Review Flags - Part 4 of 5

This script populates the Zoning RCO value for every property value.
This process must be run immediately after Parts 1-3 as it relies on the same local source data to remain in sync
"""
import datetime
import logging
import sys
import traceback
from datetime import timedelta

import arcpy
from sde_connections import DataBridge

# Step 1: Configure log file
try:
    print('Step 1: Configuring log file...')
    log_file_path = 'E:\LI_PLAN_REVIEW_FLAGS\Logs\PermitReviewFlags.log'
    log = logging.getLogger('PR Flags Part 4 - Zoning RCOs')
    log.setLevel(logging.INFO)
    hdlr = logging.FileHandler(log_file_path)
    hdlr.setLevel(logging.INFO)
    hdlrFormatter = logging.Formatter('%(name)s - %(levelname)s - %(asctime)s - %(message)s', datefmt='%m/%d/%Y  %I:%M:%S %p')
    hdlr.setFormatter(hdlrFormatter)
    log.addHandler(hdlr)
    print('SUCCESS at Step 1')

except:
    tb = sys.exc_info()[2]
    tbinfo = traceback.format_tb(tb)[0]
    pymsg = "PYTHON ERRORS:\nTraceback info:\n" + tbinfo + "\nError Info:\n" + str(sys.exc_info()[1])
    arcpy.AddError(pymsg)
    sys.exit(1)

try:
    log.info('PR Flags Part 4 Begun')
    today = datetime.datetime.today()
    oneWeekAgo = today - timedelta(days=7)

    Zoning_RCOs = DataBridge.sde_path+'\\GIS_PLANNING.Zoning_RCO'

    PWD_PARCELS_DataBridge = DataBridge.sde_path+'\\GIS_WATER.PWD_PARCELS'

    Council_Districts_2016 = DataBridge.sde_path+'\\GIS_PLANNING.Council_Districts_2016'

    zoningFC = 'ZoningRCO_'
    zoningLayer = 'ZoningLayer'
    zoningCurrent = 'ZoningCurrent'

    PR_FLAG_Temp = 'Flags_Table_Temp'
    PWD_Parcels_Working = 'PWDParcels_Working'
    Council_Districts_Local = 'Districts_'
    PPR_Assets = DataBridge.sde_path + '\\GIS_PPR.PPR_Assets'
    PPR_Assets_Temp_Pre_Dissolve = 'in_memory\\PPR_Assets_Temp_Pre_Dissolve'
    PPR_Assets_Temp = 'in_memory\\PPR_Assets_Temp'
    Park_IDs_Local = 'ParkNameIDs_'

    localWorkspace = 'E:\\LI_PLAN_REVIEW_FLAGS\\Workspace.gdb'
    inMemory = 'in_memory'

    arcpy.env.workspace = localWorkspace
    arcpy.env.overwriteOutput = True
    locallySaved = arcpy.ListFeatureClasses()

    ############################
    #Copy RCO Layer Local
    if arcpy.Exists(zoningFC):
        arcpy.Delete_management(zoningFC)
    arcpy.FeatureClassToFeatureClass_conversion(Zoning_RCOs, localWorkspace, zoningFC)

    #Append all ROC Names to a list.  Each RCO will be iterated through and added to the dictionary
    print(zoningFC)
    print([f.name for f in arcpy.ListFields(zoningFC)])

    rcoOIDCursor = arcpy.da.SearchCursor(zoningFC, 'ORGANIZATION_NAME')
    rcoIDS = list(set([row[0] for row in rcoOIDCursor]))
    del rcoOIDCursor

    #Create lookup dictionary for RCO Objectid names:
    from li_dbs_Lazy import DataBridge
    lookupSQL = "select LNI_ID, ORGANIZATION_NAME from GIS_PLANNING.ZONING_RCO"
    lookupCursor = DataBridge.DataBridge.cursor()
    lookupCursor.execute(lookupSQL)
    lookupDict = {row[1]:row[0] for row in lookupCursor.fetchall()}
    lookupCursor.close()
    del lookupCursor

    #Create a layer based off of each overlay type saved from previous list
    overlayTotal = len(rcoIDS)
    overlayCount = 0
    arcpy.MakeFeatureLayer_management(zoningFC, zoningLayer)
    parcelDict = {}
    for overlayType in rcoIDS:
        arcpy.env.workspace = localWorkspace
        overlayCount += 1
        print('Starting ' + str(overlayType) + ', ' + str(overlayCount)+ ' of ' + str(overlayTotal))
        arcpy.SelectLayerByAttribute_management(zoningLayer, 'NEW_SELECTION', "ORGANIZATION_NAME = '" + str(overlayType if "'" not in overlayType else overlayType.replace("'", "|| CHR(39) ||")) + "'")
        arcpy.MakeFeatureLayer_management(zoningLayer, zoningCurrent)
        IntersectOutput = localWorkspace + '\\' + 'Int_' + zoningFC
        arcpy.Intersect_analysis([PWD_Parcels_Working] + [zoningCurrent], IntersectOutput, 'ALL')

        # To ensure no slivers are included a thiness ratio and shape area are calculated for intersecting polygons
        arcpy.AddField_management(IntersectOutput, 'Thinness', 'FLOAT')
        # NOTE Thiness calculation was removed by request, this is designed remove small overlaps in zoning from adjacent parcels
        arcpy.AddGeometryAttributes_management(IntersectOutput, 'AREA', Area_Unit='SQUARE_FEET_US')
        """
        arcpy.AddGeometryAttributes_management(IntersectOutput, 'PERIMETER_LENGTH', 'FEET_US')
        arcpy.CalculateField_management(IntersectOutput, 'ThinessRatio',
                                        "4 * 3.14 * !POLY_AREA! / (!PERIMETER! * !PERIMETER!)", 'PYTHON_9.3')

        """
        fieldList = ['PARCELID', 'GROSS_AREA', 'POLY_AREA', 'Thinness', 'ORGANIZATION_NAME', 'ADDRESS']
        IntersectCursor = arcpy.da.SearchCursor(IntersectOutput, fieldList)
        countin = int(arcpy.GetCount_management(IntersectOutput).getOutput(0))
        count = 0
        print('Found ' + str(countin) + ' records in intersect table')
        breaks = [int(float(countin) * float(b) / 100.0) for b in range(10, 100, 10)]
        for row in IntersectCursor:
            count += 1
            if count in breaks:
                print('Parsing RCO FC ' + str(int(round(count * 100.0 / countin))) + '% complete...')
            if (float(row[fieldList.index('POLY_AREA')]) / float(row[fieldList.index('GROSS_AREA')])) > 0.01:  #To implment 3% coverage and thinness minimum: row[3] > 0.3 and (row[2] / float(row[1])) > 0.03:
                if row[fieldList.index('PARCELID')] in parcelDict:
                    tempList = parcelDict.get(row[0])
                    parcelDict[row[fieldList.index('PARCELID')]] = list(set([lookupDict[row[fieldList.index('ORGANIZATION_NAME')]]] + tempList))
                else:
                    parcelDict[row[fieldList.index('PARCELID')]] = [lookupDict[row[fieldList.index('ORGANIZATION_NAME')]]]
        arcpy.Delete_management(IntersectOutput)
        arcpy.Delete_management(zoningCurrent)
        arcpy.SelectLayerByAttribute_management(zoningLayer, 'CLEAR_SELECTION')
    del rcoIDS
    zoneCursor = arcpy.da.UpdateCursor(PR_FLAG_Temp, ['PWD_PARCEL_ID', 'ZONING_RCO'])
    print('Starting Cursor')
    for parcel in zoneCursor:
        if parcel[0] in parcelDict:
            parcel[1] = '|'.join([str(oid) for oid in parcelDict.get(parcel[0])])
            zoneCursor.updateRow(parcel)
    del zoneCursor
    log.info('PR Flags Part 4 Complete')

except:
    tb = sys.exc_info()[2]
    tbinfo = traceback.format_tb(tb)[0]
    pymsg = "PYTHON ERRORS:\nTraceback info:\n" + tbinfo + "\nError Info:\n" + str(sys.exc_info()[1])
    log.error(pymsg)
    print(pymsg)

    import smtplib
    from email.mime.text import MIMEText
    from phila_mail import server
    sender = 'LIGISTeam@phila.gov'
    recipientslist = ['DANI.INTERRANTE@PHILA.GOV', 'SHANNON.HOLM@PHILA.GOV', 'Philip.Ribbens@Phila.gov', 'LIGISTeam@phila.gov', 'Jessica.bradley@phila.gov']
    commaspace = ', '
    msg = MIMEText('AUTOMATIC EMAIL \n Plan Review Flags Update Failed during update: \n' + pymsg)
    msg['To'] = commaspace.join(recipientslist)
    msg['From'] = sender
    msg['X-Priority'] = '2'
    msg['Subject'] = 'Plan Review Flags Table Update Failure'
    server.server.sendmail(sender, recipientslist, msg.as_string())
    server.server.quit()
    sys.exit(1)


from logging import warning
from os import listdir, path

import email
import email.policy
import time

from urllib import request
try:
    import lxml.html
except ImportError:
    warning("No lxml module: can't parse html")

__all__ = ["Email", "mailLoaderGen"]

# Constantes
robotintro = "Une nouvelle offre d'emploi vient d'être postée sur le site www.sfbi.fr"
policy = email.policy.EmailPolicy(raise_on_defect=True, utf8=True)
parser = email.parser.BytesParser(policy=policy)

#
# Outils pour aider la lecture de mails
#

def decodeHeader(s):
    if "=?" in s:
        return ''.join(strbytes.decode('latin-1' if encoding is None else encoding)
                        for strbytes, encoding
                        in email.header.decode_header(s))
    else:
        return s

def extractTagsFromSubject(subject):
    tags = set()
    if subject is not None:
        for s in subject.split('['):
            i = s.find(']')
            if i != -1:
                tags.add(s[:i].strip())

    i = subject.rindex(']')
    if i != -1:
        subject = subject[i+1:]
    return tags, subject.strip()

#
# Outils pour aider la lecture de pages sfbi.fr d'annonce d'emplois
#

cachedir='cacheSFBI'
fieldtypeClass = 'field-type-'
fieldnameClass = 'field-field-emploi-'

def extractText(e, inP=False):
    text = e.text
    tail = e.tail
    tag = e.tag

    islabel = tag == "div" and "field-label" in e.attrib.get("class", "")

    acc = []

    if not islabel:
        if text:
            acc.append(text if inP else text.strip())

        # recursion
        acc.extend(extractText(c, inP=inP or tag == 'p') for c in e)

        if tag == 'br':
            acc.append('\n')
        elif tag == 'p' or tag == 'li':
            acc.append('\n\n')

    if tail:
        acc.append(tail if inP else tail.strip())

    return ''.join(acc)

class Email:

    def __init__(self, mailfile):
        self.mailfile = mailfile
        self.valid = True

    def load(self):
        with open(self.mailfile, 'rb') as fd:
            msg = parser.parse(fd)

        self.sender = decodeHeader(msg.get('From'))
        self.timestamp = time.mktime(email.utils.parsedate(msg.get('Date')))

        content = msg.get_content().lstrip()
        if content.startswith(robotintro):
            for line in content.splitlines():
                if line.startswith('http://'):
                    self._loadWebPage(line)
            self.sfbi = True
        else:
            self.sfbi = False
            self.tags, self.sujet = extractTagsFromSubject(decodeHeader(msg.get('Subject')))
            self.tags.remove('bioinfo')
            self.description = content

        return self

    def _loadWebPage(self, url, cachedir=cachedir):
        urlparts = url.split('/')
        cachefile = path.join(cachedir,'_'.join(urlparts[urlparts.index('')+1:]))

        if path.exists(cachefile):
            with open(cachefile, 'tr') as fd:
                html = fd.read()
        else:
            try:
                print("Downloading", url)
                html = request.urlopen(request.quote(url, safe=':/%')).read().decode('utf-8')
            except request.HTTPError as e:
                html = str(e.getcode())
            with open(cachefile, 'tw') as fd:
                fd.write(html)

        # Il y a quelques 404, ils sont mis en cache et repérés par une str
        if html == '404' or html == '403':
            warning('Error {}: {}.'.format(html, url))
            self.valid = False
            return

        #html = lxml.html.clean.clean_html(html)

        sectiondiv = lxml.html.fromstring(html).xpath('//div[@class="section"]')[0]
        self.sujet = sectiondiv.xpath('//h1[@class="title"]/text()')[0]
        self.sfbi_url = url

        for entry in sectiondiv.xpath('//div[contains(@class,"node-type-emploi")]/div[@class="content"]/div'):
            classes = entry.attrib['class'].split(' ')
            assert 'field' in classes

            # La classe du div renseigne sur le nom et le type de champs
            fieldname = None
            fieldtype = None
            for c in classes:
                if c.startswith(fieldtypeClass):
                    fieldtype = c[len(fieldtypeClass):]
                if c.startswith(fieldnameClass):
                    fieldname = c[len(fieldnameClass):]
            assert fieldname is not None and fieldtype is not None

            items = entry.xpath('div[@class="field-items"]/div[contains(@class, "field-item ")]')

            assert len(items) > 0
            if len(items) > 1:
                content = set(extractText(i).strip() for i in items)
            else:
                content = extractText(items[0]).replace('\xa0', ' ').strip()

            setattr(self, fieldname, content)

archivedir = 'archives_SFBI/2015_06_10-bioinfo_archives_annee_2014'
def mailLoaderGen(archivedir=archivedir):
    dedup = {}
    for mounth in listdir(archivedir):
        mounthdir = path.join(archivedir, mounth)
        for mailfn in listdir(mounthdir):
            # Passe sur les fichiers .recoded
            if mailfn.endswith(".recoded"):
                continue
            mailfile = path.join(mounthdir, mailfn)
            mail = Email(mailfile).load()
            if mail.valid:
                key = (mail.timestamp, mail.sujet)
                if key not in dedup:
                    dedup[key] = mail.mailfile
                    yield mail
                else:
                    warning('duplicates {} <-> {}'.format(dedup[key], mailfile))


import re
import warnings

import numpy as np
from astropy.coordinates.sky_coordinate import SkyCoord
from bs4 import BeautifulSoup

from lcc.db_tier.base_query import LightCurvesDb
from lcc.entities.exceptions import QueryInputError
from lcc.entities.star import Star
from lcc.utils.commons import ProxyRotatingSession

class OgleII(LightCurvesDb):
    """
    Connector to OGLEII. It is divided into two subdatabases - "phot" and "bvi".
    The first one contains light curves and metadata about coordinates, identifiers
    and V magnitude. The second one also contains information about V and I.

    Identifier of the stars in OgleII db are: field, starid and target.

    In case of cone search (if coordinates are provided), "nearest" key can
    be used. If it is True just nearest star to the target point is returned.

    Example:
    --------
    que1 = {"ra": 5.549147 * 15,
       "dec": -70.55792, "delta": 5, "nearest": True}
    que2 = {"field":"LMC_SC1","starid":"152248"}

    client = StarsProvider().getProvider(
        obtain_method="OgleII", obtain_params=[que1, que2])
    stars = client.getStarsWithCurves()
    """

    ROOT = "http://ogledb.astrouw.edu.pl/~ogle/photdb"

    BVI_TARGETS = ["lmc", "smc", "bul"]
    PHOT_TARGETS = ["lmc", "smc", "bul", "car"]

    QUERY_TYPES = ["bvi", "phot"]

    MAX_TIMEOUT = 60

    LC_META = {"xlabel": "hjd",
               "xlabel_unit": "days",
               "ylabel": "magnitude",
               "ylabel_unit": "mag",
               "color": "V",
               "origin": "OgleII"}

    COL_MAP = {"Field": "field",
               "StarID": "starid",
               "RA": "ra",
               "Decl": "dec",
               "V": "v_mag",
               "I": "i_mag",
               "B": "b_mag"}

    QUERY_OPTIONS = ["ra", "dec", "delta", "nearest", "field", "field_num", "starid", "target"]

    def __init__(self, queries, multiproc=False):
        """
        Parameters
        ----------
        queries : list, dict, iterable
            Query is list of dictionaries of query parameters or single
            dictionary

        multiproc : bool, int
            If True task will be distributed into threads by using all cores. If it is number,
            just that number of cores are used
        """
        if isinstance(queries, dict):
            queries = [queries]
        self.queries = self._parseQueries(queries)
        self.multiproc = multiproc

    def getStar(self, query, load_lc=True):
        """
        Query `Star` object

        Parameters
        ----------
        load_lc : bool
            Append light curves to star objects

        Returns
        -------
        list
            List of `Star` objects
        """
        if "starid" in query and query["starid"] < 1:
            raise ValueError("Starid has to be greater then 0")

        stars = self.postQuery(query, load_lc)
        if "ra" in query and "dec" in query and "delta" in query:
            stars = self.coneSearch(SkyCoord(float(query["ra"]), float(query["dec"]), unit="deg"),
                                    stars, float(query["delta"] / 3600.),
                                    nearest=query.get("nearest", False))
        return stars

    def postQuery(self, query, load_lc):
        PAGE_LEN = int(1e10)
        valmin_ra, valmax_ra, valmin_dec, valmax_dec = self._getRanges(query.get("ra"),
                                                                       query.get(
                                                                           "dec"),
                                                                       query.get("delta"))
        if valmax_ra and valmax_ra:
            valmax_ra = valmax_ra / 15.
            valmin_ra = valmin_ra / 15.

        params = {
            "db_target": query.get("target"),
            "dbtyp": "dia2",
            "sort": "field",
            "use_field": "field" in query,
            "val_field": query.get("field"),
            "use_starid": "starid" in query,
            "val_starid": query.get("starid"),
            "disp_starcat": "off",
            "use_starcat": "off",
            "disp_ra": "on",
            "use_ra": valmin_ra != "",
            "valmin_ra": valmin_ra,
            "valmax_ra": valmax_ra,
            "disp_decl": "on",
            "use_decl": valmin_dec != "",
            "valmin_decl": valmin_dec,
            "valmax_decl": valmax_dec,
            "disp_imean": "on",
            "use_imean": "mag_i_min" in query,
            "valmin_imean": query.get("mag_i_min"),
            "valmax_imean": query.get("mag_i_max"),
            "disp_pgood": "off",
            "disp_bmean": "on",
            "disp_vmean": "on",
            "disp_imed": "off",
            "disp_bsig": "off",
            "disp_vsig": "off",
            "disp_isig": "off",
            "disp_imederr": "off",
            "disp_ndetect": "off",
            "disp_v_i": "off",
            "disp_b_v": "off",
            "sorting": "ASC",
            "pagelen": PAGE_LEN,
        }
        # Delete unneeded parameters
        cparams = dict()
        for key, value in params.items():
            if value and value != "off":
                cparams[key] = value
        # Url for query
        url = "%s/query.php?qtype=%s&first=1" % (self.ROOT, query.get("db"))
        result = ProxyRotatingSession().post(url, cparams)
        return self._parseResult(result, lc=load_lc)

    def _parseQueries(self, queries):
        todel_queries = []
        new_queries = []
        for i, query in enumerate(queries):
            if "db" not in query:
                query["db"] = self.QUERY_TYPES[0]

            if "coo" in query and isinstance(query["coo"], SkyCoord) and "delta" in query:
                todel_queries.append(i)
                coo = query["coo"]
                query["ra"] = coo.ra.degree
                query["dec"] = coo.dec.degree

            if "ra" in query and "dec" in query and "target" not in query:
                todel_queries.append(i)

                if query["db"] == "phot":
                    targets = self.PHOT_TARGETS
                else:
                    targets = self.BVI_TARGETS

                for target in targets:
                    z = query.copy()
                    z["target"] = target
                    new_queries.append(z)

            elif "starid" in query:
                if "field" in query:
                    query["target"] = query["field"][:3].lower()
                elif "field_num" in query and "target" in query:
                    query["field"] = query[
                        "target"].upper() + "_SC" + str(query["field_num"])
                else:
                    raise QueryInputError("Unresolved target")

            if query["db"] not in self.QUERY_TYPES:
                raise QueryInputError(
                    "Invalid db. Available OgleII databases: %s" % self.QUERY_TYPES)

        return [item for i, item in enumerate(
            queries) if i not in todel_queries] + new_queries

    def _parseResult(self, result, lc=False):
        START_TABLE = "
<table"
        END_TABLE = "
"

        if result.status_code != 200:
            warnings.warn("Website has not returned 200 status")
            return []

        lc_tmp = None
        raw_table = ""
        skip = True
        for line in result.iter_lines():
            line = line.decode()
            if skip:
                if line.strip().startswith(START_TABLE):
                    raw_table += line[len("
"):]
                    skip = False
                if lc and line.strip().startswith("<input type='hidden'"):
                    tmpdir_pattern = re.compile(
                        "<input type='hidden' name='tmpdir' value='(.*)'>")
                    tmpdir = tmpdir_pattern.match(line)
                    if tmpdir:
                        lc_tmp = tmpdir.group(1)

            else:
                if line.strip().startswith(END_TABLE):
                    break
                raw_table += line

        if not raw_table:
            return []

        soup = BeautifulSoup(raw_table, "lxml")
        table = soup.find('table')
        rows = table.findAll('tr')

        res_rows = []
        for tr in rows[1:]:
            cols = tr.findAll('td')
            res_cols = []
            for td in cols:
                res_cols.append(td.find(text=True))
            res_rows.append(res_cols)

        header = [th.find(text=True) for th in table.findAll("th")]
        return self._createStars(header, res_rows, lc_tmp)

    def _createStars(self, header, rows, lc_tmp):
        # [u'No', u'Field', u'StarID', u'RA', u'Decl', u'V', u'I', u'B']
        # [u'No', u'Field', u'StarID', u'RA', u'Decl', u'I']

        cols_map = self._parseHeader(header)
        stars = []
        for row in rows:
            field = str(row[cols_map.get("field")])
            starid = int(row[cols_map.get("starid")])
            ra = float(row[cols_map.get("ra")])
            dec = float(row[cols_map.get("dec")])

            colors = ["i_mag", "b_mag", "v_mag"]
            more = {}
            for col in colors:
                if cols_map.get(col) and cols_map.get(col):
                    try:
                        more[col] = float(row[cols_map.get(col)])
                    except:
                        pass

            name = field + "_" + str(starid)
            coo = (ra * 15, dec, "deg")

            ident = {"OgleII": {"name": name,
                                "db_ident": {"field": field,
                                             "starid": starid}}}

            st = Star(ident, name, coo, more)

            if lc_tmp:
                lc = self._getLc(field, starid, lc_tmp)
                if lc and len(lc) != 0:
                    st.putLightCurve(np.array(lc), meta=self.LC_META)

            stars.append(st)
        return stars

    def _parseHeader(self, header):
        cols_map = {}
        for i, col in enumerate(header):
            if col in list(self.COL_MAP.keys()):
                cols_map[self.COL_MAP[col]] = i
        return cols_map

    def _getLc(self, field, starid, lc_tmp):
        params = {
            "field": field,
            "starid": starid,
            "tmpdir": lc_tmp,
            "db": "DIA",
            "points": "good",
        }

        _url = "%s/getobj.php" % self.ROOT
        ProxyRotatingSession().post(_url, params)

        url = "%s/data/%s/%s_i_%s.dat" % (self.ROOT,
                                          lc_tmp, field.lower(), starid)
        try:
            result = ProxyRotatingSession().get(url)

            if result.status_code == 200:
                star_curve = []
                # TODO: Can be loaded with pandas
                for line in result.iter_lines():
                    line = line.decode()
                    parts = line.strip().split(" ")
                    star_curve.append(
                        [round(float(parts[0]), 4), round(float(parts[1]), 3), round(float(parts[2]), 3)])
                return star_curve

        # TODO
        except:
            raise


from .utils import make_node

import ast
import astor

class RootCallTransformer(ast.NodeTransformer):
    def visit_Expr(self, node):
        """
        Expr(value)
        node.value is a Call
        """
        if not isinstance(node.value, ast.Call):
            return node
        src = astor.to_source(node.value)
        return make_node(
            f"ln.call({src}, text='''{src.strip()}''', line_no={node.lineno})"
        )


from PyQt5 import QtCore, QtGui
import numpy as np
from grale.constants import ANGLE_ARCSEC
import openglhelper
import tools
import os

def qImageToArray(img):
    img = img.convertToFormat(QtGui.QImage.Format_ARGB32)
    ptr = img.constBits()
    ptr.setsize(img.byteCount())
    arr = np.array(ptr).reshape(img.height(), img.bytesPerLine())
    if img.bytesPerLine() != img.width()*4:
        arr = arr[:,:img.width()*4]
    arr = arr.reshape(img.height(), img.width(), 4)
    arr = arr.astype(np.float)
    return arr

def arrayToQImage(img):
    img = img.copy()
    img = img.astype(np.uint8)
    destImg = QtGui.QImage(img, img.shape[1], img.shape[0], img.shape[1]*4, QtGui.QImage.Format_ARGB32)
    return destImg

def _retraceCPU(ip, img, center, sizes, subSample):

    import grale.images as images

    cx, cy = center
    cx *= ANGLE_ARCSEC
    cy *= ANGLE_ARCSEC
    w, h = sizes
    w *= ANGLE_ARCSEC
    h *= ANGLE_ARCSEC
    arr = qImageToArray(img)

    totalPlane = None
    for idx in range(4):
        data = arr[:,:,idx].reshape(arr.shape[:2])
        src = images.DiscreteSource(data, w, h, [cx, cy])

        plane = ip.renderImages([src], subSamples=subSample)
        if totalPlane is None:
            totalPlane = np.empty(plane.shape + (4,))
        totalPlane[:,:,idx] = plane

    return arrayToQImage(totalPlane)

def _old_backprojectAndRetrace(imgPlane, img, minXY, maxXY, bpXY,useCPU, subSample, numXYGPU):

    centerX, centerY = [ (minXY[i] + maxXY[i])*0.5 for i in range(2) ]
    widthArcsec, heightArcsec = [ (maxXY[i] - minXY[i]) for i in range(2) ]
    ip = imgPlane
    imgSrc, bl, tr = openglhelper.backProject(ip, img, [centerX, centerY],
                                      [widthArcsec, heightArcsec], [bpXY, bpXY])

    # Recalculate

    w, h = [(tr[i]-bl[i]) for i in [0,1]]
    cx, cy = [ (tr[i]+bl[i])/2 for i in [0,1]]

    if useCPU:
        img = _retraceCPU(ip, imgSrc, [cx, cy], [w, h], subSample)
    else:
        img = imgSrc.mirrored(False, True)
        dims = [ round(numXYGPU*w/h), numXYGPU ] if w < h else [ numXYGPU, round(numXYGPU*h/w) ]
        img = openglhelper.trace(ip, img, [cx, cy], [w, h], dims, subSample)
        img = img.mirrored(False, True)

    return imgSrc, bl, tr, img

def _createRGBLayerForImage(img, fn, layerName, bl, tr, yMirror):
    import imagelayer

    if fn is not None:
        img.save(fn)
        l = imagelayer.RGBImageLayer(fn, layerName)
    else:
        l = imagelayer.RGBImageLayer(img, layerName)

    if yMirror:
        mp = [  [ [0, img.height()], bl],
                [ [0, 0], [ bl[0], tr[1]] ],
                [ [img.width(), img.height()], [tr[0], bl[1]] ],
                [ [img.width(), 0], tr ] ]
    else:
        mp = [  [ [0, 0], bl],
                [ [0, img.height()], [ bl[0], tr[1]] ],
                [ [img.width(), 0], [tr[0], bl[1]] ],
                [ [img.width(), img.height()], tr ] ]

    l.matchToPoints(mp, True)
    return l

def getImageRegions(scene, layers, splitLayers, extra, numImgPix): # Uses aspect ratio

    exportTimeDelays = False
    exportGroups = False
    imgDat, usedLayers = tools.layersToImagesData(layers, splitLayers, exportGroups, exportTimeDelays, pointsLeftInfo=None, ignoreRemainingPoints=True)

    borders = [ ]
    for i in range(imgDat.getNumberOfImages()):
        border = None
        if imgDat.getNumberOfImagePoints(i) < 3: # Not enough points for triangulation or hull, use first point
            border = [ imgDat.getImagePointPosition(i, 0) ]
        else:
            for fn in [ imgDat.getBorder, imgDat.getConvexHull ]:
                try:
                    border = fn(i)
                except Exception as e:
                    print("Warning:", e)

        if not border:
            raise Exception("Unable to get a border for image {}".format(i+1))
        borders.append(border)

    import grale.images as images
    borders = [ np.array(images.enlargePolygon(b, extra))/ANGLE_ARCSEC for b in borders ]

    imgs = [ ]
    for idx in range(len(borders)):
        border = borders[idx]
        img = scene.getSceneRegionImage_minMax(border.min(0), border.max(0), [ numImgPix, None], border)
        imgs.append(img)

    return list(zip(borders, imgs)), usedLayers

def backprojectAndRetrace(imgPlane, bordersAndImages,
                       numBPPix = 1024, # same used in x and y direction, is this ok? perhaps we'd lose information otherwise?
                       numRetracePix = 1024, # Again scaled according to aspect ratio
                       numResample = 1,
                       relensSeparately = True,
                       overWriteFiles = False,
                       bpFileNameTemplate = "img_{srcidx}_backproj.png",
                       bpLayerNameTemplate = "Source shape for image {srcidx}: {fn}",
                       relensFileNameTemplate = "img_{srcidx}_to_{tgtidx}_relensed.png",
                       relensLayerNameTemplate = "Relensed source from image {srcidx} to {tgtidx}: {fn}",
                       newImageDir = None,
                       progressCallback = None):

    ip = imgPlane
    numGPUXY = numRetracePix

    if newImageDir is None:
        newImageDir = os.getcwd()
    if progressCallback is None:
        def dummyCb(msg):
            pass
        progressCallback = dummyCb

    newLayers = [ ]

    if not overWriteFiles: # Check what would be overwritten
        filesToWrite = [ ]
        if relensSeparately:
            for idx in range(len(bordersAndImages)):
                filesToWrite.append(os.path.join(newImageDir, bpFileNameTemplate.format(srcidx=idx+1, tgtidx=0)))

                for tgtidx in range(len(bordersAndImages)):
                    filesToWrite.append(os.path.join(newImageDir, relensFileNameTemplate.format(srcidx=idx+1, tgtidx=tgtidx+1)))
        else:
            for idx in range(len(bordersAndImages)):
                for tmpl in [ bpFileNameTemplate, relensFileNameTemplate ]:
                    filesToWrite.append(os.path.join(newImageDir, tmpl.format(srcidx=idx+1, tgtidx=0)))

        filesToOverwrite = [ fn for fn in filesToWrite if os.path.exists(fn) ]
        if filesToOverwrite:
            raise Exception(f"{len(filesToOverwrite)} files would be overwritten, first are:\n" + "\n".join(filesToOverwrite[:10]))

        if len(set(filesToWrite)) != len(filesToWrite):
            raise Exception("Some output files would overwrite each other")

    def addImg(img, srcidx, tgtidx, bl, tr, isSrc):
        # TODO: yMirror is unused; did I forget something or was this not needed after all?
        #yMirror = True if isSrc else False
        fnTemplate = bpFileNameTemplate if isSrc else relensFileNameTemplate
        layerNameTemplate = bpLayerNameTemplate if isSrc else relensLayerNameTemplate

        fn = None if fnTemplate is None else fnTemplate.format(srcidx=srcidx+1, tgtidx=tgtidx+1)
        newLayers.append(_createRGBLayerForImage(
            img, fn, layerNameTemplate.format(srcidx=srcidx+1, tgtidx=tgtidx+1, fn=fn),
            bl, tr, isSrc))

    srcAreas = []
    for idx in range(len(bordersAndImages)):
        border, img = bordersAndImages[idx]

        if ip:
            progressCallback(f"Creating source shape from image {idx+1}")
            centerX, centerY = 0.5*(border.max(0)+border.min(0))
            widthArcsec, heightArcsec = border.max(0)-border.min(0)
            imgSrc, srcbl, srctr = openglhelper.backProject(ip, img, [centerX, centerY],
                                              [widthArcsec, heightArcsec], [numBPPix, numBPPix])

            srcbl, srctr = np.array(srcbl), np.array(srctr)
        else:
            imgSrc = img.mirrored(False, True)
            srcbl = border.min(0)
            srctr = border.max(0)

        srcCtr = (srcbl+srctr)*0.5
        srcSize = srctr-srcbl

        srcAreas.append([srcbl, srctr])

        addImg(imgSrc, idx, -1, srcbl, srctr, True)

        if numRetracePix <= 0:
            continue

        def getDims(tgtBl, tgtTr):
            tgtW, tgtH = tgtTr-tgtBl
            return [ round(numGPUXY*tgtW/tgtH), numGPUXY ] if tgtW < tgtH else [ numGPUXY, round(numGPUXY*tgtH/tgtW) ]

        if relensSeparately:
            for tgtidx in range(len(bordersAndImages)):
                tgtBorder = bordersAndImages[tgtidx][0]
                tgtBl, tgtTr = tgtBorder.min(0), tgtBorder.max(0)

                dims = getDims(tgtBl, tgtTr)
                tmpImg = imgSrc.mirrored(False, True)

                progressCallback(f"Re-tracing image {tgtidx+1} based on source shape from image {idx+1}")
                tmpImg = openglhelper.trace(ip, tmpImg, srcCtr, srcSize, dims, numResample, tgtBl, tgtTr)
                imgRelens = tmpImg.mirrored(False, True)

                addImg(imgRelens, idx, tgtidx, tgtBl, tgtTr, False)
        else:

            tgtidx = -1

            ri = ip.getRenderInfo()
            tgtBl = np.array(ri["bottomleft"])/ANGLE_ARCSEC
            tgtTr = np.array(ri["topright"])/ANGLE_ARCSEC

            dims = getDims(tgtBl, tgtTr)
            tmpImg = imgSrc.mirrored(False, True)

            progressCallback(f"Re-tracing entire image plane based on source shape from image {idx+1}")
            tmpImg = openglhelper.trace(ip, tmpImg, srcCtr, srcSize, dims, numResample, tgtBl, tgtTr)
            imgRelens = tmpImg.mirrored(False, True)

            addImg(imgRelens, idx, tgtidx, tgtBl, tgtTr, False)

    return newLayers, srcAreas


import requests as req

def API():
    response = req.get('https://api.evaly.com.bd/core/custom/shops/?page=1&limit=200')
    if response.status_code == 200:
        content = response.json()
        cards = ''

        for dic in content['data']:
            card = """

Name: {}

Number: {}

""".format(dic['shop_image'], dic['shop_name'], dic['shop_name'], dic['contact_number'])

            cards = cards + card + "\n\t\t\t\t\t"
        return cards

    else:
        print('{} is not connect'.format(response.status_code))

def Read():
    with open('index.html', 'r', encoding="UTF-8") as file:
        doctype = file.read()
        return doctype

def Write():
    with open('index1.html', 'a+', encoding="UTF-8") as file:
        str = ""
        table = API()
        lists = list(Read())
        lists.insert(-22, table)
        for x in lists:
            str = str + x

        print(table)

        file.write(str)

Write()