import asyncio
import json
import re
import unittest
from typing import Awaitable, Optional
from unittest.mock import AsyncMock, MagicMock, patch

from aioresponses import aioresponses

from hummingbot.client.config.client_config_map import ClientConfigMap
from hummingbot.client.config.config_helpers import ClientConfigAdapter
from hummingbot.connector.exchange.kucoin import kucoin_constants as CONSTANTS, kucoin_web_utils as web_utils
from hummingbot.connector.exchange.kucoin.kucoin_api_user_stream_data_source import KucoinAPIUserStreamDataSource
from hummingbot.connector.exchange.kucoin.kucoin_auth import KucoinAuth
from hummingbot.connector.exchange.kucoin.kucoin_exchange import KucoinExchange
from hummingbot.connector.test_support.network_mocking_assistant import NetworkMockingAssistant
from hummingbot.core.api_throttler.async_throttler import AsyncThrottler

class TestKucoinAPIUserStreamDataSource(unittest.TestCase):
    # the level is required to receive logs from the data source logger
    level = 0

    @classmethod
    def setUpClass(cls) -> None:
        super().setUpClass()
        cls.ev_loop = asyncio.get_event_loop()
        cls.base_asset = "COINALPHA"
        cls.quote_asset = "HBOT"
        cls.trading_pair = f"{cls.base_asset}-{cls.quote_asset}"
        cls.api_key = "someKey"
        cls.api_passphrase = "somePassPhrase"
        cls.api_secret_key = "someSecretKey"

    def setUp(self) -> None:
        super().setUp()
        self.log_records = []
        self.listening_task: Optional[asyncio.Task] = None
        self.mocking_assistant = NetworkMockingAssistant()

        self.throttler = AsyncThrottler(CONSTANTS.RATE_LIMITS)
        self.mock_time_provider = MagicMock()
        self.mock_time_provider.time.return_value = 1000
        self.auth = KucoinAuth(
            self.api_key,
            self.api_passphrase,
            self.api_secret_key,
            time_provider=self.mock_time_provider)

        client_config_map = ClientConfigAdapter(ClientConfigMap())
        self.connector = KucoinExchange(
            client_config_map=client_config_map,
            kucoin_api_key="",
            kucoin_passphrase="",
            kucoin_secret_key="",
            trading_pairs=[],
            trading_required=False)

        self.data_source = KucoinAPIUserStreamDataSource(
            auth=self.auth,
            trading_pairs=[self.trading_pair],
            connector=self.connector,
            api_factory=self.connector._web_assistants_factory)

        self.data_source.logger().setLevel(1)
        self.data_source.logger().addHandler(self)

    def tearDown(self) -> None:
        self.listening_task and self.listening_task.cancel()
        super().tearDown()

    def handle(self, record):
        self.log_records.append(record)

    def _is_logged(self, log_level: str, message: str) -> bool:
        return any(record.levelname == log_level and record.getMessage() == message
                   for record in self.log_records)

    def async_run_with_timeout(self, coroutine: Awaitable, timeout: int = 1):
        ret = self.ev_loop.run_until_complete(asyncio.wait_for(coroutine, timeout))
        return ret

    @staticmethod
    def get_listen_key_mock():
        listen_key = {
            "code": "200000",
            "data": {
                "token": "someToken",
                "instanceServers": [
                    {
                        "endpoint": "wss://someEndpoint",
                        "encrypt": True,
                        "protocol": "websocket",
                        "pingInterval": 18000,
                        "pingTimeout": 10000,
                    }
                ]
            }
        }
        return listen_key

    @aioresponses()
    @patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock)
    @patch("hummingbot.connector.exchange.kucoin.kucoin_web_utils.next_message_id")
    def test_listen_for_user_stream_subscribes_to_orders_and_balances_events(self, mock_api, id_mock, ws_connect_mock):
        id_mock.side_effect = [1, 2]
        url = web_utils.private_rest_url(path_url=CONSTANTS.PRIVATE_WS_DATA_PATH_URL)

        resp = {
            "code": "200000",
            "data": {
                "instanceServers": [
                    {
                        "endpoint": "wss://test.url/endpoint",
                        "protocol": "websocket",
                        "encrypt": True,
                        "pingInterval": 50000,
                        "pingTimeout": 10000
                    }
                ],
                "token": "testToken"
            }
        }
        mock_api.post(url, body=json.dumps(resp))

        ws_connect_mock.return_value = self.mocking_assistant.create_websocket_mock()

        result_subscribe_trades = {
            "type": "ack",
            "id": 1
        }
        result_subscribe_diffs = {
            "type": "ack",
            "id": 2
        }

        self.mocking_assistant.add_websocket_aiohttp_message(
            websocket_mock=ws_connect_mock.return_value,
            message=json.dumps(result_subscribe_trades))
        self.mocking_assistant.add_websocket_aiohttp_message(
            websocket_mock=ws_connect_mock.return_value,
            message=json.dumps(result_subscribe_diffs))

        output_queue = asyncio.Queue()

        self.listening_task = self.ev_loop.create_task(self.data_source.listen_for_user_stream(output=output_queue))

        self.mocking_assistant.run_until_all_aiohttp_messages_delivered(ws_connect_mock.return_value)

        sent_subscription_messages = self.mocking_assistant.json_messages_sent_through_websocket(
            websocket_mock=ws_connect_mock.return_value)

        self.assertEqual(2, len(sent_subscription_messages))
        expected_orders_subscription = {
            "id": 1,
            "type": "subscribe",
            "topic": "/spotMarket/tradeOrders",
            "privateChannel": True,
            "response": False
        }
        self.assertEqual(expected_orders_subscription, sent_subscription_messages[0])
        expected_balances_subscription = {
            "id": 2,
            "type": "subscribe",
            "topic": "/account/balance",
            "privateChannel": True,
            "response": False
        }
        self.assertEqual(expected_balances_subscription, sent_subscription_messages[1])

        self.assertTrue(self._is_logged(
            "INFO",
            "Subscribed to private order changes and balance updates channels..."
        ))

    @aioresponses()
    @patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock)
    def test_listen_for_user_stream_get_listen_key_successful_with_user_update_event(self, mock_api, mock_ws):
        url = web_utils.private_rest_url(path_url=CONSTANTS.PRIVATE_WS_DATA_PATH_URL)
        regex_url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?"))

        mock_response = self.get_listen_key_mock()
        mock_api.post(regex_url, body=json.dumps(mock_response))

        mock_ws.return_value = self.mocking_assistant.create_websocket_mock()
        order_event = {
            "type": "message",
            "topic": "/spotMarket/tradeOrders",
            "subject": "orderChange",
            "channelType": "private",
            "data": {

                "symbol": "KCS-USDT",
                "orderType": "limit",
                "side": "buy",
                "orderId": "5efab07953bdea00089965d2",
                "type": "open",
                "orderTime": 1593487481683297666,
                "size": "0.1",
                "filledSize": "0",
                "price": "0.937",
                "clientOid": "1593487481000906",
                "remainSize": "0.1",
                "status": "open",
                "ts": 1593487481683297666
            }
        }
        self.mocking_assistant.add_websocket_aiohttp_message(mock_ws.return_value, json.dumps(order_event))

        msg_queue = asyncio.Queue()
        self.listening_task = self.ev_loop.create_task(
            self.data_source.listen_for_user_stream(msg_queue)
        )

        msg = self.async_run_with_timeout(msg_queue.get())
        self.assertEqual(order_event, msg)
        mock_ws.return_value.ping.assert_called()

    @aioresponses()
    @patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock)
    def test_listen_for_user_stream_does_not_queue_pong_payload(self, mock_api, mock_ws):
        url = web_utils.private_rest_url(path_url=CONSTANTS.PRIVATE_WS_DATA_PATH_URL)
        regex_url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?"))

        mock_response = self.get_listen_key_mock()

        mock_pong = {
            "id": "1545910590801",
            "type": "pong"
        }
        mock_api.post(regex_url, body=json.dumps(mock_response))

        mock_ws.return_value = self.mocking_assistant.create_websocket_mock()
        self.mocking_assistant.add_websocket_aiohttp_message(mock_ws.return_value, json.dumps(mock_pong))

        msg_queue = asyncio.Queue()
        self.listening_task = self.ev_loop.create_task(
            self.data_source.listen_for_user_stream(msg_queue)
        )

        self.mocking_assistant.run_until_all_aiohttp_messages_delivered(mock_ws.return_value)

        self.assertEqual(0, msg_queue.qsize())

    @aioresponses()
    @patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock)
    @patch("hummingbot.core.data_type.user_stream_tracker_data_source.UserStreamTrackerDataSource._sleep")
    def test_listen_for_user_stream_connection_failed(self, mock_api, sleep_mock, mock_ws):
        url = web_utils.private_rest_url(path_url=CONSTANTS.PRIVATE_WS_DATA_PATH_URL)
        regex_url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?"))

        mock_response = self.get_listen_key_mock()
        mock_api.post(regex_url, body=json.dumps(mock_response))

        mock_ws.side_effect = Exception("TEST ERROR.")
        sleep_mock.side_effect = asyncio.CancelledError  # to finish the task execution

        msg_queue = asyncio.Queue()
        try:
            self.async_run_with_timeout(self.data_source.listen_for_user_stream(msg_queue))
        except asyncio.CancelledError:
            pass

        self.assertTrue(
            self._is_logged("ERROR",
                            "Unexpected error while listening to user stream. Retrying after 5 seconds..."))

    @aioresponses()
    @patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock)
    @patch("hummingbot.core.data_type.user_stream_tracker_data_source.UserStreamTrackerDataSource._sleep")
    def test_listen_for_user_stream_iter_message_throws_exception(self, mock_api, sleep_mock, mock_ws):
        url = web_utils.private_rest_url(path_url=CONSTANTS.PRIVATE_WS_DATA_PATH_URL)
        regex_url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?"))

        mock_response = self.get_listen_key_mock()
        mock_api.post(regex_url, body=json.dumps(mock_response))

        msg_queue: asyncio.Queue = asyncio.Queue()
        mock_ws.return_value = self.mocking_assistant.create_websocket_mock()
        mock_ws.return_value.receive.side_effect = Exception("TEST ERROR")
        sleep_mock.side_effect = asyncio.CancelledError  # to finish the task execution

        try:
            self.async_run_with_timeout(self.data_source.listen_for_user_stream(msg_queue))
        except asyncio.CancelledError:
            pass

        self.assertTrue(
            self._is_logged(
                "ERROR",
                "Unexpected error while listening to user stream. Retrying after 5 seconds..."))

    @aioresponses()
    @patch("aiohttp.ClientSession.ws_connect", new_callable=AsyncMock)
    @patch("hummingbot.connector.exchange.kucoin.kucoin_web_utils.next_message_id")
    @patch("hummingbot.connector.exchange.kucoin.kucoin_api_user_stream_data_source.KucoinAPIUserStreamDataSource"
           "._time")
    def test_listen_for_user_stream_sends_ping_message_before_ping_interval_finishes(
            self,
            mock_api,
            time_mock,
            id_mock,
            ws_connect_mock):

        id_mock.side_effect = [1, 2, 3, 4]
        time_mock.side_effect = [1000, 1100, 1101, 1102]  # Simulate first ping interval is already due
        url = web_utils.private_rest_url(path_url=CONSTANTS.PRIVATE_WS_DATA_PATH_URL)

        resp = {
            "code": "200000",
            "data": {
                "instanceServers": [
                    {
                        "endpoint": "wss://test.url/endpoint",
                        "protocol": "websocket",
                        "encrypt": True,
                        "pingInterval": 20000,
                        "pingTimeout": 10000
                    }
                ],
                "token": "testToken"
            }
        }
        mock_api.post(url, body=json.dumps(resp))

        ws_connect_mock.return_value = self.mocking_assistant.create_websocket_mock()

        result_subscribe_trades = {
            "type": "ack",
            "id": 1
        }
        result_subscribe_diffs = {
            "type": "ack",
            "id": 2
        }

        self.mocking_assistant.add_websocket_aiohttp_message(
            websocket_mock=ws_connect_mock.return_value,
            message=json.dumps(result_subscribe_trades))
        self.mocking_assistant.add_websocket_aiohttp_message(
            websocket_mock=ws_connect_mock.return_value,
            message=json.dumps(result_subscribe_diffs))

        output_queue = asyncio.Queue()

        self.listening_task = self.ev_loop.create_task(self.data_source.listen_for_user_stream(output=output_queue))

        self.mocking_assistant.run_until_all_aiohttp_messages_delivered(ws_connect_mock.return_value)

        sent_messages = self.mocking_assistant.json_messages_sent_through_websocket(
            websocket_mock=ws_connect_mock.return_value)

        expected_ping_message = {
            "id": 3,
            "type": "ping",
        }
        self.assertEqual(expected_ping_message, sent_messages[-1])


""" Promoter models for ecommerce app """

import datetime
import logging

from django.contrib.contenttypes import generic
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ValidationError
from django.core.validators import MinValueValidator
from django.db import models
from django.utils.translation import ugettext_lazy as _

from common.utils import generate_guid
from ecommerce.models.order_models import Order
from ecommerce.validators import require_date_not_past, require_percent

LOG = logging.getLogger('ten.%s' % __name__)
LOG.setLevel(logging.ERROR)

PROMO_TYPE_CHOICES = (
    ('1', _('% discount')),
    ('2', _('Fixed amount discount')),
    ('3', _('Fixed amount cost')),
)
USE_METHOD_CHOICES = (
    ('1', _('No limitation')),
    ('2', _('This promo can be used once per advertiser')),
    ('3', _('This promo can only be ever used once (like a gift certificate)')),
    ('4', _('Can be used a fixed number of times per month')),
)
CODE_METHOD_CHOICES = (
    ('one', _('Everyone uses the same code')),
    ('unique', _('Everyone gets his own unique code')),
)

def now_plus_365():
    """ Return this date plus 365 days. """
    return datetime.date.today() + datetime.timedelta(days=365)

class PromoterManager(models.Manager):
    """ Default manager for Promoter class."""
    def get_by_natural_key(self, name):
        """ Sane serialization. """
        return self.get(name=name)

class Promoter(models.Model):
    """ A way to describe a user who has promotions and receives a cut. """
    name = models.CharField(max_length=48, unique=True, db_index=True)
    content_type = models.ForeignKey(ContentType, blank=True, null=True,
        help_text=_("Which kind of object is this promoter related to?"))
    object_id = models.PositiveIntegerField(_('Object id'), blank=True,
        null=True,
        help_text=_("Which instance is this promoter related to?"))
    related_object = generic.GenericForeignKey(ct_field="content_type",
        fk_field="object_id")
    guid = models.CharField(max_length=40, unique=True, null=True, blank=True,
        help_text=_("This gets generated automatically."))
    is_approved = models.BooleanField(default=False)
    is_active = models.BooleanField(default=False)
    is_paid_traffic = models.BooleanField(default=False)
    start_date = models.DateField(_('date promoter starts'),
        default=datetime.date.today)
    end_date = models.DateField(_('date promoter ends'), blank=True, null=True)
    promoter_cut_percent = models.DecimalField(max_digits=5, decimal_places=2,
        default=0, validators=[require_percent])
    create_datetime = models.DateTimeField(_('date/time created'),
        auto_now_add=True)
    objects = PromoterManager()

    class Meta:
        app_label = 'ecommerce'
        verbose_name = _("Promoter")
        verbose_name_plural = _("Promoters")

    def __unicode__(self):
        return u'%s' % self.name

    def natural_key(self):
        """ Sane serialization. """
        return (self.name,)

    def save(self, *args, **kwargs):
        if not self.id:
            self.guid = generate_guid()
        super(Promoter, self).save(*args, **kwargs)

    def clean(self):
        if not self.start_date:
            self.start_date = datetime.date.today()
        if not self.id:
            require_date_not_past(self.start_date)
            if self.end_date:
                require_date_not_past(self.end_date)
        if self.end_date and self.end_date <= self.start_date:
            raise ValidationError(_('End date must be after start date.'))

    def delete(self, *args, **kwargs):
        if PromotionCode.objects.select_related('promotion__promoter').filter(
                promotion__promoter=self,
                used_count__gt=0
            ).exists():
            error_message = \
                'Cannot delete a promoter having a promotion that has been used'
            LOG.error('%s %s' % (error_message, self.id))
            raise ValidationError(_(error_message))
        super(Promoter, self).delete(*args, **kwargs)

class PromotionManager(models.Manager):
    """ Default manager for Promotion class."""
    def get_by_natural_key(self, name):
        """ Sane serialization. """
        return self.get(name=name)

class Promotion(models.Model):
    """ A product promotion. """
    promoter = models.ForeignKey(Promoter, related_name='promotions')
    product = models.ManyToManyField('ecommerce.Product',
        related_name='promotions')
    name = models.CharField(max_length=48, unique=True, db_index=True)
    description = models.CharField(max_length=64)
    promo_type = models.CharField(max_length=1, choices=PROMO_TYPE_CHOICES)
    promo_amount = models.SmallIntegerField(max_length=3, default=0,
        validators=[MinValueValidator(0)])
    use_method = models.CharField(max_length=1, choices=USE_METHOD_CHOICES)
    monthly_usages_allowed = models.SmallIntegerField(max_length=3,
        default=0, validators=[MinValueValidator(0)],
        help_text=_("Ignored except for 'x times per month' promos"))
    code_method = models.CharField(max_length=16, choices=CODE_METHOD_CHOICES)
    is_active = models.BooleanField(default=True)
    start_date = models.DateField(_('date promotion starts'),
        default=datetime.date.today)
    end_date = models.DateField(_('date promotion ends'), default=now_plus_365,
        validators=[require_date_not_past])
    create_datetime = models.DateTimeField(_('Date/time Created'),
        auto_now_add=True)
    objects = PromotionManager()

    class Meta:
        app_label = 'ecommerce'
        verbose_name = _("Promotion")
        verbose_name_plural = _("Promotions")

    def __unicode__(self):
        return u'Promotion %s: %s' % (self.id, self.description)

    def natural_key(self):
        """ Sane serialization. """
        return (self.name,)

    def clean(self):
        if self.id:
            if self.promotion_codes.filter(used_count__gt=0).exists():
                self.check_modify_used_promo()
        else:
            # This rule only applies on create.
            require_date_not_past(self.start_date)
        if self.end_date <= self.start_date:
            raise ValidationError(_('End date must be after start date.'))
        # 0% off = $0 off. Normalize to the former.
        if self.promo_amount == 0 and self.promo_type == '2':
            self.promo_type = '1'
        if self.use_method == '3':
            if self.code_method == 'unique':
                raise ValidationError(_("""For promotions that can only ever be
                    used once (like a gift certificate), selecting unique codes
                    is not valid."""))
        if self.use_method == '4':
            if self.monthly_usages_allowed == 0:
                raise ValidationError(_("""For monthly usage promotions, specify
                    number of uses per month allowed."""))
        else:
            self.monthly_usages_allowed = 0

    def check_modify_used_promo(self):
        """ Prevent modifying significant terms of a used promo. """
        original = Promotion.objects.get(id=self.id)
        error_msg = "Cannot change %s once the promo has been used."
        if self.promoter != original.promoter:
            raise ValidationError(_(error_msg % 'promoter'))
        elif self.promo_type != original.promo_type:
            raise ValidationError(_(error_msg % 'promoter type'))
        elif self.promo_amount != original.promo_amount:
            raise ValidationError(_(error_msg % 'promo amount'))
        elif self.use_method != original.use_method:
            raise ValidationError(_(error_msg % 'use method'))
        elif self.code_method != original.code_method:
            raise ValidationError(_(error_msg % 'code method'))

    def delete(self, *args, **kwargs):
        if PromotionCode.objects.filter(
                promotion=self, used_count__gt=0).exists():
            error_message = 'Cannot delete a promotion with a used code.'
            LOG.error('%s %s' % (error_message, self.id))
            raise ValidationError(_(error_message))
        super(Promotion, self).delete(*args, **kwargs)

    def can_be_used(self):
        """ Validate promotion for instance """
        if not self.is_active:
            raise ValidationError(_('Promotion is not active.'))
        if not self.promoter.is_active:
            raise ValidationError(_('Promoter is not active.'))
        if not self.promoter.is_approved:
            raise ValidationError(_('Promoter is not approved.'))
        today = datetime.date.today()
        if today < self.start_date:
            raise ValidationError(_('This promotion has not started yet.'))
        elif today > self.end_date:
            raise ValidationError(_('This promotion is already over.'))
        if today < self.promoter.start_date:
            raise ValidationError(_('This promoter is not valid yet.'))
        elif self.promoter.end_date and today > self.promoter.end_date:
            raise ValidationError(_('This promoter is no longer valid.'))
        if not self.product.filter(is_active=True).exists():
            raise ValidationError(_(
                'This promotion is not for a product that is active.'))
        LOG.debug('promotion use method %s' % self.use_method)
        self.can_be_used_by_use_method()

    def can_be_used_by_use_method(self):
        """ Validate based on use_method. """
        if self.use_method == '3':
            # Promotion can only be used once ever, like a gift certificate.
            used_count = self.promotion_codes.aggregate(
                    sum_used=models.Sum('used_count')
                )['sum_used']
            LOG.debug('promotion used_count %s' % used_count)
            if used_count > 0:
                raise ValidationError(_('This promotion is used up.'))
        elif self.use_method == '4':
            # Promotion can be used max x times per month.
            start_month = datetime.datetime.combine(
                datetime.date.today().replace(day=1), datetime.time())
            used_count = Order.objects.filter(
                    promotion_code__in=self.promotion_codes.all(),
                    create_datetime__gt=start_month
                ).count()
            LOG.debug('promotion monthly usages allowed %s'
                % self.monthly_usages_allowed)
            LOG.debug('used count %s' % used_count)
            if used_count >= self.monthly_usages_allowed:
                error_message = 'This monthly promotion is used up.'
                LOG.debug(error_message)
                raise ValidationError(_(error_message))

    def can_be_used_by_advertiser(self, order):
        """ Validate this promotion can be used by this advertiser. """
        if self.use_method == '2':
            # Promo can be used at most once by this advertiser.
            if Order.objects.exclude(id=order.id).filter(
                        promotion_code__in=self.promotion_codes.all(),
                        order_items__business__advertiser=order.order_items.all(
                            )[0].business.advertiser
                    ).count():
                error_message = 'This promotion has already been used.'
                LOG.debug(error_message)
                raise ValidationError(_(error_message))

class PromotionCodeManager(models.Manager):
    """ Default manager for PromotionCode class."""
    def get_by_natural_key(self, code):
        """ Sane serialization. """
        return self.get(code=code)

class PromotionCode(models.Model):
    """ The code a customer enters to receive a promotion. """
    # TO DO: How is this associated with an advertiser who:
    # is eligible to use it, when code is unique per advertiser?
    # has used it, when code is for everyone and usable once per advertiser?
    promotion = models.ForeignKey(Promotion, related_name='promotion_codes')
    code = models.CharField(max_length=64, unique=True, db_index=True)
    used_count = models.PositiveSmallIntegerField(max_length=8, default=0,
        editable=False,
        help_text=_('How many times has this promo been used?'))
    objects = PromotionCodeManager()

    class Meta:
        app_label = 'ecommerce'
        verbose_name = _("Promotion Code")
        verbose_name_plural = _("Promotion Codes")

    def __unicode__(self):
        return u'%s' % self.code

    def natural_key(self):
        """ Sane serialization. """
        return (self.code,)

    def delete(self, *args, **kwargs):
        if self.used_count > 0:
            error_message = 'Cannot delete a promotion code that has been used:'
            LOG.error('%s %s' % (error_message, self.code))
            raise ValidationError(_(error_message))
        super(PromotionCode, self).delete(*args, **kwargs)

    def clean_code(self):
        """ Checks if a PromoCode exists. """
        if PromotionCode.objects.filter(code__iexact=self.code):
            return True
        else:
            return False

    def decrement_used_count(self):
        """ Decrement used_count of this promotion_code. """
        self.used_count = models.F('used_count') - 1
        self.save()

    def increment_used_count(self):
        """ Increment used_count of this promotion_code. """
        self.used_count = models.F('used_count') + 1
        self.save()
        LOG.debug("incremented promo '%s' used count" % self.code)


import conf
import errors
from model import display, autocommit
from api import get, post, delete, AdminApi, put
from api.check_params import check_params
from api.validator import IntRange, String, TokenId, List, Choose, LocalizedName, ModelId, Visibility, Integer
from api.admin.role import TokenAdmin, TokenAccount, TokenManager
from model import Category, Service, Measure
from os_interfaces.openstack_wrapper import openstack
from novaclient.exceptions import NotFound

CategoryList = List(Choose(list(Category.list().keys())))
MeasureTime = Choose({measure.measure_id: measure for measure in Measure.list(Measure.TIME)})
ServiceIdExpand = ModelId(Service, errors.ServiceNotFound)
ServiceId = ModelId(Service, errors.ServiceNotFound, expand=False)

class ServiceApi(AdminApi):
    @get("category/")
    @check_params(
        token=TokenId,
    )
    def service_category_list(self):
        """
        Returns list of categories with localized names

        :return list category_list: List of dict with category

        **Example**::

             {"category_list": [
                {"category_id": "net", "localized_name": {"ru": "Сеть", "en": "Network"}},
                {"category_id": "storage", "localized_name": {"ru": "Хранение данных", "en": "Storage"}},
                {"category_id": "vm", "localized_name": {"ru": "Виртуальные машины", "en": "Virtual machine"}},
                {"category_id": "custom", "localized_name": {"ru": "Дополнительные", "en": "Custom"}}]}

        """
        return {"category_list": display(Category.list().values())}

    @get("measure/")
    @check_params(
        token=TokenId,
        measure_type=Choose(Measure.TYPES)
    )
    def measure_list(self, measure_type=None):
        """
        Returns list of measures

        :param String measure_type: Return measures only specified type. It can be "time", "quant" or 'time_quant'
        :return list measure_list: List of dict with measure info

        **Example**::

            {
                "measure_list": [
                   {"measure_type": "time", "localized_name": {"ru": "час", "en": "hour"}, "measure_id": "hour"},
                   {"measure_type": "time", "localized_name": {"ru": "месяц", "en": "month"}, "measure_id": "month"}
                ]
            }
        """
        return {"measure_list": display(Measure.list(measure_type))}

    @post("service/custom/")
    @check_params(
        token=TokenAccount,
        localized_name=LocalizedName,
        description=LocalizedName(requires_default_language=False),
        measure=MeasureTime
    )
    @autocommit
    def new_custom_service(self, localized_name, measure, description=None):
        """
        Add new custom service.

        Parameters must be sent as json object.

        :param LocalizedName localized_name: Dict with name localization. en is mandatory key
                ``{"en": "Name", "ru": "\u0418\u043c\u044f"}``
        :param LocalizedName description: Dict with localized description. (Not mandatory)
        :param Measure measure: Measure id. Only time measure is possible.

        **Example**::

            {"service_info":
                {
                    "mutable": true,
                    "localized_name": {
                        "ru": "",
                        "en": "Test Custom Service"
                    },
                    "deleted": null,
                    "measure": {
                        "localized_name": {
                            "ru": "\u0447\u0430\u0441",
                            "en": "hour"
                        },
                        "measure_type": "time",
                        "measure_id": "hour"
                    },
                    "category": {
                        'localized_name':
                            {'ru': 'Дополнительные', 'en': 'Custom'},
                            'category_id': 'custom'
                        },
                    "service_id": 1,
                    "description": {}
                }
            }

        """
        service = Service.create_custom(localized_name, measure, description)
        return {"service_info": display(service)}

    @staticmethod
    def check_flavor_existence(flavor_info):
        try:
            flavor = openstack.get_nova_flavor(flavor_info['flavor_id'])
            if (flavor_info['vcpus'], flavor_info['ram'], flavor_info['disk']) != \
                    (flavor.vcpus, flavor.ram, flavor.disk):
                raise errors.OsFlavorExistsWithDifferentParams(
                    "Flavor already exists in OS with different parameters: vcpus: %s, ram: %s, disk: %s" % \
                    (flavor.vcpus, flavor.ram, flavor.disk)
                )
        except NotFound:
            return False

        return True

    @post("service/vm/")
    @check_params(
        token=TokenManager,
        flavor_id=String(),
        vcpus=Integer(),
        ram=Integer(),
        disk=Integer(),
        network=Integer(),
        localized_name=LocalizedName,
        description=LocalizedName(requires_default_language=False)
    )
    @autocommit
    def new_vm(self, flavor_id, vcpus, ram, disk, localized_name, network=None, description=None):
        """
        Add new flavor service.

        Parameters must be sent as json object.

        :param LocalizedName localized_name: Dict with name localization. en is mandatory key
                ``{"en": "Name", "ru": "\u0418\u043c\u044f"}``
        :param LocalizedName description: Dict with localized description. (Not mandatory)
        :param flavor_id: Flavor name
        :param vcpus: Number of flavor's vcpus
        :param ram: flavor's RAM amount
        :param disk: flavor's disk size
        :param network: flavor's network

        **Example**::

            {"service_info":
                {
                    "mutable": true,
                    "localized_name": {
                        "ru": "\u0424\u043b\u0430\u0432\u043e\u0440 TestFlavor",
                        "en": "Flavor TestFlavor"
                    },
                    "deleted": null,
                    "measure": {
                        "localized_name": {
                            "ru": "\u0447\u0430\u0441",
                            "en": "hour"
                        },
                        "measure_type": "time",
                        "measure_id": "hour"
                    },
                    "category": {
                        "localized_name": {
                            "ru": "\u0412\u0438\u0440\u0442\u0443\u0430\u043b\u044c\u043d\u044b\u0435 \u043c\u0430\u0448\u0438\u043d\u044b",
                            "en": "Virtual server"
                        },
                        "category_id": "vm"
                    },
                    "service_id": 1,
                    "description": {}
                }
            }

        """
        flavor_info = dict(flavor_id=flavor_id, vcpus=vcpus, ram=ram, disk=disk, network=network)
        mutable = True
        if self.check_flavor_existence(flavor_info):
            mutable = False
        flavor = Service.create_vm(localized_name, description, flavor_info, mutable)
        return {"service_info": display(flavor)}

    @get('service/<service_id>/')
    @check_params(
        token=TokenId,
        service_id=String)
    def get_service(self, service_id):
        """
        Return service description

        :param Id service_id: Service Id

        :return dict service_info: Dict with service parameters

        **Example**::

            {
              "service_info": {
                "service_id": "storage.volume",
                "measure": {
                  "measure_type": "time_quant",
                  "measure_id": "gigabyte*month",
                  "localized_name": {
                    "ru": "Гб*месяц",
                    "en": "Gb*month"
                  }
                },
                "category": {
                  "localized_name": {
                    "ru": "Хранение данных",
                    "en": "Storage"
                  },
                  "category_id": "storage"
                },
                "localized_name": {
                  "ru": "Диск",
                  "en": "Volume"
                }
              }
            }

        """

        service = Service.get_by_id(service_id)
        if not service:
            raise errors.ServiceNotFound()

        return {"service_info": display(service)}

    @delete('service/<service>/')
    @check_params(token=TokenAdmin, service=ServiceIdExpand)
    @autocommit
    def remove(self, service):
        """
        Archive service (only custom services can be deleted)

        :param Id service: Service Id
        :return: None
        """
        service.remove()
        return {}

    @staticmethod
    def paginate_services(services, limit, page):
        services = display(services)
        total = len(services)

        paginated_list = services[(page - 1) * limit:page * limit]

        res = {
            'per_page': limit,
            'total': total,
            'page': page,
            'items': paginated_list
        }
        return res

    # noinspection PyUnusedLocal
    @get('service/')
    @check_params(
        name=String,
        category=CategoryList,
        page=IntRange(1),
        limit=IntRange(1, conf.api.pagination.limit),
        sort=String,  # Sort(User.Meta.sort_fields),
        visibility=Visibility(),
        all_parameters=True
    )
    def list(self, name=None, category=None,
             page=1, limit=conf.api.pagination.limit,
             sort=None, visibility=Visibility.DEFAULT, all_parameters=True):
        """
        Return filtered list of services.

        :param str name: Filter for service name
        :param str category: Filter
        :param int page: page number
        :param int limit: number of items per page
        :param str visibility: Visibility options
                               *visible* - Only active services, [by default]
                               *deleted* - Only removed services.
                               *all* - All services.

        :return List service_list: List of services

        **Example**::

            {
                "service_list": {
                    "total": 20,
                    "per_page": 1,
                    "page": 1,
                    "items": [
                        {
                         "service_id":"net.associated_ip",
                         "localized_name":{ 
                            "ru":"net.associated_ip",
                            "en":"net.associated_ip"
                         },
                         "measure":{
                            "measure_id":"hour",
                            "localized_name":{
                               "ru":"\u0447\u0430\u0441",
                               "en":"hour"
                            },
                            "measure_type":"time"
                         },
                         "category":{
                            "localized_name":{
                               "ru":"\u0421\u0435\u0442\u044c",
                               "en":"Network"
                            },
                            "category_id":"net"
                        }
                    }
                ]
            }
        }

        """
        if category:
            category = frozenset(category)

        services = Service.list(only_categories=category, visibility=visibility)
        return {"service_list": self.paginate_services(services, limit, page)}

    # noinspection PyUnusedLocal
    @put('service/<service>/custom/')
    @check_params(
        token=TokenAccount,
        service=ServiceIdExpand,
        localized_name=LocalizedName,
        description=LocalizedName(requires_default_language=False),
        measure=MeasureTime
    )
    @autocommit
    def update_custom(self, service, localized_name=None, description=None, measure=None):
        """
        Update custom service. (only custom services can be updated)

        Parameters must be sent as json object.

        :param Id service: Service Id
        :param LocalizedName localized_name: Dict with name localization. en is mandatory key
                ``{"en": "Name", "ru": "\u0418\u043c\u044f"}``
        :param LocalizedName description: Dict with localized description.
        :param Measure measure: Measure id. Only time measure is possible.

        :return dict service_info: Dict as returned by
                                  :obj:`GET /0/service/\<service\>/ <view.GET /0/service/\<service\>>`
        """
        updated = service.update_custom(localized_name, description, measure)

        return {"service_info": display(service)}

    @put('service/<service>/vm/')
    @check_params(
        token=TokenAccount,
        service=ServiceIdExpand,
        localized_name=LocalizedName,
        description=LocalizedName(requires_default_language=False),
        measure=MeasureTime,
        flavor_id=String(),
        vcpus=Integer(),
        ram=Integer(),
        disk=Integer(),
        network=Integer(),
    )
    @autocommit
    def update_vm(self, token, service, localized_name=None, description=None,
                  flavor_id=None, vcpus=None, ram=None, disk=None, network=None):
        """
        Update Flavor.

        Parameters must be sent as json object.

        :param ServiceId service: Service Id
        :param LocalizedName localized_name: Dict with name localization. en is mandatory key
                ``{"en": "Name", "ru": "\u0418\u043c\u044f"}``
        :param LocalizedName description: Dict with localized description.
        :param flavor_id: Flavor name
        :param vcpus: Number of flavor's vcpus
        :param ram: flavor's RAM amount
        :param disk: flavor's disk size
        :param network: flavor's network

        :return dict service_info: Dict as returned by
                                  :obj:`GET /0/service/\<service\>/ <view.GET /0/service/\<service\>>`
        """
        flavor_info = dict(flavor_id=flavor_id, vcpus=vcpus, ram=ram, disk=disk, network=network)
        service.update_vm(localized_name, description, flavor_info)

        return {"service_info": display(service)}

    @put('service/<service>/immutable/')
    @check_params(
        token=TokenAccount,
        service=ServiceIdExpand,
    )
    @autocommit
    def immutable(self, service):
        """
        Make service immutable.

        :param Id service: Service Id.
        :return dict service_info: Dict as returned
        by :obj:`GET /0/service/\<service\>/ <view.GET /0/service/\<service\>>`
        """
        service.mark_immutable()

        return {"service_info": display(service)}


import datetime
import unittest

import vcr

from eodclient.errors import (
    InvalidExchangeCodeError
)
from eodclient.exchange import Exchange
from eodclient.symbol import Symbol

class ExchangeTests(unittest.TestCase):
    '''Tests for the exchange'''

    SYMBOL_KEYS = ['Code', 'Name', 'Country', 'Exchange', 'Currency', 'Type']

    @vcr.use_cassette(
        'eodclient/tests/vcr_cassettes/get-symbols.yml',
        filter_query_parameters=['api_token']
    )
    def test_get_symbols(self):
        '''Ensure symbols are returned in correct format from the exchange'''
        exchange_instance = Exchange('JSE')
        response = exchange_instance.get_symbols()

        self.assertIsInstance(response, list)
        self.assertIsInstance(response[0], dict)
        self.assertEqual(
            list(response[0].keys()),
            self.SYMBOL_KEYS
        )

    def test_init_without_exchange_code(self):
        '''Ensure a no code exception is raised when initialising without
        a share code'''
        with self.assertRaises(TypeError):
            exchange_instance = Exchange()

    def test_init_bad_code(self):
        '''Ensure initialisng an exchange with a positional argument fails'''
        with self.assertRaises(InvalidExchangeCodeError):
            exchange_instance = Exchange('XXYP')

    def test_bad_code(self):
        with self.assertRaises(InvalidExchangeCodeError):
            Exchange_instance = Exchange(exchange_code='XXYP')

class SymbolTests(unittest.TestCase):
    '''Tests for the symbol'''

    def test_formatted_date(self):
        '''Ensure formatted date works'''
        str_date = '2018-08-20'
        date_object = Symbol.get_date(str_date)
        self.assertEqual(
            date_object,
            datetime.datetime(2018, 8, 20)
        )

    def test_unformatted_date(self):
        '''Ensure unformatted date fails'''
        str_date = '20-08-2018'
        with self.assertRaises(Exception):
            date_object = Symbol.get_date(str_date)

    def test_int_date(self):
        '''Ensure int date fails'''
        str_date = 2018
        with self.assertRaises(Exception):
            date_object = Symbol.get_date(str_date)

    @vcr.use_cassette(
        'eodclient/tests/vcr_cassettes/ticker-not-found.yml',
        filter_query_parameters=['api_token']
    )
    def test_ticker_not_found(self):
        '''Ensure ticker not found is handled'''
        exchange_instance = Exchange('NZ')
        response = exchange_instance.get_symbols()

        self.assertIsInstance(response, dict)
        self.assertEqual(
            response,
            {'message': 'Ticker Not Found'}
        )


#!/home/hannes/.local/share/miniconda/envs/bioinfo/bin/python3.8

################################################################################
# Author: Hannes Meinert                                                       #
# Created: 08.12.2020                                                          #
# Last Modified: 14.12.2020                                                    #
################################################################################

############################## Tasks ###########################################
# 1. enable for .xml

############################## Import Modules ##################################
import argparse
import re

############################## Initialize Argparse #############################
parser = argparse.ArgumentParser(
        description="This script fetches sequences from a database file in "
                   + "fasta format. Therefore a tab separated input file is "
                   + "require with the accession number at the second position."
                   )
parser.add_argument('-i', '--input', required=True,
                    help="file with accession numbers")
parser.add_argument('-t', '--type', required=True,
                    help="dmnd: output from diamond search, tsv file with "
                        + "acc num at second position;; "
                        + "hmmer: modified output from hmmer, one acc num "
                        + "per line")
parser.add_argument('-d', '--db', required=True,
                    help="fasta file containing all sequences in the database")
parser.add_argument('-o', '--output', required=True,
                    help="name that is used for the output file in .fa format")
args = parser.parse_args()

############################## Process Data ####################################
# extract accession numbers from input file
acc_lst = []
data_dict = {}
if args.type == 'dmnd':
    with open(args.input, 'r') as in_file:
        for line in in_file:
            line = line.strip()
            p_lst = re.split('\t',line)
            acc_lst.append(p_lst[1])
elif args.type == 'hmmer':
    with open(args.input, 'r') as in_file:
        for line in in_file:
            line = line.strip()
            acc_lst.append(line)
else:
    print("The specified type is either not supported or does not exits.\n"
          + "See help for more information.")
    quit(1)
print("Accession numbers scanned.")

# extract sequences from database
total_seq = len(acc_lst)
count = 0
hit = ''
hit_check = False
with open(args.db, 'r') as db_file:
    for line in db_file:
        line = line.strip()
        if re.search('^>', line):
            if hit_check:
                print(str(count) + " of " + str(total_seq) + " fetched")
                hit_check = False
                data_dict[header] = hit
                hit = ''
            # quit when all sequences are fetched
            if count == total_seq:
                print('quit')
                break
            acc = re.split('\s', line)
            acc = acc[0]
            acc = acc.strip('>')
            for acc_num in acc_lst:
                if acc_num == acc:
                    count += 1
                    hit_check = True
                    header = line
                    break
        elif re.search(r'^[A-Z]', line) and hit_check:
            hit += line
print("Fetching sequences finished.")

############################## Generate Output File ############################
with open(args.output, 'w') as out_file:
    for header in data_dict.keys():
        out_file.write(header + '\n' + data_dict[header] + '\n')


import os

import paddle

from lib.metric_stategy import MetricStrategy
from lib.optimizer_stategy import OptimizerStrategy

class Trainer:

    def __init__(self,
                 data_module,
                 model,
                 optimizer_strategy: OptimizerStrategy,
                 metric_strategy: MetricStrategy,
                 epochs=10):

        self.epochs = epochs
        self.global_step = 0

        # 模型
        self.data_module = data_module
        self.model = model
        self.metric_strategy = metric_strategy

        # 优化器策略
        self.lr_scheduler, self.optimizer = optimizer_strategy.get_scheduler_and_optimizer()

    def train(self):
        """训练"""

        for epoch in range(1, self.epochs + 1):
            for step, batch_data in enumerate(self.data_module.train_dataloader, start=1):

                self.global_step += 1
                progress = (epoch, self.global_step, step)

                # 训练
                model_ret = self.model.training_step(batch_data)

                # 评估指标
                self.metric_strategy.compute_train_metric(batch_data, model_ret, progress)

                # 反向传播
                self.backward(model_ret["loss"])

                # 验证集上进行评估
                if self.global_step % 100 == 0:
                    self.evaluate()

        self.save_model()

    @paddle.no_grad()
    def evaluate(self, dev=True):
        """验证/评估"""

        # 进入 eval 模式
        self.model.eval()
        self.metric_strategy.reset()

        if dev:
            dataloader = self.data_module.dev_dataloader
        else:
            dataloader = self.data_module.train_dataloader

        # 在验证集/测试上跑一遍
        batch_count = len(dataloader)
        for step, batch_data in enumerate(dataloader, start=1):
            model_ret = self.model.validation_step(batch_data)
            # 评估指标
            self.metric_strategy.compute_dev_metric(batch_data, model_ret, batch_count == step)

        # 进入 train 模式
        if dev:
            self.model.train()

        # 重置
        self.metric_strategy.reset()

    @paddle.no_grad()
    def test(self):
        """测试"""
        self.evaluate(dev=False)

    def save_model(self, dir_name="ckpt"):

        # 训练结束后，存储模型参数
        save_dir = os.path.join(dir_name, "model_%d" % self.global_step)
        os.makedirs(save_dir)

        # 保存当前模型参数
        paddle.save(self.model.state_dict(), os.path.join(save_dir, 'model_state.pdparams'))
        # 保存tokenizer的词表等
        self.data_module.tokenizer.save_pretrained(save_dir)

    def backward(self, loss):
        loss.backward()
        self.optimizer.step()
        self.lr_scheduler.step()
        self.optimizer.clear_grad()


#!/usr/bin/env python
import rospy
import asyncio
import websockets
import math
from std_msgs.msg import String
import json
from nav_msgs.msg import Odometry
import socket
from geometry_msgs.msg import Twist, Vector3
from threading import Thread

class ServerSocket(Thread):
    def __init__(self):
        Thread.__init__(self)
        self.__server  = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.__server.bind(("127.0.0.1", 32233))
        self.__server.listen(1)
        self.__clients = []

    def run(self):
        print("démarrage du serveur")
        while(True):
            client, clientAddress = self.__server.accept()
            self.__clients.append(ClientProcess(client))
            self.__clients[-1].start()

class ClientProcess(Thread):
    def __init__(self, client):
        Thread.__init__(self)
        self.__client = client
    def run(self):
        print("nouveau client connecté")
        while True:
            response = self.__client.recv(1024)

            self.processRequest(response)

    def processRequest(self, msg):
        try:
            data = json.loads(msg.decode('utf8'))
            #si on reçoit une requêtte (un oprocess à éxecuter sur le robot)
            if data["type"] == "request":
                if data["request"] == "get_position":
                    #send the response

                    response = {
                        "type" : "position",
                        "position" : robot.getPosition(),
                        "diffSpeed" : robot.getVelocity(),
                        "cons" : robot.getCons()
                    }

                    self.send(response)
                elif data["request"] == "start_open_loop":
                    print(msg)
                    robotCons.publish(msg.decode('utf8'))
                elif data["request"] == "start_diff_speed":
                    robotCons.publish(msg.decode('utf8'))
                elif data["request"] == "start_vel_speed":
                    robotCons.publish(msg.decode('utf8'))
                elif data["request"] == "position_cons":
                    robotCons.publish(msg.decode('utf8'))
            #si on reçoit une rerquêtte moteur (des changement à faire sur la carte moteur)
            elif data["type"] == "motor_request":

                robotCons.publish(msg.decode('utf8'))
                pass
        except Exception:
            pass
    def send(self, objMsg):
        self.__client.send( json.dumps(objMsg).encode('utf8') )

class RosNode(Thread):
    def __init__(self):

        Thread.__init__(self)
        rospy.init_node("val_server")
    def run(self):
        print("Initialisation des subscribers")
        rospy.Subscriber("enc_velocity", Odometry, robot.setPosition)
        rospy.Subscriber("robot_consign", Twist, robot.setCons)
        rospy.Subscriber("diff_velocity", Twist, robot.setVelocity)
        rospy.spin()

class RobotCom():
    def __init__(self):
        self.__position = {
            "x" : 0,
            "y" : 0,
            "z" : 0
        }
        self.__DiffVelocity = {
            "left" : 0,
            "right" : 0
        }
        self.__cons = {
            "left" : 0,
            "right" : 0
        }

    def setPosition(self, posOdom):
        #print(posOdom)
        self.__position["x"] = posOdom.pose.pose.position.x
        self.__position["y"] = posOdom.pose.pose.position.y
        sin = posOdom.pose.pose.orientation.w
        cos = posOdom.pose.pose.orientation.z
        self.__position["th"] = math.atan2(sin, cos)*2

    def setVelocity(self, diffVel):
        self.__DiffVelocity["left"] = diffVel.linear.x
        self.__DiffVelocity["right"] = diffVel.linear.y

    def getVelocity(self):
        return self.__DiffVelocity

    def getPosition(self):
        return self.__position
    def getCons(self):
        return self.__cons
    def setCons(self, cons):
        self.__cons["left"] = cons.linear.x
        self.__cons["right"] = cons.linear.y

print(socket.gethostname())
#start server
server = ServerSocket()
server.start()
#communication interrobot
robot = RobotCom()
#start rosnode and subscriber
rosnode = RosNode()
rosnode.start()
#ros publishers
robotCons = rospy.Publisher('server_req', String, queue_size=10)
#motorReq = rospy.Publisher('motor_request', String, queue_size=10)


import argparse
import os.path
import pandas as pd

fio_input_names = [
    'bs', 'iodepth', 'threads', 'lat_min', 'lat_max', 'lat_avg', 'lat_stdev',
    'ops', 'lat_pctl_99.0', 'lat_pctl_99.9', 'lat_pctl_99.99',
    'lat_pctl_99.999', 'bw_avg', 'bw_min', 'bw_max',
    'iops_avg', 'iops_min', 'iops_max', 'cpuload']

fio_nsec_2_usec_names = [
    'lat_min', 'lat_max', 'lat_avg', 'lat_stdev', 'lat_pctl_99.0',
    'lat_pctl_99.9', 'lat_pctl_99.99', 'lat_pctl_99.999']

fio_KiBps_2_Gbps_names = ['bw_avg', 'bw_min', 'bw_max']

KiBpbs_2_Gbps = 1024 * 8 / 1000 / 1000 / 1000

fio_output_names = [
    'threads', 'iodepth', 'bs', 'ops', 'lat_min', 'lat_max', 'lat_avg',
    'lat_stdev', 'lat_pctl_99.0', 'lat_pctl_99.9', 'lat_pctl_99.99',
    'lat_pctl_99.999', 'bw_min', 'bw_max', 'bw_avg',
    'iops_min', 'iops_max', 'iops_avg', 'cpuload']

ib_lat_input_names = [
    'bs', 'ops', 'lat_min', 'lat_max', 'lat_mode', 'lat_avg', 'lat_stdev',
    'lat_pctl_99.0', 'lat_pctl_99.9']

ib_lat_output_names = [
    'bs', 'ops', 'lat_min', 'lat_max', 'lat_avg', 'lat_stdev', 'lat_pctl_99.0',
    'lat_pctl_99.9']

ib_bw_input_names = [
    'threads', 'iodepth', 'bs', 'ops', 'bw_peak', 'bw_avg', 'msg_rate']

ib_bw_output_names = [
    'threads', 'iodepth', 'bs', 'ops', 'bw_avg']

def main():
    parser = argparse.ArgumentParser(
        description='Standardize a CSV (EXPERIMENTAL)')
    parser.add_argument('csv_file', metavar='CSV_FILE',
        help='a CSV log file to process')
    parser.add_argument('--csv_type', metavar='CSV_TYPE', required=True,
        choices=['ib_lat', 'ib_bw', 'fio'], help='a type of the CSV file')
    parser.add_argument('--output_file', metavar='OUTPUT_FILE',
        default='output.csv', help='an output file')
    args = parser.parse_args()

    if args.csv_type == 'ib_lat':
        df = pd.read_csv(args.csv_file, header=0, names=ib_lat_input_names)
        df = df.reindex(columns=ib_lat_output_names)
    elif args.csv_type == 'ib_bw':
        df = pd.read_csv(args.csv_file, header=0, names=ib_bw_input_names)
        df = df.reindex(columns=ib_bw_output_names)
        df = df.apply(lambda x: round(x, 2) \
            if x.name == 'bw_avg' else x)
    else: # fio
        df = pd.read_csv(args.csv_file, header=0, names=fio_input_names)
        # convert nsec to usec
        df = df.apply(lambda x: round(x / 1000, 2) \
            if x.name in fio_nsec_2_usec_names else x)
        # convert KiB/s to Gb/s
        df = df.apply(lambda x: round(x * KiBpbs_2_Gbps, 2) \
            if x.name in fio_KiBps_2_Gbps_names else x)
        df = df.reindex(columns=fio_output_names)

    _, ext = os.path.splitext(args.output_file)
    if ext == '.csv':
        df.to_csv(args.output_file, index=False)
    elif ext == '.json':
        df.to_json(args.output_file, orient='records')
    else:
        raise Exception(f"Unsupported output file extension: {ext}")

if __name__ == "__main__":
    main()


from pydantic import BaseModel
from typing import Optional
from enum import Enum

class Purchase(str, Enum):
    standard = 'standard'
    premium = 'premium'

class SinglePurchase(BaseModel):
    id: Optional[str] = None
    userId: str
    username: str
    purchase: str
    type: str
    price: float

class Period(str, Enum):
    monthly = 'monthly'
    biannual = "biannual"
    annual = "annual"

class PlansIn(BaseModel):
    type: Purchase
    period: Period

class PlansOut(BaseModel):
    userId: str
    username: str
    planType: str
    period: str
    price: float

if __name__ == "__main__":
    ...


# -*- coding: utf -8-*-

from . import config
import uuid
import requests
import threading

def getUUID():
    mac = uuid.UUID(int=uuid.getnode()).hex[-12:]
    return ":".join([mac[e : e + 2] for e in range(0, 11, 2)])

def report(t):
    ReportThread(t).start()

class ReportThread(threading.Thread):
    def __init__(self, t):
        # 需要执行父类的初始化方法
        threading.Thread.__init__(self)
        self.t = t

    def run(self):
        to_report = config.get("statistic", True)
        if to_report:
            try:
                persona = config.get("robot_name_cn", "孙悟空")
                url = "http://livecv.hahack.com:8022/statistic"
                payload = {
                    "type": str(self.t),
                    "uuid": getUUID(),
                    "name": persona,
                    "project": "wukong",
                }
                requests.post(url, data=payload, timeout=3)
            except Exception:
                return


import django

django.setup()

import asyncio
import datetime
import json
import multiprocessing

import aiohttp
from django.db.models import Q
from django.utils import timezone
from main.models import Result, InterestCategory, Coord
from main.models.in_house import ApiKey, Pairs, PairsWithSexAndAge
from django.db.models import Sum

queries = {
    "--": 'SELECT 1 as id,sum(count_of_person) as count_of_person ,coordinate_id '
          'from "result" where age_begin is not null and age_end is not null '
          'and is_male is not NULL and count_of_person != 0 and interest_id = %s group by coordinate_id;',
    "+-": 'SELECT 1 as id,sum(count_of_person) as count_of_person ,coordinate_id '
          'from "result" where age_begin = %s and age_end = %s '
          'and is_male is not NULL and count_of_person != 0 and interest_id = %s group by coordinate_id;',
    "-+": 'SELECT 1 as id,sum(count_of_person) as count_of_person ,coordinate_id '
          'from "result" where age_begin is not null and age_end is not null '
          'and is_male = %s and count_of_person != 0 and interest_id = %s group by coordinate_id;',
    '++': 'SELECT 1 as id,sum(count_of_person) as count_of_person ,coordinate_id '
          'from "result" where age_begin = %s and age_end = %s '
          'and is_male = %s and count_of_person != 0 and interest_id = %s group by coordinate_id;'
}

def pick_points(interest_name: str, sex: str, age: str):
    interest = InterestCategory.objects.using("cache").get(interes_name=interest_name)
    key = ''
    append = []
    if age != "-":
        key += "+"
        from_, to_ = list(map(int, age.split("-")))
        append.append(from_)
        append.append(to_)
    else:
        key += "-"

    if sex != "-":
        key += "+"
        if sex == "м":
            sex = "TRUE"
        else:
            sex = "FALSE"
        append.append(sex)
    else:
        key += "-"

    append.append(interest.id)

    append = tuple(append)
    print(append)
    print(key)
    query = queries[key] % append

    print(query)

    return Result.objects.using('cache').raw(query)

    # query = Q(interest__interes_name=interest_name)
    #
    # if sex:
    #     query &= Q(is_male=sex)
    # else:
    #     query &= Q(is_male__isnull=False)
    #
    # if age:
    #     from_, to_ = list(map(int, age.split("-")))
    #     query &= Q(age_begin=from_, age_end=to_)
    # else:
    #     query &= Q(age_begin__isnull=False, age_end__isnull=True)
    #
    # print(Result.objects.using("cache").filter(query).values('coordinate').annotate(
    #     count_of_person=Sum('count_of_person')).query)
    #
    # points = Result.objects.using("cache").filter(query).values('coordinate') \
    #     .annotate(count_of_person=Sum('count_of_person'))
    #
    # print(points)
    #
    # return points

def get_points_by_interest_name_service(interest_name: str):
    date_10_days_ago = timezone.now() - datetime.timedelta(days=10)
    # left = int(batch) * 1000
    # right = left + 1000
    return Result.objects.filter(interest__interes_name=interest_name, count_of_person__gt=0,
                                 end_date__gte=date_10_days_ago).select_related(
        'coordinate')  # [left:right]

API_URL = "https://api.vk.com/method/ads.getTargetingStats"
LINK_URL = "https://vk.com/dev/ads.getTargetingStats"
LINK_DOMAIN = "vk.com"

async def parser(token, pairs):
    result = []
    for pair in pairs:
        point = pair.point
        interest = pair.interest
        token.is_taken = True
        token.save()
        results_qs = Result.objects.filter(coordinate=point, interest=interest).order_by("-end_date")
        if results_qs and (timezone.now() - results_qs[0].end_date).days < 10:
            return
        entity = Result(begin_date=timezone.now())
        criter = {
            "interest_categories": interest.interes_name,
            "geo_near": f"{point.y},{point.x},500"
        }
        # latitude - широта - y
        # longitude - долгота - x
        json_geo = json.dumps(criter)
        params_dict = {"account_id": token.acc_id, "access_token": token.key, "v": "5.131", "link_url": API_URL,
                       "link_domain": LINK_DOMAIN, "criteria": json_geo}
        # response = requests.post(API_URL, params=params_dict, headers={"User_Agent": generate_user_agent()})
        async with aiohttp.ClientSession() as session:
            async with session.post(API_URL, params=params_dict) as response:
                response_json = await response.json()
                try:
                    print(multiprocessing.current_process())
                except Exception:
                    pass
                print(token.key)
                print(response_json)
                await asyncio.sleep(5)
                if response_json.get('error'):
                    if response_json.get('error_code') == 601:
                        token.is_taken = False
                        token.expired = True
                        token.save()
                    else:
                        continue
                else:
                    response_data = response_json.get('response')
                    entity.interest = interest
                    entity.coordinate = point
                    entity.link = API_URL
                    entity.count_of_person = response_data.get('audience_count')
                    pair = Pairs.objects.get(point=point, interest=interest)
                    pair.last_executions = timezone.now()
                    token.is_taken = False
                    token.save()
                    pair.save()
                    entity.save()

async def parser_info(token, pairs_limit):
    i = pairs_limit[0] + 1000
    date_10_days_ago = timezone.now() - datetime.timedelta(days=10)
    while i <= pairs_limit[1]:
        pairs = PairsWithSexAndAge.objects.filter(
            Q(last_executions=None) | Q(last_executions__lte=date_10_days_ago))[i - 1000:i]
        for pair in pairs:
            point = pair.point
            interest = pair.interest
            token.is_taken = True
            token.save()
            results_qs = Result.objects.filter(coordinate=point, interest=interest, is_male=pair.is_male,
                                               age_begin=pair.age_begin, age_end=pair.age_end).order_by("-end_date")
            if results_qs and (timezone.now() - results_qs[0].end_date).days < 10:
                continue
            entity = Result(begin_date=timezone.now())
            sex = 1
            if pair.is_male:
                sex = 2
            print(str(interest.interes_name))
            criter = {
                "interest_categories": interest.interes_name,
                "geo_near": f"{point.y},{point.x},500",
                "sex": sex,
                "age_from": pair.age_begin,
                "age_to": pair.age_end
            }
            # latitude - широта - y
            # longitude - долгота - x
            json_geo = json.dumps(criter)
            params_dict = {"account_id": token.acc_id, "access_token": token.key, "v": "5.131", "link_url": API_URL,
                           "link_domain": LINK_DOMAIN, "criteria": json_geo}
            async with aiohttp.ClientSession() as session:
                async with session.post(API_URL, params=params_dict) as response:
                    response_json = await response.json()
                    try:
                        print(multiprocessing.current_process())
                    except Exception:
                        pass
                    print(token.key)
                    print(response_json)
                    await asyncio.sleep(5)
                    if response_json.get('error'):
                        if response_json.get('error_code') == 601:
                            token.is_taken = False
                            token.expired = True
                            token.save()
                        else:
                            continue
                    else:
                        try:
                            interest_from_chache = InterestCategory.objects.using('cache').get(
                                id=interest.id)
                            coordinate_from_cache = Coord.objects.using('cache').get(x=point.x, y=point.y)

                            response_data = response_json.get('response')
                            entity.interest = interest
                            entity.coordinate = point
                            entity.link = API_URL
                            entity.is_male = pair.is_male
                            entity.age_begin = pair.age_begin
                            entity.age_end = pair.age_end
                            entity.count_of_person = int(response_data.get('audience_count'))
                            pair.last_executions = timezone.now()

                            try:
                                update_cache = Result.objects.using('cache').get(interest=interest_from_chache,
                                                                                coordinate=coordinate_from_cache,
                                                                                is_male=pair.is_male,
                                                                                age_begin=pair.age_begin,
                                                                                age_end=pair.age_end)
                                update_cache.end_date = timezone.now()
                                update_cache.count_of_person = int(response_data.get('audience_count'))
                                update_cache.save(using='cache')
                                print('updated in cache')
                            except Exception as e:
                                if int(response_data.get('audience_count')) != 0:
                                    entity_to_cache = Result(begin_date=timezone.now())
                                    entity_to_cache.interest = interest_from_chache
                                    entity_to_cache.coordinate = coordinate_from_cache
                                    entity_to_cache.link = API_URL
                                    entity_to_cache.is_male = pair.is_male
                                    entity_to_cache.age_begin = pair.age_begin
                                    entity_to_cache.age_end = pair.age_end
                                    entity_to_cache.count_of_person = int(response_data.get('audience_count'))
                                    entity_to_cache.save(using='cache')
                                    print('cache created')
                            token.is_taken = False
                            token.save()
                            pair.save()
                            entity.save(using='default')
                            print('saved default')
                        except Exception as e:
                            print(e)
        del pairs
        i += 1000

def bridge_to_async(corteg):
    asyncio.get_event_loop().run_until_complete(parser(corteg[0], corteg[1]))

def bridge_to_async_info(corteg):
    print("async bridge")
    asyncio.get_event_loop().run_until_complete(parser_info(corteg[0], corteg[1]))

def butch_before_procs():
    num_tokens = ApiKey.objects.filter(expired=False).count()
    date_10_days_ago = timezone.now() - datetime.timedelta(days=10)
    num_points_to_check = Pairs.objects.filter(
        Q(last_executions=None) | Q(last_executions__lte=date_10_days_ago)).count()
    butch_size = int(num_points_to_check / num_tokens + 0.5)
    data = []
    i = 0
    pairs = Pairs.objects.filter(
        Q(last_executions=None) | Q(last_executions__lte=date_10_days_ago))
    for api_key in ApiKey.objects.filter(expired=False):
        data.append((api_key, pairs[i * butch_size: i * butch_size + butch_size]))
        i += 1
    return data

def butch_before_procs_info():
    num_tokens = ApiKey.objects.filter(expired=False).count()
    date_10_days_ago = timezone.now() - datetime.timedelta(days=10)
    num_points_to_check = PairsWithSexAndAge.objects.filter(
        Q(last_executions=None) | Q(last_executions__lte=date_10_days_ago)).count()
    butch_size = int(num_points_to_check / num_tokens + 0.5)
    data = []
    i = 0
    # pairs = PairsWithSexAndAge.objects.filter(
    #    Q(last_executions=None) | Q(last_executions__lte=date_10_days_ago))
    for api_key in ApiKey.objects.filter(expired=False):
        data.append((api_key, (i * butch_size, i * butch_size + butch_size)))
        i += 1
    return data


import pytest
import subprocess
import logging
from time import time

class Server():

    def __init__(self, port=2000, count=300, delay=1):
        logging.info(f'{self} init')
        self.port = port
        self.count = count
        self.delay = delay
        self.started = None
        self.stopped = None
        self.stdout = ''
        self.stderr = ''
        self.command = [
            'socat',
            '-d',
            '-d',
            '-d',
            '-v',
            f'tcp4-listen:{port},reuseaddr,fork',
            f"system:'for s in $(seq {count}); do fortune -a; sleep {delay}; done'",
        ]

    def __enter__(self):
        logging.info(f'{self} enter')
        self.started = time()
        self.process = subprocess.Popen(
            self.command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
        )
        return self

    def __exit__(self, _, exception, traceback):
        logging.info(f'{self} exit')
        self.ret = self.process.poll()
        if not self.ret:
            self.kill()
        self.stopped = time()
        return False

    def read(self, timeout=None):
        logging.info(f'{self} read')
        if self.running():
            stdout, stderr = self.process.communicate(None, timeout)
            if stdout:
                logging.info(f'{self} stdout {stdout}')
            if stderr:
                logging.info(f'{self} stdout {stdout}')
            self.stdout += stdout
            self.stderr += stderr
        return self.stdout, self.stderr

    def stop(self, timeout=None):
        logging.info(f'{self} stop')
        self.ret = self.process.poll()
        if not self.ret:
            self.process.terminate()
            self.ret = self.process.wait(timeout)
        if not self.stopped:
            self.stopped = time()

    def kill(self):
        logging.info(f'{self} kill')
        self.ret = self.process.poll()
        if not self.ret:
            self.process.kill()
        if not self.stopped:
            self.stopped = time()

    def running(self):
        ret = self.process.poll()
        return not ret

    def elapsed(self):
        if self.stopped:
            end = self.stopped()
        else:
            end = time()
        return end - self.started

@pytest.fixture(scope='session')
def server():
    logging.info('session server setup')
    with Server() as s:
        yield s
    logging.info('session server teardown')

class CallbackCatcher():

    def __init__(self):
        self.buffer = []

    def rx(self, state, data):
        logging.info(f"{str(state)} {repr(data)}")
        self.buffer.append((state, data))

@pytest.fixture()
def callback():
    return CallbackCatcher()


from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from bs4 import BeautifulSoup
import pandas as pd
import time
import random

options = Options()
options.add_argument("--headless")
header = {"user-agent" : "Mozilla/5.0"}
driver = webdriver.Chrome(executable_path=r"D:\tool\chrome_apps\chromedriver.exe", options=options)
url_base = "http://www.jjwxc.net/topten.php?orderstr={}&t={}"
for i in [0,1,2]:#种类分类
    for j in range(3, 23):#榜单分类
        url_now = url_base.format(j, i)
        data_lst = []
        time.sleep(random.random())
        driver.get(url_now)
        r = driver.page_source
        soup = BeautifulSoup(r)
        if len(soup.find_all("tbody")) < 4:
            print()
            continue
        infos = soup.find_all("tbody")[2]
        #print(infos)
        books = infos.find_all("tr")[1:]#每一本书
        #print(infos.find_all("tr")[0])
        book_info_lst = []
        print(len(books))
        for book in books:
            one_book_info = {}
            one_book_infos = book.find_all("td")
            if len(one_book_infos) < 9:
                print("出现问题，跳过")
                continue
            one_book_info["rank"] = one_book_infos[0].text
            one_book_info["author"] = one_book_infos[1].text.strip("\xa0\n")
            one_book_info["title"] = one_book_infos[2].find("a")["title"]
            one_book_info["class"] = one_book_infos[3].text.strip(" \n")
            one_book_info["being"] = one_book_infos[5].text.strip(" \n")
            one_book_info["words"] = one_book_infos[6].text.strip("\xa0")
            one_book_info["score"] = one_book_infos[7].text
            one_book_info["time"] = one_book_infos[8].text
            one_book_info["descrip"] = one_book_infos[2].find("a")["rel"]
            book_info_lst.append(one_book_info)
            #print(len(book_info_lst))
        pd.DataFrame(book_info_lst).to_csv(r"晋江\第五十九周2022.9.26~10.2\{}\{}.csv".format(i, j), encoding="utf-8-sig")
        print("完成第{}个排行榜".format(j))
    print("完成第{}种分类".format(i))


import csv

from PIL import Image, ImageDraw, ImageFont

def wrap_text(words, font, max_width):
    lines = []
    current_line = ""

    for word in words:
        if not current_line:
            temp_line = word
        else:
            temp_line = " ".join([current_line, word])
        width = font.getbbox(temp_line)[2]  # x,y,x,y
        if width >= max_width:
            lines.append(current_line)
            current_line = word
        else:
            current_line = temp_line

    lines.append(current_line)

    # lines
    return lines

def generate_bounding_boxes(canvas, font, coordinate, line, space_right):
    bboxes = []

    l_left, l_top, l_right, l_bottom = canvas.textbbox(coordinate, line, font=font, anchor="ra")

    x = l_left

    for word in line.split(" "):
        w_left, w_top, w_right, w_bottom = font.getbbox(word)
        bboxes.append((word, (w_left + x, l_top, w_right + x, l_bottom)))

        x += w_right + space_right
    return bboxes

def main():
    image = Image.new("RGB", (1080, 920), (100, 200, 150))
    canvas = ImageDraw.Draw(image)
    font = ImageFont.truetype("Raleway-VariableFont_wght.ttf", size=90)

    _, _, space_right, _ = font.getbbox(" ")

    text = "Python is a programming language that lets you work quickly and integrate systems more effectively."
    lines = wrap_text(text.split(" "), font, image.width * 0.8)

    y = 100
    x = image.width - (image.width * 0.1)

    ascender, descender = font.getmetrics()

    bboxes = []
    for line in lines:
        canvas.text((x, y), line, font=font, fill="white", anchor="ra")
        bboxes += generate_bounding_boxes(canvas, font, (x, y), line, space_right)

        y += ascender + descender

    image.save("image-without-bboxes.png")

    for word, bbox in bboxes:
        canvas.rectangle(bbox, outline="pink")

    image.save("image-with-bboxes.png")

    save_bboxes(bboxes)

def save_bboxes(bboxes):
    with open("text-bboxes.csv", "w") as text:
        writer = csv.writer(text)
        writer.writerow(["word", "coordinates"])
        for word, bbox in bboxes:
            writer.writerow([word, ",".join([str(p) for p in bbox])])

if __name__ == '__main__':
    main()


from __future__ import print_function
import pickle,gzip,time,sys,os
import numpy as np
import argparse
from enum import Enum
import pprint
import collections

resultLogPath = None

def set_resultLogPath(myPath):
    global resultLogPath
    resultLogPath = myPath

class OPTZ_METHOD(Enum):
    ilp = 1
    lp_rptdRnd = 2
    lp_clustRptdRnd = 3

def modifiedPrint(thingToPrint, prettyPrint = False):
    global resultLogPath
    if not prettyPrint:
        print (thingToPrint)
    else:
        pprint.pprint(thingToPrint)
    if (resultLogPath is not None):
        with open(resultLogPath,'a') as f:
            if not prettyPrint:
                print(thingToPrint, file = f)
            else:
                pprint.pprint(thingToPrint, f)

def prettyFormat(d, delim = "\n", keyLength = 15):
    strToReturn = ""
    dOrdered = collections.OrderedDict(sorted(d.items())) #sorting by keys
    for key, value in dOrdered.items():
        if isinstance(value, dict):
            value = prettyFormat(value, delim = ", ", keyLength = 0) #recursive call
        strToReturn += (key.ljust(keyLength) + " : " + str(value) + delim)
    if (delim != "\n"):
        strToReturn = "{" + strToReturn.strip().strip(",") + "}"
    return strToReturn

def partitionV_toPrioLevels(V, num_prio_lvl):
    # V is Interest Matrix ==> has Participants along rows & Talks along Columns
    modifiedPrint ("\n------- Partitioning V starts -------\n")
    modifiedPrint ("Example: For 3 prio-levels, V_partitioned_list contains [[Top Prio], [Mid Prio], [Low Prio]]")
    interestPerTalk = V.sum(axis=0) #column-wise sum
    Vindx_descByInterest = interestPerTalk.argsort()[::-1]
    modifiedPrint ("Total interest of each talk: {}".format(interestPerTalk))
    modifiedPrint ("Talk indices, descending by total interest: {}".format(Vindx_descByInterest))
    V_partitioned_list = []
    V_partitioned_indxList = []
    numTalksInEachPrioLvl = np.ceil(V.shape[1]/num_prio_lvl).astype('int') #equal partitions
    modifiedPrint ("numTalksInEachPrioLvl: {}".format(numTalksInEachPrioLvl))
    for prio in range(num_prio_lvl):
        strtIndx = prio * numTalksInEachPrioLvl
        endIndx = strtIndx + numTalksInEachPrioLvl if (prio != num_prio_lvl - 1) else Vindx_descByInterest.shape[0]
        V_indices_toChoose = Vindx_descByInterest[strtIndx : endIndx].tolist()
        V_partitioned_indxList.append(V_indices_toChoose)
        modifiedPrint ("--------------")
        modifiedPrint ("Talk indices, in priority-partition #{}: {}".format(prio + 1, V_indices_toChoose))
        V_partitioned = np.take(V, V_indices_toChoose, axis = 1) #choose columns whose indx is present in the list
        modifiedPrint ("Shape of Talk-matrix, for priority-partition #{}: {}".format(prio + 1, V_partitioned.shape))
        V_partitioned_list.append(V_partitioned)
    modifiedPrint ("\n------- Partitioning V ends -------\n")
    # V_partitioned_indxList contains corresponding indices
    return V_partitioned_list, V_partitioned_indxList

def IAM_baseline(dataset, V, A, resultDir):
    modifiedPrint("============================== " + dataset)
    m=V.shape[0]
    n=V.shape[1]
    l=A.shape[1]
    X=np.zeros((n,l))
    sorted_A=np.argsort(A.sum(axis=0))
    sorted_V=np.argsort(V.sum(axis=0))
    for i in range(n):
        t=sorted_V[-1-i]
        s=sorted_A[-1-i]
        X[t,s]=1.0
    np.savetxt(resultDir + "/" + dataset + "_IAM.csv", X, delimiter=",")
    modifiedPrint(dataset + ", numTalks = " + str(n) + ", numSlots = " + str(l))

def giniCoeff(x):
    # Mean absolute difference
    meanAbsDiff = np.abs(np.subtract.outer(x, x)).mean()
    # Relative mean absolute difference
    relMAD = meanAbsDiff/np.mean(x)
    # Gini coefficient
    giniCoeff = 0.5 * relMAD
    return giniCoeff

def metrics(V, A, X, num_prio_lvl = None, V_partitioned_list = None, V_partitioned_indxList = None):
    #set num_prio_lvl = None when there's no priority partitioning
    mets={}
    # m=number of participants
    m=V.shape[0]
    # n=number of talks
    n=V.shape[1]
    # l=number of slots
    l=A.shape[1]

    # ---- PARTICIPANT SIDE METRICS >>>>

    # ICGs of participants
    ICG=np.zeros(m)
    for p in range(m): #for each participant
        Interests=sorted(V[p,],reverse=True)
        Availabilities=sorted(A[p,],reverse=True)
        for t in range(n):
            ICG[p]+=(Interests[t]*Availabilities[t])

    if (num_prio_lvl is None):
        NCGs = np.diag(np.matmul(V,np.matmul(A,X.T).T))/ICG
    else:
        # global-level metric for participant (not prio-level wise)
        # gain is considered only for top-1 out of all slots
        # that is, participants attend talks only once
        CGs = np.zeros(m)
        partcpPreferredSlotsForTalks = np.full((m,n), None) # partcpPreferredSlotsForTalks[p,t] = best slot num
        for partcp in range(m):
            # each talk being offered in one/more slots
            # for each talk, consider gain only from that slot that gives highest gain for partcp
            for tlk in range(n):
                possib_slotGains = np.multiply(A[partcp,:], X[tlk,:]) #element-wise multiplication
                maxGain = V[partcp,tlk] * np.amax(possib_slotGains)
                if (np.all(possib_slotGains == 0)):
                    # if product of all A and X are 0, choose any slot where the talk is scheduled
                    partcpPreferredSlotsForTalks[partcp,tlk] = np.nonzero(X[tlk,:])[0][0]
                    assert X[tlk, partcpPreferredSlotsForTalks[partcp,tlk]] != 0
                    assert A[partcp, partcpPreferredSlotsForTalks[partcp,tlk]] == 0
                else:
                    # else choose slot that's most beneficial for partcp to attend talk tlk
                    partcpPreferredSlotsForTalks[partcp,tlk] = np.argmax(possib_slotGains) # to be used in speaker metric
                CGs[partcp] += maxGain
        NCGs = CGs / ICG
        # print ("CGs" + str(CGs.tolist()))
        # print ("ICG" + str(ICG.tolist()))
        # print ("min(NCGs)", min(NCGs))
        # print ("max(NCGs)", max(NCGs))
        # modifiedPrint ("partcpPreferredSlotsForTalks")
        # modifiedPrint (partcpPreferredSlotsForTalks)
        # for vvv in range(partcpPreferredSlotsForTalks.shape[1]):
        #      modifiedPrint (np.unique(partcpPreferredSlotsForTalks[:,vvv]))

    #modifiedPrint ("Individual Participant Satisfaction:" + str(np.sort(NCGs)))

    MeanNCG = "{:.3f}".format(np.mean(NCGs))
    PFairness = "{:.3f}".format(np.max(NCGs)-np.min(NCGs))
    PGini = "{:.3f}".format(giniCoeff(NCGs))

    # <<<< PARTICIPANT SIDE METRICS ----

    # ---- SPEAKER SIDE METRICS >>>>

    # --- IECs of speakers ---
    # IEC calculation is as per full-scale scheduling
    IEC=np.zeros(n)
    for t in range(n): #for each talk
        Interests=V[:,t] #for a particular talk
        ECs=np.matmul(Interests.T,A)
        IEC[t]=max(ECs)

    if (num_prio_lvl is None):
        NECs = np.diag(np.matmul(V.T,np.matmul(A,X.T)))/IEC
        MeanNEC = "{:.3f}".format(np.mean(NECs))
        SFairness = "{:.3f}".format(np.max(NECs)-np.min(NECs))
        SGini = "{:.3f}".format(giniCoeff(NECs))
        #modifiedPrint ("Individual Speaker Satisfaction:" + str(np.sort(NECs)))
    else:
        # local-level metric for participant (prio-level wise)
        IEC_partitioned = {}
        NECs_partitioned = {}
        MeanNEC_partitioned = {}
        SFairness_partitioned = {}
        SGini_partitioned = {}

        for prio in range(num_prio_lvl):
            IEC_partitioned["P{}".format(prio+1)] = np.take(IEC, V_partitioned_indxList[prio])
            V_partitioned = V_partitioned_list[prio]
            X_partitioned = np.take(X, V_partitioned_indxList[prio], axis = 0) #row-wise selection, bcos X is (#talks)*(#slots)
            # gain is considered only for top-1 out of all slots
            # that is, participants attend talks only once
            ECs_partitioned = np.zeros(V_partitioned.shape[1])
            for tlk in range(V_partitioned.shape[1]):
                # each talk being offered in one/more slots
                # for each talk, consider gain only from that slot that gives highest gain for partcp
                for partcp in range(m):
                    partcpPreferredSlotsForTalks_partitioned = np.take(partcpPreferredSlotsForTalks,
                                                       V_partitioned_indxList[prio], axis = 1) #col-wise selection
                    preferredSlotForTalk = int(partcpPreferredSlotsForTalks_partitioned[partcp,tlk])
                    maxGain = V_partitioned[partcp,tlk] * A[partcp, preferredSlotForTalk]
                    ECs_partitioned[tlk] += maxGain
            NECs_partitioned["P{}".format(prio+1)] = ECs_partitioned / IEC_partitioned["P{}".format(prio+1)]
            #print ("ECs_partitioned", ECs_partitioned)
            #print ("IEC_partitioned", IEC_partitioned)

            MeanNEC_partitioned["P{}".format(prio+1)] = "{:.3f}".format(np.mean(NECs_partitioned["P{}".format(prio+1)]))
            SFairness_partitioned["P{}".format(prio+1)] = "{:.3f}".format(np.max(NECs_partitioned["P{}".format(prio+1)]) - \
                                                          np.min(NECs_partitioned["P{}".format(prio+1)]))
            SGini_partitioned["P{}".format(prio+1)] = "{:.3f}".format(giniCoeff(NECs_partitioned["P{}".format(prio+1)]))

    # <<<< SPEAKER SIDE METRICS ----

    # ---- SOCIAL WELFARE METRIC >>>>

    if (num_prio_lvl is None):
        TEP = np.trace(np.matmul(V,np.matmul(A,X.T).T))
    else:
        # TEP = sum of cumulative gains of all the participants, considering they will attend only once
        TEP = np.sum(CGs)

    # <<<< SOCIAL WELFARE METRIC ----

    mets["NCG_mean"] = MeanNCG
    mets["NCG_max-NCG_min"] = PFairness
    mets["NCG_Gini"] = PGini
    mets["NEC_mean"] = MeanNEC if (num_prio_lvl is None) else MeanNEC_partitioned
    mets["NEC_max-NEC_min"] = SFairness if (num_prio_lvl is None) else SFairness_partitioned
    mets["NEC_Gini"] = SGini if (num_prio_lvl is None) else SGini_partitioned
    mets["TEP"] = TEP
    return (mets)

def print_baseline_results(V, A, dataset, resultDir, instanceSuffix, num_prio_lvl, V_partitioned_list, V_partitioned_indxList):
    modifiedPrint("============================== baseline_results")
    # Social Welfare Maximization
    X=np.loadtxt(resultDir + "/" + dataset + "_1.0_0.0_0.0" + instanceSuffix + ".csv", delimiter=",")
    modifiedPrint("------- SWM Metrics:\n" + prettyFormat(metrics(V, A, X, num_prio_lvl, V_partitioned_list, V_partitioned_indxList)))
    # Participant Fairness Maximization
    X=np.loadtxt(resultDir + "/" + dataset + "_0.0_1.0_0.0" + instanceSuffix + ".csv", delimiter=",")
    modifiedPrint("------- PFair Metrics:\n" + prettyFormat(metrics(V, A, X, num_prio_lvl, V_partitioned_list, V_partitioned_indxList)))
    # Speaker Fairness Maximization
    X=np.loadtxt(resultDir + "/" + dataset + "_0.0_0.0_1.0" + instanceSuffix + ".csv", delimiter=",")
    modifiedPrint("------- SFair Metrics:\n" + prettyFormat(metrics(V, A, X, num_prio_lvl, V_partitioned_list, V_partitioned_indxList)))
    # Interest-Availability Matching
    X=np.loadtxt(resultDir + "/" + dataset + "_IAM.csv",delimiter=",")
    modifiedPrint("------- IAM Metrics:\n" + prettyFormat(metrics(V, A, X, num_prio_lvl, V_partitioned_list, V_partitioned_indxList)))

def argPreprocess(args, dataDir):
    optzMethod = OPTZ_METHOD(args.confType)

    # ---- loading data ----
    V_path = dataDir + "/" + args.dataset + "_V.csv"
    A_path = dataDir + "/" + args.dataset + "_A.csv"
    if (os.path.exists(V_path)):
        V = np.loadtxt(V_path, delimiter=",")
    else:
        print ("Interest matrix does not exist at path " + V_path)
        sys.exit(0)
    if (os.path.exists(A_path)):
        A = np.loadtxt(A_path, delimiter=",")
    else:
        print ("Availability matrix does not exist at path " + A_path)
        sys.exit(0)
    assert V.shape[0] == A.shape[0] # number of participants same in both A and V

    # ---- refining inputs, converting all to n_clusters perspective ----
    mOrig = V.shape[0] # m=number of participants
    nOrig = V.shape[1] # n=number of talks
    lOrig = A.shape[1] # l=number of slots

    if (optzMethod == OPTZ_METHOD.ilp):
        args.n_clusters = mOrig
        args.clusterFrac = None
        instanceSuffix = "_ILP_CN" + str(args.n_clusters) + "_FB" + str(args.fairBoundIndx)
    elif (optzMethod == OPTZ_METHOD.lp_rptdRnd):
        args.n_clusters = mOrig
        args.clusterFrac = None
        instanceSuffix = "_CN" + str(args.n_clusters) + "_FB" + str(args.fairBoundIndx)
    elif (optzMethod == OPTZ_METHOD.lp_clustRptdRnd):
        if (args.clusterFrac is not None):
            if (args.clusterFrac <= 0) or (args.clusterFrac > 1):
                modifiedPrint ("clusterFrac value is either non-positive or exceeds 1. Setting it to 1.0")
                args.clusterFrac = 1.0 #no clustering
            args.n_clusters = int(args.clusterFrac * mOrig) if args.clusterFrac <= 1 else V.shape[0]
            if (args.n_clusters == 0):
                args.n_clusters = 1 #at least one cluster
            instanceSuffix = "_CN" + str(args.n_clusters) + "_FB" + str(args.fairBoundIndx)
        elif (args.n_clusters is not None):
            if (args.n_clusters <= 0) or (args.n_clusters > mOrig):
                args.n_clusters = mOrig
            instanceSuffix = "_CN" + str(args.n_clusters) + "_FB" + str(args.fairBoundIndx)
        else:
            args.n_clusters = mOrig
            instanceSuffix = "_CN" + str(args.n_clusters) + "_FB" + str(args.fairBoundIndx)
    if (args.num_prio_lvl_EVALONLY is None) and (args.num_prio_lvl is not None):
        instanceSuffix += "_PrioLvls" + str(args.num_prio_lvl)
    else:
        instanceSuffix += "_PrioLvls" + str(1)
    return (V, A, instanceSuffix)


from collections import Counter

def byte_probability(data: bytes):
    """Составить словарь «байт→вероятность».

    Аргументы:
    data -- строка байтов, на основе которо строится словарь
    """

    if not isinstance(data, bytes):
        raise ValueError("Допустимы только байтовые строки")

    if not data:
        raise ValueError("Пустая строка байтов")

    byte_count = Counter(data)
    return {byte: byte_count[byte] / len(data) for byte in byte_count}


import json
import os

from celery.result import AsyncResult
from django.conf import settings
from django.http import JsonResponse, HttpResponseBadRequest, HttpResponseNotAllowed, HttpResponse, HttpRequest
from django.shortcuts import render
from django.views.decorators.csrf import csrf_exempt

from api import models
from tasks import code_editor, task_check
from tasks.sample_tasks import create_task

def home(request: HttpRequest) -> HttpResponse:
    return render(request, 'home.html')

@csrf_exempt
def run_task(request: HttpRequest) -> HttpResponse:
    if request.method != 'POST':
        return HttpResponseNotAllowed(['POST'])

    try:
        task_type = int(json.loads(request.body).get('type'))
    except ValueError:
        return HttpResponseBadRequest()

    result = {
        'task_id': create_task.delay(task_type).id,
    }

    return JsonResponse(result, status=202)

@csrf_exempt
def start_editor(request: HttpRequest, user_id: int) -> HttpResponse:
    if request.method != 'POST':
        return HttpResponseNotAllowed(['POST'])

    try:
        models.User.objects.get(id=user_id)
    except models.User.DoesNotExist:
        return HttpResponseBadRequest()

    result = {
        'task_id': code_editor.start.delay(user_id).id,
    }

    return JsonResponse(result, status=202)

@csrf_exempt
def stop_editor(request: HttpRequest, user_id: int) -> HttpResponse:
    if request.method != 'POST':
        return HttpResponseNotAllowed(['POST'])

    try:
        models.User.objects.get(id=user_id)
    except models.User.DoesNotExist:
        return HttpResponseBadRequest()

    result = {
        'task_id': code_editor.stop.delay(user_id).id,
    }

    return JsonResponse(result, status=202)

@csrf_exempt
def editor_status(request: HttpRequest, user_id: int) -> HttpResponse:
    try:
        user = models.User.objects.get(id=user_id)
    except models.User.DoesNotExist:
        return HttpResponseBadRequest()

    result = {
        'status': code_editor.status(user_id),
        'port': user.editor_port,
    }

    return JsonResponse(result, status=202)

@csrf_exempt
def verify_task(request: HttpRequest, user_id: int, task_id: int, language_id: int) -> HttpResponse:
    try:
        models.User.objects.get(id=user_id)
    except models.User.DoesNotExist:
        return HttpResponseBadRequest()

    try:
        models.Task.objects.get(id=task_id)
    except models.Task.DoesNotExist:
        return HttpResponseBadRequest()

    try:
        models.Language.objects.get(id=language_id)
    except models.Language.DoesNotExist:
        return HttpResponseBadRequest()

    result = {
        'task_id': task_check.verify.delay(user_id, task_id, language_id).id,
    }

    return JsonResponse(result, status=202)

@csrf_exempt
def get_status(request: HttpRequest, task_id: str) -> HttpResponse:
    task_result = AsyncResult(task_id)

    result = {
        'task_id': task_id,
        'task_status': task_result.status,
        'task_result': task_result.result,
    }

    return JsonResponse(result, status=200)