from datetime import date, datetime
from functools import partial, reduce
from operator import attrgetter
from random import sample

import pytest
from django.utils.timezone import now
from freezegun import freeze_time
from rest_framework import status
from rest_framework.reverse import reverse
from reversion.models import Version

from datahub.company.test.factories import AdviserFactory, CompanyFactory, ContactFactory
from datahub.core.constants import Service
from datahub.core.reversion import EXCLUDED_BASE_MODEL_FIELDS
from datahub.core.test_utils import (
    APITestMixin,
    create_test_user,
    format_date_or_datetime,
    random_obj_for_model,
)
from datahub.event.test.factories import EventFactory
from datahub.interaction.models import CommunicationChannel, Interaction, InteractionPermission
from datahub.interaction.test.factories import (
    CompanyInteractionFactory,
    EventServiceDeliveryFactory,
    InteractionDITParticipantFactory,
)
from datahub.interaction.test.permissions import (
    NON_RESTRICTED_CHANGE_PERMISSIONS,
)
from datahub.interaction.test.utils import random_service
from datahub.interaction.test.views.utils import resolve_data
from datahub.investment.project.test.factories import InvestmentProjectFactory
from datahub.metadata.test.factories import TeamFactory

class TestAddInteraction(APITestMixin):
    """Tests for the add interaction view."""

    @pytest.mark.parametrize('kind', ('policy_feedback', 'invalid-kind'))
    def test_invalid_kind_is_rejected(self, kind):
        """Test that invalid kind values are rejected."""
        adviser = self.user
        contact = ContactFactory()
        company = contact.company

        data = {
            'kind': kind,
            'company': {
                'id': company.pk,
            },
            'contacts': [{
                'id': contact.pk,
            }],
            'date': '2017-04-18',
            'dit_participants': [
                {
                    'adviser': {
                        'id': adviser.pk,
                    },
                },
            ],
            'service': {
                'id': random_service().pk,
            },
            'subject': 'whatever',
            'was_policy_feedback_provided': False,
        }
        url = reverse('api-v3:interaction:collection')
        response = self.api_client.post(url, data)

        assert response.status_code == status.HTTP_400_BAD_REQUEST
        assert response.json() == {
            'kind': [f'"{kind}" is not a valid choice.'],
        }

    @pytest.mark.parametrize(
        'data,errors',
        (
            # empty string not allowed for theme
            (
                {
                    'kind': Interaction.Kind.INTERACTION,
                    'communication_channel': partial(random_obj_for_model, CommunicationChannel),
                    'date': date.today().isoformat(),
                    'subject': 'whatever',
                    'service': Service.inbound_referral.value.id,
                    'was_policy_feedback_provided': False,
                    'dit_participants': [
                        {'adviser': AdviserFactory},
                    ],

                    'theme': '',
                },
                {
                    'theme': ['"" is not a valid choice.'],
                },
            ),

            # invalid theme not allowed
            (
                {
                    'kind': Interaction.Kind.INTERACTION,
                    'communication_channel': partial(random_obj_for_model, CommunicationChannel),
                    'date': date.today().isoformat(),
                    'subject': 'whatever',
                    'service': Service.inbound_referral.value.id,
                    'was_policy_feedback_provided': False,
                    'dit_participants': [
                        {'adviser': AdviserFactory},
                    ],

                    'theme': 'not_valid',
                },
                {
                    'theme': ['"not_valid" is not a valid choice.'],
                },
            ),
        ),
    )
    def test_validation(self, data, errors):
        """Test validation errors."""
        company = CompanyFactory()
        resolved_data = {
            'company': company.pk,
            'contacts': [ContactFactory(company=company).pk],
            **resolve_data(data),
        }
        url = reverse('api-v3:interaction:collection')
        response = self.api_client.post(url, resolved_data)

        assert response.status_code == status.HTTP_400_BAD_REQUEST
        assert response.json() == errors

    def test_error_returned_if_contacts_dont_belong_to_company(self):
        """
        Test that an error is returned if the contacts don't belong to the specified company.
        """
        company = CompanyFactory()
        contacts = [ContactFactory(), ContactFactory(company=company)]
        communication_channel = random_obj_for_model(CommunicationChannel)

        url = reverse('api-v3:interaction:collection')
        request_data = {
            'kind': Interaction.Kind.INTERACTION,
            'communication_channel': communication_channel.pk,
            'subject': 'whatever',
            'date': date.today().isoformat(),
            'dit_participants': [
                {'adviser': self.user.pk},
            ],
            'company': {
                'id': company.pk,
            },
            'contacts': [{
                'id': contact.pk,
            } for contact in contacts],
            'service': {
                'id': random_service().pk,
            },
            'was_policy_feedback_provided': False,
        }

        api_client = self.create_api_client()
        response = api_client.post(url, request_data)
        assert response.status_code == status.HTTP_400_BAD_REQUEST
        assert response.json() == {
            'non_field_errors': ['The interaction contacts must belong to the specified company.'],
        }

    def test_multiple_participating_advisers_can_be_specified(self):
        """Test that an interaction can be created with multiple DIT participants."""
        contact = ContactFactory()
        communication_channel = random_obj_for_model(CommunicationChannel)
        advisers = AdviserFactory.create_batch(5)
        advisers.sort(key=attrgetter('pk'))

        url = reverse('api-v3:interaction:collection')
        request_data = {
            'kind': Interaction.Kind.INTERACTION,
            'communication_channel': communication_channel.pk,
            'subject': 'whatever',
            'date': date.today().isoformat(),
            'dit_participants': [
                {
                    'adviser': {
                        'id': adviser.pk,
                    },
                }
                for adviser in advisers
            ],
            'company': {
                'id': contact.company.pk,
            },
            'contacts': [{
                'id': contact.pk,
            }],
            'service': {
                'id': random_service().pk,
            },
            'was_policy_feedback_provided': False,
        }

        api_client = self.create_api_client()
        response = api_client.post(url, request_data)
        assert response.status_code == status.HTTP_201_CREATED

        response_data = response.json()
        response_data['dit_participants'].sort(
            key=lambda dit_participant: dit_participant['adviser']['id'],
        )
        assert response_data['dit_participants'] == [
            {
                'adviser': {
                    'id': str(adviser.pk),
                    'first_name': adviser.first_name,
                    'last_name': adviser.last_name,
                    'name': adviser.name,
                },
                'team': {
                    'id': str(adviser.dit_team.pk),
                    'name': adviser.dit_team.name,
                },
            }
            for adviser in advisers
        ]

    def test_error_returned_if_duplicate_participating_advisers_specified(self):
        """
        Test that an error is returned if an adviser is specified as a DIT participant
        multiple times.
        """
        contact = ContactFactory()
        communication_channel = random_obj_for_model(CommunicationChannel)
        dit_adviser = AdviserFactory()

        url = reverse('api-v3:interaction:collection')
        request_data = {
            'kind': Interaction.Kind.INTERACTION,
            'communication_channel': communication_channel.pk,
            'subject': 'whatever',
            'date': date.today().isoformat(),
            'dit_participants': [
                {
                    'adviser': {
                        'id': dit_adviser.pk,
                    },
                },
                {
                    'adviser': {
                        'id': AdviserFactory().pk,
                    },
                },
                {
                    'adviser': {
                        'id': dit_adviser.pk,
                    },
                },
            ],
            'company': {
                'id': contact.company.pk,
            },
            'contacts': [{
                'id': contact.pk,
            }],
            'service': {
                'id': random_service().pk,
            },
            'was_policy_feedback_provided': False,
        }

        api_client = self.create_api_client()
        response = api_client.post(url, request_data)
        assert response.status_code == status.HTTP_400_BAD_REQUEST

        # An error should be returned for each duplicated item in the dit_participants list in
        # the request
        assert response.json() == {
            'dit_participants': [
                {'adviser': ['You cannot add the same adviser more than once.']},
                {},
                {'adviser': ['You cannot add the same adviser more than once.']},
            ],
        }

class TestGetInteraction(APITestMixin):
    """Base tests for the get interaction view."""

    def test_fails_without_permissions(self):
        """Should return 403"""
        interaction = CompanyInteractionFactory()
        user = create_test_user(dit_team=TeamFactory())
        api_client = self.create_api_client(user=user)
        url = reverse('api-v3:interaction:item', kwargs={'pk': interaction.pk})
        response = api_client.get(url)
        assert response.status_code == status.HTTP_403_FORBIDDEN

class TestUpdateInteraction(APITestMixin):
    """Base tests for the update interaction view."""

    def test_cannot_update_read_only_fields(self):
        """Test updating read-only fields."""
        interaction = CompanyInteractionFactory(
            archived_documents_url_path='old_path',
            archived=False,
            archived_by=None,
            archived_on=None,
            archived_reason=None,
        )

        url = reverse('api-v3:interaction:item', kwargs={'pk': interaction.pk})
        # TODO: also test `archived` field once we have made it read-only
        response = self.api_client.patch(
            url,
            data={
                'archived_documents_url_path': 'new_path',
                'archived': True,
                'archived_by': 123,
                'archived_on': date.today(),
                'archived_reason': 'test',
            },
        )

        assert response.status_code == status.HTTP_200_OK
        assert response.data['archived_documents_url_path'] == 'old_path'
        assert response.data['archived'] is False
        assert response.data['archived_by'] is None
        assert response.data['archived_on'] is None
        assert response.data['archived_reason'] is None

    @pytest.mark.parametrize(
        'data,errors',
        (
            # empty string not allowed for theme
            (
                {
                    'theme': '',
                },
                {
                    'theme': ['"" is not a valid choice.'],
                },
            ),

            # invalid theme not allowed
            (
                {
                    'theme': 'not_valid',
                },
                {
                    'theme': ['"not_valid" is not a valid choice.'],
                },
            ),

            # date validation
            (
                {
                    'date': 'abcd-de-fe',
                },
                {
                    'date': [
                        'Datetime has wrong format. Use one of these formats instead: YYYY-MM-DD.',
                    ],
                },
            ),

            # cannot remove all participants
            (
                {
                    'dit_participants': [],
                },
                {
                    'dit_participants': {
                        'non_field_errors': ['This list may not be empty.'],
                    },
                },
            ),
        ),
    )
    def test_validation(self, data, errors):
        """Test validation when an invalid date is provided."""
        interaction = CompanyInteractionFactory()

        url = reverse('api-v3:interaction:item', kwargs={'pk': interaction.pk})
        response = self.api_client.patch(url, data=data)

        assert response.status_code == status.HTTP_400_BAD_REQUEST
        assert response.json() == errors

    @pytest.mark.parametrize(
        'request_data',
        (
            {
                'company': CompanyFactory,
            },
            {
                'companies': [CompanyFactory],
            },
            {
                'contacts': [ContactFactory],
            },
        ),
    )
    def test_error_returned_if_contacts_dont_belong_to_company(self, request_data):
        """
        Test that an error is returned if an update makes the contacts and company
        fields inconsistent.
        """
        interaction = CompanyInteractionFactory()

        url = reverse('api-v3:interaction:item', kwargs={'pk': interaction.pk})
        resolved_data = resolve_data(request_data)
        response = self.api_client.patch(url, data=resolved_data)

        assert response.status_code == status.HTTP_400_BAD_REQUEST
        assert response.json() == {
            'non_field_errors': ['The interaction contacts must belong to the specified company.'],
        }

    @pytest.mark.parametrize('include_company', (True, False))
    @pytest.mark.parametrize('include_contacts', (True, False))
    def test_inconsistent_interaction_can_be_updated(self, include_company, include_contacts):
        """
        Test that an interaction with inconsistent company and contact fields can still be
        updated.
        """
        interaction = CompanyInteractionFactory(
            company=CompanyFactory(),
            notes='old notes',
        )

        url = reverse('api-v3:interaction:item', kwargs={'pk': interaction.pk})
        data = {'notes': 'new notes'}

        if include_company:
            data['company'] = {'id': interaction.company.pk}

        if include_contacts:
            data['contacts'] = [{'id': contact.pk} for contact in interaction.contacts.all()]

        response = self.api_client.patch(url, data=data)

        assert response.status_code == status.HTTP_200_OK
        assert response.json()['notes'] == data['notes']

    def test_can_replace_some_participants(self):
        """Test that a subset of existing DIT participants can be replaced."""
        interaction = CompanyInteractionFactory(dit_participants=[])
        dit_participants = InteractionDITParticipantFactory.create_batch(
            3,
            interaction=interaction,
        )
        # Change the first adviser's team so that we can check that the participant's team is
        # unchanged after the update.
        dit_participants[0].adviser.dit_team = TeamFactory()
        dit_participants[0].adviser.save()

        new_advisers = [
            dit_participants[0].adviser,
            AdviserFactory(),
        ]

        request_data = {
            'dit_participants': [
                {
                    'adviser': {
                        'id': adviser.pk,
                    },
                }
                for adviser in new_advisers
            ],
        }

        url = reverse('api-v3:interaction:item', kwargs={'pk': interaction.pk})
        response = self.api_client.patch(url, data=request_data)

        assert response.status_code == status.HTTP_200_OK

        response_data = response.json()
        response_data['dit_participants'].sort(
            key=lambda dit_participant: dit_participant['adviser']['id'],
        )

        expected_advisers_and_teams = [
            (new_advisers[0], dit_participants[0].team),
            (new_advisers[1], new_advisers[1].dit_team),
        ]
        expected_advisers_and_teams.sort(key=lambda adviser_and_team: adviser_and_team[0].pk)

        assert response_data['dit_participants'] == [
            {
                'adviser': {
                    'id': str(adviser.pk),
                    'first_name': adviser.first_name,
                    'last_name': adviser.last_name,
                    'name': adviser.name,
                },
                'team': {
                    'id': str(team.pk),
                    'name': team.name,
                },
            }
            for adviser, team in expected_advisers_and_teams
        ]

    def test_can_replace_all_participants(self):
        """Test that all existing participants can be replaced with different ones."""
        interaction = CompanyInteractionFactory(dit_participants=[])
        InteractionDITParticipantFactory.create_batch(
            3,
            interaction=interaction,
        )

        new_advisers = AdviserFactory.create_batch(2)
        new_advisers.sort(key=attrgetter('pk'))

        request_data = {
            'dit_participants': [
                {
                    'adviser': {
                        'id': adviser.pk,
                    },
                }
                for adviser in new_advisers
            ],
        }

        url = reverse('api-v3:interaction:item', kwargs={'pk': interaction.pk})
        response = self.api_client.patch(url, data=request_data)
        assert response.status_code == status.HTTP_200_OK

        response_data = response.json()
        response_data['dit_participants'].sort(
            key=lambda dit_participant: dit_participant['adviser']['id'],
        )

        expected_advisers_and_teams = [(adviser, adviser.dit_team) for adviser in new_advisers]

        assert response_data['dit_participants'] == [
            {
                'adviser': {
                    'id': str(adviser.pk),
                    'first_name': adviser.first_name,
                    'last_name': adviser.last_name,
                    'name': adviser.name,
                },
                'team': {
                    'id': str(team.pk),
                    'name': team.name,
                },
            }
            for adviser, team in expected_advisers_and_teams
        ]

    def test_can_add_participants(self):
        """Test that participants can be added to an interaction without any being removed."""
        interaction = CompanyInteractionFactory(dit_participants=[])
        dit_participants = InteractionDITParticipantFactory.create_batch(
            3,
            interaction=interaction,
        )

        new_advisers = [
            *[dit_participant.adviser for dit_participant in dit_participants],
            *AdviserFactory.create_batch(2),
        ]
        new_advisers.sort(key=attrgetter('pk'))

        request_data = {
            'dit_participants': [
                {
                    'adviser': {
                        'id': adviser.pk,
                    },
                }
                for adviser in new_advisers
            ],
        }

        url = reverse('api-v3:interaction:item', kwargs={'pk': interaction.pk})
        response = self.api_client.patch(url, data=request_data)
        assert response.status_code == status.HTTP_200_OK

        response_data = response.json()
        response_data['dit_participants'].sort(
            key=lambda dit_participant: dit_participant['adviser']['id'],
        )

        expected_advisers_and_teams = [(adviser, adviser.dit_team) for adviser in new_advisers]

        assert response_data['dit_participants'] == [
            {
                'adviser': {
                    'id': str(adviser.pk),
                    'first_name': adviser.first_name,
                    'last_name': adviser.last_name,
                    'name': adviser.name,
                },
                'team': {
                    'id': str(team.pk),
                    'name': team.name,
                },
            }
            for adviser, team in expected_advisers_and_teams
        ]

    @pytest.mark.parametrize(
        'current_status,new_status',
        (
            (Interaction.Status.DRAFT, Interaction.Status.DRAFT),
            (Interaction.Status.DRAFT, Interaction.Status.COMPLETE),
            (Interaction.Status.COMPLETE, Interaction.Status.COMPLETE),
        ),
    )
    @freeze_time('2017-04-18 13:25:30.986208')
    def test_status_change_valid(self, current_status, new_status):
        """
        Test the different ways that an Interaction's status can change.
        """
        interaction = CompanyInteractionFactory(status=current_status)
        api_client = self.create_api_client()
        url = reverse('api-v3:interaction:item', kwargs={'pk': interaction.pk})
        response = api_client.patch(
            url,
            data={
                'status': new_status,
            },
        )

        assert response.status_code == status.HTTP_200_OK
        assert response.data['status'] == new_status

    @pytest.mark.parametrize(
        'current_status,new_status,response_body',
        (
            (
                Interaction.Status.DRAFT,
                None,
                {'status': ['This field may not be null.']},
            ),
            (
                Interaction.Status.COMPLETE,
                Interaction.Status.DRAFT,
                {'non_field_errors': ['The status of a complete interaction cannot change.']},
            ),
            (
                Interaction.Status.COMPLETE,
                None,
                {'status': ['This field may not be null.']},
            ),
        ),
    )
    @freeze_time('2017-04-18 13:25:30.986208')
    def test_status_change_invalid(self, current_status, new_status, response_body):
        """
        Test the different ways that an Interaction's status can change.
        """
        interaction = CompanyInteractionFactory(status=current_status)
        api_client = self.create_api_client()
        url = reverse('api-v3:interaction:item', kwargs={'pk': interaction.pk})
        response = api_client.patch(
            url,
            data={
                'status': new_status,
            },
        )

        assert response.status_code == status.HTTP_400_BAD_REQUEST
        assert response.data == response_body

class TestListInteractions(APITestMixin):
    """Tests for the list interactions view."""

    def test_filtered_by_company(self):
        """List of interactions filtered by company"""
        company1 = CompanyFactory()
        company2 = CompanyFactory()

        CompanyInteractionFactory.create_batch(3, company=company1)
        interactions = CompanyInteractionFactory.create_batch(2, company=company2)

        url = reverse('api-v3:interaction:collection')
        response = self.api_client.get(url, data={'company_id': company2.id})

        assert response.status_code == status.HTTP_200_OK
        assert response.data['count'] == 2
        assert {i['id'] for i in response.data['results']} == {str(i.id) for i in interactions}

    def test_filter_by_contact(self):
        """Test filtering interactions by contact (using contacts__id)."""
        contact1 = ContactFactory()
        contact2 = ContactFactory()

        CompanyInteractionFactory.create_batch(3, contacts=[contact1])
        interactions = CompanyInteractionFactory.create_batch(2, contacts=[contact1, contact2])

        url = reverse('api-v3:interaction:collection')
        response = self.api_client.get(url, data={'contacts__id': contact2.id})

        assert response.status_code == status.HTTP_200_OK
        assert response.data['count'] == 2
        assert {i['id'] for i in response.data['results']} == {str(i.id) for i in interactions}

    def test_filtered_by_investment_project(self):
        """List of interactions filtered by investment project"""
        contact = ContactFactory()
        project = InvestmentProjectFactory()
        company = CompanyFactory()

        CompanyInteractionFactory.create_batch(3, contacts=[contact])
        CompanyInteractionFactory.create_batch(3, company=company)
        project_interactions = CompanyInteractionFactory.create_batch(
            2, investment_project=project,
        )

        url = reverse('api-v3:interaction:collection')
        response = self.api_client.get(
            url,
            data={
                'investment_project_id': project.id,
            },
        )

        assert response.status_code == status.HTTP_200_OK
        response_data = response.json()
        assert response_data['count'] == 2
        actual_ids = {i['id'] for i in response_data['results']}
        expected_ids = {str(i.id) for i in project_interactions}
        assert actual_ids == expected_ids

    def test_filtered_by_event(self):
        """List of interactions filtered by event"""
        contact = ContactFactory()
        event = EventFactory()

        CompanyInteractionFactory.create_batch(3, contacts=[contact])
        EventServiceDeliveryFactory.create_batch(3)
        service_deliveries = EventServiceDeliveryFactory.create_batch(3, event=event)

        url = reverse('api-v3:interaction:collection')
        response = self.api_client.get(url, data={'event_id': event.id})

        assert response.status_code == status.HTTP_200_OK
        response_data = response.json()
        assert response_data['count'] == 3

        actual_ids = {result['id'] for result in response_data['results']}
        expected_ids = {str(service_delivery.id) for service_delivery in service_deliveries}
        assert actual_ids == expected_ids

    @pytest.mark.parametrize(
        'field',
        (
            'company.name',
            'created_on',
            'subject',
        ),
    )
    def test_sorting(self, field):
        """Test sorting interactions by various fields."""
        data_list = [
            {
                'created_on': datetime(2015, 1, 1),
                'company__name': 'Black Group',
                'subject': 'lorem',
            },
            {
                'created_on': datetime(2005, 4, 1),
                'company__name': 'Hicks Ltd',
                'subject': 'ipsum',
            },
            {
                'created_on': datetime(2019, 1, 1),
                'company__name': 'Sheppard LLC',
                'subject': 'dolor',
            },
        ]

        interactions = []
        for data in data_list:
            creation_time = data.pop('created_on')
            with freeze_time(creation_time):
                interactions.append(
                    EventServiceDeliveryFactory(**data),
                )

        url = reverse('api-v3:interaction:collection')
        response = self.api_client.get(
            url,
            data={
                'sortby': field.replace('.', '__'),
            },
        )

        assert response.status_code == status.HTTP_200_OK
        response_data = response.json()
        assert response_data['count'] == len(interactions)

        expected = sorted(map(attrgetter(field), interactions))
        if isinstance(expected[0], datetime):
            expected = [format_date_or_datetime(item) for item in expected]

        actual = [
            reduce(  # get nested items if needed
                lambda data, key: data.get(key),
                field.split('.'),
                result,
            )
            for result in response_data['results']
        ]
        assert expected == actual

    @pytest.mark.parametrize(
        'primary_field,secondary_field',
        (
            ('first_name', 'last_name'),
            ('last_name', 'first_name'),
        ),
    )
    def test_sort_by_first_and_last_name_of_first_contact(self, primary_field, secondary_field):
        """Test sorting interactions by the first and last names of the first contact."""
        contacts = [
            ContactFactory(**{primary_field: 'Alfred', secondary_field: 'Jones'}),
            ContactFactory(**{primary_field: 'Alfred', secondary_field: 'Terry'}),
            ContactFactory(**{primary_field: 'Thomas', secondary_field: 'Richards'}),
            ContactFactory(**{primary_field: 'Thomas', secondary_field: 'West'}),
        ]
        interactions = [
            EventServiceDeliveryFactory(contacts=[contact])
            for contact in sample(contacts, len(contacts))
        ]

        url = reverse('api-v3:interaction:collection')
        response = self.api_client.get(
            url,
            data={
                'sortby': f'{primary_field}_of_first_contact,{secondary_field}_of_first_contact',
            },
        )

        assert response.status_code == status.HTTP_200_OK

        response_data = response.json()
        assert response_data['count'] == len(interactions)

        actual_ids = [
            interaction['contacts'][0]['id'] for interaction in response_data['results']
        ]
        expected_ids = [str(person.pk) for person in contacts]
        assert actual_ids == expected_ids

class TestInteractionVersioning(APITestMixin):
    """
    Tests for versions created when interacting with the interaction endpoints.
    """

    def test_add_creates_a_new_version(self):
        """Test that creating an interaction creates a new version."""
        assert Version.objects.count() == 0

        company = CompanyFactory()
        contact = ContactFactory(company=company)
        response = self.api_client.post(
            reverse('api-v3:interaction:collection'),
            data={
                'kind': Interaction.Kind.INTERACTION,
                'communication_channel': random_obj_for_model(CommunicationChannel).pk,
                'subject': 'whatever',
                'date': date.today().isoformat(),
                'dit_participants': [
                    {'adviser': AdviserFactory().pk},
                ],
                'notes': 'hello',
                'company': company.pk,
                'contacts': [contact.pk],
                'service': Service.inbound_referral.value.id,
                'was_policy_feedback_provided': False,
            },
        )

        assert response.status_code == status.HTTP_201_CREATED
        assert response.data['subject'] == 'whatever'

        interaction = Interaction.objects.get(pk=response.data['id'])

        # check version created
        assert Version.objects.get_for_object(interaction).count() == 1
        version = Version.objects.get_for_object(interaction).first()
        assert version.revision.user == self.user
        assert version.field_dict['subject'] == 'whatever'
        assert not any(set(version.field_dict) & set(EXCLUDED_BASE_MODEL_FIELDS))

    def test_add_400_doesnt_create_a_new_version(self):
        """Test that if the endpoint returns 400, no version is created."""
        assert Version.objects.count() == 0

        response = self.api_client.post(
            reverse('api-v3:interaction:collection'),
            data={
                'kind': Interaction.Kind.INTERACTION,
            },
        )

        assert response.status_code == status.HTTP_400_BAD_REQUEST
        assert Version.objects.count() == 0

    def test_update_creates_a_new_version(self):
        """Test that updating an interaction creates a new version."""
        service_delivery = EventServiceDeliveryFactory()

        assert Version.objects.get_for_object(service_delivery).count() == 0

        response = self.api_client.patch(
            reverse('api-v3:interaction:item', kwargs={'pk': service_delivery.pk}),
            data={'subject': 'new subject'},
        )

        assert response.status_code == status.HTTP_200_OK
        assert response.data['subject'] == 'new subject'

        # check version created
        assert Version.objects.get_for_object(service_delivery).count() == 1
        version = Version.objects.get_for_object(service_delivery).first()
        assert version.revision.user == self.user
        assert version.field_dict['subject'] == 'new subject'

    def test_update_400_doesnt_create_a_new_version(self):
        """Test that if the endpoint returns 400, no version is created."""
        service_delivery = EventServiceDeliveryFactory()

        assert Version.objects.get_for_object(service_delivery).count() == 0

        response = self.api_client.patch(
            reverse('api-v3:interaction:item', kwargs={'pk': service_delivery.pk}),
            data={'kind': 'invalid'},
        )

        assert response.status_code == status.HTTP_400_BAD_REQUEST
        assert Version.objects.get_for_object(service_delivery).count() == 0

class TestArchiveViews(APITestMixin):
    """
    Tests for the archive and unarchive views.
    """

    @pytest.mark.parametrize(
        'permissions', NON_RESTRICTED_CHANGE_PERMISSIONS,
    )
    def test_archive_interaction_non_restricted_user(self, permissions):
        """
        Tests archiving an interaction for a non-restricted user.
        """
        requester = create_test_user(permission_codenames=permissions)
        api_client = self.create_api_client(user=requester)

        interaction = CompanyInteractionFactory()
        url = reverse(
            'api-v3:interaction:archive-item',
            kwargs={'pk': interaction.pk},
        )
        response = api_client.post(
            url,
            data={
                'reason': 'archive reason',
            },
        )

        assert response.status_code == status.HTTP_200_OK
        response_data = response.json()
        assert response_data['archived'] is True
        assert response_data['archived_by']['id'] == str(requester.pk)
        assert response_data['archived_reason'] == 'archive reason'

    def test_archive_interaction_restricted_user_associated_project(self):
        """
        Tests archiving an interaction for a restricted user.
        """
        project_creator = AdviserFactory()
        project = InvestmentProjectFactory(created_by=project_creator)
        requester = create_test_user(
            permission_codenames=[InteractionPermission.change_associated_investmentproject],
            dit_team=project_creator.dit_team,  # same dit team as the project creator
        )
        api_client = self.create_api_client(user=requester)
        interaction = CompanyInteractionFactory(investment_project=project)
        url = reverse(
            'api-v3:interaction:archive-item',
            kwargs={'pk': interaction.pk},
        )
        response = api_client.post(
            url,
            data={
                'reason': 'archive reason',
            },
        )

        assert response.status_code == status.HTTP_200_OK
        response_data = response.json()
        assert response_data['archived'] is True
        assert response_data['archived_by']['id'] == str(requester.pk)
        assert response_data['archived_reason'] == 'archive reason'

    def test_archive_interaction_restricted_user_non_associated_project(self):
        """
        Test that a restricted user cannot archive a non-associated interaction.
        """
        project_creator = AdviserFactory()
        project = InvestmentProjectFactory(created_by=project_creator)
        # Ensure the requester is created for a different DIT team
        requester = create_test_user(
            permission_codenames=[InteractionPermission.change_associated_investmentproject],
        )
        api_client = self.create_api_client(user=requester)
        interaction = CompanyInteractionFactory(investment_project=project)
        url = reverse(
            'api-v3:interaction:archive-item',
            kwargs={'pk': interaction.pk},
        )
        response = api_client.post(
            url,
            data={
                'reason': 'archive reason',
            },
        )

        assert response.status_code == status.HTTP_403_FORBIDDEN

    @pytest.mark.parametrize(
        'data,response_data',
        (
            (
                {},
                {'reason': ['This field is required.']},
            ),
            (
                {'reason': ''},
                {'reason': ['This field may not be blank.']},
            ),
            (
                {'reason': None},
                {'reason': ['This field may not be null.']},
            ),
        ),
    )
    def test_archive_failures(self, data, response_data):
        """
        Test archive an interaction without providing a reason.
        """
        interaction = CompanyInteractionFactory()
        url = reverse(
            'api-v3:interaction:archive-item',
            kwargs={'pk': interaction.pk},
        )
        response = self.api_client.post(url, data=data)

        assert response.status_code == status.HTTP_400_BAD_REQUEST
        assert response.data == response_data

    @pytest.mark.parametrize(
        'permissions', NON_RESTRICTED_CHANGE_PERMISSIONS,
    )
    def test_unarchive_interaction_non_restricted_user(self, permissions):
        """
        Tests un-archiving an interaction for a non-restricted user.
        """
        requester = create_test_user(permission_codenames=permissions)
        api_client = self.create_api_client(user=requester)

        interaction = CompanyInteractionFactory(
            archived=True,
            archived_by=requester,
            archived_reason='just cos',
            archived_on=now(),
        )
        url = reverse(
            'api-v3:interaction:unarchive-item',
            kwargs={'pk': interaction.pk},
        )
        response = api_client.post(url)

        assert response.status_code == status.HTTP_200_OK
        response_data = response.json()
        assert response_data['archived'] is False
        assert response_data['archived_by'] is None
        assert response_data['archived_reason'] == ''
        assert response_data['archived_on'] is None

    def test_unarchive_interaction_restricted_user_associated_project(self):
        """
        Tests archiving an interaction for a restricted user.
        """
        project_creator = AdviserFactory()
        project = InvestmentProjectFactory(created_by=project_creator)
        requester = create_test_user(
            permission_codenames=[InteractionPermission.change_associated_investmentproject],
            dit_team=project_creator.dit_team,  # same dit team as the project creator
        )
        api_client = self.create_api_client(user=requester)
        interaction = CompanyInteractionFactory(
            investment_project=project,
            archived=True,
            archived_by=project_creator,
            archived_on=now(),
            archived_reason='why not',
        )
        url = reverse(
            'api-v3:interaction:unarchive-item',
            kwargs={'pk': interaction.pk},
        )
        response = api_client.post(
            url,
        )

        assert response.status_code == status.HTTP_200_OK
        response_data = response.json()
        assert response_data['archived'] is False
        assert response_data['archived_by'] is None
        assert response_data['archived_reason'] == ''
        assert response_data['archived_on'] is None

    def test_unarchive_interaction_restricted_user_non_associated_project(self):
        """
        Test that a restricted user cannot un-archive a non-associated interaction.
        """
        project_creator = AdviserFactory()
        project = InvestmentProjectFactory(created_by=project_creator)
        # Ensure the requester is created for a different DIT team
        requester = create_test_user(
            permission_codenames=[InteractionPermission.change_associated_investmentproject],
        )
        api_client = self.create_api_client(user=requester)
        interaction = CompanyInteractionFactory(
            investment_project=project,
            archived=True,
            archived_by=project_creator,
            archived_on=now(),
            archived_reason='why not',
        )
        url = reverse(
            'api-v3:interaction:unarchive-item',
            kwargs={'pk': interaction.pk},
        )
        response = api_client.post(
            url,
        )

        assert response.status_code == status.HTTP_403_FORBIDDEN


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from scipy import stats

INPUT = "../../data"

# -------------------------------------------------------------------------------
# IMPORT DATA
# -------------------------------------------------------------------------------

df = pd.read_parquet(f"{INPUT}/efdata_clean.parquet")
df_n = pd.read_parquet(f"{INPUT}/asdata_clean.parquet")

df_n["lf"] = (df_n.capY- df_n.sY) / df_n.capY # adjust to look at coach only

main_columns = [
    "origin", "dest", "flightNum", "tdate",
    "ddate", "fare", "seats", "lf", "capacity"
]
df = df[main_columns]
df_n["seats"] = df_n.sY # adjust to look at coach only
df_n["capacity"] = df_n.capY # add capacity
df = df.append(df_n[main_columns])

cols = ["origin", "dest", "flightNum", "ddate"]
df["difS"] = df.groupby(cols).seats.shift(-1) - df.seats
df["difP"] = df.groupby(cols).fare.shift(-1) - df.fare

dfR = pd.read_csv(f"{INPUT}/airline_routes.csv", sep = "\t", header = None)
dfR[0] = dfR[0].str.strip()
dfR[1] = dfR[1].str.strip()
dfR[0] = dfR[0].astype("str")
dfR[1] = dfR[1].astype("str")
dfR.rename(
    columns = {0 : "origin", 1 : "dest", 2 : "year", 3 : "comp"},
    inplace = True
)

df = df.merge(dfR, on = ["origin", "dest"], how = "left")

df_n = df_n.merge(dfR, on = ["origin", "dest"], how="inner")
df_n = df_n.loc[df_n.comp <= 1]

# % of flights with first class
df_n.loc[df_n.capF.notnull()][["origin", "dest", "flightNum", "ddate"]].drop_duplicates().shape[0] / \
    df_n[["origin", "dest", "flightNum", "ddate"]].drop_duplicates().shape[0]
#0.5836279867138112

# how important is first class
df_n.loc[df_n.capF.notnull()].capF.mean()
#11.99129333735303
df_n["capacity"] = df_n[["capF", "capY"]].sum(axis = 1)
df_n["fracCabin"] = df_n.capF / df_n.capacity
df_n.loc[df_n.capF.notnull()].fracCabin.mean()
#0.14352479603061152

df1 = df.groupby(["origin", "dest", "flightNum", "ddate"])["capacity"].nunique().reset_index()

(df1.loc[df1.capacity > 1].shape[0]) / (df1.shape[0])
#0.030117996534367523

# df1 = df.loc[df.tdate == 0 ].groupby(["origin", "dest", "flightNum", "ddate"])["capacity"].mean().reset_index()
# df2 = df.loc[df.tdate == 59].groupby(["origin", "dest", "flightNum", "ddate"])["capacity"].mean().reset_index()
# df1 = df1.merge(df2, on=["origin", "dest", "flightNum", "ddate"], how="inner")
# (df1.capacity_x != df1.capacity_y).sum()/df1.shape[0]
# #0.020683903252710592
# # (df1.loc[(df1.capacity_x != df1.capacity_y)].capacity_x  - df1.loc[(df1.capacity_x != df1.capacity_y)].capacity_y).mean()
# # #1.7137096774193548

# (df1.loc[(df1.capacity_x > df1.capacity_y)].capacity_x  - df1.loc[(df1.capacity_x > df1.capacity_y)].capacity_y).mean()
# #8.330708661417322
# (df1.loc[(df1.capacity_x < df1.capacity_y)].capacity_x  - df1.loc[(df1.capacity_x < df1.capacity_y)].capacity_y).mean()
# #-5.231404958677686

# changeEquip = df_n.copy()
# modes 		= changeEquip.groupby(["origin", "dest", "flightNum", "ddate"]).capacity.agg(pd.Series.mode).reset_index()
# modes 		= modes.rename(columns = {"capacity" : "modeCap"})
# changeEquip = changeEquip.merge(modes)
# changeEquip["indC"] = changeEquip["modeCap"] != changeEquip["capacity"]

changeEquip1 = df.copy()
modes1 = changeEquip1.groupby(["origin", "dest", "flightNum", "ddate"]) \
    .capacity.agg(pd.Series.mode).reset_index()
modes1 = modes1.rename(columns = {"capacity" : "modeCap"})
changeEquip1 = changeEquip1.merge(modes1)
changeEquip1["indC"] = changeEquip1["modeCap"] != changeEquip1["capacity"]

#changeSM = changeEquip.loc[changeEquip.indC == 1].groupby(["origin", "dest", "flightNum", "ddate"]).tdate.max().value_counts()
changeSM = changeEquip1.loc[changeEquip1.indC == 1] \
    .groupby(["origin", "dest", "flightNum", "ddate"]).tdate.max().value_counts()
# changeSM is overwhelming far from departure.
changeSM
# >>> changeSM
# 2     201
# 1      47
# 0      39
# 60     24
# 5      22
# 10      5
# 13      5
# 9       4
# 15      4
# 3       3
# 6       3
# 11      3
# 12      2
# 42      2
# 8       1
# Name: tdate, dtype: int64
# >>>

changeSM.reset_index(drop = False).loc[changeSM.index <= 2].sum().tdate / changeSM.sum()
#0.7863013698630137

changes = df.copy()
changes = changes.sort_values(
    ["origin", "dest", "flightNum", "ddate", "tdate"],
    ascending = False
).reset_index(drop = True)
changes["difC"] = changes.groupby(["origin", "dest", "flightNum", "ddate"]) \
    .capacity.shift(1)
changes["difC_D"] = (changes.difC - changes.capacity > 0) & (changes.difC.notnull())
changes["difC_U"] = (changes.difC - changes.capacity < 0) & (changes.difC.notnull())

changes["elf"] = changes.groupby(["origin", "dest", "flightNum", "ddate"]) \
    .lf.transform("last")

means = changes[["origin", "dest", "flightNum", "ddate", "elf"]].drop_duplicates() \
    .groupby(["origin", "dest"]).elf.mean().reset_index()
means = means.rename(columns = {"elf" : "avg"})
changes = changes.merge(means, on = ["origin", "dest"])
changes["delLF"] = changes.elf - changes.avg

changes.loc[changes.difC_U == True].delLF.mean()
changes.loc[changes.difC_D == True].delLF.mean()

a = stats.ttest_rel(
    changes.loc[changes.difC_U == True].elf,
    changes.loc[changes.difC_U == True].avg
)
b = stats.ttest_rel(
    changes.loc[changes.difC_D == True].elf,
    changes.loc[changes.difC_D == True].avg
)

def t_test(x, y, alternative="both-sided"):
        _, double_p = stats.ttest_ind(x, y, equal_var = False)
        if alternative == "both-sided":
            pval = double_p
        elif alternative == "greater":
            if np.mean(x) > np.mean(y):
                pval = double_p / 2.
            else:
                pval = 1.0 - double_p / 2.
        elif alternative == "less":
            if np.mean(x) < np.mean(y):
                pval = double_p / 2.
            else:
                pval = 1.0 - double_p / 2.
        return pval

print(
    t_test(
        changes.loc[changes.difC_U == True].elf,
        changes.loc[changes.difC_U == True].avg,
        alternative = "greater"
    )
)
print(
    t_test(
        changes.loc[changes.difC_D == True].avg,
        changes.loc[changes.difC_D == True].elf,
        alternative="greater"
    )
)

# >>> changes.loc[changes.difC_U == True].delLF.mean()
# -0.0321720699993666
# >>> changes.loc[changes.difC_D == True].delLF.mean()
# -0.007989449720960586
# >>> stats.ttest_rel(changes.loc[changes.difC_U == True].elf, changes.loc[changes.difC_U == True].avg)
# Ttest_relResult(statistic=-5.509321255134468, pvalue=9.163242107189687e-08)
# >>> stats.ttest_rel(changes.loc[changes.difC_D == True].elf, changes.loc[changes.difC_D == True].avg)
# Ttest_relResult(statistic=-1.1843204139538213, pvalue=0.23747763601134167)
# >>>


from tweepy import API
from tweepy import Cursor
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
from tweepy import Stream
import joblib

import twitter_credentials

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import re

from sklearn.feature_extraction.text import TfidfVectorizer
#from sklearn.metrics import classification_report

from sklearn import svm
from sklearn.neighbors import KNeighborsClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier

trainData = pd.read_csv("train.csv")
testData = pd.read_csv("test.csv")

vectorizer = TfidfVectorizer(min_df = 5,
                             max_df = 0.8,
                             sublinear_tf = True,
                             use_idf = True)

train_vectors = vectorizer.fit_transform(trainData['Content'])
test_vectors = vectorizer.transform(testData['Content'])

# # # # Using SVM # # # #

#classifier_linear = svm.SVC(kernel='linear')
#classifier_linear.fit(train_vectors, trainData['Label'])
#joblib.dump(classifier_linear, 'svm.pkl')#this saves the trained model in a file.
classifier_linear = joblib.load("svm.pkl")

# # # # # # # # # # # # #

# # # # Using KNN # # # #

#Y = trainData[['Label']]
#Y = Y['Label'].values
#k = 8
#neigh = KNeighborsClassifier(n_neighbors = k).fit(train_vectors,Y)
#joblib.dump(neigh, 'knn.pkl')#this saves the trained model in a file.
neigh = joblib.load("knn.pkl")

# # # # # # # # # # # # #

# # # # Using logistic regression # # # #

#Y = np.asarray(trainData['Label'])
#Logistic = LogisticRegression(C=0.01, solver='liblinear').fit(train_vectors,Y)
#joblib.dump(Logistic, 'lr.pkl')#this saves the trained model in a file.
Logistic = joblib.load("lr.pkl")

# # # # # # # # # # # # #
# # # # Using Tree Classifier # # # #

#Y = trainData[['Label']]
#sentTree = DecisionTreeClassifier(criterion="entropy", max_depth = 4)
#sentTree.fit(train_vectors,Y)
#joblib.dump(sentTree, 'tc.pkl')#this saves the trained model in a file.
sentTree = joblib.load("tc.pkl")

# # # # # # # # # # # # #
# # # # TWITTER CLIENTS # # # #
class TwitterClient():
    def __init__(self, twitter_user=None):
        self.auth = TwitterAuthenticator().authenticate_twitter_app()
        self.twitter_client = API(self.auth)

        self.twitter_user = twitter_user

    def get_twitter_client_api(self):
        return self.twitter_client

    def get_tweets(self, query, count = 10):
        tweets1 = []
        try:
            # call twitter api to fetch tweets
            fetched_tweets = self.twitter_client.search(q = query, count = count)

            for tweet in fetched_tweets:

                # appending parsed tweet to tweets list
                if not hasattr(tweet,'retweeted_status'):
                    # if tweet has retweets, ensure that it is appended only once
                    tweets1.append(tweet)

            # return parsed tweets
            return tweets1

        except tweepy.TweepError as e:
            # print error (if any)
            print("Error : " + str(e))
    """
    def get_user_timeline_tweets(self, num_tweets):
        tweets = []
        for tweet in Cursor(self.twitter_client.user_timeline, id=self.twitter_user).items(num_tweets):
            tweets.append(tweet)
        return tweets

    def get_friend_list(self, num_friends):
        friend_list = []
        for friend in Cursor(self.twitter_client.friends, id=self.twitter_user).items(num_friends):
            friend_list.append(friend)
        return friend_list

    def get_home_timeline_tweets(self, num_tweets):
        home_timeline_tweets = []
        for tweet in Cursor(self.twitter_client.home_timeline, id=self.twitter_user).items(num_tweets):
            home_timeline_tweets.append(tweet)
        return home_timeline_tweets

    """
# # # # TWITTER AUTHENTICATER # # # #
class TwitterAuthenticator():

    def authenticate_twitter_app(self):
        auth = OAuthHandler(twitter_credentials.CONSUMER_KEY, twitter_credentials.CONSUMER_SECRET)
        auth.set_access_token(twitter_credentials.ACCESS_TOKEN, twitter_credentials.ACCESS_TOKEN_SECRET)
        return auth

# # # #  TWITTER ANALYZER# # # #
class TweetAnalyzer():
    """
    Functionality for analyzing and categorizing content from tweets.
    """

    def clean_tweet(self, tweet):
        return ' '.join(re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)", " ", tweet).split())

    def analyze_sentiment_svm(self, tweet):
        review_vector = vectorizer.transform([tweet])
        return classifier_linear.predict(review_vector)

    def analyze_sentiment_knn(self, tweet):
        review_vector = vectorizer.transform([tweet])
        return neigh.predict(review_vector)

    def analyze_sentiment_logisticReg(self, tweet):
        review_vector = vectorizer.transform([tweet])
        return Logistic.predict(review_vector)

    def analyze_sentiment_TreeClassifier(self, tweet):
        review_vector = vectorizer.transform([tweet])
        return sentTree.predict(review_vector)

    def tweets_to_data_frame(self, tweets):
        df = pd.DataFrame(data=[tweet.text for tweet in tweets], columns=['tweets'])

        #df['id'] = np.array([tweet.id for tweet in tweets])
        df['len'] = np.array([len(tweet.text) for tweet in tweets])
        #df['date'] = np.array([tweet.created_at for tweet in tweets])
        #df['source'] = np.array([tweet.source for tweet in tweets])
        #df['likes'] = np.array([tweet.favorite_count for tweet in tweets])
        #df['retweets'] = np.array([tweet.retweet_count for tweet in tweets])

        return df

if __name__ == '__main__':

    twitter_client = TwitterClient()
    tweet_analyzer = TweetAnalyzer()

    api = twitter_client.get_twitter_client_api()

    #tweets = api.user_timeline(screen_name = "guardiannews", count=200)
    tweets1 = twitter_client.get_tweets(query = 'Dark Phoenix', count = 200)

    df = tweet_analyzer.tweets_to_data_frame(tweets1)
    df['sent_svm'] = np.array([tweet_analyzer.analyze_sentiment_svm(tweet) for tweet in df['tweets']])
    df['sent_knn'] = np.array([tweet_analyzer.analyze_sentiment_knn(tweet) for tweet in df['tweets']])
    df['sent_lr'] = np.array([tweet_analyzer.analyze_sentiment_logisticReg(tweet) for tweet in df['tweets']])
    df['sent_treeC'] = np.array([tweet_analyzer.analyze_sentiment_TreeClassifier(tweet) for tweet in df['tweets']])

    print(df.head(14))
    print('positive: ', report['pos'])
    print('negative: ', report['neg'])


# para ter acesso as bibliotecas primeiramente instale-as através do terminal, segue comando de instalação:
# pip install pdfminer3k
from pdfminer.pdfinterp import PDFResourceManager, process_pdf
from pdfminer.converter import TextConverter
from pdfminer.layout import LAParams
from io import StringIO
from urllib.request import urlopen

def ler_pdf(arquivopdf):
    # PDFResourceManager Usado para armazenar recursos compartilhados
    # como fontes e imagens
    recursos = PDFResourceManager()
    buffer = StringIO()
    layout_params = LAParams()
    dispositivo = TextConverter(recursos, buffer, laparams=layout_params)
    process_pdf(recursos, dispositivo, arquivopdf)
    dispositivo.close()
    conteudo = buffer.getvalue()
    buffer.close()
    return conteudo

# Abrindo arquivo PDF online
pdf_lido = 'http://imagens.seplag.ce.gov.br/PDF/20210719/do20210719p01.pdf'
arquivoPDF = urlopen(pdf_lido)
stringSaida = ler_pdf(arquivoPDF)
# print(stringSaida)
arquivoPDF.close()
leitura = stringSaida.split('*** *** ***')
# print(leitura)
print(len(leitura))
lista = []
flag_acerto = 0
for i in range(len(leitura)):
    if 'Licitação' in leitura[i] or 'LICITAÇÃO' in leitura[i]:
        if flag_acerto == 0:
            texto = leitura[i].split('AVISO DE')
            lista.append('AVISO DE' + texto[-1])
        # elif i < len(leitura) - 1:
        else:
            lista.append(leitura[i])
        flag_acerto += 1
        # else:
        #     texto = leitura[i].split('*** *** ***')
        #     lista.append(texto[0])
print(len(lista))
print(f'Foram encontrados {flag_acerto} casos.')
arquivo = open('saida.txt', 'w', encoding='UTF-8')
arquivo.write('Dados retirados do pdf encontrado em: ' + pdf_lido + '\n')
arquivo.write('-' * 150)
arquivo.write('\n')
for i in lista:
    # print(i)
    # print('-' * 150)
    arquivo.write(i)
    arquivo.write('-' * 120)
arquivo.close()
# print(stringSaida)


import tempfile

import pkg_resources
from flask import g
from invenio_files_rest.views import ObjectResource

try:
    pkg_resources.get_distribution('wand')
    from wand.image import Image
    HAS_IMAGEMAGICK = True
except pkg_resources.DistributionNotFound:
    # Python module not installed
    HAS_IMAGEMAGICK = False
except ImportError:
    # ImageMagick notinstalled
    HAS_IMAGEMAGICK = False

def protect_api(uuid=None, **kwargs):
    """Retrieve object and check permissions.

    Retrieve ObjectVersion of image being requested and check permission
    using the Invenio-Files-REST permission factory.
    """
    bucket, version_id, key = uuid.split(':', 2)
    g.obj = ObjectResource.get_object(bucket, key, version_id)
    return g.obj

def image_opener(key):
    """Handler to locate file based on key.

    .. note::
        If the file is a PDF then only the first page will be
        returned as an image.

    :param key: A key encoded in the format "<bucket>:<version>:<object_key>".
    :returns: A file-like object.
    """
    if hasattr(g, 'obj'):
        obj = g.obj
    else:
        obj = protect_api(key)

    fp = obj.file.storage().open('rb')

    # If ImageMagick with Wand is installed, extract first page
    # for PDF/text.
    if HAS_IMAGEMAGICK and obj.mimetype in ['application/pdf', 'text/plain']:
        first_page = Image(Image(fp).sequence[0])
        tempfile_ = tempfile.TemporaryFile()
        with first_page.convert(format='png') as converted:
            converted.save(file=tempfile_)
        return tempfile_
    return fp


# On 1 TitanX, it takes around 22 hours for training
# metircs (pre, rec, pairwise fscore)
# test on 2 proposal params (gt iop, 0.2-0.8): (99.1, 67.29, 80.16)
# test on 20 proposal params (gt iop, 0.2-0.8): (97.83, 81.2, 88.74)

import os.path as osp
from functools import partial
from proposals import generate_proposals

use_random_seed = True
featureless = False

# model
model = dict(type='dsgcn',
             kwargs=dict(feature_dim=256,
                         featureless=featureless,
                         stage='seg',
                         reduce_method='no_pool',
                         use_random_seed=use_random_seed,
                         hidden_dims=[512, 64]))

# training args
optimizer = dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=1e-4)
optimizer_config = {}

lr_config = dict(
    policy='step',
    step=[15, 24, 28],
)

iter_size = 32
batch_size_per_gpu = 1
test_batch_size_per_gpu = 256
total_epochs = 30
workflow = [('train', 1)]

# misc
workers_per_gpu = 1

checkpoint_config = dict(interval=1)

log_level = 'INFO'
log_config = dict(interval=100, hooks=[
    dict(type='TextLoggerHook'),
])

# post_process
th_pos = -1
th_iou = 1
th_outlier = 0.5
keep_outlier = True

# testing metrics
metrics = ['pairwise', 'bcubed', 'nmi']

# data locations
prefix = './data'
train_name = 'part0_train'
test_name = 'part1_test'

th_iop_min=0.2
th_iop_max=0.8

knn_method = 'hnsw'

k_lst_i0 = [30, 60, 80]
step_i0 = 0.05
minsz_i0 = 3
maxsz_i0 = 300
th_lst_i0 = [0.6, 0.65, 0.7, 0.75]

k_th_lst_i0 = []
for k in k_lst_i0:
    for th in th_lst_i0:
        k_th_lst_i0.append((k, th))

th_i1 = 0.4
step_i1 = 0.05
minsz_i1 = 3
maxsz_i1 = 500
sv_minsz_i1 = 2
k_maxsz_lst_i1 = [(2, 8), (2, 12), (2, 16), (3, 5), (3, 10), (4, 4)]

proposal_params = [
    dict(k=k_i0,
         knn_method=knn_method,
         th_knn=th_i0,
         th_step=step_i0,
         minsz=minsz_i0,
         maxsz=maxsz_i0,
         iter1_params=[
             dict(k=k_i1,
                  knn_method=knn_method,
                  th_knn=th_i1,
                  th_step=step_i1,
                  minsz=minsz_i1,
                  maxsz=maxsz_i1,
                  sv_minsz=sv_minsz_i1,
                  sv_maxsz=sv_maxsz_i1) for k_i1, sv_maxsz_i1 in k_maxsz_lst_i1
         ]) for k_i0, th_i0 in k_th_lst_i0
]
feat_path = osp.join(prefix, 'features', '{}.bin'.format(train_name))
label_path = osp.join(prefix, 'labels', '{}.meta'.format(train_name))
proposal_path = osp.join(prefix, 'cluster_proposals')
train_data = dict(wo_weight=False,
                  feat_path=feat_path,
                  label_path=label_path,
                  use_random_seed=use_random_seed,
                  featureless=featureless,
                  th_iop_min=th_iop_min,
                  th_iop_max=th_iop_max,
                  proposal_folders=partial(generate_proposals,
                                           params=proposal_params,
                                           prefix=prefix,
                                           oprefix=proposal_path,
                                           name=train_name,
                                           dim=model['kwargs']['feature_dim'],
                                           no_normalize=False))

step_i0 = 0.05
minsz_i0 = 3
maxsz_i0 = 300
th_iter0_lst_i0 = [(0.6, True), (0.7, True), (0.75, False)]

th_i1 = 0.4
step_i1 = 0.05
minsz_i1 = 3
maxsz_i1 = 500
sv_minsz_i1 = 2
k_maxsz_lst_i1 = [(2, 8), (2, 12), (2, 16), (3, 5), (3, 10), (4, 4)]

proposal_params = [
    dict(k=k,
         knn_method=knn_method,
         th_knn=th_i0,
         th_step=step_i0,
         minsz=minsz_i0,
         maxsz=maxsz_i0,
         iter0=iter0,
         iter1_params=[
             dict(k=k_i1,
                  knn_method=knn_method,
                  th_knn=th_i1,
                  th_step=step_i1,
                  minsz=minsz_i1,
                  maxsz=maxsz_i1,
                  sv_minsz=sv_minsz_i1,
                  sv_maxsz=sv_maxsz_i1) for k_i1, sv_maxsz_i1 in k_maxsz_lst_i1
         ]) for th_i0, iter0 in th_iter0_lst_i0
]
# use the same proposal params as training
feat_path = osp.join(prefix, 'features', '{}.bin'.format(test_name))
label_path = osp.join(prefix, 'labels', '{}.meta'.format(test_name))
test_data = dict(wo_weight=True,
                 feat_path=feat_path,
                 label_path=label_path,
                 use_random_seed=use_random_seed,
                 use_max_degree_seed=True,
                 featureless=featureless,
                 th_iop_min=th_iop_min,
                 th_iop_max=th_iop_max,
                 proposal_folders=partial(generate_proposals,
                                          params=proposal_params,
                                          prefix=prefix,
                                          oprefix=proposal_path,
                                          name=test_name,
                                          dim=model['kwargs']['feature_dim'],
                                          no_normalize=False))


import cv2
import numpy as np
import mediapipe as mp
import matplotlib.pyplot as plt
import math
import os
import dlib
import pandas as pd
from localize import Localize
from cnnlstm import CNNLSTM
from evaluation import MyEvaluation
from typing import List, Mapping, Optional, Tuple, Union

df = pd.read_csv("CAS(ME)^2evaluation.csv")

test_video = [
            r"D:\CASME\CASME(2)\rawvideo\rawvideo\s15\15_0102disgustingteeth.avi"
            #   r"D:\CASME\CASME(2)\rawvideo\rawvideo\s36\36_0401girlcrashing.avi",
            #   r"D:\CASME\CASME(2)\rawvideo\rawvideo\s36\36_0505funnyinnovations.avi",

            #   r"D:\CASME\CASME(2)\rawvideo\rawvideo\s37\37_0101disgustingteeth.avi",
            #   r"D:\CASME\CASME(2)\rawvideo\rawvideo\s37\37_0402beatingpregnantwoman.avi",
            #   r"D:\CASME\CASME(2)\rawvideo\rawvideo\s37\37_0502funnyerrors.avi",
            #   r"D:\CASME\CASME(2)\rawvideo\rawvideo\s37\37_0505funnyinnovations.avi",
            #   r"D:\CASME\CASME(2)\rawvideo\rawvideo\s37\37_0507climbingthewall.avi",
            #   r"D:\CASME\CASME(2)\rawvideo\rawvideo\s37\37_0508funnydunkey.avi",

            #   r"D:\CASME\CASME(2)\rawvideo\rawvideo\s38\38_0502funnyerrors.avi",
            #   r"D:\CASME\CASME(2)\rawvideo\rawvideo\s38\38_0507climbingthewall.avi",

            #   r"D:\CASME\CASME(2)\rawvideo\rawvideo\s40\40_0401girlcrashing.avi",
            #   r"D:\CASME\CASME(2)\rawvideo\rawvideo\s40\40_0502funnyerrors.avi",
            #   r"D:\CASME\CASME(2)\rawvideo\rawvideo\s40\40_0503unnyfarting.avi"
            ]

localization_algorithm = Localize()
# Initialize Spotting Algorithm
prediction_model = CNNLSTM()
# self.prediction_model.mobilenet("core/transfer_mobilenet_cnnlstm_tfrecord_2/cp.ckpt")
prediction_model.mobilenet("D:\CodeProject2\SKRIPSI_FINAL\core\Weights\Transfer_mobilenet_cnnlstm_localize_tfrecord_pyramid_1/cp.ckpt")

# Pyramid
# "D:\CodeProject2\SKRIPSI_FINAL\core\Transfer_mobilenet_cnnlstm_localize_tfrecord_pyramid_1/cp.ckpt"
# "D:\CodeProject2\SKRIPSI_FINAL\core\Transfer_mobilenet_localize_cnnlstm_tfrecord/cp.ckpt"

eval = MyEvaluation()

for video in test_video:
    # find the detection
    raw_filename = video.split('\\')[-1][:-4]
    find_in_df = df[(df['raw_path'] == raw_filename)]

    detection = []
    for index, item in find_in_df.iterrows():
        onset = int(item['onset'])
        apex = int(item['apex'])
        offset = int(item['offset'])

        if offset == 0:
            offset = apex

        detection.append((onset, offset))

    print("Filename:", raw_filename)
    print("Detection:", detection)
    # Evaluation Loop
    cap = cv2.VideoCapture(video)

    arm_the_localization = True
    detected = True
    xleft = 0
    ytop = 0
    xright = 0
    ybot = 0

    # Read the file
    sliding_window = []
    itr = 0
    while True:
        ret, frame = cap.read()
        if not ret:
            print("Can't receive frame (stream end?). Exiting ...")
            # Process
            break

        itr += 1
        # Our operations on the frame come here
        rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
        if arm_the_localization:
            arm_the_localization = False
            detected, xleft, ytop, xright, ybot = localization_algorithm.mp_face_mesh_crop_preprocessing(frame)

        cropped = frame[ytop:ybot, xleft:xright]
        # Display the resulting frame
        cv2.imshow('frame', cropped)

        # Process the frame using CNN-LSTM
        frame = cv2.resize(cropped, (224, 224), interpolation=cv2.INTER_AREA)
        rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) / 255

        sliding_window.append(rgb)
        if len(sliding_window) > 12:
            sliding_window.pop(0)

        if len(sliding_window) == 12:
            np_sliding_window = np.expand_dims(np.array(sliding_window), axis=0)
            conf, label = prediction_model.process(np_sliding_window, conf=0.7)
            start_frame = itr - 11
            end_frame = itr

            sliding_window = []

            print("Label of frame", start_frame, "to", end_frame, "Label", label, "Confidence Level:", conf)

            eval.count_casme2(start_frame, end_frame, label, detection)

        if cv2.waitKey(1) == ord('q'):
            break
    # When everything done, release the capture
    cap.release()
    cv2.destroyAllWindows()

    eval.to_csv(raw_filename)
    eval.print_total()
    eval.reset_count()
    print("==============================")

print("Evaluation Done")


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('staff', '0016_member'),
    ]

    operations = [
        migrations.DeleteModel(
            name='USAddress',
        ),
        migrations.AlterField(
            model_name='membershipdate',
            name='membership_date',
            field=models.DateField(unique=True),
        ),
    ]


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('grafiks', '0016_remove_planotajs_one'),
    ]

    operations = [
        migrations.AlterField(
            model_name='grafiks',
            name='vietas',
            field=models.IntegerField(default=20),
        ),
    ]


########################################################################
############################# ASSIGNMENT 1 #############################
################################ Task 1 ################################
########################################################################

# %% Importing Libraries

import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

# %% Problem 1.1

# Changing the working directory
os.chdir("/Users/heinzemax/Documents/GitHub/inequality_sse/assignment_1")

# Reading in the data
income_data = pd.read_stata("/Users/heinzemax/Documents/GitHub/inequality_sse/assignment_1/psid1999.dta")

# Filtering the data so that only household heads are contained.
# Note that filtering by relationhead==10 is kind of a "best guess" after
# consulting PSID codebooks available online, since the variable in question
# is not labeled. Source for the "best guess": Page 4 of the document available at
# https://psidonline.isr.umich.edu/documents/psid/codebook/MX19REL_codebook.pdf
income_data = income_data.query("relationhead == 10")

# %% Problem 1.2

# Filtering the data set by age older than 24 and younger than 66
income_data = income_data.query("age > 24 & age < 66")

# Filtering the data set by married HHH with spouse present
income_data = income_data.query("(mls=='married' & sp_in_fu=='Spouse/Partner in FU now')")

# %% Problem 1.3

# Filtering out all household heads who worked zero hours
income_data = income_data.query("head_hrs > 0")

# Creating a logged hourly earnings variable from yearly income and yearly hrs,
# adding 1 to avoid taking the log of 0
income_data['log_hr_earnings'] = np.log((income_data['head_li'] / income_data['head_hrs']) + 1)

# Computing averages by age and education
average_wages = income_data.groupby(['age', 'edu'])['log_hr_earnings'].mean().reset_index()

# %% Problem 1.4

# Create a scatter plot showing avg log hrly earnings by age, with education
# levels represented as different colored dots
plt.figure(figsize=(8,6), dpi = 300)

education_levels = average_wages['edu'].unique()

for this_education_level in education_levels:
    subset = average_wages[average_wages['edu'] == this_education_level]
    plt.scatter(subset['age'], subset['log_hr_earnings'], label=this_education_level)

plt.title('Average Log Hourly Earnings by Age and Education')
plt.xlabel('Age')
plt.ylabel('Average Log Hourly Earnings')
plt.legend(title = "Education")
plt.grid(True, which='both', linestyle='--', linewidth=0.5)
plt.show()


import asyncio
from typing import List

import discord
from discord.ext import commands

from settings import rulebook_url, tourney_name, footer_icon, footer_note, \
    veto_timeout, newline
from smash.game import Game
from smash.player import Player
from utils.checks import playerCheck, stageCheck

class Match:
    """
    Represents a Smash match
    """

    def __init__(self, player1: Player, player2: Player, num_of_games: int):
        # players
        self.player1 = player1
        self.player2 = player2
        self.winner = None

        # match data
        self.games: List[Game] = []
        self.name: str = f"{tourney_name}: {player1.name} vs {player2.name}"
        self.description: str = f"{self.player1.mention} vs " \
                                f"{self.player2.mention} " \
                                f"\nThe rulebook can be found " \
                                f"[here]({rulebook_url})"

        # match state
        self.num_of_games: int = num_of_games
        self.current_game: int = 0

        # generate first game
        self.games.append(Game(0))

    @property
    def embed(self):
        title = f"Smash Ultimate Best-of-{self.num_of_games} Veto"
        embed = discord.Embed(title=title,
                              description=self.description,
                              color=discord.Colour.gold())
        # loop through max games times and generate embed fields
        for x in range(len(self.games)):
            embed.add_field(name=self.games[x].name,
                            value=self.games[x].winner_embed(),
                            inline=False)
            # x - 1 because its using index num
            embed.add_field(name="__Starter Stages__",
                            value=self.games[x].starters_embed(),
                            inline=True)
            embed.add_field(name="__Counterpick Stages__",
                            value=self.games[x].counters_embed(),
                            inline=True)
        # set footer
        embed.set_footer(icon_url=footer_icon,
                         text=f"{tourney_name} | {footer_note}")

        return embed

    async def veto(self, ctx: discord.ext.commands.Context,
                   bot: discord.ext.commands.Bot):
        # FIGURING OUT GAME EMBED INDEXES (ONLY IF U WANT TO REWRITE):
        # GAME NUM * 3 = embed title index, just + 1 and + 2 for stage lists
        # IF FINISH EARLY LOOP FROM CURRENT_GAME TO NUM_OF_GAMES - 1
        # AND CROSS OUT LINE

        # add DSR stages to the game if on game 3 or higher
        if self.current_game >= 2:
            for x in range(len(self.player2.dsr)):
                self.games[self.current_game].veto_stage(
                    self.player2.dsr[x].name)

        # send first embed
        await ctx.send(embed=self.embed)

        # initial veto
        if self.current_game == 0:
            # loop through 4 stage veto's/selection
            for x in range(4):
                # check which message to send
                if x == 0:
                    await ctx.send(f"{self.player1.mention}, veto a starter.")
                if x == 1:
                    await ctx.send(f"{self.player2.mention}, veto a starter.")
                if x == 2:
                    await ctx.send(
                        f"{self.player2.mention}, veto another starter.")
                if x == 3:
                    await ctx.send(
                        f"{self.player1.mention}, select the stage from "
                        f"the remaining starters.")

                # wait for players stage choice
                msg = await bot.wait_for('message',
                                         check=stageCheck(ctx, self),
                                         timeout=veto_timeout)

                # if on stage selection
                if x == 3:
                    # choose the stage
                    self.games[self.current_game].choose_stage(msg.content)
                # if on veto still
                else:
                    # veto the stage
                    self.games[self.current_game].veto_stage(msg.content)

                # regenerate and send embed
                await ctx.send(embed=self.embed)

        # subsequent veto
        else:
            # loop through 3 stage veto's/selection process
            for x in range(3):
                if x == 0:
                    await ctx.send(
                        f"{self.player1.mention}, veto a stage.")
                elif x == 1:
                    await ctx.send(
                        f"{self.player1.mention}, veto another stage.")
                elif x == 2:
                    await ctx.send(
                        f"{self.player2.mention}, select the stage.")

                # wait for players stage choice
                msg = await bot.wait_for('message',
                                         check=stageCheck(ctx, self),
                                         timeout=veto_timeout)

                # if on stage selection
                if x == 2:
                    # choose the stage
                    self.games[self.current_game].choose_stage(msg.content)
                # if on veto still
                else:
                    # veto the stage
                    self.games[self.current_game].veto_stage(msg.content)

                # regenerate and send embed
                await ctx.send(embed=self.embed)

        # get winner
        await ctx.send(f"{newline}GLHF! Once finished, "
                       f"the winner should say `me`.")
        msg = await bot.wait_for('message',
                                 check=playerCheck(ctx),
                                 timeout=veto_timeout)

        # swap player 1 if player 2 was winner
        if msg.author.id == self.player2.id:
            self.swap_players()

        # add stage to winner dsr list
        self.player1.dsr.append(
            self.games[self.current_game].selected_stage)

        # set the game winner
        self.games[self.current_game].winner = self.player1
        self.player1.wins += 1

        # check if the match has a winner
        if self.player1.wins >= (self.num_of_games // 2) + 1:
            self.winner = self.player1
            await ctx.send(embed=self.embed)
            await ctx.send('GG! Run `ve!close` to archive this channel.')

        # else prep for the next game veto
        else:
            # move to next game
            self.current_game += 1

            # add games
            self.games.append(Game(self.current_game))

            # setup next game
            await ctx.send(embed=self.embed)
            await ctx.send('Starting next game veto...')
            await asyncio.sleep(3)

    def swap_players(self):
        """
        Swap player1 and player2
        """
        # swap player objects
        p1 = self.player2
        p2 = self.player1

        # set new players
        self.player1 = p1
        self.player2 = p2


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import json
import logging
import urllib

from .tornado_testing_base import TornadoBaseTestCase

from ..models.thread import THREAD_TYPE

class APIChatOfferTestCase (TornadoBaseTestCase):

    @staticmethod
    def _get_offer_body():
        return 'chat_offer=%s' % json.dumps({
            'title': "Hello! I am a #test_offer and i have #tho_hashtags",
            'lang': ['russian', 'english']
        })

    def test10_add_offer(self):
        response = self.fetch(
            '/chat_offers',
            method='POST',
            body=self._get_offer_body()
        )
        self.assertEqual(response.code, 200)
        offers = json.loads(response.body.decode())['chat_offers']
        for o in offers.values():
            self.assertEqual(
                o['title'],
                "Hello! I am a #test_offer and i have #tho_hashtags"
            )

    def test11_add_offer_incorrect_data(self):
        response = self.fetch(
            '/chat_offers',
            method='POST',
            body="chat_offer=sdfsdf"
        )
        self.assertEqual(response.code, 406)

    def test21_get_offers(self):
        response = self.fetch(
            '/chat_offers',
            method='GET',
        )
        self.assertEqual(response.code, 200)
        offers = json.loads(response.body.decode())['chat_offers']
        self.assertEqual(len(offers.keys()), 1)
        id = list(offers.keys())[0]
        offer = offers[id]
        self.assertEqual(
            offer['title'],
            "Hello! I am a #test_offer and i have #tho_hashtags"
        )

    def test31_delete_offer(self):
        response = self.fetch(
            '/chat_offers/1',
            method='DELETE',
        )
        self.assertEqual(response.code, 200)
        response = self.fetch(
            '/chat_offers',
            method='GET',
        )
        self.assertEqual(response.code, 200)
        offers = json.loads(response.body.decode())['chat_offers']
        self.assertEqual(len(offers), 0)

    def test32_join_offer(self):
        response = self.fetch(
            '/chat_offers',
            method='POST',
            body=self._get_offer_body()
        )
        self.assertEqual(response.code, 200)
        offers = json.loads(response.body.decode())['chat_offers']
        id = list(offers.keys())[0]
        offer = offers[id]
        # response = self.fetch(
        #     '/chat_offer_join/%i' % int(offer['id']),
        #     method='POST',
        #     body='post=%s' % json.dumps({
        #         'text': 'hello'
        #     })
        # )
        body = {
            'users_id': json.dumps([offer['author_id']]),
            'offer_id': int(offer['id']),
            'post': json.dumps({'text': 'hello'})
        }
        response = self.fetch(
            '/thread',
            method='POST',
            body=urllib.parse.urlencode(body)
        )
        self.assertEqual(response.code, 200)
        thread_id = json.loads(response.body.decode())['thread_id']
        self.assertTrue(int(thread_id))


import os
from enum import Enum
from pathlib import Path
import matplotlib.pyplot as plt

# import matplotlib.pyplot as plt
import numpy as np
import scipy.fftpack
from scipy import signal
import pyedflib
import random

class States:
    rest, r_fist, l_fist, r_fist_im, l_fist_im, fists, fists_im, feet, feet_im = range(9)

class Tasks:
    T0, T1, T2 = range(3)

#structure to store signal information
class EEG_Sample:
	def __init__(self, signal, label, patient_num, record_num, sample_num):
		self.signal = signal
		self.label = label
		self.patient_num = patient_num
		self.record_num = record_num
		self.sample_num = sample_num

	def print_sample_info(self):
		print("signal shape: ", self.signal.shape, " ;label: ", self.label, " ;patient: ", self.patient_num, " ;record: ", self.record_num, " ;sample: ", self.sample_num)

#maps the record number and task type (T0, T1, T2) to the State
state_table = np.array([[States.rest,States.rest,States.rest],   		#1
						[States.rest,States.rest,States.rest],   		#2
						[States.rest,States.l_fist, States.r_fist],		#3
						[States.rest,States.l_fist_im,States.r_fist_im],#4
						[States.rest,States.fists,States.feet],    		#5
						[States.rest,States.fists_im,States.feet_im],   #6
						[States.rest,States.l_fist, States.r_fist],		#7
						[States.rest,States.l_fist_im,States.r_fist_im],#8
						[States.rest,States.fists,States.feet], 		#9
						[States.rest,States.fists_im,States.feet_im], 	#10
						[States.rest,States.l_fist, States.r_fist],		#11
						[States.rest,States.l_fist_im,States.r_fist_im],#12
						[States.rest,States.fists,States.feet], 		#13
						[States.rest,States.fists_im,States.feet_im]]	#14
						)

#returns array of EEG_Samples
def get_eeg_samples(db_dir_name, num_patients=109, num_records=14, channels=[9,11,13], sig_len=640, num_segs=2):
	# rootdir = os.path.join(str(Path.home()), 'SeniorProject/EEG_Dataset/')
	rootdir = os.path.join(str(Path.home()), db_dir_name)

	#the completed eeg data set to be sent to Tensorflow
	eeg_samples = []
	records1 = {'03', '07', '11'}
	records2 = {'04', '08', '12'}	#these are the records than contain the left/right fist data

	#iterate through every volunteer's directory (S001, S002,..., S109)
	for patient in range(1, num_patients  + 1):
		direc = rootdir + 'S%03d' % patient
		st = "VOLUNTEER #" + '%03d' % patient
		print (st)
		sample_count = 0
		#iterate through the EDF num_records in the directory (S001R01.edf, S001R02.edf,..., S001R14.edf)
		for record_num in records2:
			#create new EdfReader for the EDF record file
			edf_reader = create_edf_reader(direc, patient, record_num)

			#get the list of [starting values, states] for each task-sample
			record_data = parse_record_ann(direc, record_num)

			#iterate through the 4 second task-samples
			for sample_num in range(len(record_data)):
				start_sample = int(record_data[sample_num][0])
				sample_sig = read_eeg_signal(edf_reader, channels, start_sample, sig_len)
				segments = segment_signal(sample_sig, num_segs, sig_len)

				for segment in segments:
					# segment = segment.flatten()
					segment_data = EEG_Sample(segment, int(record_data[sample_num][1]) - 3, patient, record_num, sample_num)
					eeg_samples.append(segment_data)
					# segment_data.print_sample_info()
					sample_count += 1
		print("   sample count: ", sample_count)

	r = 0
	l = 0
	for sample in eeg_samples:
		if sample.label == 0:
			r += 1
		if sample.label == 1:
			l += 1

	print("right fist: ", r, ", left fist: ", l)

	return eeg_samples

def train_test_split_shuffle(eeg_samples, total_samples_used, train_test_percent=0.7):

	#shuffle all samples
	random.shuffle(eeg_samples)

	#shorten the list of samples
	eeg_samples = np.asarray(eeg_samples[:total_samples_used])
	# eeg_data = np.asarray([x.signal for x in eeg_samples])
	# eeg_labels = np.asarray([x.label - 1 for x in eeg_samples])

	# #split the list
	# train_test_split = np.random.rand(len(eeg_samples)) < train_test_percent
	# train_data = eeg_data[train_test_split]
	# train_labels = eeg_labels[train_test_split]
	# test_data = eeg_data[~train_test_split]
	# test_labels = eeg_labels[~train_test_split]

	train_test_split = np.random.rand(len(eeg_samples)) < train_test_percent
	train_samples = eeg_samples[train_test_split]
	test_samples = eeg_samples[~train_test_split]

	# return train_data, train_labels, test_data, test_labels
	return train_samples, test_samples

def train_test_split_patients(eeg_samples, patient_div):
	train_data = []
	train_labels = []
	test_data = []
	test_labels = []

	for sample in eeg_samples:
		if sample.patient_num <= patient_div:
			train_data.append(sample.signal)
			train_labels.append(sample.label)
		else:
			test_data.append(sample.signal)
			test_labels.append(sample.label)

	train_data, train_labels = randomize_data(train_data, train_labels)
	test_data, test_labels = randomize_data(test_data, test_labels)
	return np.asarray(train_data), np.asarray(train_labels), np.asarray(test_data), np.asarray(test_labels)

def randomize_data(data, labels):
	rng_state = np.random.get_state()
	np.random.shuffle(data)
	np.random.set_state(rng_state)
	np.random.shuffle(labels)
	return data, labels

def static_randomize_data(data, labels):
	np.random.seed(69)
	rlist = np.random.permutation(len(data))
	new_data = []
	new_labels = []

	for idx in rlist:
		new_data.append(data[idx])
		new_labels.append(labels[idx])

	return np.asarray(new_data), np.asarray(new_labels)

#create an pyedflib.EdfReader for the given record number
def create_edf_reader(direc, patient_num, record_num):
	edf_fname = 'S' + '%03d' % patient_num + 'R' + record_num + '.edf'
	edf_fname = os.path.join(direc, edf_fname)

	return pyedflib.EdfReader(edf_fname)

# parses through the ann.txt file for the given record number
# returns array of entries: [starting_sample_number, state_type]
def parse_record_ann(direc, record_num):
	record_data = []
	ann_fname = direc + '/ann' + record_num + '.txt'

	#create array of starting samples for each task(T0, T1, T2) in the ann-file (i.e sample# 0, 672, 1328,...)
	#create array of labels for each task in the ann-file (T0, T1, T2)
	with open(ann_fname) as fobj:
		for line in fobj:
			row = line.split()
			start_sample = row[1]
			state_type = state_table[int(record_num) - 1][int(row[2][1])]
			record_data.append([start_sample, state_type])

	# for i in range(len(record_data)):
	# 	if record_data[i][1] != States.l_fist and record_data[i][1] == States.r_fist:

	for sample_data in record_data:
		if sample_data[1] != States.l_fist_im and sample_data[1] != States.r_fist_im:
			record_data.remove(sample_data)

	return record_data

#returns array of signal values from EDF file
def read_eeg_signal(edf_reader, channels, start_sample, sig_len):

	#create empty numpy array of shape: [channel_num, signal_length]
	signal_arr = np.zeros((len(channels), sig_len))

	#iterate through all channels
	for l in range(len(channels)):
		signal_arr[l] = edf_reader.readSignal(int(channels[l]),start_sample, sig_len)

	return signal_arr

# divide input signal into equal length segments
# returns array of shape: [total_segments]
def segment_signal(sig, num_segs, sig_len):
	segment_len = int(sig_len / num_segs)
	segments = []

	for s in range(num_segs):
		start = s * segment_len
		end = start + segment_len
		segment = sig[:,start:end]
		segments.append(segment)

	segments = np.array(segments)
	# print("Signal Shape: ", sig.shape)
	# print("Segments Shape: ", segments.shape)
	return segments


import logging
from io import StringIO
from pathlib import Path
from typing import Any

from alembic.config import Config
from alembic.migration import MigrationContext
from alembic.runtime.environment import EnvironmentContext
from alembic.runtime.migration import MigrationContext
from alembic.script import ScriptDirectory
from sqlalchemy.engine import Engine

from pocpoc.api.microservices import Container

logger = logging.getLogger(__name__)

class CheckDatabaseMigration:
    def __init__(self, engine: Engine, db_version_path: Path) -> None:
        self.engine = engine
        self.db_version_path = db_version_path

    def __call__(self, container: Container) -> None:
        connection = self.engine.connect()
        with connection:
            output_buffer = StringIO()
            context = MigrationContext.configure(
                connection, opts={"output_buffer": output_buffer}
            )
            db_current_revision = context.get_current_revision()

            config = config = Config(str(self.db_version_path / "alembic.ini"))
            config.set_main_option("script_location", str(self.db_version_path))
            config.config_file_name = None
            script = ScriptDirectory.from_config(config)

            repository_current_revision = script.get_current_head()

            if db_current_revision != repository_current_revision:

                def on_revision_apply(
                    context: EnvironmentContext, revision: MigrationContext, **kw: Any
                ) -> None:
                    return script._upgrade_revs("head", db_current_revision)  # type: ignore

                env = EnvironmentContext(
                    config,
                    script,
                    fn=on_revision_apply,
                    starting_rev=db_current_revision,
                    as_sql=True,
                )
                env.configure(
                    output_buffer=output_buffer,
                    dialect=context.dialect,
                    dialect_opts=context.opts,
                    connection=context.connection,
                )
                with env:
                    script.run_env()

                sql = output_buffer.getvalue()

                logger.warning(
                    "Database is not up to date. Current revision on database: %s. Current revision on repository: %s. May the microservice does not perform as well. run the following SQL to update the database: \n\n %s",
                    db_current_revision,
                    repository_current_revision,
                    sql,
                )

                with open("migration.sql", "w") as f:
                    f.write(sql)

                logger.warning("SQL migration script saved to migration.sql")

            connection.close()

class CheckDatabaseConnection:
    def __init__(self, engine: Engine) -> None:
        self.engine = engine

    def __call__(self, container: Container) -> None:
        try:
            connection = self.engine.connect()
            connection.execute("SELECT 1")  # type: ignore
        except Exception as e:
            logger.fatal("Error connecting to database: %s", e)
            raise e

        connection.close()
        logger.debug("Database connection OK")


"""empty message

Revision ID: 0a66b1ea0744
Revises: aa506496a272
Create Date: 2020-12-12 15:32:13.489656

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '0a66b1ea0744'
down_revision = 'aa506496a272'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('sessions', sa.Column('removed', sa.Boolean(), nullable=False))
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('sessions', 'removed')
    # ### end Alembic commands ###


import tensorflow as tf
import numpy as np
from tensorbox.common.classes import DatasetWrapper
import tensorbox.datasets.data_utils as du
from tensorbox.datasets.datawrapper.lissajouswrapper import LissajousWrapper

from matplotlib import pyplot as plt

def change_type(data, label):
    data = tf.cast(data, tf.float32)
    label = tf.cast(label, tf.float32)
    return data, label

def get_arrays(dim, size=1024, null_space=2, rotation=-45.0, axis_scale=(2., 1.)):
    """

    :param dim:
    :param size:
    :param null_space:
    :param rotation: float, rotation in degrees
    :param axis_scale:
    :return:
    """

    radian = rotation * np.pi / 180
    rotation_matrix = np.array([[np.cos(radian), -np.sin(radian)],
                                [np.sin(radian), np.cos(radian)]])

    scale = np.array(axis_scale)

    train_x = np.zeros((size, null_space + dim))
    fake_y = np.zeros(size)

    array = np.matmul(np.random.normal(0.0, 1.0, (size, dim)) * scale, rotation_matrix)
    train_x[:, :dim] = array

    return train_x, fake_y

def create_unsupervised_gaussian_dataset(batch_size=64, dim=2, normalize=True, **kwargs):
    """ create dataset and wrap with tf.data.Dataset()
    :param batch_size:
    :param normalize: bool
    :param kwargs:
    :return:
    """
    train_x, fake_y = get_arrays(dim=dim)

    ds = DatasetWrapper(x_train=train_x,
                        y_train=train_x,
                        x_test=None,
                        y_test=None,
                        batch_size=batch_size,
                        wrapped_class=None,
                        mappings=(du.type_cast_sp,),
                        name='Unsupervised Gaussian')
    ds.normalize_data() if normalize else None  # do not normalize labels to see contractions
    ds.build_tf_dataset()
    return ds

if __name__ == '__main__':
    print('testing..')
    ds = create_unsupervised_gaussian_dataset(normalize=False)

    xs = ds.x_train[:, 0]
    ys = ds.x_train[:, 1]
    fig = plt.figure()

    u, s, v = np.linalg.svd(ds.x_train)

    plt.scatter(xs, ys)
    print('Right-sided vectors\n', v.T)
    plt.show()