import logging
import json
import os
import random
from typing import Dict
from typing import List
from typing import Iterable
from typing import Optional
from typing import Tuple
from pathlib import Path

from .analyses import AnalysisResultsStore
from .advisers import AdvisersResultsStore
from .dependency_monkey_reports import DependencyMonkeyReportsStore
from .inspections import InspectionStore
from .provenance import ProvenanceResultsStore
from .revsolvers import RevSolverResultsStore
from .security_indicators import SIAggregatedStore, SecurityIndicatorsResultsStore
from .solvers import SolverResultsStore

from .graph import GraphDatabase

_LOGGER = logging.getLogger(__name__)
_RANDOMIZE_LISTING = bool(int(os.getenv("THOTH_STORAGES_RANDOMIZE_LISTING", 0)))

def sync_adviser_documents(
    document_ids: Optional[Iterable[str]] = None,
    force: bool = False,
    graceful: bool = False,
    graph: Optional[GraphDatabase] = None,
    is_local: bool = False,
) -> Tuple[int, int, int, int]:
    """Sync adviser documents into graph."""
    if is_local and not document_ids:
        raise ValueError(
            "Cannot sync documents from local directory without explicitly specifying a list of documents to be synced"
        )

    if not graph:
        graph = GraphDatabase()
        graph.connect()

    if not is_local:
        adviser_store = AdvisersResultsStore()
        adviser_store.connect()

    listing = document_ids or adviser_store.get_document_listing()

    if _RANDOMIZE_LISTING:
        random.shuffle(list(listing))

    processed, synced, skipped, failed = 0, 0, 0, 0
    for document_id in listing:
        processed += 1

        if force or not graph.adviser_document_id_exist(os.path.basename(document_id)):
            try:
                if is_local:
                    _LOGGER.debug("Loading document from a local file: %r", document_id)
                    with open(document_id, "r") as document_file:
                        document = json.loads(document_file.read())
                else:
                    _LOGGER.info(
                        "Syncing adviser document from %r with id %r to graph", adviser_store.ceph.host, document_id
                    )
                    document = adviser_store.retrieve_document(document_id)

                graph.sync_adviser_result(document)
                synced += 1
            except Exception:
                if not graceful:
                    raise

                _LOGGER.exception("Failed to sync adviser result with document id %r", document_id)
                failed += 1
        else:
            _LOGGER.info(f"Sync of adviser document with id {document_id!r} skipped - already synced")
            skipped += 1

    return processed, synced, skipped, failed

def sync_solver_documents(
    document_ids: Optional[Iterable[str]] = None,
    force: bool = False,
    graceful: bool = False,
    graph: Optional[GraphDatabase] = None,
    is_local: bool = False,
) -> Tuple[int, int, int, int]:
    """Sync solver documents into graph."""
    if is_local and not document_ids:
        raise ValueError(
            "Cannot sync documents from local directory without explicitly specifying a list of documents to be synced"
        )

    if not graph:
        graph = GraphDatabase()
        graph.connect()

    if not is_local:
        solver_store = SolverResultsStore()
        solver_store.connect()

    listing = document_ids or solver_store.get_document_listing()

    if _RANDOMIZE_LISTING:
        random.shuffle(list(listing))

    processed, synced, skipped, failed = 0, 0, 0, 0
    for document_id in listing:
        processed += 1
        if force or not graph.solver_document_id_exists(os.path.basename(document_id)):
            try:
                if is_local:
                    _LOGGER.debug("Loading document from a local file: %r", document_id)
                    with open(document_id, "r") as document_file:
                        document = json.loads(document_file.read())
                else:
                    _LOGGER.info(
                        "Syncing solver document from %r with id %r to graph", solver_store.ceph.host, document_id
                    )
                    document = solver_store.retrieve_document(document_id)

                graph.sync_solver_result(document, force=force)
                synced += 1
            except Exception:
                if not graceful:
                    raise

                _LOGGER.exception("Failed to sync solver result with document id %r", document_id)
                failed += 1
        else:
            _LOGGER.info(f"Sync of solver document with id {document_id!r} skipped - already synced")
            skipped += 1

    return processed, synced, skipped, failed

def sync_revsolver_documents(
    document_ids: Optional[Iterable[str]] = None,
    force: bool = False,
    graceful: bool = False,
    graph: Optional[GraphDatabase] = None,
    is_local: bool = False,
) -> Tuple[int, int, int, int]:
    """Sync reverse solver documents into graph."""
    if is_local and not document_ids:
        raise ValueError(
            "Cannot sync documents from local directory without explicitly specifying a list of documents to be synced"
        )

    if not graph:
        graph = GraphDatabase()
        graph.connect()

    if not is_local:
        revsolver_store = RevSolverResultsStore()
        revsolver_store.connect()

    listing = document_ids or revsolver_store.get_document_listing()

    if _RANDOMIZE_LISTING:
        random.shuffle(list(listing))

    processed, synced, skipped, failed = 0, 0, 0, 0
    for document_id in listing:
        processed += 1
        try:
            if is_local:
                _LOGGER.debug("Loading document from a local file: %r", document_id)
                with open(document_id, "r") as document_file:
                    document = json.loads(document_file.read())
            else:
                _LOGGER.info(
                    "Syncing reverse solver document from %r with id %r to graph",
                    revsolver_store.ceph.host,
                    document_id,
                )
                document = revsolver_store.retrieve_document(document_id)

            graph.sync_revsolver_result(document)
            synced += 1
        except Exception:
            if not graceful:
                raise

            _LOGGER.exception("Failed to sync reverse solver result with document id %r", document_id)
            failed += 1

    return processed, synced, skipped, failed

def sync_analysis_documents(
    document_ids: Optional[Iterable[str]] = None,
    force: bool = False,
    graceful: bool = False,
    graph: Optional[GraphDatabase] = None,
    is_local: bool = False,
) -> Tuple[int, int, int, int]:
    """Sync image analysis documents into graph."""
    if is_local and not document_ids:
        raise ValueError(
            "Cannot sync documents from local directory without explicitly specifying a list of documents to be synced"
        )

    if not graph:
        graph = GraphDatabase()
        graph.connect()

    if not is_local:
        analysis_store = AnalysisResultsStore()
        analysis_store.connect()

    listing = document_ids or analysis_store.get_document_listing()

    if _RANDOMIZE_LISTING:
        random.shuffle(list(listing))

    processed, synced, skipped, failed = 0, 0, 0, 0
    for document_id in listing:
        processed += 1

        if force or not graph.analysis_document_id_exist(os.path.basename(document_id)):
            try:
                if is_local:
                    _LOGGER.debug("Loading document from a local file: %r", document_id)
                    with open(document_id, "r") as document_file:
                        document = json.loads(document_file.read())
                else:
                    _LOGGER.info(
                        "Syncing analysis document from %r with id %r to graph", analysis_store.ceph.host, document_id
                    )
                    document = analysis_store.retrieve_document(document_id)

                graph.sync_analysis_result(document)
                synced += 1
            except Exception:
                if not graceful:
                    raise

                _LOGGER.exception("Failed to sync analysis result with document id %r", document_id)
                failed += 1
        else:
            _LOGGER.info(f"Sync of analysis document with id {document_id!r} skipped - already synced")
            skipped += 1

    return processed, synced, skipped, failed

def sync_provenance_checker_documents(
    document_ids: Optional[Iterable[str]] = None,
    force: bool = False,
    graceful: bool = False,
    graph: Optional[GraphDatabase] = None,
    is_local: bool = False,
) -> Tuple[int, int, int, int]:
    """Sync provenance check documents into graph."""
    if is_local and not document_ids:
        raise ValueError(
            "Cannot sync documents from local directory without explicitly specifying a list of documents to be synced"
        )

    if not graph:
        graph = GraphDatabase()
        graph.connect()

    if not is_local:
        provenance_check_store = ProvenanceResultsStore()
        provenance_check_store.connect()

    listing = document_ids or provenance_check_store.get_document_listing()

    if _RANDOMIZE_LISTING:
        random.shuffle(list(listing))

    processed, synced, skipped, failed = 0, 0, 0, 0
    for document_id in listing:
        processed += 1

        if force or not graph.provenance_checker_document_id_exist(os.path.basename(document_id)):
            try:
                if is_local:
                    _LOGGER.debug("Loading document from a local file: %r", document_id)
                    with open(document_id, "r") as document_file:
                        document = json.loads(document_file.read())
                else:
                    _LOGGER.info(
                        "Syncing provenance-checker document from %r with id %r to graph",
                        provenance_check_store.ceph.host,
                        document_id,
                    )
                    document = provenance_check_store.retrieve_document(document_id)

                graph.sync_provenance_checker_result(document)
                synced += 1
            except Exception:
                if not graceful:
                    raise

                _LOGGER.exception("Failed to sync provenance-checker result with document id %r", document_id)
                failed += 1
        else:
            _LOGGER.info(f"Sync of provenance-checker document with id {document_id!r} skipped - already synced")
            skipped += 1

    return processed, synced, skipped, failed

def sync_dependency_monkey_documents(
    document_ids: Optional[Iterable[str]] = None,
    force: bool = False,
    graceful: bool = False,
    graph: Optional[GraphDatabase] = None,
    is_local: bool = False,
) -> Tuple[int, int, int, int]:
    """Sync dependency monkey reports into graph database."""
    if is_local and not document_ids:
        raise ValueError(
            "Cannot sync documents from local directory without explicitly specifying a list of documents to be synced"
        )

    if not graph:
        graph = GraphDatabase()
        graph.connect()

    if not is_local:
        dependency_monkey_reports_store = DependencyMonkeyReportsStore()
        dependency_monkey_reports_store.connect()

    listing = document_ids or dependency_monkey_reports_store.get_document_listing()

    if _RANDOMIZE_LISTING:
        random.shuffle(list(listing))

    processed, synced, skipped, failed = 0, 0, 0, 0
    for document_id in listing:
        processed += 1

        if force or not graph.dependency_monkey_document_id_exists(os.path.basename(document_id)):
            try:
                if is_local:
                    _LOGGER.debug("Loading document from a local file: %r", document_id)
                    with open(document_id, "r") as document_file:
                        document = json.loads(document_file.read())
                else:
                    _LOGGER.info(
                        f"Syncing dependency monkey report document from "
                        f"{dependency_monkey_reports_store.ceph.host} with id "
                        f"{document_id}, to graph"
                    )
                    document = dependency_monkey_reports_store.retrieve_document(document_id)

                graph.sync_dependency_monkey_result(document)
                synced += 1
            except Exception:
                if not graceful:
                    raise

                _LOGGER.exception("Failed to sync dependency-monkey result with document id %r", document_id)
                failed += 1
        else:
            _LOGGER.info(f"Sync of dependency-monkey document with id {document_id!r} skipped - already synced")
            skipped += 1

    return processed, synced, skipped, failed

def sync_inspection_documents(
    document_ids: Optional[Iterable[str]] = None,
    force: bool = False,
    graceful: bool = False,
    graph: Optional[GraphDatabase] = None,
    is_local: bool = False,
) -> Tuple[int, int, int, int]:
    """Sync observations made on Amun into graph database."""
    if is_local and not document_ids:
        raise ValueError(
            "Cannot sync documents from local directory without explicitly specifying a list of documents to be synced"
        )

    if not graph:
        graph = GraphDatabase()
        graph.connect()

    listing = document_ids or InspectionStore.iter_inspections()

    if _RANDOMIZE_LISTING:
        random.shuffle(list(listing))

    processed, synced, skipped, failed = 0, 0, 0, 0
    for inspection_document_id in listing:

        if not is_local:
            results = []
            inspection_store = InspectionStore(inspection_id=inspection_document_id)
            inspection_store.connect()

            number_results = inspection_store.results.get_results_count()
        else:
            main_repo = Path(f"{inspection_document_id}/results")
            results = [int(repo.name) for repo in main_repo.iterdir()]
            # dir entry should be numbers

        if number_results > 0:

            for inspection_result_number in results or range(number_results):

                processed += 1
                try:
                    if force or not graph.inspection_document_id_result_number_exists(
                        inspection_document_id=inspection_document_id, inspection_result_number=inspection_result_number
                    ):
                        if is_local:

                            inspection_specification_path = f"{inspection_document_id}/build/specification"
                            _LOGGER.debug(
                                "Loading specification document from a local file: %r", inspection_specification_path
                            )

                            inspection_result_path = (
                                f"{inspection_document_id}/results/{inspection_result_number}/result"
                            )
                            _LOGGER.debug("Loading result document from a local file: %r", inspection_result_path)

                            with open(inspection_specification_path, "r") as document_file:
                                inspection_specification_document = json.loads(document_file.read())

                            with open(inspection_result_path, "r") as document_file:
                                inspection_result_document = json.loads(document_file.read())
                        else:
                            _LOGGER.info(
                                "Syncing analysis document from %r with id %r and number %r to graph",
                                inspection_store.results.ceph.host,
                                inspection_document_id,
                                inspection_result_number,
                            )

                            inspection_specification_document = inspection_store.retrieve_specification()
                            inspection_result_document = inspection_store.results.retrieve_result(
                                inspection_result_number
                            )

                        inspection_document = {
                            "document_id": inspection_document_id,
                            "result_number": inspection_result_number,
                            "specification": inspection_specification_document,
                            "result": inspection_result_document,
                        }

                        graph.sync_inspection_result(inspection_document)
                        synced += 1

                    else:
                        _LOGGER.info(
                            f"Sync of results n.{inspection_result_number!r}"
                            f" from inspection id {inspection_document_id!r} skipped - already synced"
                        )
                        skipped += 1

                except Exception:
                    if not graceful:
                        raise

                    _LOGGER.exception(
                        f"Failed to sync results n.{inspection_result_number!r}"
                        f" from inspection id {inspection_document_id!r}"
                    )
                    failed += 1

        else:
            _LOGGER.info(f"inspection_document_id: {inspection_document_id!r} - does not have any results.")

    return processed, synced, skipped, failed

def sync_security_indicators_documents(
    document_ids: Optional[Iterable[str]] = None,
    force: bool = False,
    graceful: bool = False,
    graph: Optional[GraphDatabase] = None,
    is_local: bool = False,
) -> Tuple[int, int, int, int]:
    """Sync security indicators results into graph."""
    if is_local and not document_ids:
        raise ValueError(
            "Cannot sync documents from local directory without explicitly specifying a list of documents to be synced"
        )

    if not graph:
        graph = GraphDatabase()
        graph.connect()

    listing = document_ids or SecurityIndicatorsResultsStore.iter_security_indicators()

    if _RANDOMIZE_LISTING:
        random.shuffle(list(listing))

    processed, synced, skipped, failed = 0, 0, 0, 0
    for security_indicator_id in listing:

        processed += 1

        if force or not graph.si_aggregated_document_id_exists(security_indicator_id):
            try:
                if is_local:
                    _LOGGER.debug("Loading document from a local file: %r", security_indicator_id)
                    with open(security_indicator_id, "r") as document_file:
                        aggregated_document = json.loads(document_file.read())
                else:
                    si_aggregated_store = SIAggregatedStore(security_indicator_id=security_indicator_id)
                    si_aggregated_store.connect()
                    _LOGGER.info(
                        "Syncing analysis document from %r with id %r to graph",
                        si_aggregated_store.ceph.host,
                        security_indicator_id,
                    )

                    aggregated_document = si_aggregated_store.retrieve_document()

                graph.sync_security_indicator_aggregated_result(aggregated_document)
                synced += 1
            except Exception:
                if not graceful:
                    raise

                _LOGGER.exception("Failed to sync analysis result with document id %r", security_indicator_id)
                failed += 1
        else:
            _LOGGER.info(f"Sync of analysis document with id {security_indicator_id!r} skipped - already synced")
            skipped += 1

    return processed, synced, skipped, failed

# Corresponding mapping of document prefix (before the actual unique document identifier part) to
# functions which can handle the given document sync.

HANDLERS_MAPPING = {
    "adviser": sync_adviser_documents,
    "dependency": sync_dependency_monkey_documents,
    "inspection": sync_inspection_documents,
    "package": sync_analysis_documents,
    "provenance": sync_provenance_checker_documents,
    "solver": sync_solver_documents,
    "revsolver": sync_revsolver_documents,
    "security": sync_security_indicators_documents,
}

def sync_documents(
    document_ids: Optional[Iterable[str]] = None,
    force: bool = False,
    graceful: bool = False,
    graph: Optional[GraphDatabase] = None,
    is_local: bool = False,
) -> Dict[str, Tuple[int, int, int, int]]:
    """Sync documents based on document type.

    If no list of document ids is provided, all documents will be synced.
    >>> from thoth.storages.sync import sync_documents
    >>> sync_documents(["adviser-efa7213babd12911", "package-extract-f8e354d9597a1203"])
    """
    handlers: Dict[str, List[str]] = dict()
    if document_ids:
        handlers = {key: [] for key in HANDLERS_MAPPING.keys()}
        assert handlers is not None
        for doc in document_ids:
            try:
                handlers[doc[doc.rfind("/") + 1 : doc.find("-")]].append(doc)
                # Basename for local syncs, document_id should not have slash otherwise.
            except KeyError:
                error_msg = f"No handler defined for document identifier {doc}"
                if not graceful:
                    raise ValueError(error_msg)
                _LOGGER.error(error_msg)

    else:
        static_handlers = dict.fromkeys(HANDLERS_MAPPING, None)

    stats = dict.fromkeys(HANDLERS_MAPPING, (0, 0, 0, 0))
    for handler, documents in (handlers or static_handlers).items():
        if documents != []:
            stats[handler] = HANDLERS_MAPPING[handler](
                documents, force=force, graceful=graceful, graph=graph, is_local=is_local
            )

    return stats


# -*- encoding: utf-8 -*-

from django.test import TestCase

from caffe.models import Caffe
from employees.models import Employee

from .forms import CashReportForm, CompanyForm, ExpenseForm, FullExpenseForm
from .models import Company, Expense

class CompanyFormTest(TestCase):
    """Tests Company form."""

    def setUp(self):
        """Test data setup."""

        self.caffe = Caffe.objects.create(
            name='kafo',
            city='Gliwice',
            street='Wieczorka',
            house_number='14',
            postal_code='44-100'
        )
        self.filtry = Caffe.objects.create(
            name='filtry',
            city='Warszawa',
            street='Filry',
            house_number='14',
            postal_code='44-100'
        )

    def test_company(self):
        """Check validation."""

        bad_company = CompanyForm(
            {'name': ''},
            caffe=self.caffe
        )

        self.assertFalse(bad_company.is_valid())

        good_company = CompanyForm(
            {'name': 'very good'},
            caffe=self.caffe
        )

        self.assertTrue(good_company.is_valid())
        good_company.save()

        copycat_company = CompanyForm(
            {'name': 'very good'},
            caffe=self.caffe
        )
        self.assertFalse(copycat_company.is_valid())

        with self.assertRaises(Exception):
            CompanyForm({'bakery'})

    def test_name_validation(self):
        """Check name validation."""

        # same name, different caffes
        Company.objects.create(name='Bakery', caffe=self.filtry)

        form_correct = CompanyForm(
            {'name': 'Bakery'},
            caffe=self.caffe
        )
        self.assertTrue(form_correct.is_valid())

        # same name, same caffes
        Company.objects.create(name='Bakery', caffe=self.caffe)

        form_incorrect = CompanyForm(
            {'name': 'Bakery'},
            caffe=self.caffe
        )
        self.assertFalse(form_incorrect.is_valid())

class ExpenseFormTest(TestCase):
    """Tests Expense form."""

    def setUp(self):
        """Prepare objects for tests."""

        self.caffe = Caffe.objects.create(
            name='kafo',
            city='Gliwice',
            street='Wieczorka',
            house_number='14',
            postal_code='44-100'
        )
        self.filtry = Caffe.objects.create(
            name='filtry',
            city='Warszawa',
            street='Filry',
            house_number='14',
            postal_code='44-100'
        )

        self.company = Company.objects.create(
            name='GoodCake',
            caffe=self.caffe
        )

        self.company_f = Company.objects.create(
            name='GoodCake',
            caffe=self.filtry
        )

    def test_validation(self):
        """Check validation."""

        empty_name_expense = ExpenseForm(
            {'name': '', 'company': self.company},
            caffe=self.caffe
        )
        self.assertFalse(empty_name_expense.is_valid())

        empty_company_expense1 = ExpenseForm(
            {'name': 'newspapers'},
            caffe=self.caffe
        )
        self.assertTrue(empty_company_expense1.is_valid())

        empty_company_expense2 = ExpenseForm(
            {'name': 'newspapers', 'company': None},
            caffe=self.caffe
        )
        self.assertTrue(empty_company_expense2.is_valid())

        empty_company_expense3 = ExpenseForm(
            {'name': 'newspapers', 'company': ''},
            caffe=self.caffe
        )
        self.assertTrue(empty_company_expense3.is_valid())

        complete_expense = ExpenseForm(
            {'name': 'cakes', 'company': self.company.pk},
            caffe=self.caffe
        )
        self.assertTrue(complete_expense.is_valid())

        different_caffe = ExpenseForm(
            {'name': 'cakes', 'company': self.company.pk},
            caffe=self.filtry
        )
        with self.assertRaises(Exception):
            different_caffe.save()

        with self.assertRaises(Exception):
            ExpenseForm({'name': 'cakes', 'company': self.company.pk})

    def test_name_validation(self):
        """Check name validation."""

        Expense.objects.create(
            name='cakes',
            company=self.company_f,
            caffe=self.filtry
        )

        form_correct = ExpenseForm(
            {'name': 'cakes', 'company': self.company.pk},
            caffe=self.caffe
        )
        self.assertTrue(form_correct.is_valid())

        # invalid name
        Expense.objects.create(
            name='cakes',
            company=self.company,
            caffe=self.caffe
        )

        form_incorrect = ExpenseForm(
            {'name': 'cakes', 'company': self.company.pk},
            caffe=self.caffe
        )
        self.assertFalse(form_incorrect.is_valid())

class FullExpenseFormTest(TestCase):
    """Tests FullExpense form."""

    def setUp(self):
        """Prepare objects for tests."""

        self.caffe = Caffe.objects.create(
            name='kafo',
            city='Gliwice',
            street='Wieczorka',
            house_number='14',
            postal_code='44-100'
        )
        self.filtry = Caffe.objects.create(
            name='filtry',
            city='Warszawa',
            street='Filry',
            house_number='14',
            postal_code='44-100'
        )

        self.company = Company.objects.create(
            name='GoodCake',
            caffe=self.caffe
        )

        self.expense = Expense.objects.create(
            name='cakes',
            company=self.company,
            caffe=self.caffe
        )

    def test_validation(self):
        """Test validation."""

        cakes_for_10 = FullExpenseForm(
            {'expense': self.expense.pk, 'amount': 10},
            caffe=self.caffe
        )
        self.assertTrue(cakes_for_10.is_valid())

        cakes_for_nothing = FullExpenseForm(
            {'expense': self.expense.pk, 'amount': None},
            caffe=self.caffe
        )
        self.assertFalse(cakes_for_nothing.is_valid())

        money_for_nothing = FullExpenseForm(
            {'expense': None, 'amount': 10},
            caffe=self.caffe
        )
        self.assertFalse(money_for_nothing.is_valid())

        different_caffe = FullExpenseForm(
            {'expense': self.expense.pk, 'amount': 1},
            caffe=self.filtry
        )

        with self.assertRaises(Exception):
            different_caffe.save()

class CashReportFormTest(TestCase):
    """Tests CashReport form."""

    def setUp(self):
        """Test data setup."""

        self.caffe = Caffe.objects.create(
            name='kafo',
            city='Gliwice',
            street='Wieczorka',
            house_number='14',
            postal_code='44-100'
        )
        self.filtry = Caffe.objects.create(
            name='filtry',
            city='Warszawa',
            street='Filry',
            house_number='14',
            postal_code='44-100'
        )

        self.kate = Employee.objects.create(
            username='KateT',
            first_name='Kate',
            last_name='Tempest',
            telephone_number='12345678',
            email='kate@tempest.com',
            favorite_coffee='flat white',
            caffe=self.caffe
        )

    def test_validation(self):
        """Test validation of CashReport form."""

        no_cash_report = CashReportForm(
            {'card_payments': 500, 'amount_due': 1700},
            creator=self.kate,
            caffe=self.caffe,
        )
        self.assertFalse(no_cash_report.is_valid())

        no_cards_report = CashReportForm(
            {
                'cash_before_shift': 1000,
                'cash_after_shift': 2000,
                'amount_due': 1700
            },
            creator=self.kate,
            caffe=self.caffe
        )
        self.assertFalse(no_cards_report.is_valid())

        no_due_report = CashReportForm(
            {
                'card_payments': 500,
                'cash_before_shift': 1000,
                'cash_after_shift': 2000,
            },
            creator=self.kate,
            caffe=self.caffe
        )
        self.assertFalse(no_due_report.is_valid())

        perfectly_fine_report = CashReportForm(
            {
                'card_payments': 500,
                'cash_before_shift': 1000,
                'cash_after_shift': 2000,
                'amount_due': 1700
            },
            creator=self.kate,
            caffe=self.caffe
        )
        self.assertTrue(perfectly_fine_report.is_valid())

        different_caffe = CashReportForm(
            {
                'card_payments': 500,
                'cash_before_shift': 1000,
                'cash_after_shift': 2000,
                'amount_due': 1700
            },
            creator=self.kate,
            caffe=self.filtry
        )
        with self.assertRaises(Exception):
            different_caffe.save()

        with self.assertRaises(Exception):
            CashReportForm(
                {
                    'card_payments': 500,
                    'cash_before_shift': 1000,
                    'cash_after_shift': 2000,
                    'amount_due': 1700
                },
                creator=self.kate
            )

        with self.assertRaises(Exception):
            CashReportForm(
                {
                    'card_payments': 500,
                    'cash_before_shift': 1000,
                    'cash_after_shift': 2000,
                    'amount_due': 1700
                },
                caffe=self.caffe
            )


import matplotlib
matplotlib.use('TkAgg')
from matplotlib import pyplot as plt
from math import *

xf = []
yf = []
xF = []
yFr = []
yFi = []

min_xf = -2.0
max_xf = 4.0

min_xF = -9.0
max_xF = 9.0

tf = min_xf
tF = min_xF
for i in range(0, 1000):
    xf.append(tf)
    xF.append(tF)

    yf.append(exp(-tf))
    yFr.append((complex(1,0)/ complex(1,tF)).real)
    yFi.append((complex(1,0)/ complex(1,tF)).imag)

    tf = tf + (max_xf - min_xf)/1000
    tF = tF + (max_xF - min_xF)/1000

plt.subplot(121)
plt.plot(xf, yf, label = 'f(t) = exp(-t)')
plt.xlabel = 't'
plt.ylabel = 'f(t)'
plt.legend()

plt.subplot(122)
plt.plot(xF, yFr, 'b', label='f(t) = Re(1/(1+jt))')
plt.plot(xF, yFi, 'r', label='f(t) = Im(1/(1+jt))')
plt.xlabel = 't'
plt.ylabel = 'f(t)'
plt.legend()

#plt.savefig('plottedPic.png')
plt.savefig('plottedPic.png', bbox_inches='tight')

plt.show()


import os, sys
sys.path.append("../utils/")

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable

is_GPU=torch.cuda.is_available()

def gather_2d(xyz, index):
    """
    ues 2d-indices to gather value in 3d-tensor
    :param xyz: [B,N1,C]
    :param index: [B,N2]
    :return: [B,N2,C]
    """
    ndim = xyz.size(2)
    index = torch.unsqueeze(index, dim=2).repeat(1, 1, ndim)
    new_xyz = torch.gather(xyz, dim=1, index=index)

    return new_xyz

def random_sample(xyz, num_sample):
    """
     xyz: [bs,P1,3]
    :return:
     idx_sample: [bs,P2]
     new xyz:[bs,P2,3]
    """
    bs, P1, _ = xyz.size()
    idx = []
    for i in range(bs):
        a = torch.randperm(P1)[:num_sample]
        idx.append(a)
    idx_sample = torch.stack(idx, dim=0)
    if is_GPU:
        idx_sample = idx_sample.cuda()
    new_xyz = gather_2d(xyz, idx_sample)

    return idx_sample, new_xyz

def get_distance_matrix(A, B):
    r_A = torch.unsqueeze(torch.sum(A * A, dim=2), dim=2)  ##[N,Pa,1]
    r_B = torch.unsqueeze(torch.sum(B * B, dim=2), dim=2)  ##[N,Pb,1]
    m = torch.bmm(A, torch.transpose(B, 2, 1))  ##[N,Pa,Pb]
    D = r_A - 2 * m + torch.transpose(r_B, 2, 1)

    return D  ##[N,Pa,Pb]

def KNN(new_xyz, input_xyz, num_nn):
    """
    get KNN pts for each key pts and normalize the coordinate
     new_xyz:  [bs,P2,3]
     input_xyz: [bs,P1,3]
     num_nn: int
    :return:
     nn_distances: [bs,P2,num_nn]
     nn_indices: [bs,P2,num_nn]
     knn_xyz: [bs,P2,num_nn,3]
    """
    bs, P1, _ = input_xyz.size()
    P2 = new_xyz.size(1)

    D = get_distance_matrix(new_xyz, input_xyz)  ##[bs,P2,P1]

    nn_distances, nn_indices = torch.topk(-D, k=num_nn)  ##[bs,P2,num_nn]

    nn_indices_patten = nn_indices.view(bs, P2 * num_nn)  ##[bs,P2*num_nn]
    knn_xyz = gather_2d(input_xyz, nn_indices_patten)  ##[bs,P2*num_nn,3]
    knn_xyz = knn_xyz.view(bs, P2, num_nn, 3)  ##[bs,P2,num_nn,3]
    knn_xyz -= torch.unsqueeze(new_xyz, dim=2).repeat(1, 1, num_nn, 1)

    return nn_distances,nn_indices, knn_xyz

##################################################################################

class SA_module(nn.Module):
    def __init__(self, num_sample, num_nn, mlp_list, input_dim, grouping_all=False,use_FPS=False):
        super(SA_module, self).__init__()
        self.num_sample = num_sample
        self.num_nn = num_nn
        self.grouping_all = grouping_all
        self.fps=use_FPS
        self.mlp = mlp_list

        mlp1, mlp2, mlp3 = mlp_list
        self.conv1 = nn.Sequential(
            nn.Conv1d(input_dim, mlp1, 1, 1),
            nn.BatchNorm1d(mlp1),
            nn.ReLU(),
        )
        self.conv2 = nn.Sequential(
            nn.Conv1d(mlp1, mlp2, 1, 1),
            nn.BatchNorm1d(mlp2),
            nn.ReLU(),
        )
        self.conv3 = nn.Sequential(
            nn.Conv1d(mlp2, mlp3, 1, 1),
            nn.BatchNorm1d(mlp3),
            nn.ReLU(),
        )

    def forward(self, input_xyz, input_feature):
        """
         input_xyz: [bs,P1,3]
         input_feat: [bs,P1,C1]
        :return:
         new_xyz:[bs,P2,3]
         new_feat: [bs,P2,C1]
         idx: [bs,P1,num_nn]
        """
        batch_size, P1, C1 = input_feature.size()

        if not self.grouping_all:
            ###########################################
            # sampling
            if self.fps:
                from sample_utils import furthest_point_sample
                idx=furthest_point_sample(input_xyz.contiguous(),self.num_sample)
                new_xyz=gather_2d(input_xyz,idx.long())
            else:
                idx, new_xyz = random_sample(input_xyz, self.num_sample)

            ###########################################
            # grouping
            _,nn_indices, grouped_xyz = KNN(new_xyz, input_xyz, self.num_nn)
            ##[bs,P2,num_nn] [bs,P2,num_nn,3]

            nn_indices_patten = nn_indices.view(batch_size, self.num_sample * self.num_nn)
            grouped_feature = gather_2d(input_feature, nn_indices_patten)
            grouped_feature = grouped_feature.contiguous().view(batch_size, self.num_sample, self.num_nn, C1)
            ##[bs,P2,num_nn,C1]
            grouped_feature = torch.cat([grouped_feature, grouped_xyz], dim=3)

            grouped_feature = grouped_feature.view(batch_size,
                            self.num_sample * self.num_nn, -1).transpose(2,1)  ##[bs, C1+3, P2*num_nn]
            knn_feature = self.conv3(self.conv2(self.conv1(grouped_feature)))  ##[bs, C, P2*num_nn]

            knn_feature = knn_feature.transpose(2, 1)
            knn_feature = knn_feature.contiguous().view(batch_size, self.num_sample, self.num_nn,-1)  ##[bs, P2, num_nn, C]

        else:
            ## all pts grouping to origin, now  P2=1 num_nn=P1
            new_xyz = torch.zeros([batch_size,1,3])

            nn_indices = torch.from_numpy(np.tile(np.array(range(P1)).reshape((1, 1, P1)), (batch_size, 1, 1)))
            grouped_xyz = input_xyz.contiguous().view(batch_size, 1, P1, 3)
            ## [bs,1,P1] [bs,1,P1,3]

            grouped_feature = input_feature.contiguous().view(batch_size, 1, P1, -1) ## [bs,1,P1,C1]

            if is_GPU:
                new_xyz = new_xyz.cuda()
                nn_indices = nn_indices.cuda()

            grouped_feature = torch.cat([grouped_feature, grouped_xyz], dim=3) ##[bs,1,P1,C1+3]
            grouped_feature = grouped_feature.view(batch_size,
                                                P1, -1).transpose(2,1)  ##[bs, C1+3, P1*1]
            knn_feature = self.conv3(self.conv2(self.conv1(grouped_feature)))  ##[bs, C, P1*1]
            knn_feature = knn_feature.transpose(2, 1)
            knn_feature = knn_feature.contiguous().view(batch_size, 1, P1,-1)  ##[bs, 1, P1, C]

        ##########################################
        ## pooling
        knn_feature = knn_feature.contiguous().view(batch_size * self.num_sample, self.num_nn, -1).transpose(2, 1)
        ##[bs*P2, C, num_nn]
        if self.grouping_all:
            new_feature = F.max_pool1d(knn_feature, kernel_size=P1)
        else:
            new_feature = F.max_pool1d(knn_feature, kernel_size=self.num_nn)
        new_feature = torch.squeeze(new_feature).contiguous().view(batch_size, self.num_sample, -1)

        return nn_indices,new_xyz, new_feature

######################################################################
class FP_module(nn.Module):
    def __init__(self, mlp_list, input_dim,fp_nn=3):
        super(FP_module, self).__init__()
        self.fp_nn=fp_nn

        num_layer = len(mlp_list)
        if num_layer == 2:
            mlp1, mlp2 = mlp_list
            mlp3 = None
        else:
            mlp1, mlp2, mlp3 = mlp_list

        self.conv1 = nn.Sequential(
            nn.Conv1d(input_dim, mlp1, 1, 1),
            nn.BatchNorm1d(mlp1),
            nn.ReLU(),
        )
        self.conv2 = nn.Sequential(
            nn.Conv1d(mlp1, mlp2, 1, 1),
            nn.BatchNorm1d(mlp2),
            nn.ReLU(),
        )
        self.conv3 = None
        if mlp3 is not None:
            self.conv3 = nn.Sequential(
                nn.Conv1d(mlp2, mlp3, 1, 1),
                nn.BatchNorm1d(mlp3),
                nn.ReLU(),
            )
    def forward(self,xyz1,xyz2,pts1,pts2):
        """
         xyz1: [N,P1,3] -> pts of previous layer
         xyz2: [N,P2,3] -> pts of late layer, here P1 > P2
         pts1: [N.P1,C1]
         pts2: [N,P2,C2]
        :return:
         new_points: (N, P1, mlp[-1]) -> P1 pts with new feature
        """
        batchsize, P1, C1 = pts1.size()
        _,P2,C2 = pts2.size()

        fp_disc, fp_indice, fp_xyz = KNN(xyz1, xyz2, num_nn=self.fp_nn)
        ##[bs,P1,3] [bs,P1,3] [bs,P1,3,3]
        fp_disc = torch.clamp(fp_disc, min=1e-10,max=10000)
        norm = torch.unsqueeze(torch.sum(1.0 / fp_disc, dim=2), dim=2).repeat(1,1,self.fp_nn) ##[bs,P1,3]
        weight = (1.0/fp_disc) / norm ##[bs,P1,3]
        weight = torch.unsqueeze(weight, dim=3).repeat(1,1,1,C2)

        ## feature propagation: [bs,P1,nn] + [bs,P2,C]2 -> [bs,P1,nn,C2]
        fp_indice_platten = fp_indice.contiguous().view(batchsize, P1 * self.fp_nn) ##[bs,P1*nn]
        fp_row_feature = gather_2d(pts2,fp_indice_platten)
        fp_row_feature = fp_row_feature.contiguous().view(batchsize, P1, self.fp_nn, C2) ##[bs,P1,3,C2]
        fp_weight_feature = fp_row_feature * weight ##[bs,P1,3,C1]

        fp_weight_feature = fp_weight_feature.contiguous().view(batchsize * P1,self.fp_nn, -1).transpose(2,1) ##[bs*P1,C2,3]
        fp_group_feature = torch.squeeze(F.max_pool1d(fp_weight_feature,kernel_size=self.fp_nn)) ##[bs*P1,C2]
        fp_group_feature = fp_group_feature.contiguous().view(batchsize, P1, C2) ## [bs,P1,C2]

        ## unet archtecture
        fp_group_feature = torch.cat([fp_group_feature,pts1], dim=2) ## [bs,P1,C2+C1]
        fp_group_feature = fp_group_feature.transpose(1,2) ## [bs,C2+C1,P1]

        new_feature = self.conv2(self.conv1(fp_group_feature))
        if self.conv3 is not None:
            new_feature = self.conv3(new_feature)
        new_feature = new_feature.transpose(2,1) ## [bs,P1,mlp[-1]]

        return new_feature

if __name__ == '__main__':
    N = 9
    P1 = 200
    P2 = 50
    num_nn = 5

    pts_feature = torch.randn(N, P1, 6)
    if is_GPU:
        pts_feature=pts_feature.cuda()
    pts = pts_feature[:,:,:3]

    ## test SA module
    SA1 = SA_module(num_sample=P2, num_nn=num_nn, mlp_list=[8, 32, 64],
                    input_dim=(3+6),use_FPS=False)
    if is_GPU:
        SA1 = SA1.cuda()
    _,_, new_feature = SA1(pts, pts_feature)
    print (new_feature.size())

    ## test SA grouping all
    SA2=SA_module(num_sample=1, num_nn=P1, mlp_list=[8, 32, 64],
                  input_dim=(3+6),use_FPS=False,grouping_all=True)
    if is_GPU:
        SA2 = SA2.cuda()
    _,_, new_feature = SA2(pts, pts_feature)
    print (new_feature.size())
    print ('\n')
    ###############################################################################

    xyz1 = torch.randn(N, P1, 3)
    xyz2 = torch.randn(N, P2, 3)
    pts1 = torch.randn(N, P1, 6)
    pts2 = torch.randn(N, P2, 19)
    if is_GPU:
        xyz1,xyz2,pts1,pts2 = xyz1.cuda(),xyz2.cuda(),pts1.cuda(),pts2.cuda()

    ## test FP module
    FP1 = FP_module(input_dim=(6+19),mlp_list=[8,32,64])
    if is_GPU:
        FP1 = FP1.cuda()
    new_feat=FP1(xyz1,xyz2,pts1,pts2)
    print (new_feat.size())


# -*- coding: utf-8 -*-
from __future__ import absolute_import
import logging
import collections

from twisted.internet.defer import Deferred
from scrapy import signals

logger = logging.getLogger(__name__)

def scrape_items(crawler_runner, crawler_or_spidercls, *args, **kwargs):
    """
    Start a crawl and return an object (ItemCursor instance)
    which allows to retrieve scraped items and wait for items
    to become available.

    Example:

    .. code-block:: python

        @inlineCallbacks
        def f():
            runner = CrawlerRunner()
            async_items = scrape_items(runner, my_spider)
            while (yield async_items.fetch_next):
                item = async_items.next_item()
                # ...
            # ...

    This convoluted way to write a loop should become unnecessary
    in Python 3.5 because of ``async for``.
    """
    crawler = crawler_runner.create_crawler(crawler_or_spidercls)
    d = crawler_runner.crawl(crawler, *args, **kwargs)
    return ItemCursor(d, crawler)

class ItemCursor(object):
    def __init__(self, crawl_d, crawler):
        self.crawl_d = crawl_d
        self.crawler = crawler

        crawler.signals.connect(self._on_item_scraped, signals.item_scraped)
        crawler.signals.connect(self._on_error, signals.spider_error)

        crawl_d.addCallback(self._on_finished)
        crawl_d.addErrback(self._on_error)

        self.closed = False
        self._items_available = Deferred()
        self._items = collections.deque()

    def _on_item_scraped(self, item):
        # logger.info("_on_item_scraped")
        self._items.append(item)
        self._items_available.callback(True)
        self._items_available = Deferred()

    def _on_finished(self, result):
        # logger.info("_on_finished")
        self.closed = True
        self._items_available.callback(False)

    def _on_error(self, failure):
        self.closed = True
        self._items_available.errback(failure)

    @property
    def fetch_next(self):
        """
        A Deferred used with ``inlineCallbacks`` or ``gen.coroutine`` to
        asynchronously retrieve the next item, waiting for an item to be
        crawled if necessary. Resolves to ``False`` if the crawl is finished,
        otherwise :meth:`next_item` is guaranteed to return an item
        (a dict or a scrapy.Item instance).
        """
        if self.closed:
            # crawl is finished
            d = Deferred()
            d.callback(False)
            # logger.info("fetch_next: crawl is finished")
            return d

        if self._items:
            # result is ready
            d = Deferred()
            d.callback(True)
            # logger.info("fetch_next: result is ready")
            return d

        # We're active, but item is not ready yet. Return a Deferred which
        # resolves to True if item is scraped or to False if crawl is stopped.
        # logger.info("fetch_next: active, but items are not ready")
        return self._items_available

    def next_item(self):
        """Get a document from the most recently fetched batch, or ``None``.
        See :attr:`fetch_next`.
        """
        if not self._items:
            # logger.info("no next_item")
            return None
        # logger.info("got next_item")
        return self._items.popleft()


# genetic algorithm search for continuous function optimization
from numpy.random import randint
from numpy.random import rand
from algorithms.functions import beale_function
import methods.mutation as mutation
import methods.cross as cross
import methods.selection as select
from ui.UserInputs import UserInputs
from statistics import stdev, mean

def inversion(bitstring: list, r_inve):
    c1 = bitstring.copy()
    if rand() < r_inve:
        pt1 = randint(1, (len(bitstring) - 2) / 2)
        pt2 = randint((len(bitstring) - 2) / 2, len(bitstring) - 2)
        x = bitstring[pt1:pt2]
        x.reverse()
        c1 = bitstring[:pt1] + x + bitstring[pt2:]
    return c1

# decode bitstring to numbers
def decode(bounds, n_bits, bitstring):
    decoded = list()
    largest = 2 ** n_bits
    for i in range(len(bounds)):
        start, end = i * n_bits, (i * n_bits) + n_bits
        substring = bitstring[start:end]
        chars = ''.join([str(s) for s in substring])
        integer = int(chars, 2)
        value = bounds[i][0] + (integer / largest) * (bounds[i][1] - bounds[i][0])
        decoded.append(value)
    return decoded

def genetic_algorithm(user_input):
    bounds = [[user_input.begin_range_a, user_input.end_range_b], [user_input.begin_range_a, user_input.end_range_b]]
    pop = [randint(0, 2, user_input.number_of_bits * len(bounds)).tolist() for _ in range(user_input.population_amount)]
    best, best_eval = 0, beale_function(decode(bounds, user_input.number_of_bits, pop[0]))
    gen_b_rows = []
    gen_avg_rows = []
    gen_std_dev_rows = []
    for gen in range(user_input.epochs_amount):
        decoded = [decode(bounds, user_input.number_of_bits, p) for p in pop]
        scores = [beale_function(d) for d in decoded]
        best_iter = scores[0]
        for i in range(user_input.population_amount):
            if user_input.maximum:
                if scores[i] > best_eval:
                    best, best_eval = pop[i], scores[i]
                    print(">%d, new best f(%s) = %f" % (gen, decoded[i], scores[i]))
                if scores[i] > best_iter:
                    best_iter = scores[i]
            else:
                if scores[i] < best_eval:
                    best, best_eval = pop[i], scores[i]
                    print(">%d, new best f(%s) = %f" % (gen, decoded[i], scores[i]))
                if scores[i] < best_iter:
                    best_iter = scores[i]
        gen_b_rows.append([str(gen), str(best_iter)])
        gen_avg_rows.append([str(gen), mean(scores)])
        gen_std_dev_rows.append([str(gen), str(stdev(scores))])
        if user_input.selection_method == "Tournament":
            selected = [select.tournament(pop, scores, user_input.best_and_tournament_chromosome_amount) for _ in
                        range(user_input.population_amount)]
        if user_input.selection_method == "Roulette":
            selected = [select.roulette(pop, scores) for _ in range(user_input.population_amount)]
        if user_input.selection_method == "Best":
            selected = [select.best(pop, scores, _) for _ in range(user_input.population_amount)]
        children = list()
        if user_input.elite_strategy:
            children.append(best)
        for i in range(0, user_input.population_amount, 2):
            p1, p2 = selected[i], selected[i + 1]
            if user_input.cross_method == "One Point Cross":
                cross_result = cross.crossover(p1, p2, user_input.cross_probability)
            if user_input.cross_method == "Two Point Cross":
                cross_result = cross.crossover2(p1, p2, user_input.cross_probability)
            if user_input.cross_method == "Three Point Cross":
                cross_result = cross.crossover3(p1, p2, user_input.cross_probability)
            if user_input.cross_method == "Uniform Cross":
                cross_result = cross.uniformCrossover(p1, p2)
            for c in cross_result:
                if user_input.mutation_method == "Edge":
                    c = mutation.edge_mutation(c, user_input.mutation_probability)
                if user_input.mutation_method == "One Point":
                    c = mutation.op_mutation(c, user_input.mutation_probability)
                if user_input.mutation_method == "Two Point":
                    c = mutation.tp_mutation(c, user_input.mutation_probability)
                c = inversion(c, user_input.inversion_probability)
                children.append(c)
        pop = children

    print('Done!')
    decoded = decode(bounds, user_input.number_of_bits, best)
    print('f(%s) = %f' % (decoded, best_eval))
    return [decoded, best_eval, gen_b_rows, gen_avg_rows, gen_std_dev_rows]

defaultUser = UserInputs(
    begin_range_a=-4.5,
    end_range_b=4.5,
    population_amount=100,
    number_of_bits=16,
    epochs_amount=1000,
    best_and_tournament_chromosome_amount=3,
    cross_probability=0.5,
    mutation_probability=0.1,
    inversion_probability=0.1,
    selection_method="Best",
    cross_method="cross",
    mutation_method="mutate",
    maximum=False,
    elite_strategy=True,
)


import socket
from urllib.parse import unquote
from concurrent import futures
import os
import random

import pbb_pb2
import pbb_pb2_grpc
import grpc
from test import cleanup

class RegistryServicer(pbb_pb2_grpc.RegistryServicer):
    replica_list = []
    quorum_data = {
        "N": 0,
        "N_R": 0,
        "N_W": 0,
    }

    def is_quorum_satisfied(self):
        return (
            (self.quorum_data["N"] < self.quorum_data["N_R"] + self.quorum_data["N_W"])
            and (0 < self.quorum_data["N_R"] <= self.quorum_data["N"])
            and (0 < self.quorum_data["N_W"] <= self.quorum_data["N"])
        )

    def __init__(self):
        cleanup()

        while not self.is_quorum_satisfied():
            self.quorum_data["N"] = int(input("Number of replicas: "))
            self.quorum_data["N_R"] = int(input("Read quorum size: "))
            self.quorum_data["N_W"] = int(input("Write quorum size: "))
            if not self.is_quorum_satisfied():
                print("Ensure N < N_R + N_W, and 0 < N_R, N_W <= N.")

    def Register(self, request, context):
        address = request.address
        print(f"[.] REGISTER REQUEST FROM {address}")
        if any(r_address == address for r_address in self.replica_list):
            return pbb_pb2.RegisterResponse(
                status=pbb_pb2.Status.ERROR,
                message=f"Server {address} already registered.",
            )
        self.replica_list.append(address)
        replica_id = len(self.replica_list)
        path = os.path.join(os.getcwd(), f"data/replica_{replica_id}")
        if not os.path.exists(path):
            os.mkdir(path)
        return pbb_pb2.RegisterResponse(
            status=pbb_pb2.Status.OK,
            message=f"Server {address} registered.",
            replica_id=replica_id,
        )

    def GetReplicaList(self, request, context):
        quorum_size_map = {
            "READ": self.quorum_data["N_R"],
            "WRITE": self.quorum_data["N_W"],
            "ALL": self.quorum_data["N"],
        }
        replicas = self.replica_list
        if request.purpose is not None:
            quorum_size = quorum_size_map[request.purpose]
            replicas = random.sample(self.replica_list, quorum_size)
        return pbb_pb2.GetReplicaListResponse(replicas=replicas)

def serve():
    PORT = 56149
    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
        s.bind(("", PORT))
    registry_address = f"[::]:{PORT}"
    registry = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
    pbb_pb2_grpc.add_RegistryServicer_to_server(RegistryServicer(), registry)
    registry.add_insecure_port(registry_address)
    registry.start()
    print(f"[.] Registry node started on {registry_address}")
    registry.wait_for_termination()

if __name__ == "__main__":
    serve()


import torch
torch.set_float32_matmul_precision('high')
import argparse

## Parser Arguments
parser = argparse.ArgumentParser(description='Masterarbeit Segformer Parser')

parser.add_argument('--project', type=str, default='Masterarbeit Segformer')
parser.add_argument('--run', type=str, default='Segformer_train')
parser.add_argument('--backbone', type=str, default='b2')
parser.add_argument('--epochs', type=int, default=250)  # gpu: 250, local:1
parser.add_argument('--lr', type=float, default=6e-5) # gpu: default=6e-5, local:2e-1
parser.add_argument('--dataset', type=str, default='TLESS')
parser.add_argument('--root', type=str, default='./data/tless')
parser.add_argument('--train_split', type=str, default='train_pbr')
parser.add_argument('--val_split', type=str, default='train_pbr')
parser.add_argument('--val_size', type=str, default=200)
parser.add_argument('--checkpoints', type=str, default='./checkpoints')
parser.add_argument('--load_checkpoints', type=str, default=None)

parser.add_argument('--use_scaling', type=bool, default=True)
parser.add_argument('--use_cropping', type=bool, default=True)
parser.add_argument('--use_flipping', type=bool, default=True)

args = parser.parse_args()

## W&B Logging (optional)
ENTITY = 'gaolilli'   # change this to your W&B username
PROJECT = args.project
RUN_NAME = args.run

## Training & Hyperparameters
DEVICES = [0, 1, 2, 3]      # use 4 GPUs on the HPC. For local training or ipf server, just use one GPU.
#DEVICES = 1    #cpu
NUM_EPOCHS = args.epochs
BACKBONE = args.backbone
MOMENTUM = 0.9
WEIGHT_DECAY = 0.01
PRECISION = '16-mixed'   #gpu: '16-mixed', cpu: bf16-mixed
LEARNING_RATE = args.lr

## Data Augmentation
USE_SCALING = args.use_scaling
USE_CROPPING = args.use_cropping
USE_FLIPPING = args.use_flipping

## Dataset & Dataloader
DATASET = args.dataset
ROOT = args.root
TRAIN_SPLIT = args.train_split
VAL_SPLIT = args.val_split
VAL_SIZE = args.val_size
CHECKPOINTS_DIR = args.checkpoints
LOAD_CHECKPOINTS = args.load_checkpoints

if DATASET == 'TLESS':
    NUMBER_TRAIN_IMAGES = 50000 # 50000 total
    NUMBER_VAL_IMAGES = 10080  # 10080 total
    BATCH_SIZE = 8  # 8 for train on server with gpu, 2 for cpu
    NUM_CLASSES = 30
    IGNORE_INDEX = 255
    NUM_WORKERS = 8


import datetime
import math
from concurrent.futures import ProcessPoolExecutor, wait, as_completed

def fun_factor(num:int):
    return math.factorial(num)

big_nums = [[1550]*100]*100
if __name__ == '__main__':
    start = datetime.datetime.utcnow()
    executor = ProcessPoolExecutor()
    futures = []
    for lst in big_nums:
        for num in lst:
            future = executor.submit(fun_factor, num)
            futures.append(future)
    end1 = datetime.datetime.utcnow()
    for future in as_completed(futures):
        print(future.result())
    end = datetime.datetime.utcnow()
    print(f"Time took for calaulting: {(end1 - start).total_seconds()}s---------------------------------------")
    print(f"Time took for all: {(end - start).total_seconds()}s---------------------------------------")


import itertools
from pathlib import Path
from typing import Any, Callable, Dict, List

from ztrack.datatype import Event

class EventFileReader(object):
    """It is responsible for read one event file"""

    def __init__(self, name: str, event_file_path: Path) -> None:
        self._name = name
        self._event_file_path = event_file_path
        self._filters = []

    def add_filter(self, cb: Callable[[Event], bool]):
        self._filters.append(cb)
        return self

    def iter_read(self):
        with self._event_file_path.open('r') as _event_file:
            while True:
                line = _event_file.readline()
                # eof
                if not line:
                    return None

                event = Event.from_json(line)

                filtered = True
                for filter in self._filters:
                    if not filter(event):
                        filtered = False
                        break

                if not filtered:
                    continue

                yield event

    def read_all(self) -> List[Event]:
        events = []

        for event in self.iter_read():
            events.append(event)
        return events

    def datas_to_pandas(self, stop=None, normalize_json=False):
        import pandas as pd
        datas = []

        it = itertools.islice(self.iter_read(), stop)
        datas = [evt.data for evt in it]

        if normalize_json:
            return pd.json_normalize(datas)
        return pd.DataFrame(datas)

class Reader(object):
    """Reader used to read tracked events directories and files"""

    def __init__(self, result_dir: Path) -> None:
        self._result_dir = result_dir

    def event_reader(self, reporter: str = 'default') -> EventFileReader:
        event_file_path = self._result_dir / f"{reporter}.event.json"
        if not event_file_path.exists():
            raise ValueError(
                f"event reporter file, reporter={reporter}, path={event_file_path.as_posix()} not exists")
        return EventFileReader(reporter, event_file_path)


from typing import Literal
from binarySearchTree import Node

# O(hm)
# h = height of the tree
# m = length of list (list.copy())
def parentList(root: Node, node: Node) -> list[int] | Literal[-1]:
    '''
        returns list of parents in order for node
    '''
    queue: list[Node] = [ root ]
    parent: dict = { root.data: [] }

    while queue:
        curr = queue[0]
        del queue[0]

        if curr == node:
            return parent[curr.data]

        # iteration
        if curr.left:
            queue.append(curr.left)
            parent[curr.left.data] = parent[curr.data].copy()
            parent[curr.left.data].append(curr.data)

            if curr.left == node:
                return parent[curr.left.data]

        if curr.right:
            queue.append(curr.right)
            parent[curr.right.data] = parent[curr.data].copy()
            parent[curr.right.data].append(curr.data)

            if curr.right == node:
                return parent[curr.right.data]

    return -1

# O(nm+2h)
# n > m
# n = lenght of list (set conversation)
# m = length of list (iterating each element)
# h = height of the tree (h = log(n) to n)
def commonAncestor(root: Node, node1: Node, node2: Node) -> int:
    '''
        returns the first common parent for node1 and node2
    '''
    parent1: list[int] = parentList(root, node1)
    parent2: list[int] = parentList(root, node2)

    if parent1 == -1 or parent2 == -1:
        return -1

    if len(parent1) > len(parent2):
        parent1 = set(parent1)

        for parent in parent2:
            if parent in parent1:
                return parent

    else:
        parent2 = set(parent2)

        for parent in parent1:
            if parent in parent2:
                return parent

    return -1

'''
        (3)
       ↙    ↘
     (6)     (8)
   ↙  ↘      ↙   ↘
(10) (21)   (13) (11)
           ↙   ↘
        (14)   (24)
'''

if __name__ == '__main__':
    # binary Tree
    root: Node = Node(3)
    root.left = Node(6)
    root.right = Node(8)

    root.left.left = Node(10)
    root.left.right = Node(21)

    root.right.left = Node(13)
    root.right.right = Node(11)

    root.right.left.left = Node(14)
    root.right.left.right = Node(24)

    #                          21               24
    print(commonAncestor(root, root.left.right, root.right.left.right))


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
import os

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

###

NLAGS = 20

###

## plot single plots
ylimdict = {\
	'absh': [-0.1, 0.07],
	't2m': [-0.08, 0.15],
	'utci': [-0.06, 0.02],
	'tp': [-0.02, 0.02],
	'ssrd': [-0.03, 0.03],
	'r': [-0.1, 0.1]
}

variablesdict = {
	'absh': 'Absolute humidity [g m-3]',
	'r': 'Relative humidity [\%]',
	't2m': 'Temperature [degree K]',
	'utci': 'UTCI [-]',
	'tp': 'Precipitation [mm day-1]',
	'ssrd': 'Solar radiation [W m-2]'
}

variables = ['t2m', 'utci', 'tp', 'ssrd'		]

for i, variable in enumerate(variables):

	fig, ax = plt.subplots(figsize=(6,4))

	xi = 0.5

	## plot coeffs of lags
	for lag in range(1, NLAGS+1, 1):
		resultfile = 'coeffs_firstdiff_L20_pol1_nomob_10.csv'
		try:
			df = pd.read_csv(os.path.join('./results', resultfile))
			if df.iloc[:, 0].apply(lambda x: x[:2] == variable[:2]).sum() < 1:
				continue
		except:
			print('File not found: ', os.path.join('./results', resultfile))
			continue
		x = np.array([xi, xi+3-0.1])
		y = np.array([float(df.loc[df.iloc[:, 0] == (variable+'_L'+str(lag)), 'Estimate'].values)]*2)
		yerr = np.array([float(df.loc[df.iloc[:, 0] == variable+'_L'+str(lag), 'Cluster.s.e.'].values)]*2)
		ax.plot(x, y, color='b', lw=1.)
		ax.fill_between(x, y1=y-1.96*yerr, y2=y+1.96*yerr, color='b', alpha=0.3, lw=0.)
		xi += 3.

	## plot cumulative coeffs
	resultfile = 'coeffscum_firstdiff_L20_pol1_L-cum5_nomob_10_{0:s}.csv'.format(variable)
	try:
		df = pd.read_csv(os.path.join('./results', resultfile))
	except:
		print('File not found: ', os.path.join('./results', resultfile))
		continue
	x = -4.
	y = df['coeffs'].values
	yerr = df['se'].values
	ax.plot(x, y, 'o', color='b', lw=1., markersize=5.)
	ax.errorbar(x, y, yerr=1.96*yerr, fmt='o', capthick=2, markersize=0., lw=1., color='b')

	resultfile = 'coeffscum_firstdiff_L20_pol1_L-cum_nomob_10_{0:s}.csv'.format(variable)
	df = pd.read_csv(os.path.join('./results', resultfile))
	x = -9.
	y = df['coeffs'].values
	yerr = df['se'].values
	ax.plot(x, y, 'o', color='b', lw=1., markersize=5.)
	ax.errorbar(x, y, yerr=1.96*yerr, fmt='o', capthick=2, markersize=0., lw=1., color='b')

	ax.set_xlim(-14., NLAGS * 3 + 1)
	ax.set_xticks([-9., -4.] + list(np.arange(1, NLAGS * 3 + 1, 6)))
	ax.set_xticklabels(['c', 'c5'] + [str(i) for i in np.arange(1, NLAGS * 3 + 1, 6)])
	ax.set_xlabel('Days before reporting')

	ax.annotate(s=variablesdict[variable], xy=(0.95, 0.99), xycoords='axes fraction', ha='right', va='top')
	ax.plot([-14., NLAGS * 3.], [0., 0.], 'k-', lw=0.5)
	ax.set_ylabel('Coefficient of lag')
	sns.despine(ax=ax, offset=1., right=True, top=True)
	fig.savefig(os.path.join('./figures/', 'coeffs_distlags_{0:s}.png'.format(variable)), bbox_inches='tight')


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import spacy # nlp
import pdfminer # pdf2txt
import re # regex
import os # file manipulation
import pandas as pd # csv - tabular

# In[2]:

import pdf2txt

# In[3]:

def convert_pdf(file_name):
    filename_txt = os.path.basename(os.path.splitext(file_name)[0]) + ".txt" # basename but with txt instead of pdf
    filepath_txt = os.path.join("output/txt/", filename_txt) # output path
    pdf2txt.main(args=[file_name, "--outfile", filepath_txt]) # pdf to txt and save it in the given location
    print(f"{filepath_txt} saved successfully!")
    with open(filepath_txt) as to_read:
        return to_read.read()
    # return open(filepath_txt).read()

# In[4]:

nlp = spacy.load("en_core_web_sm") # load the language model

# In[5]:

result_dict = {'name': [], 'phone': [], 'email': [], 'skills': []}
names = []
phones = []
emails = []
skills = []

# In[6]:

def parse_content(text):
    skillset = re.compile("python|java|sql|hadoop|tableau")
    phone_num = re.compile(
    "(\d{3}[-\.\s]??\d{3}[-\.\s]??\d{4}|\(\d{3}\)\s*\d{3}[-\.\s]??\d{4}|\d{3}[-\.\s]??\d{4})"
    )
    nlp_text = nlp(text)
    name = [entity.text for entity in nlp_text.ents if entity.label_ == "PERSON"][0]
    email = [word for word in nlp_text if word.like_email is True][0]
    print(f"Name: {name}, E-mail: {email}")
    phone = re.findall(phone_num, text.lower())
    skills_list = re.findall(skillset, text.lower())
    unique_skills_list = list(set(skills_list))

    names.append(name.strip())
    emails.append(email)
    phones.append(phone)
    skills.append(unique_skills_list)

    print("Extraction completed successfully!\n")

# In[7]:

def for_file_in_ls():
    for file_name in os.listdir('resumes/'):
        if file_name.endswith('.pdf'):
            print(f"Reading {file_name}")
            txt = convert_pdf(os.path.join('resumes/', file_name))
            parse_content(txt)

# In[8]:

for_file_in_ls()

# In[13]:

result_dict["name"] = names
result_dict["phone"] = phones
result_dict["email"] = emails
result_dict["skills"] = skills

# In[15]:

result_df = pd.DataFrame(result_dict)
result_df

# In[28]:

result_df.to_csv("output/csv/parsed_resumes.csv")


import os
import random
import torch
import math
import glob

from PIL import Image
import numpy as np

from .dataloader_base import DLBase
import datasets.daugm_video as tf

class DataVideo(DLBase):

    def __init__(self, cfg, split, val=False):
        super(DataVideo, self).__init__()

        self.cfg = cfg
        self.split = split
        self.val = val

        self.cfg_frame_gap = cfg.DATASET.VAL_FRAME_GAP if val else cfg.DATASET.FRAME_GAP

        self._init_palette(cfg.TRAIN.BATCH_SIZE * cfg.MODEL.GRID_SIZE**2)

        # train/val/test splits are pre-cut
        split_fn = os.path.join(cfg.DATASET.ROOT, "filelists", self.split + ".txt")
        assert os.path.isfile(split_fn)

        self.images = []

        token = None # sequence token (new video when it changes)

        subsequence = []
        ignored = [0]
        num_frames = [0]

        def add_sequence():

            if cfg.DATASET.VIDEO_LEN > len(subsequence):
                # found a very short sequence
                ignored[0] += 1
            else:
                # adding the subsequence
                self.images.append(tuple(subsequence))
                num_frames[0] += len(subsequence)

            del subsequence[:]

        with open(split_fn, "r") as lines:
            for line in lines:
                _line = line.strip("\n").split(' ')

                assert len(_line) > 0, "Expected at least one path"
                _image = _line[0]

                # each sequence may have a different length
                # do some book-keeping e.g. to ensure we have
                # sequences long enough for subsequent sampling
                _token = _image.split("/")[-2] # parent directory

                # sequence ID is in the filename
                #_token = os.path.basename(_image).split("_")[0]
                if token != _token:
                    if not token is None:
                        add_sequence()
                    token = _token

                # image 1
                _image = os.path.join(cfg.DATASET.ROOT, _image.lstrip('/'))
                #assert os.path.isfile(_image), '%s not found' % _image
                subsequence.append(_image)

        # update the last sequence
        # returns the total amount of frames
        add_sequence()
        print("Dataloader: {}".format(split), " / Frame Gap: ", self.cfg_frame_gap)
        print("Loaded {} sequences / {} ignored / {} frames".format(len(self.images), \
                                                                    ignored[0], \
                                                                    num_frames[0]))

        self._num_samples = num_frames[0]
        self._init_augm(cfg)

    def _init_augm(self, cfg):

        # general (unguided) affine transformations
        tfs_pre = [tf.CreateMask()]
        self.tf_pre = tf.Compose(tfs_pre)

        # photometric noise
        tfs_affine = []

        # guided affine transformations
        tfs_augm = []

        # 1.
        # general affine transformations
        #
        tfs_pre.append(tf.MaskScaleSmallest(cfg.DATASET.SMALLEST_RANGE))

        if cfg.DATASET.RND_CROP:
            tfs_pre.append(tf.MaskRandCrop(cfg.DATASET.CROP_SIZE, pad_if_needed=True))
        else:
            tfs_pre.append(tf.MaskCenterCrop(cfg.DATASET.CROP_SIZE))

        if cfg.DATASET.RND_HFLIP:
            tfs_pre.append(tf.MaskRandHFlip())

        # 2.
        # Guided affine transformation
        #
        if cfg.DATASET.GUIDED_HFLIP:
            tfs_affine.append(tf.GuidedRandHFlip())

        # this will add affine transformation
        if cfg.DATASET.RND_ZOOM:
            tfs_affine.append(tf.MaskRandScaleCrop(*cfg.DATASET.RND_ZOOM_RANGE))

        self.tf_affine = tf.Compose(tfs_affine)
        self.tf_affine2 = tf.Compose([tf.AffineIdentity()])

        tfs_post = [tf.ToTensorMask(),
                    tf.Normalize(mean=self.MEAN, std=self.STD),
                    tf.ApplyMask(-1)]

        # image to the teacher will have no noise
        self.tf_post = tf.Compose(tfs_post)

    def set_num_samples(self, n):
        print("Re-setting # of samples: {:d} -> {:d}".format(self._num_samples, n))
        self._num_samples = n

    def __len__(self):
        return len(self.images) #self._num_samples

    def denorm(self, image):

        if image.dim() == 3:
            assert image.dim() == 3, "Expected image [CxHxW]"
            assert image.size(0) == 3, "Expected RGB image [3xHxW]"

            for t, m, s in zip(image, self.MEAN, self.STD):
                t.mul_(s).add_(m)
        elif image.dim() == 4:
            # batch mode
            assert image.size(1) == 3, "Expected RGB image [3xHxW]"

            for t, m, s in zip((0,1,2), self.MEAN, self.STD):
                image[:, t, :, :].mul_(s).add_(m)

        return image

    def _get_affine(self, params):

        N = len(params)

        # construct affine operator
        affine = torch.zeros(N, 2, 3)

        aspect_ratio = float(self.cfg.DATASET.CROP_SIZE[0]) / \
                            float(self.cfg.DATASET.CROP_SIZE[1])

        for i, (dy,dx,alpha,scale,flip) in enumerate(params):

            # R inverse
            sin = math.sin(alpha * math.pi / 180.)
            cos = math.cos(alpha * math.pi / 180.)

            # inverse, note how flipping is incorporated
            affine[i,0,0], affine[i,0,1] = flip * cos, sin * aspect_ratio
            affine[i,1,0], affine[i,1,1] = -sin / aspect_ratio, cos

            # T inverse Rinv * t == R^T * t
            affine[i,0,2] = -1. * (cos * dx + sin * dy)
            affine[i,1,2] = -1. * (-sin * dx + cos * dy)

            # T
            affine[i,0,2] /= float(self.cfg.DATASET.CROP_SIZE[1] // 2)
            affine[i,1,2] /= float(self.cfg.DATASET.CROP_SIZE[0] // 2)

            # scaling
            affine[i] *= scale

        return affine

    def _get_affine_inv(self, affine, params):

        aspect_ratio = float(self.cfg.DATASET.CROP_SIZE[0]) / \
                            float(self.cfg.DATASET.CROP_SIZE[1])

        affine_inv = affine.clone()
        affine_inv[:,0,1] = affine[:,1,0] * aspect_ratio**2
        affine_inv[:,1,0] = affine[:,0,1] / aspect_ratio**2
        affine_inv[:,0,2] = -1 * (affine_inv[:,0,0] * affine[:,0,2] + affine_inv[:,0,1] * affine[:,1,2])
        affine_inv[:,1,2] = -1 * (affine_inv[:,1,0] * affine[:,0,2] + affine_inv[:,1,1] * affine[:,1,2])

        # scaling
        affine_inv /= torch.Tensor(params)[:,3].view(-1,1,1)**2

        return affine_inv

    def __getitem__(self, index):

        # searching for the video clip ID
        sequence = self.images[index] # % len(self.images)]
        seqlen = len(sequence)

        assert self.cfg_frame_gap > 0, "Frame gap should be positive"
        t_window = self.cfg_frame_gap * self.cfg.DATASET.VIDEO_LEN

        # reduce sampling gap for short clips
        t_window = min(seqlen, t_window)
        frame_gap = t_window // self.cfg.DATASET.VIDEO_LEN

        # strided slice
        frame_ids = torch.arange(t_window)[::frame_gap]
        frame_ids = frame_ids[:self.cfg.DATASET.VIDEO_LEN]
        assert len(frame_ids) == self.cfg.DATASET.VIDEO_LEN

        # selecting a random start
        index_start = random.randint(0, seqlen - frame_ids[-1] - 1)
        # permuting the frames in the batch
        random_ids = torch.randperm(self.cfg.DATASET.VIDEO_LEN)
        # adding the offset
        frame_ids = frame_ids[random_ids] + index_start

        # forward sequence
        images = []
        for frame_id in frame_ids:
            fn = sequence[frame_id]
            images.append(Image.open(fn).convert('RGB'))

        # 1. general transforms
        frames, valid = self.tf_pre(images)

        # 1.1 creating two sequences in forward/backward order
        frames1, valid1 = frames[:], valid[:]

        # second copy
        frames2 = [f.copy() for f in frames]
        valid2 = [v.copy() for v in valid]

        # 2. guided affine transforms
        frames1, valid1, affine_params1 = self.tf_affine(frames1, valid1)
        frames2, valid2, affine_params2 = self.tf_affine2(frames2, valid2)

        # convert to tensor, zero out the values
        frames1 = self.tf_post(frames1, valid1)
        frames2 = self.tf_post(frames2, valid2)

        # converting the affine transforms
        aff_reg = self._get_affine(affine_params1)
        aff_main = self._get_affine(affine_params2)

        aff_reg_inv = self._get_affine_inv(aff_reg, affine_params1)

        aff_reg = aff_main # identity affine2_inv
        aff_main = aff_reg_inv

        frames1 = torch.stack(frames1, 0)
        frames2 = torch.stack(frames2, 0)

        assert frames1.shape == frames2.shape

        return frames2, frames1, aff_main, aff_reg

class DataVideoKinetics(DataVideo):

    def __init__(self, cfg, split):
        super(DataVideo, self).__init__()

        self.cfg = cfg
        self.split = split

        self._init_palette(cfg.TRAIN.BATCH_SIZE * cfg.MODEL.GRID_SIZE**2)

        # train/val/test splits are pre-cut
        split_fn = os.path.join(cfg.DATASET.ROOT, "filelists", self.split + ".txt")
        assert os.path.isfile(split_fn)

        self.videos = []

        with open(split_fn, "r") as lines:
            for line in lines:
                _line = line.strip("\n").split(' ')
                assert len(_line) > 0, "Expected at least one path"

                _vid = _line[0]

                # image 1
                _vid = os.path.join(cfg.DATASET.ROOT, _vid.lstrip('/'))
                #assert os.path.isdir(_vid), "{} does not exist".format(_vid)
                self.videos.append(_vid)

        # update the last sequence
        # returns the total amount of frames
        print("DataloaderKinetics: {}".format(split))
        print("Loaded {} sequences".format(len(self.videos)))
        self._init_augm(cfg)

    def __len__(self):
        return len(self.videos)

    def __getitem__(self, index):

        C = self.cfg.DATASET
        path = self.videos[index]

        # filenames
        fns = sorted(glob.glob(path + "/*.jpg"))
        total_len = len(fns)

        # temporal window to consider
        temp_window = min(total_len, C.VIDEO_LEN * C.FRAME_GAP)
        gap = temp_window / C.VIDEO_LEN
        start_frame = random.randint(0, total_len - temp_window)

        images = []
        for idx in range(C.VIDEO_LEN):
            frame_id = start_frame + int(idx * gap)
            fn = fns[frame_id]
            images.append(Image.open(fn).convert('RGB'))

        # 1. general transforms
        frames, valid = self.tf_pre(images)

        # 1.1 creating two sequences in forward/backward order
        frames1, valid1 = frames[:], valid[:]

        # second copy
        frames2 = [f.copy() for f in frames]
        valid2 = [v.copy() for v in valid]

        # 2. guided affine transforms
        frames1, valid1, affine_params1 = self.tf_affine(frames1, valid1)
        frames2, valid2, affine_params2 = self.tf_affine2(frames2, valid2)

        # convert to tensor, zero out the values
        frames1 = self.tf_post(frames1, valid1)
        frames2 = self.tf_post(frames2, valid2)

        # converting the affine transforms
        affine1 = self._get_affine(affine_params1)
        affine2 = self._get_affine(affine_params2)

        affine1_inv = self._get_affine_inv(affine1, affine_params1)

        affine1 = affine2 # identity affine2_inv
        affine2 = affine1_inv

        frames1 = torch.stack(frames1, 0)
        frames2 = torch.stack(frames2, 0)

        assert frames1.shape == frames2.shape

        return frames2, frames1, affine2, affine1


import numpy as np
import matplotlib.pyplot as plt
import problem_1

def create_matrix(n: int) -> np.array:
    A = np.zeros((n, n))
    for i in range (n):
        for j in range(n):
            if i == j:
                A[i, j] =  2.0 + (np.float64(i) / np.float64(n)) ** 2.0
            elif j == i - 1 or j == i + 1:
                A[i, j] =  -1.0
            elif (i == 0 and j == n - 1) or (i == n - 1 and j == 0):
                A[i, j] = -1.0
            else:
                A[i, j] = 0.0

    return A

def create_f(n: int) -> np.array:
    f = np.zeros(n)
    for i in range(n):
        f[i] = (1.0 + n**2.0 * np.sin(np.pi / n) ** 2.0) * np.sin( (2.0 * np.pi * i) / np.float64(n))

    return f

def gershgorin_circles(a: np.array):
    for i, row in enumerate(a):
        a_ii = a[i, i]
        r_i = np.abs(row).sum() - a_ii
        print(f"D({a_ii:5.3f}; {r_i:5.3f})")

def LUP_decomposition(A):
    n = A.shape[0]
    P = np.eye(n)
    L = np.zeros((n, n))
    U = np.copy(A)

    for j in range(n):
        max_row = np.argmax(np.abs(U[j:, j])) + j

        if j != max_row:
            P[[j, max_row]] = P[[max_row, j]]
            L[[j, max_row], :j] = L[[max_row, j], :j]
            U[[j, max_row], j:] = U[[max_row, j], j:]

        L[j, j] = 1.0
        for i in range(j + 1, n):
            L[i, j] = U[i, j] / U[j, j]
            U[i, j:] -= L[i, j] * U[j, j:]

    return P, L, U

def LUP_solve(A: np.array, f: np.array):
    P, L, U = LUP_decomposition(A)

    z = np.matmul(P, f)
    x = problem_1.LU_solve(L, U, z)

    return x

def get_eigvals_krylov(A):
    size = A.shape[0]
    y_matrix = np.zeros((size, size))
    y = np.ones(n)

    for i in range(size - 1, -1, -1):
        y_matrix[:, i] = y
        y = np.matmul(A, y)

    ps = LUP_solve(y_matrix, y)

    coeffs = np.concatenate([np.ones(1), -1.0 * ps])

    return np.array(sorted(np.roots(coeffs)))

def solve(A, f, tau, eps=1e-6, iter_limit = 10000):
    size = A.shape[0]
    tau_f = f * tau
    cur = np.zeros(size)
    b = np.eye(size) - tau * A
    err_list = []

    for i in range(iter_limit):
        cur = np.matmul(b, cur) + tau_f
        err = f - np.matmul(A, cur)
        norm_err = problem_1.m_3(err)
        err_list.append(norm_err)

        if norm_err < eps:
            break

    solution = cur
    return solution, err_list

def output_solve(A, f, tau, eps=1e-6):
    solution, errors = solve(A, f, tau=tau, eps=eps)
    exact_sol = np.linalg.solve(A, f)
    delta = solution - exact_sol
    print("---------------------------------------------------")
    print("solution")
    print(f"tau = {tau}; res = {solution}")
    print(f"|numpy_sol - sol|_3 = {problem_1.m_3(delta)}")
    print("---------------------------------------------------")

    num_errors = np.arange(len(errors))
    line, = plt.plot(num_errors, errors, marker='.')
    line.set_label(f"tau = {tau}")

if __name__ == '__main__':
    n = 6
    A = create_matrix(n)
    f = create_f(n)

    print("---------------------------------------------------")
    print("eigenvalue estimation using Gershgorin circles")
    gershgorin_circles(A)
    print("---------------------------------------------------")

    krylov_eigvals = get_eigvals_krylov(A)
    krylov_eigvals.sort()
    numpy_eigvals = np.linalg.eigvals(A)
    numpy_eigvals.sort()

    print("---------------------------------------------------")
    print("eigenvals (krylov vs numpy)")
    for i in range (len(krylov_eigvals)):
        l_k = krylov_eigvals[i]
        l_n = numpy_eigvals[i]
        print(f"lambda_{i}: {l_k:10.6f} vs {l_n:10.6f}; delta = {abs(l_n - l_k)}")
    print("---------------------------------------------------")

    tau_opt_krylov = 2.0 / (krylov_eigvals.min() + krylov_eigvals.max())
    tau_opt_numpy = 2.0 / (numpy_eigvals.min() + numpy_eigvals.max())

    print(f"tau_opt_numpy  = {tau_opt_numpy}")
    print(f"tau_opt_krylov = {tau_opt_krylov}")

    output_solve(A, f, tau_opt_numpy)
    output_solve(A, f, tau_opt_krylov)
    output_solve(A, f, tau=0.4)
    output_solve(A, f, tau=0.5)

    plt.yscale("log")
    plt.ylabel('mismatch rate')
    plt.xlabel('number of iterations')
    plt.grid()
    plt.legend()
    plt.show()


from flask import session, render_template, redirect, request
from webViews.view import normalView
from webViews.dockletrequest import dockletRequest
import time, re, json

class cloudView(normalView):
    template_path = "cloud.html"

    @classmethod
    def post(self):
        settings = dockletRequest.post_to_all('/cloud/setting/get/')
        return self.render(self.template_path, settings = settings)

    @classmethod
    def get(self):
        return self.post()

class cloudSettingModifyView(normalView):
    @classmethod
    def post(self):
        dockletRequest.post('/cloud/setting/modify/', request.form, self.masterip)
        return redirect('/cloud/')

class cloudNodeAddView(normalView):
    @classmethod
    def post(self):
        data = {}
        dockletRequest.post('/cloud/node/add/', data, self.masterip)
        return redirect('/hosts/')

    @classmethod
    def get(self):
        return self.post()


from collections import deque

def isright(sli):
    dic = {'[':']','{':'}','(':')'}
    stack = deque([])
    for x in sli:
        if x in dic.keys():
            stack.append(x)
        if x in dic.values():
            if stack:
                if dic[stack[-1]] == x:
                    stack.pop()
            else:
                return 0
    if stack:
        return 0
    else:
        return 1

def solution(s):
    s = deque([x for x in s])
    cnt = 0
    for x in range(len(s)):
        s.append(s.popleft())
        cnt += isright(s)
    return cnt

프로그래머스/lv2/76502. 괄호 회전하기/괄호 회전하기.py


# -*-coding:Latin-1 -*

import configparser
from pathlib import Path

from logPrint import logPrint

class internalConfig():
    # This class will parse the .ini config file

    def __init__(self):
        logPrint.printLog("Parsing the config file config.ini")
        self.config = configparser.ConfigParser()
        self.config.read('config.ini')

        self.outputFilePath = self.config['timeSnip']['outputFilePath']
        readVerbosityLevel = self.config['timeSnip']['verbosityLevel']

        # outputFilePath
        if self.outputFilePath == "":
            print(
                "[ERR] In internalConfig.py, outputFilePath is not defined")
            exit(-7)
        outputFile = Path(self.outputFilePath)
        outputFileParent = outputFile.parent
        if not outputFileParent.is_dir():
            print(
                "[ERR] In internalConfig.py, outputFilePath is bad defined: its parent directory does not exist")
            exit(-8)

        # verbosityLevel
        if readVerbosityLevel == "":
            print(
                "[ERR] In internalConfig.py, readVerbosityLevel is not defined")
            exit(-7)
        if readVerbosityLevel == "debug":
            self.verbosityLevel = 0
        elif readVerbosityLevel == "log":
            self.verbosityLevel = 1
        elif readVerbosityLevel == "error":
            self.verbosityLevel = 2
        else:
            print("[ERR] In internalConfig.py, verbosityLevel is not/bad defined")
            exit(-9)

    def getOutputFilePath(self):
        return self.outputFilePath

    def getVerbosityLevel(self):
        return self.verbosityLevel


import collections

def cipher_simple(ordinal, steps):
    shifted = ordinal + (steps % (128 - 32))
    if shifted > 127:
        return shifted - (128 - 32)
    if shifted < 0x20:
        return shifted + (128 - 32)
    return shifted

__file_cipher = list(range(0x30, 0x3A)) + list(range(0x61, 0x7B)) + list(map(ord, '_- '))
def cipher_file(ordinal, steps):

    shifted = __file_cipher.index(ordinal) + (steps % len(__file_cipher))
    if shifted >= len(__file_cipher):
        shifted -= len(__file_cipher)
    if shifted < 0:
        shifted += len(__file_cipher)

    return __file_cipher[shifted]

class Cipher:

    def __init__(self, index: int, shifter: callable = None):
        """
        Create a new cipher disk.

        :index int: the key of the cipher
        :disk list optional: a list of all available values (default ascii)
        """

        self.index = index
        if shifter is None:
            self._shifter = cipher_simple
        else:
            self._shifter = shifter

    def encode(self, text: collections.Iterable) -> list:
        if isinstance(text, str):
            text = map(ord, text)

        return [self._shifter(i, self.index) for i in text]

    def encodeStr(self, text: str) -> str:
        return ''.join(map(chr, self.encode(text)))

    def decode(self, text: collections.Iterable) -> list:
        if isinstance(text, str):
            text = map(ord, text)

        return [self._shifter(i, -self.index) for i in text]

    def decodeStr(self, text: str) -> str:
        return ''.join(map(chr, self.decode(text)))


import os
import re
from typing import List, Union

import numpy as np
import paddle
from paddlehub.env import MODULE_HOME
from paddlehub.module.module import moduleinfo, serving
from paddlehub.utils.log import logger
from paddlenlp.transformers import ErnieTokenizer, ErnieForTokenClassification
from paddlenlp.data import Pad

@moduleinfo(
    name="auto_punc",
    version="1.0.0",
    summary="",
    author="KPatrick",
    author_email="",
    type="text/punctuation_restoration")
class Ernie(paddle.nn.Layer):
    def __init__(self):
        super(Ernie, self).__init__()
        res_dir = os.path.join(MODULE_HOME, 'auto_punc')
        punc_vocab_file = os.path.join(res_dir, 'assets', 'punc_vocab.txt')
        ckpt_dir = os.path.join(res_dir, 'assets', 'ckpt')

        self.punc_vocab = self._load_dict(punc_vocab_file)
        self.punc_list = list(self.punc_vocab.keys())
        self.model = ErnieForTokenClassification.from_pretrained(ckpt_dir)
        self.model.eval()
        self.tokenizer = ErnieTokenizer.from_pretrained('ernie-1.0')

    @staticmethod
    def _load_dict(dict_path):
        vocab = {}
        i = 0
        with open(dict_path, 'r', encoding='utf-8') as fin:
            for line in fin:
                key = line.strip('\n')
                vocab[key] = i
                i += 1
        return vocab

    @staticmethod
    def _clean_text(text, punc_list):
        text = text.lower()
        text = re.sub('[^A-Za-z0-9\u4e00-\u9fa5]', '', text)
        text = re.sub(f'[{"".join([p for p in punc_list][1:])}]', '', text)
        return text

    def forward(self, text: str):
        wav = None
        input_ids = self.frontend.get_input_ids(text, merge_sentences=True)
        phone_ids = input_ids["phone_ids"]
        for part_phone_ids in phone_ids:
            with paddle.no_grad():
                mel = self.fastspeech2_inference(part_phone_ids)
                temp_wav = self.pwg_inference(mel)
                if wav is None:
                    wav = temp_wav
                else:
                    wav = paddle.concat([wav, temp_wav])
        return wav

    @serving
    def add_puncs(self, texts: Union[str, List[str]], max_length=256, device='cpu'):
        assert isinstance(texts, str) or (isinstance(texts, list) and isinstance(texts[0], str)), \
            'Input data should be str or List[str], but got {}'.format(type(texts))

        if isinstance(texts, str):
            texts = [texts]

        input_ids = []
        seg_ids = []
        seq_len = []
        for i in range(len(texts)):
            clean_text = self._clean_text(texts[i], self.punc_list)
            assert len(clean_text) > 0, f'Invalid input string: {texts[i]}'

            tokenized_input = self.tokenizer(
                list(clean_text), return_length=True, is_split_into_words=True, max_seq_len=max_length)

            input_ids.append(tokenized_input['input_ids'])
            seg_ids.append(tokenized_input['token_type_ids'])
            seq_len.append(tokenized_input['seq_len'])

        paddle.set_device(device)
        with paddle.no_grad():
            pad_func_for_input_ids = Pad(axis=0, pad_val=self.tokenizer.pad_token_id, dtype='int64')
            pad_func_for_seg_ids = Pad(axis=0, pad_val=self.tokenizer.pad_token_type_id, dtype='int64')
            input_ids = paddle.to_tensor(pad_func_for_input_ids(input_ids))
            seg_ids = paddle.to_tensor(pad_func_for_seg_ids(seg_ids))
            logits = self.model(input_ids, seg_ids)
            preds = paddle.argmax(logits, axis=-1)

        tokens = []
        labels = []
        for i in range(len(input_ids)):
            tokens.append(self.tokenizer.convert_ids_to_tokens(input_ids[i, 1:seq_len[i] - 1].tolist()))
            labels.append(preds[i, 1:seq_len[i] - 1].tolist())  # Remove predictions of special tokens.

        punc_texts = []
        for token, label in zip(tokens, labels):
            assert len(token) == len(label)
            text = ''
            for t, l in zip(token, label):
                text += t
                if l != 0:  # Non punc.
                    text += self.punc_list[l]
            punc_texts.append(text)

        return punc_texts


import cv2 as cv
import numpy as np
import mediapipe as mp

mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles
mp_pose = mp.solutions.pose

def annotate_still_image(image_path):
    with mp_pose.Pose(
            static_image_mode=True,
            model_complexity=2,
            enable_segmentation=True,
            min_detection_confidence=0.5) as pose:
        image = cv.imread(image_path)
        image_height, image_width, _ = image.shape
        # Convert the BGR image to RGB before processing.
        results = pose.process(cv.cvtColor(image, cv.COLOR_BGR2RGB))

        print(
            f'Nose coordinates: ('
            f'{results.pose_landmarks.landmark[mp_pose.PoseLandmark.NOSE].x * image_width}, '
            f'{results.pose_landmarks.landmark[mp_pose.PoseLandmark.NOSE].y * image_height})'
        )

        annotated_image = image.copy()

        mp_drawing.draw_landmarks(
            annotated_image,
            results.pose_landmarks,
            mp_pose.POSE_CONNECTIONS,
            landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style())
        cv.imwrite('annotated_image.jpg', annotated_image)

# Webcam for debugging
def annotate_webcam():
    # For webcam input:
    cap = cv.VideoCapture(0)
    with mp_pose.Pose(
            min_detection_confidence=0.5,
            min_tracking_confidence=0.5) as pose:
        while cap.isOpened():
            success, image = cap.read()
            if not success:
                print("Ignoring empty camera frame.")
                # If loading a video, use 'break' instead of 'continue'.
                continue

            # To improve performance, optionally mark the image as not writeable to
            # pass by reference.
            image.flags.writeable = False
            image = cv.cvtColor(image, cv.COLOR_BGR2RGB)
            results = pose.process(image)

            # Draw the pose annotation on the image.
            image.flags.writeable = True
            image = cv.cvtColor(image, cv.COLOR_RGB2BGR)
            mp_drawing.draw_landmarks(
                image,
                results.pose_landmarks,
                mp_pose.POSE_CONNECTIONS,
                landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style())
            # Flip the image horizontally for a selfie-view display.
            cv.imshow('MediaPipe Pose', cv.flip(image, 1))
            if cv.waitKey(33) == 32:
                break
    cap.release()

def main():
    image_path = 'media/samp.jpg'
    # annotate_still_image(image_path)
    annotate_webcam()

if __name__ == '__main__':
    main()


import json
from datetime import datetime

nowdatetime = datetime.now().strftime('%m-%d-%Y %H:%M:%S')
todays_date = datetime.now().strftime('%m-%d-%Y')

class PoolTable:
    def __init__(self, pool_table_number):
        self.pool_table_number = pool_table_number
        self.occupied = False
        self.start_date_time = nowdatetime
        self.end_date_time = nowdatetime
        self.total_time_played = 0
        # self.cost = pass

table_lists = []
table_dicts = []

user_input = ""

def menu():
    print("\n1 - Book a Table\t2 - Close a Table\t3 - Close the App")

# def creating_tables_intial_12_tables():
tables_created = 1
while tables_created < 13:
    p = PoolTable(tables_created)
    table_lists.append(p)
    tables_created += 1

# creating_tables_intial_12_tables()

#see all tables and availability
def view_all_tables():
    print("\n{:*^30}".format(" INSTAPOOL "))
    print('{:=^30}'.format(""))
    for table in range(0, len(table_lists)):
        table_info = table_lists[table]
        if table_info.occupied == False:
            table_info.occupied = "Not Occupied"
        else:
            table_info.occupied = "Occupied"
        print(f"Table {table_info.pool_table_number} - {table_info.occupied}")

# converting and appending table objects to array as dictionaries
for table in table_lists:
    table_dicts.append(table.__dict__)

# print(table_dicts)

# writing/saving table dictionary array to JSON
with open("pool_table_info.json", "w") as f:
    json.dump(table_dicts, f)

#

while user_input != "3":
    view_all_tables()
    menu()
    user_input = input("\nEnter your choice: ")
    if user_input == "1":
        # LATER WRITE TABLE INFORMATION HERE
        print("\nTABLE 8 BOOKED!")
    if user_input == "2":
        # LATER CREATE A CREATE TABLE FUNCTION IN THE POOL CLASS
        print("\nTABLE 8 CLOSED!")
    elif user_input == "3":
        print("\n\nBYE!\n\n")


from django.urls import path

from data_consumption.views import UploadAndSaveView, ListBuildingsView, ListMeterView, ListMeterHourlyDataView, \
    ShowChartView

urlpatterns = [
    path("upload/", UploadAndSaveView.as_view()),
    path("buildings/", ListBuildingsView.as_view()),
    path("buildings/<int:pk>/meters/", ListMeterView.as_view()),
    path("buildings/<int:pk>/meters/<int:meter_id>/half-hourly-data/", ListMeterHourlyDataView.as_view()),
    path("meter/<int:pk>/show-chart/", ShowChartView.as_view())
]