#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
PowerMeter
---------
Main Python class or entrypoint to monitor the power consumption of
an algorithm.
"""

__all__ = ["PowerMeter"]

import datetime
import getpass
import json
import logging
import os
import shutil
import sys
import traceback
import warnings
from pathlib import Path

import pandas as pd  # type: ignore
import requests

from .nvidia_power import NoGpuPower, NvidiaPower
from .power_gadget import (
    NoPowerGadget,
    PowerGadgetLinuxMSR,
    PowerGadgetLinuxRAPL,
    PowerGadgetMac,
    PowerGadgetWin,
)
from .utils import (
    AVAILABLE_STEPS,
    COUNTRY_CODE_COLUMN,
    COUNTRY_NAME_COLUMN,
    ENERGY_MIX_COLUMN,
    ENERGY_MIX_DATABASE,
    LOGGING_FILE,
    MSR_PATH_LINUX_TEST,
    PACKAGE_PATH,
    POWERLOG_PATH_LINUX,
    TOTAL_CPU_TIME,
    TOTAL_ENERGY_ALL,
    TOTAL_ENERGY_CPU,
    TOTAL_ENERGY_GPU,
    TOTAL_ENERGY_MEMORY,
    TOTAL_ENERGY_PROCESS_CPU,
    TOTAL_ENERGY_PROCESS_MEMORY,
    TOTAL_GPU_TIME,
    match,
    normalize,
)

LOGGER = logging.getLogger(__name__)

class PowerMeter:
    """
    PowerMeter is a general tool to monitor and log the power consumption of
    any given function.

    Depending on the platform and hardware used, it allows to measure the
    power usage of the CPU,the DRAM and the GPU. The measure is, then,
    converted to CO2 emissions depending on the country set and the PUE
    of the machine (background on the PUE `here`_). You can chose to log the
    results locally or send it to an endpoint.

    Parameters
    ----------
    project_name : str, default current_directory_name
        Name of the project you are working on.
    program_name : str, optional
        Name of the program you are working on.
    client_name : str, optional
        Name of the client you are working for.
    user_name : str, default computer_user_name
        The name of the user using the tool (for logging purpose).
    cpu_power_log_path : str, optional
        The path to the tool "PowerLog" or "powercap" or "IntelPowerGadget".
    powerlog_save_path: str, optional
        Path to the folder where we should find the logs provided by
        IntelPowerGadget.
    get_country : bool, default True
        Whether to retrieve user country location or not (uses the user IP).
    location : str, optional
        Country ISO Code available `here
        <https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Officially_assigned_code_elements>`_

        Must be set if get_country is False.
    is_online : boolean, default True
        Whether the machine is connected to the internet or not.
    filepath : str, default .
        Path of the file where all the carbon logs will be written.
    output_format : {'csv', 'excel'}, default 'csv'
        Format of the carbon logs file produced.
    api_endpoint : str, optional
        Endpoint of the API to upload the collected data to

        Note: we provide an endpoint to collect data and contribute, with the
        community, towards greener algorithms.
        Here is the url :
        https://ngji0jx9dc.execute-api.eu-west-3.amazonaws.com/post_new_item

    See Also
    --------
    PowerMeter.from_config : Create a power meter from a config file.

    Examples
    --------
    Create a PowerMeter for the project `MNIST Classifier` while
    not being online.

    >>> power_meter = PowerMeter(project_name="MNIST classifier",
    ...     is_online=False, location="FR")

    Create a PowerMeter for the project `Test` and send the collected data to
    our endpoint.

    >>> power_meter = PowerMeter(project_name="Test",
    ...     api_endpoint="https://ngji0jx9dc.execute-
    api.eu-west-3.amazonaws.com/post_new_item")

    Notes
    -----
    This package may log private data (username, country, project_name).
    If you do not provide any api_endpoint, we will never have access
    to this data.

    On the other hand, if you chose to share your data with us
    (by using our endpoint:
    https://ngji0jx9dc.execute-api.eu-west-3.amazonaws.com/post_new_item
    ), we commit to anonymize any data shared.
    """

    LAPTOP_PUE = 1.3  # pue for my laptop
    SERVER_PUE = 1.58  # pue for a server
    DEFAULT_LOCATION = "FR"
    DATETIME_FORMAT = "%m/%d/%Y %H:%M:%S"  # "%c"

    # ----------------------------------------------------------------------
    # Constructors
    def __init__(
        self,
        project_name="",
        program_name="",
        client_name="",
        user_name="",
        cpu_power_log_path="",
        powerlog_save_path="",
        get_country=True,
        location="",
        is_online=True,
        filepath=None,
        output_format="csv",
        api_endpoint=None,
    ):

        self.platform = sys.platform
        powergadget_platform = {
            "darwin": PowerGadgetMac,
            "win32": PowerGadgetWin,
            "linux": self.__set_powergadget_linux,
            "linux2": self.__set_powergadget_linux,
            "": NoPowerGadget,
        }
        self.power_gadget = powergadget_platform[self.platform](
            powerlog_path=cpu_power_log_path,
            powerlog_save_path=powerlog_save_path,
        )

        self.pue = self.__set_pue()

        self.cuda_available = self.__check_gpu()
        self.gpu_power = self.__set_gpu_power()

        self.user = self.__set_username(user_name)

        self.project = self.__set_project_name(project_name)
        self.program_name = self.__set_project_entity(program_name)
        self.client_name = self.__set_project_entity(client_name)

        self.is_online = is_online
        if api_endpoint:
            LOGGER.info("Api endpoint given, will save data online")
            self.api_endpoint = api_endpoint
        else:
            LOGGER.info("No current api endpoint, will save data locally")
            self.api_endpoint = ""

        self.location = self.__set_location(location, get_country)

        self.energy_mix_db = self.__load_energy_mix_db()
        self.energy_mix = self.__get_energy_mix()  # kgCO2e/kWh
        self.location_name = self.__get_location_name()

        self.used_package = ""
        self.used_algorithm = ""
        self.used_data_type = ""
        self.used_data_shape = ""
        self.used_algorithm_params = ""
        self.used_comments = ""
        self.used_step = ""

        if not filepath:
            LOGGER.info("No current filepath, will use the default")
            self.filepath = Path.cwd() / "emissions.csv"
        else:
            LOGGER.info("Filepath given, will save there")
            self.filepath = Path(filepath)

        self.output_format = output_format

        self.logging_filename = PACKAGE_PATH / LOGGING_FILE

    @staticmethod
    def __load_energy_mix_db():
        return pd.read_csv(
            PACKAGE_PATH / ENERGY_MIX_DATABASE, encoding="utf-8"
        )

    @staticmethod
    def __extract_env_name():
        env = "unknown"
        try:
            env = os.environ["CONDA_DEFAULT_ENV"]
        except KeyError:
            pass
        if hasattr(sys, "base_prefix") and sys.base_prefix != sys.prefix:
            env = sys.base_prefix.split("/")[-1]
        return env

    @staticmethod
    def __check_gpu():
        cuda_available = False
        if shutil.which("nvidia-smi"):
            cuda_available = True

        return cuda_available

    @staticmethod
    def __get_country():
        """
        Retrieve the ISO code country
        Beware of the encoding
        cf. from https://stackoverflow.com/questions/40059654/python-convert-
        a-bytes-array-into-json-format
        """
        request = requests.get("http://ipinfo.io/json")
        response = request.content.decode("utf8").replace("'", '"')
        user_info = json.loads(response)
        return user_info["country"]

    @classmethod
    def from_config(cls, path):
        """
        Create a PowerMeter from a json config file.

        The json config file may contain any key named after the arguments of
        the :class:`carbonai.PowerMeter` constructor.

        Parameters
        ----------
        path : str
            Path to the json config file

        Returns
        -------
        PowerMeter

        See also
        --------
        PowerMeter : Instantiate a PowerMeter by declaring every argument

        Examples
        --------
        Example of config file used. This file is named `config.json`

        .. code-block:: JSON

            {
                "project_name": "Project X",
                "program_name": "Program X",
                "client_name": "Client X",
                "get_country": true,
                "is_online": false,
                "user_name": "customUsername",
                "filepath": "",
                "api_endpoint": "...",
                "location":"FR"
            }

        To load this file as a power meter.

        >>> power_meter = PowerMeter.from_config("config.json")
        """
        with open(path) as file:
            args = json.load(file)
        return cls(**args)

    def __set_pue(self):
        if self.platform in ["darwin", "win32"]:
            return self.LAPTOP_PUE  # pue for my laptop
        return self.SERVER_PUE  # pue for a server

    def __set_powergadget_linux(
        self, powerlog_path=None, powerlog_save_path=None
    ):
        if POWERLOG_PATH_LINUX.exists():
            power_gadget = PowerGadgetLinuxRAPL()
        # The user needs to be root to use MSR interface
        elif MSR_PATH_LINUX_TEST.exists() and os.getuid() == 0:
            power_gadget = PowerGadgetLinuxMSR()
        else:
            LOGGER.warning("No power reading interface was found")
            power_gadget = NoPowerGadget()
        return power_gadget

    def __set_gpu_power(self):
        if self.cuda_available:
            LOGGER.info("Found a GPU")
            gpu_power = NvidiaPower()
        else:
            LOGGER.info("Found no GPU")
            gpu_power = NoGpuPower()
        return gpu_power

    def __set_username(self, user_name):
        if user_name:
            return user_name
        return getpass.getuser()

    def __set_project_name(self, project_name):
        if project_name:
            return project_name
        return self.__extract_env_name()

    def __set_project_entity(self, entity_name):
        if entity_name:
            return entity_name
        return "--"

    def __set_location(self, provided_location, get_country):
        # Set the location used to convert energy usage to carbon emissions
        # if the location is provided, we use it
        # if it's not and we can use the internet and the user authorize us to
        # #do so then we retrieve it from the IP address
        # otherwise set the location to default
        if provided_location:
            location = provided_location
        elif (self.is_online or self.api_endpoint) and get_country:
            location = self.__get_country()
        else:
            warnings.warn(
                "No location was set, we will fallback to \
                the default location: {}".format(
                    self.DEFAULT_LOCATION
                )
            )
            location = self.DEFAULT_LOCATION
        return location

    def __get_energy_mix(self):
        if not (
            self.energy_mix_db[COUNTRY_CODE_COLUMN] == self.location
        ).any():
            raise NameError(
                "The location input was not found, make sure you wrote "
                "the isocode of your country. You used " + self.location
            )
        return self.energy_mix_db.loc[
            self.energy_mix_db[COUNTRY_CODE_COLUMN] == self.location,
            ENERGY_MIX_COLUMN,
        ].values[0]

    def __get_location_name(self):
        return self.energy_mix_db.loc[
            self.energy_mix_db[COUNTRY_CODE_COLUMN] == self.location,
            COUNTRY_NAME_COLUMN,
        ].values[0]

    def __aggregate_power(self, cpu_record, gpu_record):
        """
        Implement the cO2 emission value

        Parameters:
        -----------
        cpu_record, gpu_record (dict):
            Respectively CPU and GPU's records

        Returns
        -------
        co2_emitted (float)
        """
        used_energy = self.pue * (
            cpu_record[TOTAL_ENERGY_PROCESS_CPU]
            + cpu_record[TOTAL_ENERGY_PROCESS_MEMORY]
            + gpu_record[TOTAL_ENERGY_GPU]
        )  # mWh
        co2_emitted = used_energy * self.energy_mix * 1e-3
        LOGGER.info(
            "This process emitted %.3fg of CO2 (using the energy mix of %s)",
            co2_emitted,
            self.location_name,
        )

        return co2_emitted

    def measure_power(
        self,
        package,
        algorithm,
        step="other",
        data_type="",
        data_shape="",
        algorithm_params="",
        comments="",
    ):
        """
        A decorator to measure the power consumption of a given function

        Parameters
        ----------
        package : str
            A string describing the package used by this function
            (e.g. sklearn, Pytorch, ...)
        algorithm : str
            A string describing the algorithm used in the function monitored
            (e.g. RandomForestClassifier, ResNet121, ...)
        step : {'inference', 'training', 'other', 'test', 'run', \
            'preprocessing'}, optional
            A string to provide useful information on the current stage
            of the algorithm
        data_type : {'tabular', 'image', 'text', 'time series', 'other'},\
             optional
            A string describing the type of data used for training
        data_shape : str or tuple, optional
            A string or tuple describing the quantity of data used
        algorithm_params : str, optional
            A string describing the parameters used by the algorithm
        comments : str, optional
            A string to provide any useful information

        Returns
        -------

        See also
        --------
        PowerMeter.from_config : Create a PowerMeter object from a config file
        PowerMeter.__call__ : Measure the power usage using a with statement

        Examples
        --------
        First, create a PowerMeter (you only do to this step once).

        >>> power_meter = PowerMeter.from_config("config.json")

        Decorate the function you wish to monitor.

        >>> @power_meter.measure_power(
        ...     package="pandas, numpy",
        ...     algorithm="data cleaning",
        ...     step="preprocessing",
        ...     data_type="tabular",
        ...     comments="Cleaning of csv files + train-test splitting"
        ... )
        ... def example_func():
        ...     # do something

        Each time this function will be called, the PowerMeter will monitor
        the power usage and log the function's carbon footprint.

        >>> example_func()
        result_of_your_function
        """
        if not algorithm or not package:
            raise SyntaxError(
                "Please input a description for the function you are trying "
                "to monitor. Pass in the algorithm and the package you are "
                "trying to monitor"
            )

        def decorator(func):
            def wrapper(*args, **kwargs):
                self.start_measure(
                    package,
                    algorithm,
                    data_type=data_type,
                    data_shape=data_shape,
                    algorithm_params=algorithm_params,
                    comments=comments,
                    step=step,
                )
                try:
                    results = func(*args, **kwargs)
                finally:
                    self.stop_measure()
                return results

            return wrapper

        return decorator

    def __set_used_arguments(
        self,
        package,
        algorithm,
        data_type="",
        data_shape="",
        algorithm_params="",
        comments="",
        step="other",
    ):
        self.used_package = normalize(package)
        self.used_algorithm = normalize(algorithm)
        self.used_data_type = normalize(data_type)
        self.used_data_shape = normalize(data_shape)
        self.used_algorithm_params = normalize(algorithm_params)
        self.used_comments = normalize(comments)
        self.used_step = normalize(match(step, AVAILABLE_STEPS))

    def __call__(
        self,
        package,
        algorithm,
        step="other",
        data_type="",
        data_shape="",
        algorithm_params="",
        comments="",
    ):
        """
        Measure the power usage using a with statement.

        Parameters
        ----------
        package : str
            A string describing the package used by this function (e.g.
            sklearn, Pytorch, ...)
        algorithm : str
            A string describing the algorithm used in the function monitored
            (e.g. RandomForestClassifier, ResNet121, ...)
        step : {'inference', 'training', 'other', 'test', 'run', \
            'preprocessing'}, optional
            A string to provide useful information on the current stage
            of the alogrithm
        data_type : {'tabular', 'image', 'text', 'time series', 'other'}, \
            optional
            A string describing the type of data used for training
        data_shape : str or tuple, optional
            A string or tuple describing the quantity of data used
        algorithm_params : str, optional
            A string describing the parameters used by the algorithm
        comments : str, optional
            A string to provide any useful information

        Returns
        -------

        See also
        --------
        PowerMeter.from_config : Create a PowerMeter object from a config file
        PowerMeter.measure_power : Measure the power usage using a function \
            decorator

        Examples
        --------
        First, create a PowerMeter (you only do to this step once).

        >>> power_meter = PowerMeter.from_config("config.json")

        Use a with statement to encapsulate the code you want to monitor

        >>> with power_meter(
        ...     package="pandas, numpy",
        ...     algorithm="data cleaning",
        ...     step="preprocessing",
        ...     data_type="tabular",
        ...     comments="Cleaning of csv files + train-test splitting"
        ... ):
        ...     # do something
        result_of_your_code
        """
        self.__set_used_arguments(
            package,
            algorithm,
            data_type=data_type,
            data_shape=data_shape,
            algorithm_params=algorithm_params,
            comments=comments,
            step=step,
        )
        return self

    def __enter__(
        self,
    ):
        self.start_measure(
            self.used_package,
            self.used_algorithm,
            data_type=self.used_data_type,
            data_shape=self.used_data_shape,
            algorithm_params=self.used_algorithm_params,
            comments=self.used_comments,
        )

    def __exit__(self, exit_type, value, traceback):
        self.stop_measure()

    def start_measure(
        self,
        package,
        algorithm,
        step="other",
        data_type="",
        data_shape="",
        algorithm_params="",
        comments="",
    ):
        """
        Starts mesuring the power consumption of a given sample of code

        Parameters
        ----------
        package : str
            A string describing the package used by this function
            (e.g. sklearn, Pytorch, ...)
        algorithm : str
            A string describing the algorithm used in the function monitored
            (e.g. RandomForestClassifier, ResNet121, ...)
        step : {'inference', 'training', 'other', 'test', 'run', \
            'preprocessing'}, optional
            A string to provide useful information on the current stage
            of the algorithm
        data_type : {'tabular', 'image', 'text', 'time series', 'other'},\
            optional
            A string describing the type of data used for training
        data_shape : str or tuple, optional
            A string or tuple describing the quantity of data used
        algorithm_params : str, optional
            A string describing the parameters used by the algorithm
        comments : str, optional
            A string to provide any useful information

        Returns
        -------

        See also
        --------
        PowerMeter.stop_measure : Stop the measure started with start_measure
        PowerMeter.from_config : Create a PowerMeter object from a config file
        PowerMeter.measure_power : Measure the power usage using a function \
            decorator
        PowerMeter.__call__ : Measure the power usage using a with statement

        Notes
        -----
        We do not recommend using this method to monitor the energy usage of
        your code because it won't automatically stop if an error is raised
        at some point while running. You will then have to stop the measure
        manually with :func:`PowerMeter.stop_measure`.

        Examples
        --------
        First, create a PowerMeter (you only do to this step once).

        >>> power_meter = PowerMeter.from_config("config.json")

        Start measuring the code you wish to monitor

        >>> power_meter.start_measure(
        ...     package="pandas, numpy",
        ...     algorithm="data cleaning",
        ...     step="preprocessing",
        ...     data_type="tabular",
        ...     comments="Cleaning of csv files + train-test splitting"
        ... )
        ... # do something
        result_of_your_code

        **Do not forget to stop measuring**

        >>> power_meter.stop_measure()

        """
        self.gpu_power.start()
        self.power_gadget.start()
        self.__set_used_arguments(
            package,
            algorithm,
            data_type=data_type,
            data_shape=data_shape,
            algorithm_params=algorithm_params,
            comments=comments,
            step=step,
        )

    def stop_measure(self):
        """
        Stops the measure started with :func:`PowerMeter.start_measure`

        Parameters
        ----------

        See also
        --------
        PowerMeter.start_measure : Stop the measure started with start_measure
        PowerMeter.from_config : Create a PowerMeter object from a config file
        PowerMeter.measure_power : Measure the power usage using a function \
            decorator
        PowerMeter.__call__ : Measure the power usage using a with statement

        Notes
        -----
        We do not recommend using this method to monitor the energy usage of
        your code because it won't automatically stop if an error is raised at
        some point while running. You will then have to stop the measure
        manually with :func:`PowerMeter.stop_measure`.

        Examples
        --------
        First, create a PowerMeter (you only do to this step once).

        >>> power_meter = PowerMeter.from_config("config.json")

        Start measuring the code you wish to monitor

        >>> power_meter.start_measure(
        ...     package="pandas, numpy",
        ...     algorithm="data cleaning",
        ...     step="preprocessing",
        ...     data_type="tabular",
        ...     comments="Cleaning of csv files + train-test splitting"
        ... )
        ... # do something
        result_of_your_code

        **Do not forget to stop measuring**

        >>> power_meter.stop_measure()
        """
        self.power_gadget.stop()
        self.gpu_power.stop()
        self.gpu_power.parse_log()
        self.__log_records(
            self.power_gadget.record,  # must be a dict
            self.gpu_power.record,  # must be a dict
            algorithm=self.used_algorithm,
            package=self.used_package,
            data_type=self.used_data_type,
            data_shape=self.used_data_shape,
            algorithm_params=self.used_algorithm_params,
            comments=self.used_comments,
            step=self.used_step,
        )

    def __record_data_to_server(self, info):
        headers = {"Content-Type": "application/json"}
        data = json.dumps(info)
        try:
            response = requests.request(
                "POST",
                self.api_endpoint,
                headers=headers,
                data=data,
                timeout=1,
            )
            return response.status_code
        except requests.exceptions.Timeout:
            return 408

    def __record_data_to_csv_file(self, info):
        try:
            data = pd.DataFrame(info, index=[0])
            if Path(self.filepath).exists():
                data.to_csv(self.filepath, mode="a", index=False, header=False)
            else:
                data.to_csv(self.filepath, index=False)
            return True
        except Exception:
            LOGGER.error("* error during the csv writing process *")
            LOGGER.error(traceback.format_exc())
            return False

    def __record_data_to_excel_file(self, info):
        try:

            if Path(self.filepath).exists():
                data = pd.read_excel(self.filepath).append(
                    info, ignore_index=True
                )
                data.to_excel(self.filepath, index=False)
            else:
                data = pd.DataFrame(info, index=[0])
                data.to_excel(self.filepath, index=False)
            return True
        except Exception:
            LOGGER.error(
                "* error during the writing process in an excel file *"
            )
            LOGGER.error(traceback.format_exc())
            return False

    def __record_data_to_file(self, info):
        """
        Only two options so far: CSV or EXCEL
        """
        if self.filepath.suffix == ".csv":
            return self.__record_data_to_csv_file(info)
        elif self.filepath.suffix == ".xls" or self.filepath.suffix == ".xlsx":
            return self.__record_data_to_excel_file(info)
        LOGGER.info("unknown format: it should be either .csv, .xls or .xlsx")
        return self.__record_data_to_excel_file(info)

    def __log_records(
        self,
        cpu_recorded_power,
        gpu_recorded_power,
        algorithm="",
        package="",
        data_type="",
        data_shape="",
        algorithm_params="",
        comments="",
        step="other",
    ):
        co2_emitted = self.__aggregate_power(
            self.power_gadget.record, self.gpu_power.record
        )
        payload = {
            "Datetime": datetime.datetime.now().strftime(self.DATETIME_FORMAT),
            "Country": self.location_name,
            "Platform": self.platform,
            "User ID": self.user,
            "ISO": self.location,
            "Project name": self.project,
            "Program name": self.program_name,
            "Client name": self.client_name,
            "Total Elapsed CPU Time (sec)": cpu_recorded_power[TOTAL_CPU_TIME],
            "Total Elapsed GPU Time (sec)": gpu_recorded_power[TOTAL_GPU_TIME],
            "Cumulative Package Energy (mWh)": cpu_recorded_power[
                TOTAL_ENERGY_ALL
            ],
            "Cumulative IA Energy (mWh)": cpu_recorded_power[TOTAL_ENERGY_CPU],
            "Cumulative GPU Energy (mWh)": gpu_recorded_power[
                TOTAL_ENERGY_GPU
            ],
            "Cumulative DRAM Energy (mWh)": cpu_recorded_power[
                TOTAL_ENERGY_MEMORY
            ],
            "Cumulative process CPU Energy (mWh)": cpu_recorded_power[
                TOTAL_ENERGY_PROCESS_CPU
            ],
            "Cumulative process DRAM Energy (mWh)": cpu_recorded_power[
                TOTAL_ENERGY_PROCESS_MEMORY
            ],
            "PUE": self.pue,
            "CO2 emitted (gCO2e)": co2_emitted,
            "Package": package,
            "Algorithm": algorithm,
            "Algorithm's parameters": algorithm_params,
            "Data type": data_type,
            "Data shape": data_shape,
            "Comment": comments,
            "Step": step,
        }
        written = self.__record_data_to_file(payload)
        LOGGER.info("* recorded into a file? %s*", written)

        if self.is_online and self.api_endpoint:
            response_status_code = self.__record_data_to_server(payload)
            if response_status_code >= 400:
                LOGGER.warning(
                    "We couldn't upload the recorded data to the server, we"
                    "are going to record it for a later upload"
                )
                # can't upload we'll record the data
                data = pd.DataFrame(payload, index=[0])
                if self.logging_filename.exists():
                    data.to_csv(
                        self.logging_filename,
                        mode="a",
                        index=False,
                        header=False,
                    )
                else:
                    data.to_csv(self.logging_filename, index=False)

            if response_status_code == 200:
                # we successfully uploaded,
                # check if there are other locally recorded data
                if self.logging_filename.exists():
                    data = pd.read_csv(self.logging_filename, index_col=None)
                    payloads = data.to_dict(orient="records")
                    for i, payload in enumerate(payloads):
                        res_status_code = self.__record_data_to_server(payload)
                        if res_status_code != 200:
                            data.iloc[i:].to_csv(
                                self.logging_filename, index=False
                            )
                            break
                    else:
                        self.logging_filename.unlink()


from qiskit import (
    QuantumCircuit,
    execute,
    Aer,
    converters,
    IBMQ)
from qiskit.transpiler import CouplingMap, Layout

provider = IBMQ.load_account()

#simulated_backend = provider.get_backend('ibmq_qasm_simulator')
simulated_backend = provider.get_backend('ibmqx5')
coupling_map_list = simulated_backend.configuration().coupling_map  # Get coupling map from backend
coupling_map = CouplingMap(coupling_map_list)
print(coupling_map_list)


from pyglet.window import key
from .constants import *

keymap = {
    ord('A'): key.A,
    ord('B'): key.B,
    ord('C'): key.C,
    ord('D'): key.D,
    ord('E'): key.E,
    ord('F'): key.F,
    ord('G'): key.G,
    ord('H'): key.H,
    ord('I'): key.I,
    ord('J'): key.J,
    ord('K'): key.K,
    ord('L'): key.L,
    ord('M'): key.M,
    ord('N'): key.N,
    ord('O'): key.O,
    ord('P'): key.P,
    ord('Q'): key.Q,
    ord('R'): key.R,
    ord('S'): key.S,
    ord('T'): key.T,
    ord('U'): key.U,
    ord('V'): key.V,
    ord('W'): key.W,
    ord('X'): key.X,
    ord('Y'): key.Y,
    ord('Z'): key.Z,
    ord('0'): key._0,
    ord('1'): key._1,
    ord('2'): key._2,
    ord('3'): key._3,
    ord('4'): key._4,
    ord('5'): key._5,
    ord('6'): key._6,
    ord('7'): key._7,
    ord('8'): key._8,
    ord('9'): key._9,
    ord('\b'): key.BACKSPACE,

    # By experiment:
    0x14: key.CAPSLOCK,
    0x5d: key.MENU,

    #    VK_LBUTTON: ,
    #    VK_RBUTTON: ,
    VK_CANCEL: key.CANCEL,
    #    VK_MBUTTON: ,
    #    VK_BACK: ,
    VK_TAB: key.TAB,
    #    VK_CLEAR: ,
    VK_RETURN: key.RETURN,
    VK_SHIFT: key.LSHIFT,
    VK_CONTROL: key.LCTRL,
    VK_MENU: key.LALT,
    VK_PAUSE: key.PAUSE,
    #    VK_CAPITAL: ,
    #    VK_KANA: ,
    #    VK_HANGEUL: ,
    #    VK_HANGUL: ,
    #    VK_JUNJA: ,
    #    VK_FINAL: ,
    #    VK_HANJA: ,
    #    VK_KANJI: ,
    VK_ESCAPE: key.ESCAPE,
    #    VK_CONVERT: ,
    #    VK_NONCONVERT: ,
    #    VK_ACCEPT: ,
    #    VK_MODECHANGE: ,
    VK_SPACE: key.SPACE,
    VK_PRIOR: key.PAGEUP,
    VK_NEXT: key.PAGEDOWN,
    VK_END: key.END,
    VK_HOME: key.HOME,
    VK_LEFT: key.LEFT,
    VK_UP: key.UP,
    VK_RIGHT: key.RIGHT,
    VK_DOWN: key.DOWN,
    #    VK_SELECT: ,
    VK_PRINT: key.PRINT,
    #    VK_EXECUTE: ,
    #    VK_SNAPSHOT: ,
    VK_INSERT: key.INSERT,
    VK_DELETE: key.DELETE,
    VK_HELP: key.HELP,
    VK_LWIN: key.LWINDOWS,
    VK_RWIN: key.RWINDOWS,
    #    VK_APPS: ,
    VK_NUMPAD0: key.NUM_0,
    VK_NUMPAD1: key.NUM_1,
    VK_NUMPAD2: key.NUM_2,
    VK_NUMPAD3: key.NUM_3,
    VK_NUMPAD4: key.NUM_4,
    VK_NUMPAD5: key.NUM_5,
    VK_NUMPAD6: key.NUM_6,
    VK_NUMPAD7: key.NUM_7,
    VK_NUMPAD8: key.NUM_8,
    VK_NUMPAD9: key.NUM_9,
    VK_MULTIPLY: key.NUM_MULTIPLY,
    VK_ADD: key.NUM_ADD,
    #    VK_SEPARATOR: ,
    VK_SUBTRACT: key.NUM_SUBTRACT,
    VK_DECIMAL: key.NUM_DECIMAL,
    VK_DIVIDE: key.NUM_DIVIDE,
    VK_F1: key.F1,
    VK_F2: key.F2,
    VK_F3: key.F3,
    VK_F4: key.F4,
    VK_F5: key.F5,
    VK_F6: key.F6,
    VK_F7: key.F7,
    VK_F8: key.F8,
    VK_F9: key.F9,
    VK_F10: key.F10,
    VK_F11: key.F11,
    VK_F12: key.F12,
    VK_F13: key.F13,
    VK_F14: key.F14,
    VK_F15: key.F15,
    VK_F16: key.F16,
    #    VK_F17: ,
    #    VK_F18: ,
    #    VK_F19: ,
    #    VK_F20: ,
    #    VK_F21: ,
    #    VK_F22: ,
    #    VK_F23: ,
    #    VK_F24: ,
    VK_NUMLOCK: key.NUMLOCK,
    VK_SCROLL: key.SCROLLLOCK,
    VK_LSHIFT: key.LSHIFT,
    VK_RSHIFT: key.RSHIFT,
    VK_LCONTROL: key.LCTRL,
    VK_RCONTROL: key.RCTRL,
    VK_LMENU: key.LALT,
    VK_RMENU: key.RALT,
    #    VK_PROCESSKEY: ,
    #    VK_ATTN: ,
    #    VK_CRSEL: ,
    #    VK_EXSEL: ,
    #    VK_EREOF: ,
    #    VK_PLAY: ,
    #    VK_ZOOM: ,
    #    VK_NONAME: ,
    #    VK_PA1: ,
    #    VK_OEM_CLEAR: ,
    #    VK_XBUTTON1: ,
    #    VK_XBUTTON2: ,
    #    VK_VOLUME_MUTE: ,
    #    VK_VOLUME_DOWN: ,
    #    VK_VOLUME_UP: ,
    #    VK_MEDIA_NEXT_TRACK: ,
    #    VK_MEDIA_PREV_TRACK: ,
    #    VK_MEDIA_PLAY_PAUSE: ,
    #    VK_BROWSER_BACK: ,
    #    VK_BROWSER_FORWARD: ,
}

# Keys that must be translated via MapVirtualKey, as the virtual key code
# is language and keyboard dependent.
chmap = {
    ord('!'): key.EXCLAMATION,
    ord('"'): key.DOUBLEQUOTE,
    ord('#'): key.HASH,
    ord('$'): key.DOLLAR,
    ord('%'): key.PERCENT,
    ord('&'): key.AMPERSAND,
    ord("'"): key.APOSTROPHE,
    ord('('): key.PARENLEFT,
    ord(')'): key.PARENRIGHT,
    ord('*'): key.ASTERISK,
    ord('+'): key.PLUS,
    ord(','): key.COMMA,
    ord('-'): key.MINUS,
    ord('.'): key.PERIOD,
    ord('/'): key.SLASH,
    ord(':'): key.COLON,
    ord(';'): key.SEMICOLON,
    ord('<'): key.LESS,
    ord('='): key.EQUAL,
    ord('>'): key.GREATER,
    ord('?'): key.QUESTION,
    ord('@'): key.AT,
    ord('['): key.BRACKETLEFT,
    ord('\\'): key.BACKSLASH,
    ord(']'): key.BRACKETRIGHT,
    ord('\x5e'): key.ASCIICIRCUM,
    ord('_'): key.UNDERSCORE,
    ord('\x60'): key.GRAVE,
    ord('`'): key.QUOTELEFT,
    ord('{'): key.BRACELEFT,
    ord('|'): key.BAR,
    ord('}'): key.BRACERIGHT,
    ord('~'): key.ASCIITILDE,
}


# 去除html标签

import re
from typing import Text

# 在此实现代码
def remove_html(content):
    pattern = re.compile(r'<[^>]+>', re.S)
    result = pattern.sub('', content)
    return result

if __name__ == '__main__':
    html = '''

                <title>这是一个简单的测试页面</title>

body 元素的内容会显示在浏览器中。

title 元素的内容会显示在浏览器的标题栏中。

        '''
    Text = remove_html(html)
    print(Text)


import boto3
from botocore.exceptions import ClientError
import os

from app import SUCCESS, S3_ERROR

from dotenv import load_dotenv
load_dotenv()

bucket = os.environ.get('S3_BUCKET')
region = os.environ.get('AWS_REGION')
access_key_id = os.environ.get('AWS_ACCESS_KEY_ID')
secret_access_key = os.environ.get('AWS_SECRET_ACCESS_KEY')

s3_client = boto3.client(service_name='s3',
                         region_name=region,
                         aws_access_key_id=access_key_id,
                         aws_secret_access_key=secret_access_key)

def upload_file(file_name: str, object_name: str = None) -> int:
    if object_name is None:
        object_name = os.path.basename(file_name)
    try:
        s3_client.upload_file(file_name, bucket, object_name)
    except ClientError:
        return S3_ERROR
    return SUCCESS

def list_backups() -> int | list:
    try:
        s3_resource = boto3.resource(service_name='s3',
                                     region_name=region,
                                     aws_access_key_id=access_key_id,
                                     aws_secret_access_key=secret_access_key)
        s3_bucket = s3_resource.Bucket(bucket)
        backups = s3_bucket.objects.all()
        backups = sorted(
            backups, key=lambda backup: backup.last_modified, reverse=True)
        return [backup.key for backup in backups]
    except ClientError:
        return S3_ERROR

def restore_backup(object_name: str, file_name: str = None) -> int:
    if file_name is None:
        file_name = os.path.basename(object_name)
    try:
        s3_client.download_file(bucket, object_name, file_name)
        return SUCCESS
    except ClientError:
        return S3_ERROR


from web3.contract import Contract
from web3.logs import DISCARD, IGNORE
from web3.datastructures import AttributeDict
from web3._utils.events import EventLogErrorFlags

import logging

swapDict = {
    "KLAYTN": {
        "Klayswap": {
            "event": [-1, "ExchangePos"],
            "amount0_in": ["amountA"],
            "amount1_out": ["amountB"],
        },
    },
    "MATIC": {
        "Meshswap": {
            "event": [-1, "ExchangePos"],
            "amount0_in": ["amount0"],
            "amount1_out": ["amount1"],
        },
    },
}

bridgeDict = {
    "Orbitbridge": {
        "Vault": {"event": [-1, "Deposit"], "amount0_in": ["amount"]},
        "Minter": {"event": [-1, "SwapRequest"], "amount0_in": ["amount"]},
    },
}

transferDict = {
    "Orbitbridge": {
        "event": [0, "Transfer"],
        "amount0_in": ["value"],
    },
}

add_liquidityDict = {
    "Traderjoe": {
        "event": [0, "Transfer"],
        "amount0_in": ["value"],
    },
}

txDict = {
    "transaction_hash": None,
    "status": None,
    "block": None,
    "timestamp": None,
    "function": None,
    "from": None,
    "amountIn": None,
    "tokenA": None,
    "to": None,
    "amountOut": None,
    "tokenB": None,
    "transaction_fee:": None,
}

class Event:
    def __init__(self, swapDict, bridgeDict, txDict):
        """
        Initialize the Event class with the swap, bridge, and tx dictionaries.

        Parameters:
        ----------------
        - swapDict (dict): dictionary of swap events
        - bridgeDict (dict): dictionary of bridge events
        - txDict (dict): dictionary of tx events
        """

        self.swapDict = swapDict
        self.bridgeDict = bridgeDict
        self.txDict = txDict

    # @classmethod
    async def swap(
        chainsName: str,
        marketName: str,
        tx_receipt: AttributeDict,
        routerContract: Contract,
    ):
        """
        Extract swap event data from a transaction receipt.

        Parameters:
        --------------
        - chainsName (str): name of the chain
        - marketName (str): name of the market
        - tx_receipt (AttributeDict): transaction receipt
        - routerContract (Contract): contract instance

        Returns:
        --------------
        - tuple: tuple containing the amount of token A in and token B out

        """

        if chainsName not in swapDict:

            events_param_list = [0, "Transfer"]
            amount_in_params = ["value"]
            amount_out_params = ["value"]

        else:

            if marketName not in swapDict[chainsName]:

                events_param_list = [0, "Transfer"]
                amount_in_params = ["value"]
                amount_out_params = ["value"]

            else:

                events_param_list = swapDict[chainsName][marketName]["event"]
                amount_in_params = swapDict[chainsName][marketName]["amount0_in"]
                amount_out_params = swapDict[chainsName][marketName]["amount1_out"]

        swap = routerContract.events[events_param_list[1]]()

        events = swap.process_receipt(tx_receipt, errors=DISCARD)

        amount0_in = max(
            [events[events_param_list[0]]["args"][param] for param in amount_in_params]
        )

        amount1_out = max([events[-1]["args"][param] for param in amount_out_params])

        return amount0_in, amount1_out

    # @classmethod
    async def bridge(
        bridgeName: str, tx_receipt: AttributeDict, routerContract: Contract, *args, **kwargs
    ):
        """
        Extract bridge event data from a transaction receipt.

        Parameters:
        - bridgeName (str): name of the bridge
        - versionName (str): name of the version
        - tx_receipt (AttributeDict): transaction receipt
        - routerContract (Contract): contract instance

        Returns:
        - int: amount of token in

        """

        if kwargs["version"]:

            bridge = routerContract.events[bridgeDict[bridgeName][kwargs["version"]]["event"][1]]()
            events = bridge.process_receipt(tx_receipt, errors=DISCARD)
            amount_in_params = bridgeDict[bridgeName][kwargs["version"]]["amount0_in"]

            amount0_in = max(
                [
                    events[bridgeDict[bridgeName][kwargs["version"]]["event"][0]]["args"][param]
                    for param in amount_in_params
                ]
            )

        else:

            bridge = routerContract.events[bridgeDict[bridgeName]["event"][1]]()
            events = bridge.process_receipt(tx_receipt, errors=DISCARD)
            amount_in_params = bridgeDict[bridgeName]["amount0_in"]
            amount0_in = max(
                [
                    events[bridgeDict[bridgeName]["event"][0]]["args"][param]
                    for param in amount_in_params
                ]
            )

        return amount0_in

    async def transfer(
        bridgeName: str,
        tx_receipt: AttributeDict,
        routerContract: Contract,
    ):
        """
        Extract bridge event data from a transaction receipt.

        Parameters:
        - bridgeName (str): name of the bridge
        - versionName (str): name of the version
        - tx_receipt (AttributeDict): transaction receipt
        - routerContract (Contract): contract instance

        Returns:
        - int: amount of token in

        """

        bridge = routerContract.events[transferDict[bridgeName]["event"][1]]()

        events = bridge.process_receipt(tx_receipt, errors=DISCARD)

        amount_in_params = transferDict[bridgeName]["amount0_in"]

        amount0_in = max(
            [
                events[transferDict[bridgeName]["event"][0]]["args"][param]
                for param in amount_in_params
            ]
        )

        return amount0_in


from rest_framework import serializers
from .models import Follow, UserSetting, UserFeedPosition
from utils.serializers import UnixTimestampModelSerializer
from common.serializers import CustomUserSerializer
from django.contrib.auth import password_validation
from django.contrib.auth.hashers import make_password
from rest_framework import serializers
from authentication.models import CustomUser

class NotificationSettingsDataSerializer(serializers.Serializer):
    account = serializers.BooleanField(required=False)
    marketing = serializers.BooleanField(required=False)
    updates = serializers.BooleanField(required=False)

class SettingNotificationSerializer(serializers.Serializer):
    email = NotificationSettingsDataSerializer(required=False)

class AccountSettingsDataSerializer(serializers.ModelSerializer):
    class Meta:
        model = CustomUser
        fields = ["display_name", "username", "email"]

    def validate_email(self, value):
        if CustomUser.objects.exclude(pk=self.instance.pk).filter(email=value).exists():
            raise serializers.ValidationError("This email is already in use.")
        return value

    def validate_username(self, value):
        if (
            CustomUser.objects.exclude(pk=self.instance.pk)
            .filter(username=value)
            .exists()
        ):
            raise serializers.ValidationError("This username is already in use.")
        return value

class SystemSettingsDataSerializer(serializers.Serializer):
    theme = serializers.CharField(required=False)
    language = serializers.CharField(required=False)

class PersonalSettingsDataSerializer(serializers.Serializer):
    favorite_categories = serializers.ListField(
        child=serializers.CharField(), required=False
    )

class UserSettingSerializer(UnixTimestampModelSerializer):
    user_id = serializers.UUIDField(read_only=True)
    system_settings = SystemSettingsDataSerializer()
    account_settings = AccountSettingsDataSerializer(source="user")
    notification_settings = SettingNotificationSerializer()
    personal_settings = PersonalSettingsDataSerializer()

    class Meta:
        model = UserSetting
        fields = "__all__"

    def validate_settings_data(data):
        serializer = UserSettingSerializer(data=data)
        if serializer.is_valid(raise_exception=True):
            return serializer.validated_data
        return None

    def update(self, instance, validated_data):
        # Extract nested settings data
        personal_settings_data = validated_data.get("personal_settings", {})

        print("Current personal_settings:", instance.personal_settings)
        print("Incoming personal_settings_data:", personal_settings_data)

        # Merge old and new data for personal settings
        instance.personal_settings = {
            **instance.personal_settings,
            **personal_settings_data,
        }

        print("Merged personal_settings:", instance.personal_settings)

        # Extract nested settings data
        # personal_settings_data = validated_data.get('personal_settings', {})
        system_settings_data = validated_data.get("system_settings", {})
        account_settings_data = validated_data.get("account_settings", {})
        notification_settings_data = validated_data.get("notification_settings", {})

        print("Current notification_settings:", instance.notification_settings)
        print("Incoming notification_settings_data:", notification_settings_data)

        # Merge old and new data for each setting
        # instance.personal_settings = {**instance.personal_settings, **personal_settings_data}
        instance.system_settings = {**instance.system_settings, **system_settings_data}
        instance.account_settings = {
            **instance.account_settings,
            **account_settings_data,
        }

        # Special handling for nested dictionary in notification settings
        email_settings_data = notification_settings_data.get("email", {})
        if "email" in instance.notification_settings:
            instance.notification_settings["email"] = {
                **instance.notification_settings["email"],
                **email_settings_data,
            }
        else:
            instance.notification_settings["email"] = email_settings_data

        print("Merged notification_settings:", instance.notification_settings)

        instance.save()

        # Update CustomUser fields
        if account_settings_data:
            user = instance.user
            for attr, value in account_settings_data.items():
                if hasattr(user, attr):
                    setattr(user, attr, value)
            user.save()

        return instance

class FollowSerializer(serializers.ModelSerializer):
    follower = CustomUserSerializer(read_only=True)
    followed = CustomUserSerializer(read_only=True)

    class Meta:
        model = Follow
        fields = ("id", "follower", "followed")

class UserFeedPositionSerializer(serializers.ModelSerializer):
    class Meta:
        model = UserFeedPosition
        fields = ["last_story_read"]

# users/serializers.py


import matplotlib.pyplot as plt
from bs4 import BeautifulSoup
import requests
import re
from collections import defaultdict, Counter
import random

##################
#                #
#   WORD CLOUD   #
#                #
##################

def plot_resumes(data):
    def text_size(total):
        """equals 8 if total is 0, 28 if total is 200"""
        return 8 + total / 200 * 20

    for word, job_popularity, resume_popularity in data:
        plt.text(job_popularity, resume_popularity, word, ha='center', va='center',
                 size=text_size(job_popularity + resume_popularity))

    plt.xlabel("Popularity on Job Postings")
    plt.ylabel("Popularity on Resumes")
    plt.axis([0, 100, 0, 100])
    plt.xticks([])
    plt.yticks([])
    plt.show()

###################
#                 #
#     N-GRAMS     #
#                 #
###################

def fix_unicode(text):
    return text.replace(u"\u2019", "'")

def get_document():
    url = "http://radar.oreilly.com/2010/06/what-is-data-science.html"
    html = requests.get(url).text
    soup = BeautifulSoup(html, 'html5lib')

    content_ = soup.find("div", "article-body")
    regex = r"[\w']+|[\.]"

    document = []

    for paragraph in content_("p"):
        words = re.findall(regex, fix_unicode(paragraph.text))
        document.extend(words)

    return document

def get_bigrams_transitions(document):
    bigrams = zip(document, document[1:])
    transitions = defaultdict(list)
    for prev, current in bigrams:
        transitions[prev].append(current)
    return transitions

def generate_using_bigrams(transitions):
    current = "."           # this means the next word will start a sequence
    result = []
    while True:
        next_word_candidates = transitions[current]         # bigrams (current, )
        current = random.choice(next_word_candidates)       # choose one at random
        result.append(current)                              # append it to the results
        if current == ".": return " ".join(result)          # if "." we're done

def get_trigrams_transitions(document):
    trigrams = zip(document, document[1:], document[2:])
    trigrams_transitions = defaultdict(list)
    starts = []
    for prev, current, next in trigrams:
        if prev == ".":
            starts.append(current)
        trigrams_transitions[(prev, current)].append(next)
    return starts, trigrams_transitions

def generate_using_trigrams(starts, trigram_transitions):
    current = random.choice(starts)    # choose a random starting word
    prev = "."                         # and precede it with a '.'
    result = [current]
    while True:

        next_word_candidates = trigram_transitions[(prev, current)]
        next = random.choice(next_word_candidates)
        prev, current = current, next
        result.append(current)
        if current == ".":
            return " ".join(result)

###################
#                 #
#     GRAMMAR     #
#                 #
###################

def is_terminal(token):
    return token[0] != "_"

def expand(grammar, tokens):
    for i, token in enumerate(tokens):

        # skip over terminals
        if is_terminal(token): continue

        # if we get here, we found a non-terminal token
        # so we need to choose a replacement at random
        replacement = random.choice(grammar[token])

        if is_terminal(replacement):
            tokens[i] = replacement
        else:
            tokens = tokens[:i] + replacement.split() + tokens[(i+1):]

        # now call expand on the new list of tokens
        return expand(grammar, tokens)

    # if we get here we had all terminals and are done
    return " ".join(tokens)

def generate_sentence(grammar):
    return expand(grammar, ["_S"])

######################
#                    #
#   GIBBS SAMPLING   #
#                    #
######################

def roll_a_die():
    return random.choice([1, 2, 3, 4, 5, 6])

def direct_sample():
    d1 = roll_a_die()
    d2 = roll_a_die()
    return d1, d1 + d2

def random_y_given_x(x):
    """equally likely to be a x + 1, x + 2, x + 3, ... , x + 6"""
    return x + roll_a_die()

def random_x_given_y(y):
    if y <= 7:
        # if the total is 7 or less, the first die is equally likely to be
        # 1, 2, ..., (total - 1)
        return random.randrange(1, y)
    else:
        # if the total is 7 or more, the first die is equally likely to be
        # (total - 6), (total - 5), ..., 6
        return random.randrange(y - 6, 7)

def gibbs_sample(num_iters=100):
    x, y = 1, 2         # doesn't really matter
    for _ in range(num_iters):
        x = random_x_given_y(y)
        y = random_y_given_x(x)
    return x, y

def compare_samples(num_iters=100):
    counts = defaultdict(lambda: [0, 0])
    for _ in range(num_iters):
        counts[gibbs_sample()][0] += 1
        counts[direct_sample()][1] += 1
    return counts

######################
#                    #
#   TOPIC MODELING   #
#                    #
######################

def sample_from(weights):
    """ returns i with the probability weights[i] / sum(weights) """
    total = sum(weights)
    rnd = total * random.random()           # uniform between 0 and total
    for i, w in enumerate(weights):
        rnd -= w                            # return the smallest i such that
        if rnd <= 0: return i               # weights[0] + ... + weights[i] >= rnd

def p_topic_given_document(topic, d, alpha=0.1):
    """the fraction of words in document _d_
    that are assigned to _topic_ (plus some smoothing)"""

    return ((document_topic_counts[d][topic] + alpha) / (document_lengths[d] + K * alpha))

def p_word_given_topic(word, topic, beta=0.1):
    """the fraction of words assigned to _topic_
    that equal _word_ (plus some smoothing)"""
    return ((topic_word_counts[topic][word] + beta) / (topic_counts[topic] + W * beta))

def topic_weight(d, word, k):
    """given a document and a word in that document,
    return the weight for the kth topic"""
    return p_word_given_topic(word, k) * p_topic_given_document(k, d)

def choose_new_topic(d, word):
    return sample_from([topic_weight(d, word, k) for k in range(K)])

if __name__ == '__main__':

    data = [ ("big data", 100, 15), ("Hadoop", 95, 25), ("Python", 75, 50),
             ("R", 50, 40), ("machine learning", 80, 20), ("statistics", 20, 60),
             ("data science", 60, 70), ("analytics", 90, 3),
             ("team player", 85, 85), ("dynamic", 2, 90), ("synergies", 70, 0),
             ("actionable insights", 40, 30), ("think out of the box", 45, 10),
             ("self-starter", 30, 50), ("customer focus", 65, 15),
             ("thought leadership", 35, 35)]

    run_wordcloud = 0
    run_bigram = 0
    run_trigram = 0
    run_grammar = 0
    run_topic_modeling = 1

    if run_wordcloud == 1:
        plot_resumes(data)

    if run_bigram == 1:
        document_list = get_document()
        bigram_transitions = get_bigrams_transitions(document_list)
        generated_document_bigram = generate_using_bigrams(bigram_transitions)
        print(generated_document_bigram)

    if run_trigram == 1:
        document_list = get_document()
        starts, trigram_transitions = get_trigrams_transitions(document_list)
        generated_document_trigram = generate_using_trigrams(starts, trigram_transitions)
        print(generated_document_trigram)

    if run_grammar == 1:
        grammar = {
            "_S"  : ["_NP _VP"],
            "_NP" : ["_N",
                     "_A _NP _P _A _N"],
            "_VP" : ["_V _NP",
                     "_V _NP"],
            "_N"  : ["data science", "Python", "regression", "NLP", "matrix", "machine learning", "clustering", "Hadoop", "Tensorflow"],
            "_A"  : ["big", "linear", "logistic", "dense", "sparse", "accurate", "advanced"],
            "_P"  : ["about", "near", "with", "before"],
            "_V"  : ["learns", "trains", "tests", "is", "grows", "sets", "expands", "validate", "results"]
        }

        sentence = generate_sentence(grammar)
        print(sentence)

    if run_topic_modeling == 1:
        documents = [
            ["Hadoop", "Big Data", "HBase", "Java", "Spark", "Storm", "Cassandra"],
            ["NoSQL", "MongoDB", "Cassandra", "HBase", "Postgres"],
            ["Python", "scikit-learn", "scipy", "numpy", "statsmodels", "pandas"],
            ["R", "Python", "statistics", "regression", "probability"],
            ["machine learning", "regression", "decision trees", "libsvm"],
            ["Python", "R", "Java", "C++", "Haskell", "programming languages"],
            ["statistics", "probability", "mathematics", "theory"],
            ["machine learning", "scikit-learn", "Mahout", "neural networks"],
            ["neural networks", "deep learning", "Big Data", "artificial intelligence"],
            ["Hadoop", "Java", "MapReduce", "Big Data"],
            ["statistics", "R", "statsmodels"],
            ["C++", "deep learning", "artificial intelligence", "probability"],
            ["pandas", "R", "Python"],
            ["databases", "HBase", "Postgres", "MySQL", "MongoDB"],
            ["libsvm", "regression", "support vector machines"]
        ]

        K = 4

        # a list of Counters, one for each document // how many times each topic is assigned to each document
        document_topic_counts = [Counter() for _ in documents]

        # a list of Counters, one for each topic // how many times each word is assigned to each topic
        topic_word_counts = [Counter() for _ in range(K)]

        # a list of numbers, one for each topic // the total number of words assigned to each topic
        topic_counts = [0 for _ in range(K)]

        # a list of numbers, one for each document // the total number of words contained in each document
        document_lengths = [len(d) for d in documents]

        # the number of distinct words
        distinct_words = set(word for document in documents for word in document)
        W = len(distinct_words)

        # the number of documents
        D = len(documents)

        random.seed(0)
        document_topics = [[random.randrange(K) for word in document] for document in documents]

        for d in range(D):
            for word, topic in zip(documents[d], document_topics[d]):
                document_topic_counts[d][topic] += 1
                topic_word_counts[topic][word] += 1
                topic_counts[topic] += 1

        for iter in range(1000):
            for d in range(D):
                for i, (tord, topic) in enumerate(zip(documents[d], document_topics[d])):

                    # remove this word / topic from the counts
                    # so that it doesn't influence the weights
                    document_topic_counts[d][topic] -= 1
                    topic_word_counts[topic][word] -= 1
                    topic_counts[topic] -= 1
                    document_lengths[d] -= 1

                    # choose a new topic based on the weights
                    new_topic = choose_new_topic(d, word)
                    document_topics[d][i] = new_topic

                    # and now add it back to the counts
                    document_topic_counts[d][new_topic] += 1
                    topic_word_counts[new_topic][word] += 1
                    topic_counts[new_topic] += 1
                    document_lengths[d] += 1

        for k, word_counts in enumerate(topic_word_counts):
            for word, count in word_counts.most_common():
                if count > 0: print(k, word, count)
            print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")

        topic_names = ["Big Data and programming languages",
                       "databases",
                       "machine learning",
                       "statistics"]

        for document, topic_counts in zip(documents, document_topic_counts):
            current_topic_counts = topic_counts.most_common()
            for topic, count in current_topic_counts:
                if count > 0:
                    print(topic_names[topic], count)
            print()


"""Add description column to class materials.

Revision ID: 37e6d71d08ba
Revises: d9faf0b21d6e
Create Date: 2021-07-18 12:41:42.842215

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '37e6d71d08ba'
down_revision = 'd9faf0b21d6e'
branch_labels = None
depends_on = None

def upgrade():
    op.add_column('class_materials', sa.Column('description', sa.Text, nullable=False, server_default=""))

def downgrade():
    op.drop_column('class_materials', 'description')


'''
Created with love by Sigmoid
@Author - Clefos Alexandru - clefos.alexandru@isa.utm.md
'''

# Importing all libraries
import pandas as pd
import numpy as np
from sklearn.base import BaseEstimator, TransformerMixin

class FrequencyImputationTransformer(BaseEstimator, TransformerMixin):

    def __init__(self, categorical_index='auto', min_int_freq=5):
        '''
            Setting up the algorithm
        :param categorical_index: list, default = 'auto'
            A parameter that specifies the list of indexes of categorical columns that should be transformed.
        :param min_int_freq: int, default = 5
            A parameter that indicates the number of minimal values in a categorical column for the transformer
            to be applied.
        '''
        self.__categorical_index = categorical_index
        self.__min_int_freq = min_int_freq

    def fit(self, X, y=None, **fit_params):
        '''
            Fit function
        :param X: 2-d numpy array
            A parameter that stores the data set without the target vector.
        :param y: 1-d numpy array, default = None
            A parameter that stores the target vector.
        :param fit_params: dict
            Additional fit parameters.
        :return: FrequencyImputationTransformer
            The fitted transformer.
        '''
        # Creating a dictionary with all mappers.
        self.__mappers = dict()

        # Defining the list with categorical data types.
        categorical = [str]

        # Checking the value of __categorical_index.
        if self.__categorical_index == 'auto':

            # If __categorical_index is set as default, transformer finds all the categorical columns
            # for which mapper should be created.
            for i in range(len(X[0])):
                if (type(X[0, i]) in categorical or type(X[0, i]) == int) and len(set(X[:, i])) < self.__min_int_freq and len(set(X[:, i])) != 2:
                    self.__mappers[i] = dict()

                    # Mapping out all the categorical values in the selected column.
                    for element in np.unique(X[:, i]):
                        self.__mappers[i][element] = len(X[:, i][np.where(X[:, i] == element)]) / len(X)
        else:

            # If __categorical_index is set by the user, transformer iterates through the passed list of indexes.
            for i in self.__categorical_index:
                if i >= len(X[0]):
                    raise ValueError('Passed index list contains a invalid index!')
                self.__mappers[i] = dict()

                # Mapping out all the categorical values in the selected column.
                for element in np.unique(X[:, i]):
                    self.__mappers[i][element] = len(X[:, i][np.where(X[:, i] == element)]) / len(X)

        # Returning the fitted instance of FrequencyImputationTransformer class.
        return self

    def transform(self, X, **fit_transform):
        '''
            Function that transforms the new given data
        :param X: 2-d numpy array
            A parameter that stores the data set without the target vector.
        :param fit_transform: dict
            Additional fit parameters.
        :return: 2-d numpy array
            The transformed 2-d numpy array.
        '''
        for key in self.__mappers:

            # If a new value is passed than it is replaced with 0.
            X[:, key] = [self.__mappers[key][value] if value in self.__mappers[key] else 0 for value in X[:, key]]
        return X

    def fit_transform(self, X, y=None, **fit_params):
        '''
            Function that fits and transform the data
        :param X: 2-d numpy array
            A parameter that stores the data set without the target vector.
        :param y: 1-d numpy array
            A parameter that stores the target vector.
        :param fit_params: dict
            Additional fit parameters.
        :return: 2-d numpy array
            The transformed 2-d numpy array.
        '''
        return self.fit(X, y).transform(X)

    def apply(self, df, target):
        '''
            Function that apply the transformer on the pandas data frame.
        :param df: pandas data frame
            The pandas data frame on which the transformer should be applied.
        :param target: str
            The column name of the value that we want to predict.
        :return: pandas data frame
            The pandas data frame on which the transformer was applied.
        '''
        # Separating the feature matrix from the target vector and transforming them in numpy arrays.
        feature_columns = [column for column in df.columns if column != target]
        X = df[feature_columns].values
        y = df[target].values

        # Transforming the passed data.
        new_x = self.fit_transform(X, y)

        # Creating a new data frame with the processed categorical columns.
        new_df = pd.DataFrame(new_x, columns=feature_columns)
        new_df[target] = y

        # Returning the new data frame.
        return new_df

df = pd.read_csv('stroke.csv')
df = df.fillna(0)
print(df.dtypes)

X, y = df.drop(['stroke', 'id'], axis=1).values, df['stroke'].values

tgi = FrequencyImputationTransformer()
print()
print(X)
print(tgi.fit_transform(X, y))
print(tgi._FrequencyImputationTransformer__mappers)
print(X[:, 1].dtype.char)
print(tgi.apply(df, 'stroke'))


from pymongo import MongoClient
import pickle

client = MongoClient(host='localhost', port=27017)
db = client.coordinates
collection = db.predict

x_list = []
y_dic = {1:'C', 2:'O', 3:'L', 4:'U', 5:'M', 6:'B', 7:'I', 8:'A'}
latest = collection.find().sort([('_id', -1)]).limit(20)
for i in range(19,-1,-1):
	# x_list.extend([item['x1'], item['x2'], item['y1'], item['y2'], item['z1'], item['z2']])
	x_list.extend([latest[i]['x1']/255, latest[i]['x2']/255, latest[i]['y1']/255, latest[i]['y2']/255, latest[i]['z1']/255, latest[i]['z2']/255])

# load model and do predict
filename = 'finalized_model_6axis.sav'
loaded_model = pickle.load(open(filename, 'rb'))
result = loaded_model.predict([x_list])
print(y_dic[result[0]])


import sys
try:
    import pydot
except:
    pass
class NFA:
    def __init__(self, states, finals, transitions, start=0):
        self.states = states
        self.start = start
        self.finals = set(finals)
        self.map = transitions
        self.vocabulary = set()
        self.transitions = { state: {} for state in range(states) }

        for (origin, symbol), destinations in transitions.items():
            assert hasattr(destinations, '__iter__'), 'Invalid collection of states'
            self.transitions[origin][symbol] = destinations
            self.vocabulary.add(symbol)

        self.vocabulary.discard('')

    def epsilon_transitions(self, state):
        assert state in self.transitions, 'Invalid state'
        try:
            return self.transitions[state]['']
        except KeyError:
            return ()

    def graph(self):
        G = pydot.Dot(rankdir='LR', margin=0.1)
        G.add_node(pydot.Node('start', shape='plaintext', label='', width=0, height=0))

        for (start, tran), destinations in self.map.items():
            tran = 'ε' if tran == '' else tran
            G.add_node(pydot.Node(start, shape='circle', style='bold' if start in self.finals else ''))
            for end in destinations:
                G.add_node(pydot.Node(end, shape='circle', style='bold' if end in self.finals else ''))
                G.add_edge(pydot.Edge(start, end, label=tran, labeldistance=2))

        G.add_edge(pydot.Edge('start', self.start, label='', style='dashed'))
        return G

    def _repr_svg_(self):
        try:
            return self.graph().create_svg().decode('utf8')
        except:
            pass

class DFA(NFA):

    def __init__(self, states, finals, transitions, start=0):
        assert all(isinstance(value, int) for value in transitions.values())
        assert all(len(symbol) > 0 for origin, symbol in transitions)

        transitions = { key: [value] for key, value in transitions.items() }
        NFA.__init__(self, states, finals, transitions, start)
        self.current = start

    def _move(self, symbol):
        if symbol not in self.vocabulary:
            print(f"El simbolo: '{symbol}', no pertenece al vocabulario del automata.", file=sys.stderr)
            return False
        current_transitions = self.transitions[self.current]
        if symbol in current_transitions.keys():
            self.current = self.transitions[self.current][symbol][0]
            return True
        print(f"No hay transicion desde el estado '{self.current}' " + \
                f" para el simbolo: '{symbol}'")
        return False

    def _reset(self):
        self.current = self.start

    def recognize(self, string):
        self._reset()
        for i in string:
            if not self._move(i):
                return False
        if self.current in self.finals:
            return True
        return False

def move(automaton, states, symbol):
    '''
    Dado un conjunto de estados Q' y un simbolo se computan todos los posibles
    nodos a los que se puede llegar desde los estados de Q', utilizando al simbolo
    en cuestion como paso de transicion

    Sea Q' subconjunto de los estados Q:
    GOTO(Q', c) = {q_j de Q | q_i en Q', q_j este en t(q_i, c)}

    Parametros
    ------------
    `automaton`: Automata
    `states`: Conjunto de estados
    `symbol`: Simbolo de transicion

    Return
    -----------
    `new_states`: Conjunto de estados de `automaton` resultantes
    '''
    moves = set()
    for state in states:
        next_states = automaton.transitions[state].get(symbol, [])
        moves.update(next_states)
    return moves

def epsilon_closure(automaton, states):
    '''
    Funcion recursiva que calcula al epsilon-Clasura de un conjunto de estados
    en un automata. Este nuevo conjunto esta formado por todos los estados del
    conjunto actual, mas los estados a los que se puede llegar haciendo tantas
    transiciones con epsilon como sean posibles.

    Parametros
    ------------
    `automaton`: Automata
    `states`: Conjunto de estados

    Return
    -----------
    `new_states`: Conjunto de estados de `automaton` resultantes
    '''
    pending = list(states)
    closure = set(states)

    while pending:
        state = pending.pop()
        news_states = set(automaton.transitions[state].get('', []))
        difference = news_states.difference(closure)
        pending.extend(difference)
        closure.update(difference)

    return set(closure)

def nfa_to_dfa(automaton):
    '''
    Funcion para convertir un automata finito no determinista en un automata finito determinista.
    Usa las funciones `move` (Goto) y `epsilon_closure` (epsilonClausura).

    Parametros
    ------------
    `automaton`: Automata

    Return
    -----------
    `new_automaton`: Nuevo automata finito determinista
    '''
    class Container(set):
        def __init_subclass__(cls) -> None:
            return super().__init_subclass__()
        id = -1
        is_final = False

    transitions = {}

    start = Container(epsilon_closure(automaton, [automaton.start]))
    start.id = 0
    start.is_final = any(s in automaton.finals for s in start)
    states = [ start ]

    pending = [ start ]
    while pending:
        state = pending.pop()

        for symbol in automaton.vocabulary:
            _move = move(automaton= automaton, states= state, symbol= symbol)
            _closure = Container(epsilon_closure(automaton, _move))

            if len(_closure) == 0:
                continue
            if _closure not in states:
                _closure.id = len(states)
                _closure.is_final = any(s in automaton.finals for s in _closure)
                pending.append(_closure)
                states.append(_closure)
            else:
                _closure.id = states.index(_closure)

            try:
                transitions[state.id, symbol]
                assert False, 'Invalid DFA!!!'
            except KeyError:
                transitions[state.id, symbol] = _closure.id

    finals = [ state.id for state in states if state.is_final ]
    dfa = DFA(len(states), finals, transitions)
    return dfa

class State:
    def __init__(self, state, final=False, formatter=lambda x: str(x), shape='circle'):
        self.state = state
        self.final = final
        self.transitions = {}
        self.epsilon_transitions = set()
        self.tag = None
        self.formatter = formatter
        self.shape = shape

    # The method name is set this way from compatibility issues.
    def set_formatter(self, value, attr='formatter', visited=None):
        if visited is None:
            visited = set()
        elif self in visited:
            return

        visited.add(self)
        self.__setattr__(attr, value)
        for destinations in self.transitions.values():
            for node in destinations:
                node.set_formatter(value, attr, visited)
        for node in self.epsilon_transitions:
            node.set_formatter(value, attr, visited)
        return self

    def has_transition(self, symbol):
        return symbol in self.transitions

    def add_transition(self, symbol, state):
        try:
            self.transitions[symbol].append(state)
        except:
            self.transitions[symbol] = [state]
        return self

    def add_epsilon_transition(self, state):
        self.epsilon_transitions.add(state)
        return self

    def recognize(self, string):
        states = self.epsilon_closure
        for symbol in string:
            states = self.move_by_state(symbol, *states)
            states = self.epsilon_closure_by_state(*states)
        return any(s.final for s in states)

    def to_deterministic(self, formatter=lambda x: str(x)):
        closure = self.epsilon_closure
        start = State(tuple(closure), any(s.final for s in closure), formatter)

        closures = [ closure ]
        states = [ start ]
        pending = [ start ]

        while pending:
            state = pending.pop()
            symbols = { symbol for s in state.state for symbol in s.transitions }

            for symbol in symbols:
                move = self.move_by_state(symbol, *state.state)
                closure = self.epsilon_closure_by_state(*move)

                if closure not in closures:
                    new_state = State(tuple(closure), any(s.final for s in closure), formatter)
                    closures.append(closure)
                    states.append(new_state)
                    pending.append(new_state)
                else:
                    index = closures.index(closure)
                    new_state = states[index]

                state.add_transition(symbol, new_state)

        return start

    @staticmethod
    def from_nfa(nfa, get_states=False):
        states = []
        for n in range(nfa.states):
            state = State(n, n in nfa.finals)
            states.append(state)

        for (origin, symbol), destinations in nfa.map.items():
            origin = states[origin]
            origin[symbol] = [ states[d] for d in destinations ]

        if get_states:
            return states[nfa.start], states
        return states[nfa.start]

    @staticmethod
    def move_by_state(symbol, *states):
        return { s for state in states if state.has_transition(symbol) for s in state[symbol]}

    @staticmethod
    def epsilon_closure_by_state(*states):
        closure = { state for state in states }

        l = 0
        while l != len(closure):
            l = len(closure)
            tmp = [s for s in closure]
            for s in tmp:
                for epsilon_state in s.epsilon_transitions:
                        closure.add(epsilon_state)
        return closure

    @property
    def epsilon_closure(self):
        return self.epsilon_closure_by_state(self)

    @property
    def name(self):
        return self.formatter(self.state)

    def get(self, symbol):
        target = self.transitions[symbol]
        assert len(target) == 1
        return target[0]

    def __getitem__(self, symbol):
        if symbol == '':
            return self.epsilon_transitions
        try:
            return self.transitions[symbol]
        except KeyError:
            return None

    def __setitem__(self, symbol, value):
        if symbol == '':
            self.epsilon_transitions = value
        else:
            self.transitions[symbol] = value

    def __repr__(self):
        return str(self)

    def __str__(self):
        return str(self.state)

    def __hash__(self):
        return hash(self.state)

    def __iter__(self):
        yield from self._visit()

    def _visit(self, visited=None):
        if visited is None:
            visited = set()
        elif self in visited:
            return

        visited.add(self)
        yield self

        for destinations in self.transitions.values():
            for node in destinations:
                yield from node._visit(visited)
        for node in self.epsilon_transitions:
            yield from node._visit(visited)

    def graph(self):
        G = pydot.Dot(rankdir='LR', margin=0.1)
        G.add_node(pydot.Node('start', shape='plaintext', label='', width=0, height=0))

        visited = set()
        def visit(start):
            ids = id(start)
            if ids not in visited:
                visited.add(ids)
                G.add_node(pydot.Node(ids, label=start.name, shape=self.shape, style='bold' if start.final else ''))
                for tran, destinations in start.transitions.items():
                    for end in destinations:
                        visit(end)
                        G.add_edge(pydot.Edge(ids, id(end), label=tran, labeldistance=2))
                for end in start.epsilon_transitions:
                    visit(end)
                    G.add_edge(pydot.Edge(ids, id(end), label='ε', labeldistance=2))

        visit(self)
        G.add_edge(pydot.Edge('start', id(self), label='', style='dashed'))

        return G

    def _repr_svg_(self):
        try:
            return self.graph().create_svg().decode('utf8')
        except:
            pass

    def write_to(self, fname):
        return self.graph().write_svg(fname)

def multiline_formatter(state):
    return '\n'.join(str(item) for item in state)

def lr0_formatter(state):
    try:
        return '\n'.join(str(item)[:-4] for item in state)
    except TypeError:
        return str(state)[:-4]

def automata_union(a1, a2):
    '''
    Union de Automatas. Se genera un Automata Finito No Determinista Union
    de 2 automatas

    Parametros:
    ------------
        `a1`: Automata 1, debe ser un `NFA`
        `a2`: Automata 2, debe ser un `NFA`

    Retorna:
    ---------
        `union`: Automata Union (`NFA`)
    '''
    transitions = {}

    start = 0
    d1 = 1
    d2 = a1.states + d1
    final = a2.states + d2

    for (origin, symbol), destinations in a1.map.items():
        transitions[origin + d1, symbol] = [dest + d1 for dest in destinations]

    for (origin, symbol), destinations in a2.map.items():
        transitions[origin + d2, symbol] = [dest + d2 for dest in destinations]

    trans = transitions.get((start, ''), [])
    transitions[start, ''] = trans + [d1, d2]

    for f1 in a1.finals:
        trans = transitions.get((f1 + d1, ''), [])
        transitions[f1 + d1, ''] = trans + [final]
    for f2 in a2.finals:
        trans = transitions.get((f2 + d2, ''), [])
        transitions[f2 + d2, ''] = trans + [final]

    states = a1.states + a2.states + 2
    finals = { final }

    return NFA(states, finals, transitions, start)

def automata_concatenation(a1, a2):
    '''
    Concatenacion de Automatas. Se genera un Automata Finito No Determinista Concatenacion
    de 2 automatas

    Parametros:
    ------------
        `a1`: Automata 1, debe ser un `NFA`
        `a2`: Automata 2, debe ser un `NFA`

    Retorna:
    ---------
        `concat`: Automata Concatenacion (`NFA`)
    '''
    transitions = {}

    start = 0
    d1 = 0
    d2 = a1.states + d1
    final = a2.states + d2

    for (origin, symbol), destinations in a1.map.items():
        transitions[origin + d1, symbol] = [dest + d1 for dest in destinations]

    for (origin, symbol), destinations in a2.map.items():
        transitions[origin + d2, symbol] = [dest + d2 for dest in destinations]

    for f1 in a1.finals:
        trans = transitions.get((f1 + d1, ''), [])
        transitions[f1 + d1, ''] = trans + [d2]
    for f2 in a2.finals:
        trans = transitions.get((f2 + d2, ''), [])
        transitions[f2 + d2, ''] = trans + [final]

    states = a1.states + a2.states + 1
    finals = { final }

    return NFA(states, finals, transitions, start)

def automata_closure(a1):
    '''
    Clausura de Automata. Se genera un Automata Finito No Determinista Clausura
    de un automata

    Parametros:
    ------------
        `a1`: Automata 1, debe ser un `NFA`

    Retorna:
    ---------
        `closure`: Automata Clausura (`NFA`)
    '''
    transitions = {}

    start = 0
    d1 = 1
    final = a1.states + d1

    for (origin, symbol), destinations in a1.map.items():
        transitions[origin + d1, symbol] = [dest + d1 for dest in destinations]

    transitions[start, ''] = [d1, final]

    for f1 in a1.finals:
        trans = transitions.get((f1 + d1, ''), [])
        transitions[f1 + d1, ''] = trans + [final]
    transitions[final, ''] = [start]

    states = a1.states +  2
    finals = { final }

    return NFA(states, finals, transitions, start)

def automata_positive_closure(a1):
    '''
    Clausura Positiva de Automata. Se genera un Automata Finito No Determinista
    Clausura Positiva de un automata

    Parametros:
    ------------
        `a1`: Automata 1, debe ser un `NFA`

    Retorna:
    ---------
        `closure`: Automata Clausura Positiva (`NFA`)
    '''
    transitions = {}

    start = 0
    d1 = 1
    final = a1.states + d1

    for (origin, symbol), destinations in a1.map.items():
        transitions[origin + d1, symbol] = [dest + d1 for dest in destinations]

    transitions[start, ''] = [d1]

    for f1 in a1.finals:
        transitions[f1 + d1, ''] = [final]
    transitions[final, ''] = [start]

    states = a1.states +  2
    finals = { final }

    return NFA(states, finals, transitions, start)


from django.shortcuts import render
from . import scrapy
from time import time
import numpy as np
import pandas as pd
import json

# Create your views here.
def index(request):
    template = 'index.html'
    startTime = time()
    data = scrapy.conexao()
    acesso12 = scrapy.acesso12(data)
    acesso6 = scrapy.acesso6(data)
    acesso3 = scrapy.acesso3(data)
    acesso1 = scrapy.acesso1(data)
    dfResult = pd.concat([acesso12,acesso6,acesso3,acesso1], axis=1)
    dfResult["Acessos12"] = dfResult["Acessos12"].astype(int)
    dfResult["Acessos6"] =  dfResult["Acessos6"].astype(int)
    dfResult["Acessos3"] =  dfResult["Acessos3"].astype(int)
    dfResult["Acessos1"] =  dfResult["Acessos1"].astype(int)
    dfResult = dfResult[(dfResult["Acessos12"] >= 900) &(dfResult["Acessos6"] >= 900)&(dfResult["Acessos3"] >= 900)&(dfResult["Acessos1"] >= 900)]
    names = []
    acesso12 =  []
    acesso6 =  []
    acesso3 =  []
    acesso1 =  []
    for index, row in dfResult.iterrows():
       names.append(row['Nomes'])
       acesso12.append(row['Acessos12'])
       acesso6.append(row['Acessos6'])
       acesso3.append(row['Acessos3'])
       acesso1.append(row['Acessos1'])
    menssagem = False
    tempoExec = "{:.2f}".format(time() - startTime)
    context = {
        'names': json.dumps(names),
        'acesso12': json.dumps(acesso12),
        'acesso6': json.dumps(acesso6),
        'acesso3': json.dumps(acesso3),
        'acesso1': json.dumps(acesso1),
        'names': json.dumps(names),
        'menssagem':menssagem,
        "tempoExec": tempoExec
        }

    return render(request,template,context)


import pytest

from terra_sdk.client.lcd import LCDClient
from terra_sdk.exceptions import LCDResponseError

terra = LCDClient(
    url="https://pisco-lcd.terra.dev/",
    chain_id="pisco-1",
)

def test_contract_info():
    result = terra.wasm.contract_info(
        "terra19xa33fjdjlz9qkafrw8qnrzrawc8h0vhxvfdhh6yk3f5qxuh2fps9e49zt"
    )
    assert result is not None

def test_code_info():
    result = terra.wasm.code_info(72)

    assert result["code_id"] == 72
    assert result["creator"] == "terra1mzhc9gvfyh9swxed7eaxn2d6zzc3msgftk4w9e"
    assert (
        result["data_hash"]
        == "CD686878A33E62CBCDAF7620E776096E4D15856CC03B0F12EDE66A1D5699D39D"
    )

def test_code_info_with_params():
    with pytest.raises(LCDResponseError):
        terra.wasm.code_info(72, {"height": 100})

def test_contract_query():
    result = terra.wasm.contract_query(
        "terra19xa33fjdjlz9qkafrw8qnrzrawc8h0vhxvfdhh6yk3f5qxuh2fps9e49zt",
        {"get_count": {}},
    )
    assert result is not None

def test_contract_query_with_params():
    result = terra.wasm.contract_query(
        "terra19xa33fjdjlz9qkafrw8qnrzrawc8h0vhxvfdhh6yk3f5qxuh2fps9e49zt",
        {"get_count": {}},
        {"height": 61027},
    )
    assert result == {"count": 0}

    result = terra.wasm.contract_query(
        "terra19xa33fjdjlz9qkafrw8qnrzrawc8h0vhxvfdhh6yk3f5qxuh2fps9e49zt",
        {"get_count": {}},
        {"height": 61028},
    )
    assert result == {"count": 1}

def test_pinned_codes():
    result = terra.wasm.pinned_codes()
    assert result["code_ids"] is not None


import os
import nltk
import torch
import pandas as pd
import numpy as np
from torch.utils.data import Dataset, DataLoader
from tqdm import tqdm
from transformers import AutoTokenizer, AutoModelForSeq2SeqLM

folder = "nonEnglish_2023/"
os.environ['TORCH_HOME'] = os.getcwd()+'/cache'
lang_1 = 'pt'
lang_2 = 'gl'
model_name_1 = 'Helsinki-NLP/opus-mt-' + lang_1 + '-' +lang_2
model_name_2 = 'Helsinki-NLP/opus-mt-' + lang_2 + '-en'
tokenizer_1 = AutoTokenizer.from_pretrained(model_name_1, cache_dir=os.getcwd()+'/cache')
model_1 = AutoModelForSeq2SeqLM.from_pretrained(model_name_1, cache_dir=os.getcwd()+'/cache')

tokenizer_2 = AutoTokenizer.from_pretrained(model_name_2, cache_dir=os.getcwd()+'/cache')
model_2 = AutoModelForSeq2SeqLM.from_pretrained(model_name_2, cache_dir=os.getcwd()+'/cache')

df_name = "global_" + lang_1+'_sent.csv'

df = pd.read_csv(folder + df_name)
df_inter = pd.read_csv("resuslt_inter_pt_tmp.csv")
df = df.iloc[len(df_inter):].reset_index()
print((lang_1, lang_2), df_name, df.shape, (model_name_1, model_name_2))

class Datapre(Dataset):
    def __init__(self, dataframe, tokenizer, max_len):
        self.tokenizer = tokenizer
        self.df = dataframe
        self.text = self.df.text
        self.max_len = max_len
        self.label = dataframe.Id.tolist()
        # check same length
        if (len(self.label) != len(self.text)):
            print('length does not match!')

    def __len__(self):
        return len(self.text)

    def __getitem__(self, index):
        text = str(self.text[index])
        text = " ".join(text.split())
        label = self.label[index]

        inputs = self.tokenizer.encode_plus(
            text,
            max_length=self.max_len,
            pad_to_max_length=True,
        )
        ids = inputs['input_ids']
        mask = inputs['attention_mask']
        return  {
            'input_ids': torch.tensor(ids, dtype=torch.long).to(device),
            'attention_mask': torch.tensor(mask, dtype=torch.long).to(device),
        }, label

test = False
if test == True:
    df = df.iloc[:50]

# !!!!!!! important
device = 'cuda:3'
batch = 80
print(device)
model_1= torch.nn.DataParallel(model_1)
model_1.to(device)
model_1.eval()

model_2= torch.nn.DataParallel(model_2)
model_2.to(device)
model_2.eval()

params = {'batch_size': batch,
          'shuffle': False,
          'num_workers': 0}

#prepare dataset
df_ = Datapre(df, tokenizer_1, max_len = 300)
loader_ = DataLoader(df_, **params)

listoftransids = df_inter.Id.tolist()
listoftext = df_inter.text.tolist()
with torch.no_grad():
    for encoded, Ids in tqdm(loader_):
        listoftransids.extend(Ids.numpy())
        translated = model_1.module.generate(**encoded).to(device)
        translated_texts = tokenizer_1.batch_decode(translated, skip_special_tokens=True)
        listoftext.extend(translated_texts)
        if len(listoftransids) % 10000 == 0:
            print('Partly finished ',len(listoftransids))
            pd.DataFrame({'Id': listoftransids, 'text':listoftext}).to_csv(folder + 'resuslt_inter_' + lang_1 + '_tmp.csv', index= False)
pd.DataFrame({'Id': listoftransids, 'text':listoftext}).to_csv(folder + 'resuslt_inter_' + lang_1 + '.csv', index= False)
print('Intermediate Translation finished!!!')
df_inter = pd.DataFrame({'Id': listoftransids, 'text':listoftext})

df_inter_ = Datapre(df_inter, tokenizer_2, max_len = 300)
loader_2 = DataLoader(df_inter_, **params)

listoftransids = []
listoftext = []
with torch.no_grad():
    for encoded, Ids in tqdm(loader_2):
        listoftransids.extend(Ids.numpy())
        translated = model_2.module.generate(**encoded).to(device)
        translated_texts = tokenizer_2.batch_decode(translated, skip_special_tokens=True)
        listoftext.extend(translated_texts)
        if len(listoftransids) % 10000 == 0:
            print('Partly finished ',len(listoftransids))
            pd.DataFrame({'Id': listoftransids, 'text':listoftext}).to_csv(folder + 'resuslt_' + lang_2 + '_tmp.csv', index= False)

pd.DataFrame({'Id': listoftransids, 'text':listoftext}).to_csv(folder + 'resuslt_' + lang_1 + '.csv', index= False)


# ## From https://stackoverflow.com/questions/30988033/sending-live-video-frame-over-network-in-python-opencv

import asyncio
import time
import cv2
import numpy as np
from numpysocket import NumpySocket

THREADS = 3

frames = [None] * THREADS

async def rec(ser, cli, i):
    global frames
    frames[i] = ser.receive_numpy()
    await cli.send_ack()

async def main():
    global frames

    # host_ip = '172.27.3.5'
    host_ip = 'localhost'

    receivers = []
    ack_senders = []

    for i in range(THREADS):
        tmp = NumpySocket()
        tmp.start_server(9999 - i)
        receivers.append(tmp)

    for i in range(THREADS):
        tmp = NumpySocket()
        await tmp.start_client(host_ip, 8000 + i)
        ack_senders.append(tmp)

    # Read until video is completed
    start = time.time()
    fps = 0

    while True:
        # Capture frame-by-frame
        await asyncio.gather(*[rec(receivers[i], ack_senders[i], i) for i in range(THREADS)], return_exceptions=False)
        frame = np.hstack(tuple(frames))
        cv2.imshow('Frame', frame)

        fps += 1

        # Press Q on keyboard to  exit
        if cv2.waitKey(2) & 0xFF == ord('q'):
            break
        # if keyboard.is_pressed('command'):
        #     break

    # print(fps)
    print(frame.shape)
    print(fps / (time.time() - start))

    for i in range(THREADS):
        receivers[i].end_server()
    print("Closing")

asyncio.run(main())


import base64

import cv2
import numpy as np
from skimage import feature

def fd_hu_moments(image):
    return cv2.HuMoments(cv2.moments(image)).flatten()

def quantify_image(image):
    # For Speed and pressure of signature image
    # compute the histogram of oriented gradients feature vector for
    # the input image
    features = feature.hog(image, orientations=9,
                           pixels_per_cell=(10, 10), cells_per_block=(2, 2),
                           transform_sqrt=True, block_norm="L1")

    # return the feature vector
    return features

def read_image_in_greyscale_as_np_array(img_path):
    img = cv2.imread(img_path)
    gray_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    return gray_img

def write_np_array_as_image(img_array, out_img_path):
    is_saved = cv2.imwrite(out_img_path, img_array)
    if is_saved:
        return out_img_path
    else:
        raise Exception("Failed to save output image")

def opencv_img_from_base64(im_base64):
    im_bytes = base64.b64decode(im_base64)
    im_arr = np.frombuffer(im_bytes, dtype=np.uint8)
    img = cv2.imdecode(im_arr, flags=cv2.IMREAD_COLOR)
    return img

def opencv_img_to_base64(img):
    _, im_arr = cv2.imencode('.jpg', img)
    im_bytes = im_arr.tobytes()
    im_base64 = base64.b64encode(im_bytes)
    return im_base64

def calculate_metrics_from_3x3_confusion_matrix(cm):
    tp = cm[0][0] + cm[1][1] + cm[2][2]
    tn = (cm[1][1] + cm[1][2] + cm[2][1] + cm[2][2]) + (cm[0][0] + cm[0][2] + cm[2][0] + cm[2][2]) + (
                cm[0][0] + cm[0][1] + cm[1][0] + cm[1][1])
    fp = (cm[1][0] + cm[2][0]) + (cm[0][1] + cm[2][1]) + (cm[0][2] + cm[1][2])
    fn = (cm[0][1] + cm[0][2]) + (cm[1][0] + cm[1][2]) + (cm[2][0] + cm[2][1])

    return int(tp), int(tn), int(fp), int(fn)


from __future__ import annotations

from dataclasses import MISSING, fields, is_dataclass
from typing import Any, Generic

from typing_extensions import TypeGuard, get_type_hints

from polyfactory.factories.base import BaseFactory, T
from polyfactory.field_meta import FieldMeta, Null

class DataclassFactory(Generic[T], BaseFactory[T]):
    """Dataclass base factory"""

    __is_base_factory__ = True

    @classmethod
    def is_supported_type(cls, value: Any) -> TypeGuard[type[T]]:
        """Determine whether the given value is supported by the factory.

        :param value: An arbitrary value.
        :returns: A typeguard
        """
        return bool(is_dataclass(value))

    @classmethod
    def get_model_fields(cls) -> list["FieldMeta"]:
        """Retrieve a list of fields from the factory's model.

        :returns: A list of field MetaData instances.

        """
        fields_meta: list["FieldMeta"] = []

        model_type_hints = get_type_hints(cls.__model__, include_extras=True)

        for field in fields(cls.__model__):  # type: ignore[arg-type]
            if field.default_factory and field.default_factory is not MISSING:
                default_value = field.default_factory()
            elif field.default is not MISSING:
                default_value = field.default
            else:
                default_value = Null

            fields_meta.append(
                FieldMeta.from_type(
                    annotation=model_type_hints[field.name],
                    name=field.name,
                    default=default_value,
                    random=cls.__random__,
                ),
            )

        return fields_meta


import pandas as pd
import numpy as np
from datetime import datetime
import json
import os
import sys
import gc

# df_train.shape = (184903890, 9)
# test_supplement_partial_processed.shape = (57536872, 8)

input_dir = '../data/input/'
feature_dir = '../data/input/features/'
config = 'feature_list.json'
fmt = '%Y-%m-%d %H:%M:%S' # Time string format.

is_extract_label = 0

# Extract labels from original training set and save.
if is_extract_label:
    print('Loading "train.feather" ...')
    t0 = datetime.now()
    df_train = pd.read_feather(os.path.join(input_dir, 'train.feather'))
    t1 = datetime.now()
    print('Done. Time elapsed: {:.2f} minutes.'.format((t1-t0).total_seconds()/60))
    print('df_train.shape = {}\n'.format(df_train.shape))
    # Extract label and save.
    df_train_label = df_train['is_attributed'].to_frame()
    df_train_label.to_feather(os.path.join(input_dir, 'df_train_label.feather'))

    # Extract downsampled label.
    df_train_balanced_index_10_fold = pd.read_feather(os.path.join(input_dir, 'df_train_balanced_index_10_fold.feather'))
    for sample_set_column in df_train_balanced_index_10_fold:
        sample_index = df_train_balanced_index_10_fold[sample_set_column]
        new_file_name = sample_set_column + '_train_label.feather'
        print('\n{} - Saving {}...'.format(datetime.now().strftime(fmt), new_file_name))
        df_train_label_tmp = df_train_label.loc[sample_index].reset_index().to_feather(os.path.join(input_dir, new_file_name))

# Load feature config file.
with open(os.path.join(input_dir,'feature_config.json'),'r',encoding='utf-8') as data_file:
    feature_dict = json.load(data_file)

# Flatten feature_dict to feature_list.
feature_list = []
for k, v in feature_dict.items():
    feature_list.extend(v)

# Generate rand0_train_set, rand1_train_set, ... rand9_train_set, with corresponding labels.
for i in range(10):
    prefix = 'rand' + str(i) + '_'
    print('\n{} - Generating {} train and label set...'.format(datetime.now().strftime(fmt), 'rand' + str(i)))
    # Check the existence of all feature files.
    for feature in feature_list:
        feature_file_name = os.path.join(input_dir, prefix + feature + '.feather')
        if os.path.isfile(feature_file_name):
            print('\nOK! {}.feather exists!'.format(prefix + feature))
        else:
            print('\nError! {}.feather does not exist!'.format(prefix + feature))
            sys.exit()
    # Create combined train and label dataframe.
    df_train_and_label_tmp = pd.DataFrame({})
    # Load label.
    df_label_tmp = pd.read_feather(os.path.join(input_dir, prefix + 'train_label.feather'))
    label_index_values = df_label_tmp['index'].values
    # Load downsampled feather.
    for feature in feature_list:
        print('\n{} - Combining feature {}...'.format(datetime.now().strftime(fmt), feature))
        df_feature = pd.read_feather(os.path.join(input_dir, prefix + feature + '.feather'))
        # Check the index of label and feather.
        if (df_feature['index'].values - label_index_values).sum():
            print('\n{} - Error! The indices of labels and training set are not same!'.format(datetime.now().strftime(fmt)))
            sys.exit()
        else:
            print('\n{} - OK! The indices of labels and training set are  same!'.format(datetime.now().strftime(fmt)))
        # Concatenate.
        df_train_and_label_tmp =  pd.concat([df_train_and_label_tmp, df_feature[feature]], axis=1)
        print('\n{} - Columns after combining: {}'.format(datetime.now().strftime(fmt), len(df_train_and_label_tmp.columns)))
    # After all feathers are combined, append label as the last column.
    df_train_and_label_tmp =  pd.concat([df_train_and_label_tmp, df_label_tmp['is_attributed']], axis=1)
    print('\n{} - Columns after combining: {}'.format(datetime.now().strftime(fmt), len(df_train_and_label_tmp.columns)))
    # Save.
    print('\n{} - Saving combined train and label file...'.format(datetime.now().strftime(fmt)))
    df_train_and_label_tmp.reset_index(drop=True).to_feather(os.path.join(input_dir, prefix + 'train_and_label.feather'))
    print('\n{} - Done.'.format(datetime.now().strftime(fmt)))

    del df_label_tmp
    del label_index_values
    del df_feature
    del df_train_and_label_tmp
    gc.collect()