"""
Setup and launch a CloudMan instance.
"""
import datetime
import socket
from http.client import (
    BadStatusLine,
    HTTPConnection,
    HTTPException,
)
from urllib.parse import urlparse

import boto
import yaml
from boto.ec2.regioninfo import RegionInfo
from boto.exception import EC2ResponseError, S3ResponseError
from boto.s3.connection import OrdinaryCallingFormat, S3Connection, SubdomainCallingFormat

import bioblend
from bioblend.util import Bunch

# Uncomment the following line if no logging from boto is desired
# bioblend.logging.getLogger('boto').setLevel(bioblend.logging.CRITICAL)
# Uncomment the following line if logging at the prompt is desired
# bioblend.set_stream_logger(__name__)
def instance_types(cloud_name='generic'):
    """
    Return a list of dictionaries containing details about the available
    instance types for the given `cloud_name`.

    :type cloud_name: str
    :param cloud_name: A name of the cloud for which the list of instance
                       types will be returned. Valid values are: `aws`,
                       `nectar`, `generic`.

    :rtype: list
    :return: A list of dictionaries describing instance types. Each dict will
             contain the following keys: `name`, `model`, and `description`.
    """
    instance_list = []
    if cloud_name.lower() == 'aws':
        instance_list.append({"model": "c3.large",
                              "name": "Compute optimized Large",
                              "description": "2 vCPU/4GB RAM"})
        instance_list.append({"model": "c3.2xlarge",
                              "name": "Compute optimized 2xLarge",
                              "description": "8 vCPU/15GB RAM"})
        instance_list.append({"model": "c3.8xlarge",
                              "name": "Compute optimized 8xLarge",
                              "description": "32 vCPU/60GB RAM"})
    elif cloud_name.lower() in ['nectar', 'generic']:
        instance_list.append({"model": "m1.small",
                              "name": "Small",
                              "description": "1 vCPU / 4GB RAM"})
        instance_list.append({"model": "m1.medium",
                              "name": "Medium",
                              "description": "2 vCPU / 8GB RAM"})
        instance_list.append({"model": "m1.large",
                              "name": "Large",
                              "description": "4 vCPU / 16GB RAM"})
        instance_list.append({"model": "m1.xlarge",
                              "name": "Extra Large",
                              "description": "8 vCPU / 32GB RAM"})
        instance_list.append({"model": "m1.xxlarge",
                              "name": "Extra-extra Large",
                              "description": "16 vCPU / 64GB RAM"})
    return instance_list

class CloudManLauncher:

    def __init__(self, access_key, secret_key, cloud=None):
        """
        Define the environment in which this instance of CloudMan will be launched.

        Besides providing the credentials, optionally provide the ``cloud``
        object. This object must define the properties required to establish a
        `boto <https://github.com/boto/boto/>`_ connection to that cloud. See
        this method's implementation for an example of the required fields.
        Note that as long the as provided object defines the required fields,
        it can really by implemented as anything (e.g., a Bunch, a database
        object, a custom class). If no value for the ``cloud`` argument is
        provided, the default is to use the Amazon cloud.
        """
        self.access_key = access_key
        self.secret_key = secret_key
        if cloud is None:
            # Default to an EC2-compatible object
            self.cloud = Bunch(id='1',  # for compatibility w/ DB representation
                               name="Amazon",
                               cloud_type="ec2",
                               bucket_default="cloudman",
                               region_name="us-east-1",
                               region_endpoint="ec2.amazonaws.com",
                               ec2_port="",
                               ec2_conn_path="/",
                               cidr_range="",
                               is_secure=True,
                               s3_host="s3.amazonaws.com",
                               s3_port="",
                               s3_conn_path='/')
        else:
            self.cloud = cloud
        self.ec2_conn = self.connect_ec2(
            self.access_key,
            self.secret_key,
            self.cloud)
        self.vpc_conn = self.connect_vpc(
            self.access_key,
            self.secret_key,
            self.cloud)
        # Define exceptions that we want to catch and retry
        self.http_exceptions = (
            HTTPException,
            socket.error,
            socket.gaierror,
            BadStatusLine
        )

    def __repr__(self):
        return f"Cloud: {self.cloud.name}; acct ID: {self.access_key}"

    def launch(self, cluster_name, image_id, instance_type, password,
               kernel_id=None, ramdisk_id=None, key_name='cloudman_key_pair',
               security_groups=None, placement='', subnet_id=None,
               ebs_optimized=False, **kwargs):
        """
        Check all the prerequisites (key pair and security groups) for
        launching a CloudMan instance, compose the user data based on the
        parameters specified in the arguments and the cloud properties as
        defined in the object's ``cloud`` field.

        For the current list of user data fields that can be provided via
        ``kwargs``, see `<https://galaxyproject.org/cloudman/userdata/>`_

        Return a dict containing the properties and info with which an instance
        was launched, namely: ``sg_names`` containing the names of the security
        groups, ``kp_name`` containing the name of the key pair, ``kp_material``
        containing the private portion of the key pair (*note* that this portion
        of the key is available and can be retrieved *only* at the time the key
        is created, which will happen only if no key with the name provided in
        the ``key_name`` argument exists), ``rs`` containing the
        `boto <https://github.com/boto/boto/>`_ ``ResultSet`` object,
        ``instance_id`` containing the ID of a started instance, and
        ``error`` containing an error message if there was one.
        """
        if security_groups is None:
            security_groups = ['CloudMan']
        ret = {'sg_names': [],
               'sg_ids': [],
               'kp_name': '',
               'kp_material': '',
               'rs': None,
               'instance_id': '',
               'error': None}
        # First satisfy the prerequisites
        for sg in security_groups:
            # Get VPC ID in case we're launching into a VPC
            vpc_id = None
            if subnet_id:
                try:
                    sn = self.vpc_conn.get_all_subnets(subnet_id)[0]
                    vpc_id = sn.vpc_id
                except (EC2ResponseError, IndexError):
                    bioblend.log.exception("Trouble fetching subnet %s", subnet_id)
            cmsg = self.create_cm_security_group(sg, vpc_id=vpc_id)
            ret['error'] = cmsg['error']
            if ret['error']:
                return ret
            if cmsg['name']:
                ret['sg_names'].append(cmsg['name'])
                ret['sg_ids'].append(cmsg['sg_id'])
                if subnet_id:
                    # Must setup a network interface if launching into VPC
                    security_groups = None
                    interface = boto.ec2.networkinterface.NetworkInterfaceSpecification(
                        subnet_id=subnet_id, groups=[cmsg['sg_id']],
                        associate_public_ip_address=True)
                    network_interfaces = (boto.ec2.networkinterface.
                                          NetworkInterfaceCollection(interface))
                else:
                    network_interfaces = None
        kp_info = self.create_key_pair(key_name)
        ret['kp_name'] = kp_info['name']
        ret['kp_material'] = kp_info['material']
        ret['error'] = kp_info['error']
        if ret['error']:
            return ret
        # If not provided, try to find a placement
        # TODO: Should placement always be checked? To make sure it's correct
        # for existing clusters.
        if not placement:
            placement = self._find_placement(
                cluster_name).get('placement', None)
        # Compose user data for launching an instance, ensuring we have the
        # required fields
        kwargs['access_key'] = self.access_key
        kwargs['secret_key'] = self.secret_key
        kwargs['cluster_name'] = cluster_name
        kwargs['password'] = password
        kwargs['cloud_name'] = self.cloud.name
        ud = self._compose_user_data(kwargs)
        # Now launch an instance
        try:

            rs = None
            rs = self.ec2_conn.run_instances(image_id=image_id,
                                             instance_type=instance_type,
                                             key_name=key_name,
                                             security_groups=security_groups,
                                             # The following two arguments are
                                             # provided in the network_interface
                                             # instead of arguments:
                                             # security_group_ids=security_group_ids,
                                             # subnet_id=subnet_id,
                                             network_interfaces=network_interfaces,
                                             user_data=ud,
                                             kernel_id=kernel_id,
                                             ramdisk_id=ramdisk_id,
                                             placement=placement,
                                             ebs_optimized=ebs_optimized)
            ret['rs'] = rs
        except EC2ResponseError as e:
            err_msg = f"Problem launching an instance: {e} (code {e.error_code}; status {e.status})"
            bioblend.log.exception(err_msg)
            ret['error'] = err_msg
            return ret
        else:
            if rs:
                try:
                    bioblend.log.info("Launched an instance with ID %s", rs.instances[0].id)
                    ret['instance_id'] = rs.instances[0].id
                    ret['instance_ip'] = rs.instances[0].ip_address
                except EC2ResponseError as e:
                    err_msg = f"Problem with the launched instance object: {e} (code {e.error_code}; status {e.status})"
                    bioblend.log.exception(err_msg)
                    ret['error'] = err_msg
            else:
                ret['error'] = ("No response after launching an instance. Check "
                                "your account permissions and try again.")
        return ret

    def create_cm_security_group(self, sg_name='CloudMan', vpc_id=None):
        """
        Create a security group with all authorizations required to run CloudMan.

        If the group already exists, check its rules and add the missing ones.

        :type sg_name: str
        :param sg_name: A name for the security group to be created.

        :type vpc_id: str
        :param vpc_id: VPC ID under which to create the security group.

        :rtype: dict
        :return: A dictionary containing keys ``name`` (with the value being the
                 name of the security group that was created), ``error``
                 (with the value being the error message if there was an error
                 or ``None`` if no error was encountered), and ``ports``
                 (containing the list of tuples with port ranges that were
                 opened or attempted to be opened).

        .. versionchanged:: 0.6.1
            The return value changed from a string to a dict
        """
        ports = (('20', '21'),  # FTP
                 ('22', '22'),  # SSH
                 ('80', '80'),  # Web UI
                 ('443', '443'),  # SSL Web UI
                 ('8800', '8800'),  # NodeJS Proxy for Galaxy IPython IE
                 ('9600', '9700'),  # HTCondor
                 ('30000', '30100'))  # FTP transfer
        progress = {'name': None,
                    'sg_id': None,
                    'error': None,
                    'ports': ports}
        cmsg = None
        filters = None
        if vpc_id:
            filters = {'vpc-id': vpc_id}
        # Check if this security group already exists
        try:
            sgs = self.ec2_conn.get_all_security_groups(filters=filters)
        except EC2ResponseError as e:
            err_msg = f"Problem getting security groups. This could indicate a problem with your account credentials or permissions: {e} (code {e.error_code}; status {e.status})"
            bioblend.log.exception(err_msg)
            progress['error'] = err_msg
            return progress
        for sg in sgs:
            if sg.name == sg_name:
                cmsg = sg
                bioblend.log.debug("Security group '%s' already exists; will add authorizations next.", sg_name)
                break
        # If it does not exist, create security group
        if cmsg is None:
            bioblend.log.debug("Creating Security Group %s", sg_name)
            try:
                cmsg = self.ec2_conn.create_security_group(sg_name, 'A security '
                                                           'group for CloudMan',
                                                           vpc_id=vpc_id)
            except EC2ResponseError as e:
                err_msg = f"Problem creating security group '{sg_name}': {e} (code {e.error_code}; status {e.status})"
                bioblend.log.exception(err_msg)
                progress['error'] = err_msg
        if cmsg:
            progress['name'] = cmsg.name
            progress['sg_id'] = cmsg.id
            # Add appropriate authorization rules
            # If these rules already exist, nothing will be changed in the SG
            for port in ports:
                try:
                    if not self.rule_exists(
                            cmsg.rules, from_port=port[0], to_port=port[1]):
                        cmsg.authorize(
                            ip_protocol='tcp',
                            from_port=port[0],
                            to_port=port[1],
                            cidr_ip='0.0.0.0/0')
                    else:
                        bioblend.log.debug("Rule (%s:%s) already exists in the SG", port[0], port[1])
                except EC2ResponseError as e:
                    err_msg = f"A problem adding security group authorizations: {e} (code {e.error_code}; status {e.status})"
                    bioblend.log.exception(err_msg)
                    progress['error'] = err_msg
            # Add ICMP (i.e., ping) rule required by HTCondor
            try:
                if not self.rule_exists(
                        cmsg.rules, from_port='-1', to_port='-1', ip_protocol='icmp'):
                    cmsg.authorize(
                        ip_protocol='icmp',
                        from_port=-1,
                        to_port=-1,
                        cidr_ip='0.0.0.0/0')
                else:
                    bioblend.log.debug(
                        f"ICMP rule already exists in {sg_name} SG.")
            except EC2ResponseError as e:
                err_msg = f"A problem with security ICMP rule authorization: {e} (code {e.error_code}; status {e.status})"
                bioblend.log.exception(err_msg)
                progress['err_msg'] = err_msg
            # Add rule that allows communication between instances in the same
            # SG
            # A flag to indicate if group rule already exists
            g_rule_exists = False
            for rule in cmsg.rules:
                for grant in rule.grants:
                    if grant.name == cmsg.name:
                        g_rule_exists = True
                        bioblend.log.debug(
                            "Group rule already exists in the SG.")
                if g_rule_exists:
                    break
            if not g_rule_exists:
                try:
                    cmsg.authorize(
                        src_group=cmsg,
                        ip_protocol='tcp',
                        from_port=0,
                        to_port=65535)
                except EC2ResponseError as e:
                    err_msg = f"A problem with security group group authorization: {e} (code {e.error_code}; status {e.status})"
                    bioblend.log.exception(err_msg)
                    progress['err_msg'] = err_msg
            bioblend.log.info("Done configuring '%s' security group", cmsg.name)
        else:
            bioblend.log.warning(
                f"Did not create security group '{sg_name}'")
        return progress

    def rule_exists(
            self, rules, from_port, to_port, ip_protocol='tcp', cidr_ip='0.0.0.0/0'):
        """
        A convenience method to check if an authorization rule in a security group
        already exists.
        """
        for rule in rules:
            if rule.ip_protocol == ip_protocol and rule.from_port == from_port and \
               rule.to_port == to_port and cidr_ip in [ip.cidr_ip for ip in rule.grants]:
                return True
        return False

    def create_key_pair(self, key_name='cloudman_key_pair'):
        """
        If a key pair with the provided ``key_name`` does not exist, create it.

        :type sg_name: str
        :param sg_name: A name for the key pair to be created.

        :rtype: dict
        :return: A dictionary containing keys ``name`` (with the value being the
                 name of the key pair that was created), ``error``
                 (with the value being the error message if there was an error
                 or ``None`` if no error was encountered), and ``material``
                 (containing the unencrypted PEM encoded RSA private key if the
                 key was created or ``None`` if the key already eixsted).

        .. versionchanged:: 0.6.1
            The return value changed from a tuple to a dict
        """
        progress = {'name': None,
                    'material': None,
                    'error': None}
        kp = None
        # Check if a key pair under the given name already exists. If it does not,
        # create it, else return.
        try:
            kps = self.ec2_conn.get_all_key_pairs()
        except EC2ResponseError as e:
            err_msg = f"Problem getting key pairs: {e} (code {e.error_code}; status {e.status})"
            bioblend.log.exception(err_msg)
            progress['error'] = err_msg
            return progress
        for akp in kps:
            if akp.name == key_name:
                bioblend.log.info("Key pair '%s' already exists; reusing it.", key_name)
                progress['name'] = akp.name
                return progress
        try:
            kp = self.ec2_conn.create_key_pair(key_name)
        except EC2ResponseError as e:
            err_msg = f"Problem creating key pair '{key_name}': {e} (code {e.error_code}; status {e.status})"
            bioblend.log.exception(err_msg)
            progress['error'] = err_msg
            return progress
        bioblend.log.info("Created key pair '%s'", kp.name)
        progress['name'] = kp.name
        progress['material'] = kp.material
        return progress

    def assign_floating_ip(self, ec2_conn, instance):
        try:
            bioblend.log.debug("Allocating a new floating IP address.")
            address = ec2_conn.allocate_address()
        except EC2ResponseError:
            bioblend.log.exception("Exception allocating a new floating IP address")
        bioblend.log.info("Associating floating IP %s to instance %s", address.public_ip, instance.id)
        ec2_conn.associate_address(instance_id=instance.id,
                                   public_ip=address.public_ip)

    def get_status(self, instance_id):
        """
        Check on the status of an instance. ``instance_id`` needs to be a
        ``boto``-library copatible instance ID (e.g., ``i-8fehrdss``).If
        ``instance_id`` is not provided, the ID obtained when launching
        *the most recent* instance is used. Note that this assumes the instance
        being checked on was launched using this  class. Also note that the same
        class may be used to launch multiple instances but only the most recent
        ``instance_id`` is kept while any others will  to be explicitly specified.

        This method also allows the required ``ec2_conn`` connection object to be
        provided at invocation time. If the object is not provided, credentials
        defined for the class are used (ability to specify a custom ``ec2_conn``
        helps in case of stateless method invocations).

        Return a ``state`` dict containing the following keys: ``instance_state``,
        ``public_ip``, ``placement``, and ``error``, which capture CloudMan's
        current state. For ``instance_state``, expected values are: ``pending``,
        ``booting``, ``running``, or ``error`` and represent the state of the
        underlying instance. Other keys will return an empty  value until the
        ``instance_state`` enters ``running`` state.
        """
        ec2_conn = self.ec2_conn
        rs = None
        state = {'instance_state': "",
                 'public_ip': "",
                 'placement': "",
                 'error': ""}

        # Make sure we have an instance ID
        if instance_id is None:
            err = "Missing instance ID, cannot check the state."
            bioblend.log.error(err)
            state['error'] = err
            return state
        try:
            rs = ec2_conn.get_all_instances([instance_id])
            if rs is not None:
                inst_state = rs[0].instances[0].update()
                public_ip = rs[0].instances[0].ip_address
                state['public_ip'] = public_ip
                if inst_state == 'running':
                    # if there's a private ip, but no public ip
                    # attempt auto allocation of floating IP
                    if rs[0].instances[0].private_ip_address and not public_ip:
                        self.assign_floating_ip(ec2_conn, rs[0].instances[0])
                    # Wait until the CloudMan URL is accessible to return
                    # the data - this may only be correct initially at
                    # bootup for instances that configure themselves for
                    # https - they may ultimately block port 80. However,
                    # we don't have a good way to determine whether to
                    # check using http or https.
                    cm_url = f"http://{public_ip}/cloud"
                    if self._checkURL(cm_url) is True:
                        state['instance_state'] = inst_state
                        state['placement'] = rs[0].instances[0].placement
                    else:
                        state['instance_state'] = 'booting'
                else:
                    state['instance_state'] = inst_state
        except Exception as e:
            err = f"Problem updating instance '{instance_id}' state: {e}"
            bioblend.log.error(err)
            state['error'] = err
        return state

    def get_clusters_pd(self, include_placement=True):
        """
        Return *persistent data* of all existing clusters for this account.

        :type include_placement: bool
        :param include_placement: Whether or not to include region placement for
                                  the clusters. Setting this option will lead
                                  to a longer function runtime.

        :rtype: dict
        :return: A dictionary containing keys ``clusters`` and ``error``. The
                 value of ``clusters`` will be a dictionary with the following keys
                 ``cluster_name``, ``persistent_data``, ``bucket_name`` and optionally
                 ``placement`` or an empty list if no clusters were found or an
                 error was encountered. ``persistent_data`` key value is yet
                 another dictionary containing given cluster's persistent data.
                 The value for the ``error`` key will contain a string with the
                 error message.

        .. versionadded:: 0.3
        .. versionchanged:: 0.7.0
            The return value changed from a list to a dictionary.
        """
        clusters = []
        response = {'clusters': clusters, 'error': None}
        s3_conn = self.connect_s3(self.access_key, self.secret_key, self.cloud)
        try:
            buckets = s3_conn.get_all_buckets()
        except S3ResponseError as e:
            response['error'] = f"S3ResponseError getting buckets: {e}"
        except self.http_exceptions as ex:
            response['error'] = f"Exception getting buckets: {ex}"
        if response['error']:
            bioblend.log.exception(response['error'])
            return response
        for bucket in [b for b in buckets if b.name.startswith('cm-')]:
            try:
                # TODO: first lookup if persistent_data.yaml key exists
                pd = bucket.get_key('persistent_data.yaml')
            except S3ResponseError:
                # This can fail for a number of reasons for non-us and/or
                # CNAME'd buckets but it is not a terminal error
                bioblend.log.warning("Problem fetching persistent_data.yaml from bucket %s", bucket)
                continue
            if pd:
                # We are dealing with a CloudMan bucket
                pd_contents = pd.get_contents_as_string()
                pd = yaml.safe_load(pd_contents)
                if 'cluster_name' in pd:
                    cluster_name = pd['cluster_name']
                else:
                    for key in bucket.list():
                        if key.name.endswith('.clusterName'):
                            cluster_name = key.name.split('.clusterName')[0]
                cluster = {'cluster_name': cluster_name,
                           'persistent_data': pd,
                           'bucket_name': bucket.name}
                # Look for cluster's placement too
                if include_placement:
                    placement = self._find_placement(cluster_name, cluster)
                    cluster['placement'] = placement
                clusters.append(cluster)
        response['clusters'] = clusters
        return response

    def get_cluster_pd(self, cluster_name):
        """
        Return *persistent data* (as a dict) associated with a cluster with the
        given ``cluster_name``. If a cluster with the given name is not found,
        return an empty dict.

        .. versionadded:: 0.3
        """
        cluster = {}
        clusters = self.get_clusters_pd().get('clusters', [])
        for c in clusters:
            if c['cluster_name'] == cluster_name:
                cluster = c
                break
        return cluster

    def connect_ec2(self, a_key, s_key, cloud=None):
        """
        Create and return an EC2-compatible connection object for the given cloud.

        See ``_get_cloud_info`` method for more details on the requirements for
        the ``cloud`` parameter. If no value is provided, the class field is used.
        """
        if cloud is None:
            cloud = self.cloud
        ci = self._get_cloud_info(cloud)
        r = RegionInfo(name=ci['region_name'], endpoint=ci['region_endpoint'])
        ec2_conn = boto.connect_ec2(aws_access_key_id=a_key,
                                    aws_secret_access_key=s_key,
                                    is_secure=ci['is_secure'],
                                    region=r,
                                    port=ci['ec2_port'],
                                    path=ci['ec2_conn_path'],
                                    validate_certs=False)
        return ec2_conn

    def connect_s3(self, a_key, s_key, cloud=None):
        """
        Create and return an S3-compatible connection object for the given cloud.

        See ``_get_cloud_info`` method for more details on the requirements for
        the ``cloud`` parameter. If no value is provided, the class field is used.
        """
        if cloud is None:
            cloud = self.cloud
        ci = self._get_cloud_info(cloud)
        if ci['cloud_type'] == 'amazon':
            calling_format = SubdomainCallingFormat()
        else:
            calling_format = OrdinaryCallingFormat()
        s3_conn = S3Connection(
            aws_access_key_id=a_key, aws_secret_access_key=s_key,
            is_secure=ci['is_secure'], port=ci['s3_port'], host=ci['s3_host'],
            path=ci['s3_conn_path'], calling_format=calling_format)
        return s3_conn

    def connect_vpc(self, a_key, s_key, cloud=None):
        """
        Establish a connection to the VPC service.

        TODO: Make this work with non-default clouds as well.
        """
        if cloud is None:
            cloud = self.cloud
        ci = self._get_cloud_info(cloud)
        r = RegionInfo(name=ci['region_name'], endpoint=ci['region_endpoint'])
        vpc_conn = boto.connect_vpc(
            aws_access_key_id=a_key,
            aws_secret_access_key=s_key,
            is_secure=ci['is_secure'],
            region=r,
            port=ci['ec2_port'],
            path=ci['ec2_conn_path'],
            validate_certs=False)
        return vpc_conn

    def _compose_user_data(self, user_provided_data):
        """
        A convenience method used to compose and properly format the user data
        required when requesting an instance.

        ``user_provided_data`` is the data provided by a user required to identify
        a cluster and user other user requirements.
        """
        form_data = {}
        # Do not include the following fields in the user data but do include
        # any 'advanced startup fields' that might be added in the future
        excluded_fields = ['sg_name', 'image_id', 'instance_id', 'kp_name',
                           'cloud', 'cloud_type', 'public_dns', 'cidr_range',
                           'kp_material', 'placement', 'flavor_id']
        for key, value in user_provided_data.items():
            if key not in excluded_fields:
                form_data[key] = value
        # If the following user data keys are empty, do not include them in the
        # request user data
        udkeys = [
            'post_start_script_url',
            'worker_post_start_script_url',
            'bucket_default',
            'share_string']
        for udkey in udkeys:
            if udkey in form_data and form_data[udkey] == '':
                del form_data[udkey]
        # If bucket_default was not provided, add a default value to the user data
        # (missing value does not play nicely with CloudMan's ec2autorun.py)
        if not form_data.get(
                'bucket_default', None) and self.cloud.bucket_default:
            form_data['bucket_default'] = self.cloud.bucket_default
        # Reuse the ``password`` for the ``freenxpass`` user data option
        if 'freenxpass' not in form_data and 'password' in form_data:
            form_data['freenxpass'] = form_data['password']
        # Convert form_data into the YAML format
        ud = yaml.dump(form_data, default_flow_style=False, allow_unicode=False)
        # Also include connection info about the selected cloud
        ci = self._get_cloud_info(self.cloud, as_str=True)
        return ud + "\n" + ci

    def _get_cloud_info(self, cloud, as_str=False):
        """
        Get connection information about a given cloud
        """
        ci = {}
        ci['cloud_type'] = cloud.cloud_type
        ci['region_name'] = cloud.region_name
        ci['region_endpoint'] = cloud.region_endpoint
        ci['is_secure'] = cloud.is_secure
        ci['ec2_port'] = cloud.ec2_port if cloud.ec2_port != '' else None
        ci['ec2_conn_path'] = cloud.ec2_conn_path
        # Include cidr_range only if not empty
        if cloud.cidr_range != '':
            ci['cidr_range'] = cloud.cidr_range
        ci['s3_host'] = cloud.s3_host
        ci['s3_port'] = cloud.s3_port if cloud.s3_port != '' else None
        ci['s3_conn_path'] = cloud.s3_conn_path
        if as_str:
            ci = yaml.dump(ci, default_flow_style=False, allow_unicode=False)
        return ci

    def _get_volume_placement(self, vol_id):
        """
        Returns the placement of a volume (or None, if it cannot be determined)
        """
        try:
            vol = self.ec2_conn.get_all_volumes(volume_ids=[vol_id])
        except EC2ResponseError as ec2e:
            bioblend.log.error(f"EC2ResponseError querying for volume {vol_id}: {ec2e}")
            vol = None
        if vol:
            return vol[0].zone
        else:
            bioblend.log.error("Requested placement of a volume '%s' that does not exist.", vol_id)
            return None

    def _find_placement(self, cluster_name, cluster=None):
        """
        Find a placement zone for a cluster with the name ``cluster_name``.

        By default, this method will search for and fetch given cluster's
        *persistent data*; alternatively, *persistent data* can be provided via
        the ``cluster`` parameter. This dict needs to have ``persistent_data``
        key with the contents of cluster's *persistent data*.
        If the cluster or the volume associated with the cluster cannot be found,
        cluster placement is set to ``None``.

        :rtype: dict
        :return: A dictionary with ``placement`` and ``error`` keywords.

        .. versionchanged:: 0.7.0
            The return value changed from a list to a dictionary.
        """
        placement = None
        response = {'placement': placement, 'error': None}
        cluster = cluster or self.get_cluster_pd(cluster_name)
        if cluster and 'persistent_data' in cluster:
            pd = cluster['persistent_data']
            try:
                if 'placement' in pd:
                    response['placement'] = pd['placement']
                elif 'data_filesystems' in pd:
                    # We have v1 format persistent data so get the volume first and
                    # then the placement zone
                    vol_id = pd['data_filesystems']['galaxyData'][0]['vol_id']
                    response['placement'] = self._get_volume_placement(vol_id)
                elif 'filesystems' in pd:
                    # V2 format.
                    for fs in [fs for fs in pd['filesystems'] if fs.get(
                            'kind', None) == 'volume' and 'ids' in fs]:
                        # All volumes must be in the same zone
                        vol_id = fs['ids'][0]
                        response['placement'] = self._get_volume_placement(
                            vol_id)
                        # No need to continue to iterate through
                        # filesystems, if we found one with a volume.
                        break
            except Exception as exc:
                response['error'] = \
                    f"Exception while finding placement for cluster '{cluster_name}'. This can indicate malformed instance data. Or that this method is broken: {exc}"
                bioblend.log.error(response['error'])
                response['placement'] = None
        else:
            bioblend.log.debug(
                f"Insufficient info about cluster {cluster_name} to get placement."
            )
        return response

    def find_placements(
            self, ec2_conn, instance_type, cloud_type, cluster_name=None):
        """
        Find a list of placement zones that support the specified instance type.

        If ``cluster_name`` is given and a cluster with the given name exist,
        return a list with only one entry where the given cluster lives.

        Searching for available zones for a given instance type is done by
        checking the spot prices in the potential availability zones for
        support before deciding on a region:
        http://blog.piefox.com/2011/07/ec2-availability-zones-and-instance.html

        Note that, currently, instance-type based zone selection applies only to
        AWS. For other clouds, all the available zones are returned (unless a
        cluster is being recreated, in which case the cluster's placement zone is
        returned sa stored in its persistent data.

        :rtype: dict
        :return: A dictionary with ``zones`` and ``error`` keywords.

        .. versionchanged:: 0.3
            Changed method name from ``_find_placements`` to ``find_placements``.
            Also added ``cluster_name`` parameter.

        .. versionchanged:: 0.7.0
            The return value changed from a list to a dictionary.
        """
        # First look for a specific zone a given cluster is bound to
        zones = []
        response = {'zones': zones, 'error': None}
        if cluster_name:
            placement = self._find_placement(cluster_name)
            if placement.get('error'):
                response['error'] = placement['error']
                return response
            response['zones'] = placement.get('placement', [])
        # If placement is not found, look for a list of available zones
        if not response['zones']:
            in_the_past = datetime.datetime.now() - datetime.timedelta(hours=1)
            back_compatible_zone = "us-east-1e"
            for zone in [
                    z for z in ec2_conn.get_all_zones() if z.state == 'available']:
                # Non EC2 clouds may not support get_spot_price_history
                if instance_type is None or cloud_type != 'ec2':
                    zones.append(zone.name)
                elif ec2_conn.get_spot_price_history(instance_type=instance_type,
                                                     end_time=in_the_past.isoformat(),
                                                     availability_zone=zone.name):
                    zones.append(zone.name)
            # Higher-lettered zones seem to have more availability currently
            zones.sort(reverse=True)
            if back_compatible_zone in zones:
                zones = [back_compatible_zone] + \
                    [z for z in zones if z != back_compatible_zone]
            if len(zones) == 0:
                response['error'] = f"Did not find availabilty zone for {instance_type}"
                bioblend.log.error(response['error'])
                zones.append(back_compatible_zone)
        return response

    def _checkURL(self, url):
        """
        Check if the ``url`` is *alive* (i.e., remote server returns code 200(OK)
        or 401 (unauthorized)).
        """
        try:
            p = urlparse(url)
            h = HTTPConnection(p[1])
            h.putrequest('HEAD', p[2])
            h.endheaders()
            r = h.getresponse()
            # CloudMan UI is pwd protected so include 401
            return r.status in (200, 401)
        except Exception:
            # No response or no good response
            return False


from functools import partial
import numpy as np
from . import augmentor_utils, database_sampler
from ...utils import common_utils

class DataAugmentor(object):
    def __init__(self, root_path, augmentor_configs, class_names, logger=None):
        self.root_path = root_path
        self.class_names = class_names
        self.logger = logger

        self.data_augmentor_queue = []
        aug_config_list = augmentor_configs if isinstance(augmentor_configs, list) \
            else augmentor_configs.AUG_CONFIG_LIST

        for cur_cfg in aug_config_list:
            if not isinstance(augmentor_configs, list):
                if cur_cfg.NAME in augmentor_configs.DISABLE_AUG_LIST:
                    continue
            cur_augmentor = getattr(self, cur_cfg.NAME)(config=cur_cfg)
            self.data_augmentor_queue.append(cur_augmentor)

    def gt_sampling(self, config=None):
        db_sampler = database_sampler.DataBaseSampler(
            root_path=self.root_path,
            sampler_cfg=config,
            class_names=self.class_names,
            logger=self.logger
        )
        return db_sampler

    def __getstate__(self):
        d = dict(self.__dict__)
        del d['logger']
        return d

    def __setstate__(self, d):
        self.__dict__.update(d)

    def random_world_flip(self, data_dict=None, config=None):
        if data_dict is None:
            return partial(self.random_world_flip, config=config)
        gt_boxes, points = data_dict['gt_boxes'], data_dict['points']
        for cur_axis in config['ALONG_AXIS_LIST']:
            assert cur_axis in ['x', 'y']
            gt_boxes, points = getattr(augmentor_utils, 'random_flip_along_%s' % cur_axis)(
                gt_boxes, points,
            )

        data_dict['gt_boxes'] = gt_boxes
        data_dict['points'] = points
        return data_dict

    def random_world_rotation(self, data_dict=None, config=None):
        if data_dict is None:
            return partial(self.random_world_rotation, config=config)
        rot_range = config['WORLD_ROT_ANGLE']
        if not isinstance(rot_range, list):
            rot_range = [-rot_range, rot_range]
        gt_boxes, points = augmentor_utils.global_rotation(
            data_dict['gt_boxes'], data_dict['points'], rot_range=rot_range
        )

        data_dict['gt_boxes'] = gt_boxes
        data_dict['points'] = points
        return data_dict

    def random_world_scaling(self, data_dict=None, config=None):
        if data_dict is None:
            return partial(self.random_world_scaling, config=config)
        gt_boxes, points = augmentor_utils.global_scaling(
            data_dict['gt_boxes'], data_dict['points'], config['WORLD_SCALE_RANGE']
        )
        data_dict['gt_boxes'] = gt_boxes
        data_dict['points'] = points
        return data_dict

    def filter_by_min_points(self, data_dict=None, config=None):
        if data_dict is None:
            return partial(self.filter_by_min_points, config=config)
        gt_boxes = data_dict['gt_boxes']
        if gt_boxes.size == 0:  # empty
            return data_dict

        points_num = config['POINTS_NUM']
        assert isinstance(points_num, list)

        points_num_dict = {}
        for name_num in points_num:
            name, min_num = name_num.split(':')
            min_num = int(min_num)
            points_num_dict[name] = min_num
        if 'gt_boxes_points_num' in data_dict:
            gt_boxes = data_dict['gt_boxes']
            gt_names = data_dict['gt_names']
            gt_boxes_points_num = data_dict['gt_boxes_points_num']
            keeped = np.zeros_like(gt_boxes_points_num, dtype=np.bool)
            for index, (name, num) in enumerate(zip(gt_names, gt_boxes_points_num)):
                if num > points_num_dict[name]:
                    keeped[index] = True
            data_dict['gt_boxes'] = gt_boxes[keeped]
            data_dict['gt_names'] = gt_names[keeped]
            data_dict['gt_boxes_points_num'] = gt_boxes_points_num[keeped]
            if 'gt_boxes_mask' in data_dict:
                gt_boxes_mask = data_dict['gt_boxes_mask']
                data_dict['gt_boxes_mask'] = gt_boxes_mask[keeped]

        else:
            raise Exception
        return data_dict

    def filter_by_camera_fov(self, data_dict=None, config=None):
        if data_dict is None:
            return partial(self.filter_by_camera_fov, config=config)

        fov = config['FOV']
        assert isinstance(fov, list)
        fov_neg_angle, fov_pos_angle = fov[0], fov[1]
        fov_neg_cos, fov_pos_cos = np.cos(fov_neg_angle / 180 * np.pi), np.cos(fov_pos_angle / 180 * np.pi)
        # filter points
        points = data_dict['points']
        x, y = points[:, 0], points[:, 1]
        fov_flag = np.logical_or(
            (x / np.linalg.norm((x, y), ord=2, axis=0) > fov_neg_cos) & (y < 0),
            (x / np.linalg.norm((x, y), ord=2, axis=0) > fov_pos_cos) & (y > 0)
        )
        data_dict['points'] = points[fov_flag]
        # filter box
        if 'gt_boxes' in data_dict:
            # filter box which center is not in fov
            gt_boxes = data_dict['gt_boxes']
            if gt_boxes.size == 0:  # empty
                return data_dict

            center_x, center_y = gt_boxes[:, 0], gt_boxes[:, 1]
            x, y = center_x, center_y
            fov_flag = np.logical_or(
                (x / np.linalg.norm((x, y), ord=2, axis=0) > fov_neg_cos) & (y < 0),
                (x / np.linalg.norm((x, y), ord=2, axis=0) > fov_pos_cos) & (y > 0)
            )
            data_dict['gt_boxes'] = gt_boxes[fov_flag]
            if 'gt_names' in data_dict:
                gt_names = data_dict['gt_names']
                data_dict['gt_names'] = gt_names[fov_flag]
            if 'gt_boxes_points_num' in data_dict:
                gt_boxes_points_num = data_dict['gt_boxes_points_num']
                data_dict['gt_boxes_points_num'] = gt_boxes_points_num[fov_flag]
            if 'gt_boxes_mask' in data_dict:
                gt_boxes_mask = data_dict['gt_boxes_mask']
                data_dict['gt_boxes_mask'] = gt_boxes_mask[fov_flag]
        return data_dict

    def forward(self, data_dict):
        """
        Args:
            data_dict:
                points: (N, 3 + C_in)
                gt_boxes: optional, (N, 7) [x, y, z, dx, dy, dz, heading]
                gt_names: optional, (N), string
                ...

        Returns:
        """
        for cur_augmentor in self.data_augmentor_queue:
            data_dict = cur_augmentor(data_dict=data_dict)
        if data_dict['gt_boxes'].size > 0:
            data_dict['gt_boxes'][:, 6] = common_utils.limit_period(
                data_dict['gt_boxes'][:, 6], offset=0.5, period=2 * np.pi
            )
        if 'calib' in data_dict:
            data_dict.pop('calib')
        if 'road_plane' in data_dict:
            data_dict.pop('road_plane')
        if 'gt_boxes_mask' in data_dict:
            gt_boxes_mask = data_dict['gt_boxes_mask']
            data_dict['gt_boxes'] = data_dict['gt_boxes'][gt_boxes_mask]
            data_dict['gt_names'] = data_dict['gt_names'][gt_boxes_mask]
            data_dict.pop('gt_boxes_mask')
        return data_dict


import numpy
import gudhi

from PySide2.QtCore import Slot

from PySide2.QtWidgets import (QStackedWidget, QWidget, QVBoxLayout, QRadioButton, QHBoxLayout)

from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas
from matplotlib.figure import Figure

class PersistenceGraphsTabWidget(QWidget):
    def __init__(self, main_ui):
        super().__init__()
        self.main_ui = main_ui

        buttons = QWidget()
        button_layout = QHBoxLayout()

        self.barcode_button = QRadioButton("Barcode")
        self.barcode_button.setChecked(True)
        self.barcode_button.clicked.connect(self.barcode_show)
        self.diagram_button = QRadioButton("Diagram")
        self.diagram_button.clicked.connect(self.diagram_show)
        self.density_button = QRadioButton("Density")
        self.density_button.clicked.connect(self.density_show)

        button_layout.addWidget(self.barcode_button)
        button_layout.addWidget(self.diagram_button)
        button_layout.addWidget(self.density_button)

        buttons.setLayout(button_layout)

        self.figures = QStackedWidget()

        self.add_diagram_widgets()

        self.figures.setCurrentIndex(0)

        layout = QVBoxLayout()
        layout.addWidget(buttons)
        layout.addWidget(self.figures)
        self.setLayout(layout)

    @Slot()
    def barcode_show(self):
        self.figures.setCurrentIndex(0)

    @Slot()
    def diagram_show(self):
        self.figures.setCurrentIndex(1)

    @Slot()
    def density_show(self):
        self.figures.setCurrentIndex(2)

    def add_diagram_widgets(self):
        plot = gudhi.plot_persistence_barcode(persistence=self.main_ui.alpha_complex.persistence,
                                              legend=False)
        barcode_canvas = FigureCanvas(plot.figure(1))
        plot.close()

        plot = gudhi.plot_persistence_diagram(persistence=self.main_ui.alpha_complex.persistence,
                                              legend=False)
        diagram_canvas = FigureCanvas(plot.figure(1))
        plot.close()

        density_canvas = FigureCanvas(plot.figure(1))
        try:
            plot = gudhi.plot_persistence_density(persistence=self.main_ui.alpha_complex.persistence, dimension = 1,
                                          legend=True)
            plot.close()
        except:
            print("Density computation not possible")

        self.figures.addWidget(barcode_canvas)
        self.figures.addWidget(diagram_canvas)
        self.figures.addWidget(density_canvas)

    def update(self):
        self.figures.removeWidget(self.figures.currentWidget())
        self.figures.removeWidget(self.figures.currentWidget())
        self.figures.removeWidget(self.figures.currentWidget())
        self.add_diagram_widgets()


import numpy as np
import matplotlib.pyplot as plt
import imageio as imio
import os
plt.rcParams['image.cmap'] = 'gray'

imsize = (192,168)
nums = ['B' + (n < 10)*'0' + str(n) for n in range(1,40)]
nums.remove('B14') # for some reason the dataset is missing 'B14'
path = 'yalefaces_cropped/CroppedYale/yale{}/'

# load face matrix
X = []
for n in nums:
    for d in os.listdir(path.format(n)):
        im = imio.imread(path.format(n)+d)
        X.append(im.flatten().astype('float64'))

# each face flattened into column
X = np.array(X).T

def imshow(ax,im):
    ax.imshow(im)
    ax.set_xticks([]), ax.set_yticks([])

X_ave = X.mean(axis=1).reshape((X.shape[0],1))
X0 = X - X_ave # remove mean from each row

u,s,v = np.linalg.svd(X,full_matrices=False)
u0,s0,v0 = np.linalg.svd(X0,full_matrices=False)

fig,ax = plt.subplots()
imshow(ax,X_ave.reshape(imsize))
ax.set_title('The "average" face')

# plot singular values
def plot_singvals(s,title):
    cumu = []
    tot = np.sum(s)
    for i in range(len(s)):
        cumu.append(np.sum(s[:i])/tot)

    fig,axes = plt.subplots(2,1,sharex=True)
    axes[0].plot(s/tot,'ro',markersize=2)
    axes[0].set_ylabel('$\sigma_j$ normalized')
    axes[1].plot(cumu,'r',linewidth=3)
    axes[1].axhline(0.95,label='95%')
    axes[1].legend()
    axes[1].set_ylabel('cumulative variance')
    axes[0].set_title(title)

plot_singvals(s0,'Singular values of cropped faces')

# plot principle components of the face matrix
def plot_PCs(u,s,title,rows=3,cols=5):
    fig,axes = plt.subplots(rows,cols)

    for r in range(rows):
        for c in range(cols):
            n = r*cols+c
            im = u[:,n].reshape(imsize)
            imshow(axes[r,c],im)
            axes[r,c].set_xlabel('$\sigma_{' + str(n+1) +
                '}$ = ' + str(int(s[n])))

    fig.text(0.5,0.95,title,ha='center',fontsize=12)

plot_PCs(u0,s0,'The first 15 principle components of the cropped faces')

# projection of face matrix
def plot_proj(X,u,n,seed,title):
    fig,axes = plt.subplots(3,4)
    np.random.seed(seed)
    for i in range(3):
        face = np.random.randint(X.shape[1])
        print(face)
        for j in range(axes.shape[1]-1):
            im = X[:,face+j].reshape(imsize)
            proj = u[:,:n].T @ X[:,face+j]
            imshow(axes[i,j],im)
            axes[i,j].set_xlabel('shot ' + str(j+1))
            axes[i,axes.shape[1]-1].plot(proj)
            axes[i,axes.shape[1]-1].set_xticks([])
            axes[i,axes.shape[1]-1].set_yticks([])
    fig.text(0.5,0.95,title,ha='center',fontsize=12)

plot_proj(X0,u0,15,10,
          'Projection of 3 faces onto the first 15 principle componenets')

# low rank approximations of faces
fig,axes = plt.subplots(2,5)
for row in range(len(axes)):
    face = np.random.randint(X.shape[1])
    face_full = X[:,face].reshape(imsize)
    axes[row,0].imshow(face_full)
    axes[row,0].set_xlabel('full rank')
    face_r = np.zeros(face_full.size)
    rank = [0,10,100,250,500]
    for i in range(1,len(rank)):
        for r in range(rank[i-1],rank[i]):
            face_r += s[r] * u[:,r] * v[r,face]
        axes[row,i].imshow(face_r.reshape(imsize))
        axes[row,i].set_xlabel('rank {}'.format(rank[i]))
    for ax in axes[row]:
        ax.set_xticks([]), ax.set_yticks([])

fig.text(0.5,0.95,'Low rank reconstructions of faces',
         ha='center',fontsize=12)

# uncropped images
path = 'yalefaces/'
imsize = (243,320)
subjs = ['subject' + (n < 10)*'0' + str(n) for n in range(1,16)]
X = []
for pic in os.listdir(path):
    im = imio.imread(os.path.join(path,pic))
    X.append(im.flatten())
X = np.array(X).T

X = X - np.mean(X,axis=1).reshape((X.shape[0],1))
u,s,v = np.linalg.svd(X,full_matrices=False)

# plot singular values
plot_singvals(s,'Singular Values of uncropped faces')

# plot PCs of uncropped faces
plot_PCs(u,s,'The first 15 principle components of the cropped faces')

plot_proj(X,u,25,70,
          'Projection of 3 faces onto the first 15 priciple components')

SVD and ML - Facial images and music classification/faces.py


from django.shortcuts import render, get_object_or_404
from tab.models import Tab

def index(request):
    zadnje_tabele = Tab.objects.raw('SELECT * FROM tab_Tab GROUP BY groupname')
    context = {'zadnje_tabele': zadnje_tabele}
    return render(request, 'tab/index.html', context)

def detail(request, id):
    tab = get_object_or_404(Tab, pk=id)
    gn = tab.groupname
    tb = Tab.objects.raw('SELECT * FROM tab_Tab WHERE groupname = %s', [gn])
    context = {'group': tb}
    return render(request, 'tab/detail.html', context)


#
# Common routines for create-topology script
#
import sys
import os
from jinja2 import Environment, FileSystemLoader

LOGGING=False
VERBOSE=False

def fatal(text):
  print(text,file=sys.stderr)
  sys.exit(1)

err_count = 0

def error(text):
  global err_count
  print(text,file=sys.stderr)
  err_count = err_count + 1

def exit_on_error():
  global err_count
  if err_count > 0:
    sys.exit(1)

def template(j2,data,path):
  ENV = Environment(loader=FileSystemLoader(path),trim_blocks=True,lstrip_blocks=True)
  template = ENV.get_template(j2)
  return template.render(**data)

def get_value(data,path=[],default=None):
  for k in path:
    if not(type(data) is dict):
      return data
    if not k in data:
      return default
    data = data.get(k)
  return data

def merge_defaults(data,defaults):
  if not data:
    return defaults

  if type(data) is dict and type(defaults) is dict:
    for (k,v) in defaults.items():
      if not k in data or isinstance(data.get(k),dict):
        data[k] = merge_defaults(data.get(k),defaults[k])
  return data

def set_verbose():
  VERBOSE=True

def print_verbose(t):
  if VERBOSE:
    print(t)


#!/usr/bin/python3
# -*- coding: utf-8 -*-

# 导入Flask套件
from flask import Flask, request, send_from_directory

# 处理上传的文件
import os
from werkzeug.utils import secure_filename

# 创建Flask app物件
app = Flask(__name__)

# 创建 output 输出内容
output = [
    {
        "pid": "1",
        "title": "Example01",
        "price": 10,
        "img": "https://picsum.photos/id/999/1200/600",
        "isAvailable": True
    },
    {
        "id": "2",
        "title": "Example02",
        "price": 60,
        "img": "https://picsum.photos/id/1070/1200/600",
        "isAvailable": True
    }
]

# 测接口 get
@app.route("/get", methods=['GET'])
def getTest():
    name = request.args.get('name')  # 获取 url 上传过来的参数,对 post请求也适用
    if name is None:
        name = "Noname"
    print(name)
    item = {'name': name, 'output': output}
    return item, 200

# 测接口 post
@app.route("/post", methods=['POST'])
def postParamAsJson():
    name1 = request.args.get('name')  # 获取 url 上传过来的参数,对 post请求也适用

    print("args name1:", name1)
    print("request.data:", request.data)
    params_json = request.get_json()  # 获取 json 格式数据
    print("request.get_json():\n", params_json)
    name = params_json["name"]  # 取其中的参数
    if name1 is not None:
        name = name1 + name
    item = {'name': name, 'output': output}
    return item, 200

# 测接口 上传文件
@app.route('/upload', methods=['GET', 'POST'])
def upload():
    if request.method == 'POST':
        f = request.files['file']
        # print(request.files)
        # 参数: 1.保存路径    2.文件大小
        f.save(os.path.join("files", secure_filename(f.filename)))
        return 'file uploaded successfully', 200

    if request.method == 'GET':
        return "上传文件需要用 post!", 203

# 测接口 显示图片
@app.route("/<filename>", methods=['GET', 'POST'])
def download2(filename):
    # as_attachment : True 下载文件, False 不下载文件
    return send_from_directory(path="./files/", directory="./files/", filename=filename, as_attachment=False)

# 测接口 下载文件
@app.route("/download", methods=['GET', 'POST'])
def download():
    name = request.args.get('name')  # 获取 url 上传过来的参数,对 post请求也适用
    if name is None:
        name = "test.ipa"
    return send_from_directory(path="./files/", directory="./files/", filename=name, as_attachment=True)

if __name__ == "__main__":
    app.run(host="0.0.0.0", port=8090, debug=True)


from __future__ import unicode_literals

from django.db.models import Manager, Q

class WebHookTargetManager(Manager):
    """Manages WebHookTarget models.

    This provides a utility function for querying WebHookTargets for a
    given event.
    """
    def for_event(self, event, local_site_id=None, repository_id=None):
        """Returns a list of matching webhook targets for the given event."""
        if event == self.model.ALL_EVENTS:
            raise ValueError('"%s" is not a valid event choice' % event)

        q = Q(enabled=True) & Q(local_site=local_site_id)

        if repository_id is None:
            q &= (Q(apply_to=self.model.APPLY_TO_ALL) |
                  Q(apply_to=self.model.APPLY_TO_NO_REPOS))
        else:
            q &= (Q(apply_to=self.model.APPLY_TO_ALL) |
                  (Q(apply_to=self.model.APPLY_TO_SELECTED_REPOS) &
                   Q(repositories=repository_id)))

        return [
            target
            for target in self.filter(q)
            if event in target.events or self.model.ALL_EVENTS in target.events
        ]


import sklearn.linear_model as SKL
from sklearn.linear_model import LogisticRegression as LR
from sklearn.ensemble import RandomForestClassifier as RFC
from sklearn.ensemble import AdaBoostClassifier as ABC
from sklearn.neural_network import MLPClassifier as MLPC
from sklearn.neighbors import KNeighborsClassifier as KNC
from sklearn.svm import SVC
# from sklearn.gaussian_process import GaussianProcessClassifier as GPC
# from sklearn.gaussian_process.kernels import RBF
from sklearn.tree import DecisionTreeClassifier as DTC
from sklearn.naive_bayes import GaussianNB as GNB
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis as QDA
import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import percentileofscore
import copy
import pandas as pd

model_strings = ["LR",
                 "RFC",
                 #"ABC",
                 "MLPC",
                 "KNC",
                 "SVC",
                 #"DTC",
                 #"GNB",
                 "QDA"]
model_types = [LR,
               RFC,
               #ABC,
               MLPC,
               KNC,
               SVC,
               #DTC,
               #GNB,
               QDA]
models = [LR(),
          RFC(n_estimators=30),
          #ABC(),
          MLPC(),
          KNC(),
          SVC(probability=True),
          #DTC(),
          #GNB(),
          QDA()]
models2 = copy.deepcopy(models)

### experiment bright students math finance
N = 10000 ## 1000 of each group (groups S and T)

minority_percent = 0.3
MIN = int(minority_percent * N)
MAJ = int((1 - minority_percent) * N)
# print(MIN, MAJ)
# p_S_brightmath = 0.9
# p_T_brightmath = 0.1
bright_percent = 0.2

d = 0.2 # this is percent shift of added bias (of MIN and MAJ groups)
r = 0.15

a = 7
b = 2

quota = 1700

## first attribute is 1 if in group S, otherwise 0 for group T

## Generate Group S, Group T data
### X columns:
### Group Label, Math Grades, Writing Grades, Bright (or not)

def generate_X(biased = False):
	### S
	S = np.zeros((MIN, 4))
	S[ :,0] = 1
	brightS = np.random.choice(MIN, int(bright_percent * MIN), replace = False)

	S[ :,1] = [np.random.beta(a,b) if i in brightS else np.random.beta(b,a) for i in range(MIN)]
	S[ :,2] = [np.random.beta(b,a) if i in brightS else np.random.beta(a,b) for i in range(MIN)]

	removed = set()
	if biased == True:
		removed = set(np.random.choice(brightS, int(d/2 * MIN), replace = False))
	brightS = set(brightS)
	brightS = brightS.difference(removed)
	# print("Bright S ",len(brightS))

	S[ :,3] = [1 if i in brightS else 0 for i in range(MIN)]

	### T
	T = np.zeros((MAJ, 4))
	T[ :,0] = 0
	brightT = np.random.choice(MAJ, int(bright_percent * MAJ), replace = False)
	notbrightT = list(set(range(MAJ)).difference(set(brightT)))

	T[ :,1] = [np.random.beta(b,a) if i in brightT else np.random.beta(a,b) for i in range(MAJ)]
	T[ :,2] = [np.random.beta(a,b) if i in brightT else np.random.beta(b,a) for i in range(MAJ)]

	added = set()
	if biased == True:
		added = set(np.random.choice(notbrightT, int(d/2 * MAJ), replace = False))
	brightT = set(brightT)
	brightT = brightT.union(added)
	# print("Bright T ",len(brightT))

	T[ :,3] = [1 if i in brightT else 0 for i in range(MAJ)]

	return S,T

X_S, X_T = generate_X(biased = True)
X_B = np.append(X_S, X_T, axis=0)

### test data of ideal world

X_S_test, X_T_test = generate_X(biased = False)
X_test = np.append(X_S_test, X_T_test, axis=0) ## use this for testing

def scatter(X, name):
	df1 = pd.DataFrame(X)
	dfSBright = df1[(df1[3] == 1) & (df1[0] == 1)]
	dfSNot = df1[(df1[3] == 0) & (df1[0] ==1) ]
	# print(dfSNot.shape)
	dfTBright = df1[(df1[3] == 1) & (df1[0] == 0)]
	dfTNot = df1[df1[3] == 0  & (df1[0] == 0)]

	# plt.scatter(dfSNot[1], dfSNot[2], c = "pink")
	# plt.scatter(dfSBright[1], dfSBright[2], c = "blue")

	# plt.scatter(dfTNot[1], dfTNot[2], c = "orange")
	# plt.scatter(dfTBright[1], dfTBright[2], c = "red")

	data = (dfSNot, dfSBright, dfTNot, dfTBright)
	colors = ("black", "blue", "yellow", "red")
	groups = ("SNot", "SBright", "TNot", "TBright")
	fig = plt.figure()
	ax = fig.add_subplot(1,1,1)
	for data, color, group in zip(data, colors, groups):
		ax.scatter(data[1],data[2], c= color, edgecolors = None, label = group, s = 1)
	plt.legend(loc = 3)
	# plt.show()
	plt.savefig(name)
	plt.close()

scatter(X_test, "IdealV2")
scatter(X_B, "BiasedV2")

### fit regular biased world
reg_LR = SKL.LogisticRegression()
reg_RFC = RFC(n_estimators = 30)

reg_LR.fit(X_B[:, :-1], X_B[ : ,-1])
reg_RFC.fit(X_B[:, :-1], X_B[ : ,-1])

pred_LR = reg_LR.predict(X_test[:, :-1])
pred_RFC = reg_RFC.predict(X_test[:, :-1])

print("Score for Biased LR", reg_LR.score(X_test[:, :-1], X_test[:, -1]))
print("Score for Biased RFC", reg_RFC.score(X_test[:, :-1], X_test[:, -1]))

### fit perturbed biased world
## predict on the training data to get some probabilities

# create perturbed data
def generate_X_P(model, perturb_ratio=0.5):
	X_B_copy = copy.deepcopy(X_B)
	ranks = model.predict_proba(X_B_copy[ : , :-1])

	newData = np.append(X_B_copy, np.array([[x] for x in ranks[ :,1]]), axis = 1)
	newDf = pd.DataFrame(newData)

	S = newDf[(newDf[0]  == 1) & (newDf[3]  == 0)]
	sorted_S = S.sort_values(4)

	# ### first do the perturbing
	not_bright_S  = []
	for idx,x in enumerate(X_S[:, -1]):
		if x < 1:
			not_bright_S += [idx]

	bright_T  = []
	for idx,x in enumerate(X_T[:, -1]):
		if x > 0:
			bright_T += [idx]

	# perturb_ratio = 0.5 #len(not_bright_S) / (len(not_bright_S) + len(bright_T))

	X_P = copy.deepcopy(X_B_copy)
	X_P = pd.DataFrame(X_P)
	num_perturb = int(r * MIN * (perturb_ratio))
	to_perturb = sorted_S.index.tolist()[-num_perturb:]
	X_P.at[to_perturb, 3] = 1

	# bright in T
	T = newDf[(newDf[0]  == 0) & (newDf[3]  == 1)]
	sorted_T = T.sort_values(4)
	# 1-perturb ratio works for 0.5, fix for other values
	num_perturb_T = int(r * MAJ * (1- perturb_ratio))
	to_perturb_T = sorted_T.index.tolist()[:num_perturb_T]
	X_P.at[to_perturb_T,3] = 0

	# print(X_P.shape)
	X_P = np.array(X_P)
	return X_P

# scatter(X_P, "PerturbedV2")
X_P = generate_X_P(reg_RFC, perturb_ratio=0.5)

reg1_LR = SKL.LogisticRegression()
reg1_RFC = RFC(n_estimators = 30)

reg1_LR.fit(X_P[:, :-1], X_P[:,-1])
reg1_RFC.fit(X_P[:, :-1], X_P[:,-1])

pred1_LR = reg1_LR.predict(X_test[:, :-1])
pred1_RFC = reg1_RFC.predict(X_test[:, :-1])

print("Score for Biased LR perturbed" ,reg1_LR.score(X_test[:, :-1], X_test[:, -1]))
print("Score for Biased RFC perturbed" ,reg1_RFC.score(X_test[:, :-1], X_test[:, -1]))

def predict(X, C):
	ranks = C.predict_proba(X[ : , :-1])
	newData = np.append(X, np.array([[x] for x in ranks[ :,1]]), axis = 1)
	newDf = pd.DataFrame(newData)
	sorted_Df = newDf.sort_values(4)
	# quota = int(sum(pred1_RFC))
	top_indices = set(sorted_Df.index.tolist()[-quota:])
	# new_indices =
	# print(newDf_P.iloc[top_indices[0]])
	# for indices: setting classifications to 1 and rest to 0
	pred = np.array([1 if i in top_indices else 0 for i in range(N)])
	return pred

pred2_LR = predict(X_test, reg1_LR)
pred2_RFC = predict(X_test, reg1_RFC)

results_LR = [int((a and not b) or (b and not a)) for a, b in zip(pred2_LR, X_test[:, -1])]
score_LR = (N - sum(results_LR)) / N

results_RFC = [int((a and not b) or (b and not a)) for a, b in zip(pred2_RFC, X_test[:, -1])]
score_RFC = (N - sum(results_RFC)) / N

print("Score for Biased LR perturbed quota", score_LR)
print("Score for Biased RFC perturbed quota", score_RFC)

# print("Score for Biased perturbed quota " ,reg2.score(X_test[:, :-1], X_test[:, -1]))

## prediction is different
# diff = pred1 + pred
# diff_people = []
# for idx,i in enumerate(diff):
# 	if i == 1:
# 		diff_people += [idx]

# F = []
# for i in diff_people:
# 	## if true value is also different
# 	F = F + [X_I[i][2]]
# len(diff_people)

# plt.hist(F)
		# print(X_B[i], X_P[i])
# plt.show()

# for i in range(2*N):
# 	## if true value is also different
# 	if (X_B[i][2] != X_P[i][2]):
# 		print(X_B[i], X_P[i])

# ---------------------------------------------------------------------------------
# The Four Fundamental Numbers (TN, FN, TP, FP)

# TN : Classified as 0 (pred) and really 0 (X_I_test[:, 2])

# FN : Classified as 0 but really 1

# FP : Classified as 1 but really 0

# TP : Classified as 1 and really 1

def get_fundamentals(pred, actual):
	num_tests = len(pred)
	fundamentals = [0] * 4  # TN, FN, FP, TP
	for i in range(num_tests):
		fundamentals[int(2*pred[i] + actual[i])] += 1
	return fundamentals

def get_stats(pred, actual):
	fundamentals = get_fundamentals(pred, actual)
	TN, FN, FP, TP = fundamentals
	PPV = TP / (0.001 + TP + FP)
	FPR = FP / (FP + TN)
	FNR = FN / (FN + TP)
	return PPV, FPR, FNR

PPV_S, FPR_S, FNR_S = get_stats(pred_RFC[:MIN], X_test[:MIN, -1]) # biased
PPV_T, FPR_T, FNR_T = get_stats(pred_RFC[MIN:], X_test[MIN:, -1]) # biased
PPV1_S, FPR1_S, FNR1_S = get_stats(pred1_RFC[:MIN], X_test[:MIN, -1]) # biased perturbed
PPV1_T, FPR1_T, FNR1_T = get_stats(pred1_RFC[MIN:], X_test[MIN:, -1]) # biased perturbed
PPV2_S, FPR2_S, FNR2_S = get_stats(pred2_RFC[:MIN], X_test[:MIN, -1]) # biased perturbed quota
PPV2_T, FPR2_T, FNR2_T = get_stats(pred2_RFC[MIN:], X_test[MIN:, -1]) # biased perturbed quota

print("b = biased, p = perturbed, q = quota")
print("S:")
print("       b       bp      bpq")
print("PPVs = {:.4f}, {:.4f}, {:.4f}".format(PPV_S, PPV1_S, PPV2_S))
print("FPRs = {:.4f}, {:.4f}, {:.4f}".format(FPR_S, FPR1_S, FPR2_S))
print("FNRs = {:.4f}, {:.4f}, {:.4f}".format(FNR_S, FNR1_S, FNR2_S))
print("T:")
print("       b       bp      bpq")
print("PPVs = {:.4f}, {:.4f}, {:.4f}".format(PPV_T, PPV1_T, PPV2_T))
print("FPRs = {:.4f}, {:.4f}, {:.4f}".format(FPR_T, FPR1_T, FPR2_T))
print("FNRs = {:.4f}, {:.4f}, {:.4f}".format(FNR_T, FNR1_T, FNR2_T))

preds, b_scores, bp_scores = [], [], []
for i, model in enumerate(models):
	## biased
	print(model_strings[i])
	model.fit(X_B[:, :-1], X_B[ : ,-1])
	pred = model.predict(X_test[:, :-1])
	b_score = model.score(X_test[:, :-1], X_test[:, -1])
	preds.append(pred)
	b_scores.append(b_score)

	## biased perturbed
	X_P = generate_X_P(model, perturb_ratio=0.5)
	model2 = models2[i]
	model2.fit(X_P[:, :-1], X_P[:,-1])
	bp_score = model2.score(X_test[:, :-1], X_test[:, -1])
	bp_scores.append(bp_score)

print(model_strings)
print(b_scores)
print(bp_scores)

## Testing individual fairness
### Similar individuals should be classified similarly

## create custom data
M = 100
X_IF = []
for i in range(2):
	for j in range(M):
		for k in range(M):
			X_IF.append([i, j/M, k/M])
X_IF = np.array(X_IF)

def produce_graphs(X, name_S, name_T):
	df = pd.DataFrame(X)
	dfSBright = df[(df[3] == 1) & (df[0] == 1)]
	dfSNot = df[(df[3] == 0) & (df[0] ==1) ]
	dfTBright = df[(df[3] == 1) & (df[0] == 0)]
	dfTNot = df[df[3] == 0  & (df[0] == 0)]

	data = (dfSNot, dfSBright)
	colors = ("blue", "red")
	groups = ("SNot", "SBright")

	fig = plt.figure()
	ax = fig.add_subplot(1,1,1)
	for data, color, group in zip(data, colors, groups):
		ax.scatter(data[1],data[2], c= color, edgecolors = None, label = group, s = 1)
	plt.legend(loc = 3)
	plt.savefig(name_S)
	plt.close()

	data = (dfTNot, dfTBright)
	colors = ("blue", "red")
	groups = ("TNot", "TBright")

	fig = plt.figure()
	ax = fig.add_subplot(1,1,1)
	for data, color, group in zip(data, colors, groups):
		ax.scatter(data[1],data[2], c= color, edgecolors = None, label = group, s = 1)
	plt.legend(loc = 3)
	plt.savefig(name_T)
	plt.close()

pred_IF, pred2_IF = [], []
for i, model in enumerate(models):
	X_IF_new = copy.deepcopy(X_IF)
	## predict using biased model + produce graph
	pred_IF.append(model.predict(X_IF[:,:3]))
	X_IF_new = np.c_[X_IF_new, np.array(pred_IF[i])]
	name_S = "individual_fairness/%s_S_M-%04.f_b" % (model_strings[i], M)
	name_T = "individual_fairness/%s_T_M-%04.f_b" % (model_strings[i], M)
	produce_graphs(X_IF_new, name_S, name_T)

	X_IF_new = copy.deepcopy(X_IF)
	## predict using biased, perturbed model + produce graph
	pred2_IF.append(models2[i].predict(X_IF[:,:3]))
	X_IF_new = np.c_[X_IF_new, np.array(pred2_IF[i])]
	name_S = "individual_fairness/%s_S_M-%04.f_bp" % (model_strings[i], M)
	name_T = "individual_fairness/%s_T_M-%04.f_bp" % (model_strings[i], M)
	produce_graphs(X_IF_new, name_S, name_T)


import logging
import datetime

class LoggingService(logging.Logger):
    def __init__(self, name, level=logging.NOTSET):
        self.name = name
        return super(LoggingService, self).__init__(name, level)

    def default(self):
        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        fileName = "logs//{0}_{1}.log".format(self.name, datetime.datetime.now().strftime("%Y-%m-%d"))
        handler = logging.FileHandler(fileName)
        handler.setFormatter(formatter)
        self.addHandler(handler)


import warnings
from typing import Any, Dict, List, Optional, Tuple

from ConfigSpace.configuration_space import Configuration, ConfigurationSpace

import numpy as np

from sklearn.base import RegressorMixin

from autoPyTorch.constants import STRING_TO_TASK_TYPES
from autoPyTorch.pipeline.base_pipeline import BasePipeline
from autoPyTorch.pipeline.components.base_choice import autoPyTorchChoice
from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.TabularColumnTransformer import (
    TabularColumnTransformer
)
from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.encoding.base_encoder_choice import (
    EncoderChoice
)
from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.imputation.SimpleImputer import SimpleImputer
from autoPyTorch.pipeline.components.preprocessing.tabular_preprocessing.scaling.base_scaler_choice import ScalerChoice
from autoPyTorch.pipeline.components.setup.early_preprocessor.EarlyPreprocessing import EarlyPreprocessing
from autoPyTorch.pipeline.components.setup.lr_scheduler.base_scheduler_choice import SchedulerChoice
from autoPyTorch.pipeline.components.setup.network.base_network_choice import NetworkChoice
from autoPyTorch.pipeline.components.setup.network_initializer.base_network_init_choice import (
    NetworkInitializerChoice
)
from autoPyTorch.pipeline.components.setup.optimizer.base_optimizer_choice import OptimizerChoice
from autoPyTorch.pipeline.components.training.data_loader.feature_data_loader import FeatureDataLoader
from autoPyTorch.pipeline.components.training.trainer.base_trainer_choice import (
    TrainerChoice
)

class TabularRegressionPipeline(RegressorMixin, BasePipeline):
    """This class is a proof of concept to integrate AutoSklearn Components

    It implements a pipeline, which includes as steps:

        ->One preprocessing step
        ->One neural network

    Contrary to the sklearn API it is not possible to enumerate the
    possible parameters in the __init__ function because we only know the
    available classifiers at runtime. For this reason the user must
    specifiy the parameters by passing an instance of
    ConfigSpace.configuration_space.Configuration.

    Args:
        config (Configuration)
            The configuration to evaluate.
        random_state (Optional[RandomState): random_state is the random number generator

    Attributes:
    Examples
    """

    def __init__(
        self,
        config: Optional[Configuration] = None,
        steps: Optional[List[Tuple[str, autoPyTorchChoice]]] = None,
        dataset_properties: Optional[Dict[str, Any]] = None,
        include: Optional[Dict[str, Any]] = None,
        exclude: Optional[Dict[str, Any]] = None,
        random_state: Optional[np.random.RandomState] = None,
        init_params: Optional[Dict[str, Any]] = None
    ):
        super().__init__(
            config, steps, dataset_properties, include, exclude,
            random_state, init_params)

    def fit_transformer(
        self,
        X: np.ndarray,
        y: np.ndarray,
        fit_params: Optional[Dict[str, Any]] = None
    ) -> Tuple[np.ndarray, Optional[Dict[str, Any]]]:
        """Fits the pipeline given a training (X,y) pair

        Args:
            X (np.ndarray): features from which to guess targets
            y (np.ndarray): classification targets for this task
            fit_params (Optional[Dict[str, Any]]]): handy communication dictionary,
                so that inter-stages of the pipeline can share information

        Returns:
            np.ndarray: the transformed features
            Optional[Dict[str, Any]]]: A dictionary to share fit informations
                within the pipeline stages
        """

        if fit_params is None:
            fit_params = {}

        X, fit_params = super().fit_transformer(
            X, y, fit_params=fit_params)

        return X, fit_params

    def score(self, X: np.ndarray, y: np.ndarray, batch_size: Optional[int] = None) -> np.ndarray:
        """score.

                Args:
                    X (np.ndarray): input to the pipeline, from which to guess targets
                    batch_size (Optional[int]): batch_size controls whether the pipeline
                        will be called on small chunks of the data. Useful when calling the
                        predict method on the whole array X results in a MemoryError.
                Returns:
                    np.ndarray: coefficient of determination R^2 of the prediction
                """
        from autoPyTorch.pipeline.components.training.metrics.utils import get_metrics, calculate_score
        metrics = get_metrics(self.dataset_properties, ['r2'])
        y_pred = self.predict(X, batch_size=batch_size)
        r2 = calculate_score(y, y_pred, task_type=STRING_TO_TASK_TYPES[self.dataset_properties['task_type']],
                             metrics=metrics)['r2']
        return r2

    def _get_hyperparameter_search_space(
            self,
            dataset_properties: Dict[str, Any],
            include: Optional[Dict[str, Any]] = None,
            exclude: Optional[Dict[str, Any]] = None,
    ) -> ConfigurationSpace:
        """Create the hyperparameter configuration space.

        For the given steps, and the Choices within that steps,
        this procedure returns a configuration space object to
        explore.

        Args:
            include (Optional[Dict[str, Any]]): what hyper-parameter configurations
                to honor when creating the configuration space
            exclude (Optional[Dict[str, Any]]): what hyper-parameter configurations
                to remove from the configuration space
            dataset_properties (Optional[Dict[str, Union[str, int]]]): Characteristics
                of the dataset to guide the pipeline choices of components

        Returns:
            cs (Configuration): The configuration space describing
                the SimpleRegressionClassifier.
        """
        cs = ConfigurationSpace()

        if dataset_properties is None or not isinstance(dataset_properties, dict):
            if not isinstance(dataset_properties, dict):
                warnings.warn('The given dataset_properties argument contains an illegal value.'
                              'Proceeding with the default value')
            dataset_properties = dict()

        if 'target_type' not in dataset_properties:
            dataset_properties['target_type'] = 'tabular_regression'
        if dataset_properties['target_type'] != 'tabular_regression':
            warnings.warn('Tabular classification is being used, however the target_type'
                          'is not given as "tabular_regression". Overriding it.')
            dataset_properties['target_type'] = 'tabular_regression'
        # get the base search space given this
        # dataset properties. Then overwrite with custom
        # classification requirements
        cs = self._get_base_search_space(
            cs=cs, dataset_properties=dataset_properties,
            exclude=exclude, include=include, pipeline=self.steps)

        # Here we add custom code, like this with this
        # is not a valid configuration

        self.configuration_space = cs
        self.dataset_properties = dataset_properties
        return cs

    def _get_pipeline_steps(self, dataset_properties: Optional[Dict[str, Any]],
                            ) -> List[Tuple[str, autoPyTorchChoice]]:
        """
        Defines what steps a pipeline should follow.
        The step itself has choices given via autoPyTorchChoice.

        Returns:
            List[Tuple[str, autoPyTorchChoice]]: list of steps sequentially exercised
                by the pipeline.
        """
        steps = []  # type: List[Tuple[str, autoPyTorchChoice]]

        default_dataset_properties = {'target_type': 'tabular_regression'}
        if dataset_properties is not None:
            default_dataset_properties.update(dataset_properties)

        steps.extend([
            ("imputer", SimpleImputer()),
            ("encoder", EncoderChoice(default_dataset_properties)),
            ("scaler", ScalerChoice(default_dataset_properties)),
            ("tabular_transformer", TabularColumnTransformer()),
            ("preprocessing", EarlyPreprocessing()),
            ("network", NetworkChoice(default_dataset_properties)),
            ("network_init", NetworkInitializerChoice(default_dataset_properties)),
            ("optimizer", OptimizerChoice(default_dataset_properties)),
            ("lr_scheduler", SchedulerChoice(default_dataset_properties)),
            ("data_loader", FeatureDataLoader()),
            ("trainer", TrainerChoice(default_dataset_properties)),
        ])
        return steps

    def _get_estimator_hyperparameter_name(self) -> str:
        """
        Returns the name of the current estimator.

        Returns:
            str: name of the pipeline type
        """
        return "tabular_regresser"


import numpy as np
import math
import uuid
class Agent:
    """
    Class agent, an agent can perform some actions and can be of different types (cop, comunist or fascist)
    """
    #legitimacy=np.random.rand()#legitimacy
    #max_jail_sentence=5

    def __init__(self,type_agent,legitimacy, view_range, k, threshold, jail_max):
        self.type = type_agent
        self.threshold = threshold
        self.legitimacy = legitimacy
        self.view_range = view_range
        self.k = k
        self.hardship = np.random.rand()
        self.grievance=self.hardship*(1-self.legitimacy)
        self.risk_aversion=np.random.rand()
        self.jail_years=0
        self.n_cops=0
        self.n_agents=1
        #self.cop_to_agents_ratio=math.floor(self.n_cops/self.n_agents)
        self.cop_to_agents_ratio=self.n_cops/self.n_agents
        self.arrest_probability=1-np.exp(-self.k*self.cop_to_agents_ratio)
        self.net_risk = self.risk_aversion*self.arrest_probability #n
        self.jail_max = jail_max
        self.id = uuid.uuid4()

    def get_type(self):
        return self.type

    def get_status(self):
        return self.status

    def set_status(self,status):
        self.status = status

    def set_type(self,type):
        self.type=type

    def get_color(self):
        if self.type=="cop":
            return 3
        if self.type=="active":
            return 2
        if self.type=="quite":
            return 1

    def print(self):
        if self.type=="cop":
            print(self.type,end=" ")
            return
        print(self.type + " g: "+str(self.grievance) + " n: " + str(self.net_risk)+ "t: " + str(self.threshold),
        "p: " + str(self.arrest_probability), "ctar: " + str(self.cop_to_agents_ratio),end=" ")

    def get_types(self):
        return ["cop","active","quite"]

    def get_attributes(self):
        if(self.type=="cop"):
            attributes={
                "vision" : self.view_range,
                "id" : self.id,
                "type": self.get_type()
            }
            return attributes
        attributes = {
            "vision" :  self.view_range,
            "hardship": self.hardship,
            "grievance" : self.grievance,
            "risk_aversion" : self.risk_aversion,
            "n_cops" : self.n_cops,
            "n_agents" : self.n_agents,
            "cop_to_agents_ratio" : self.cop_to_agents_ratio,
            "arrest_probability" : self.arrest_probability,
            "net_risk" : self.net_risk,
            "threshold" : self.threshold,
            "id" : self.id,
            "type": self.get_type()
        }
        return attributes

    def get_remaining_sentence_years(self):
        return self.jail_years

    def set_sentence_years(self,years):
        self.jail_years= years

    def set_external_attributes(self,n_cops,n_agents):
        self.n_cops = n_cops
        self.n_agents = n_agents
        self.cop_to_agents_ratio=math.floor(n_cops/n_agents)
        #self.cop_to_agents_ratio=n_cops/n_agents
        self.arrest_probability=1-np.exp(-self.k*self.cop_to_agents_ratio)
        self.net_risk = self.risk_aversion*self.arrest_probability #n

    def evolve(self):
        if not self.type=="cop":
            if self.grievance-self.net_risk>self.threshold:
                self.type = "active"

            if self.grievance-self.net_risk<=self.threshold:
                self.type = "quite"


from os import makedirs
from os.path import join
from os.path import exists

import sys
import logging
from rainbow_logging_handler import RainbowLoggingHandler

def configure_logging(name, base=None):
    log = logging.getLogger()
    log.setLevel(logging.DEBUG)
    formatter = logging.Formatter("%(asctime)s %(name)-38s %(levelname)-7s - %(message)s")

    #ch = logging.StreamHandler(sys.stderr)
    ch = RainbowLoggingHandler(sys.stderr, color_name=('red', 'black', True))
    ch.setFormatter(formatter)
    log.addHandler(ch)

    if base is not None:
        fh = logging.handlers.RotatingFileHandler(
                join(base, '%s.log' % name),
                maxBytes=(262144),
                backupCount=10)

        fh.setFormatter(formatter)
        log.addHandler(fh)


# Licensed under a 3-clause BSD style license - see LICENSE.rst
# -*- coding: utf-8 -*-
"""
fiberassign.gfa
===============

Functions for computing coverage of GFAs
"""

import numpy as np
import fitsio
from astropy.table import Table
import desimodel.focalplane.gfa

from ._internal import Tiles
from .utils import Logger, Timer

def isolated(ra, dec, mindist=5.0):
    '''
    Returns bool array for whether targets are isolated
    Args:
        ra: array, RA in degrees
        dec: array, dec in degrees
        mindist: minimum separation distance in arcsec
    '''
    import astropy.units as u
    from astropy.coordinates import SkyCoord
    cx = SkyCoord(ra*u.deg, dec*u.deg)
    index, separation, dist3d = cx.match_to_catalog_sky(cx, nthneighbor=2)
    return separation.to(u.arcsec) > mindist*u.arcsec

def get_gfa_targets(tiles, gfafile,faintlim=99):

    """Returns a list of tables of GFA targets on each tile

    Args:
        tiles: table with columns TILEID, RA, DEC; or Tiles object
        targets: table of targets with columsn RA, DEC

    Returns:
        list of tables (one row per input tile) with the subset of targets
        that are covered by GFAs on each tile.  Each table has additional
        `GFA_LOC` column indicating 0-9 which GFA was covered.

    Note that a given target could be covered by GFAs on more than one tile.

    Output is a list of astropy Tables; inputs can be numpy structured arrays
    or astropy Tables
    """
    log = Logger.get()
    tm = Timer()
    tm.start()

    # Convert tiles to vanilla numpy array if needed
    if isinstance(tiles, Tiles):
        tx = np.zeros(len(tiles.ra),
                      dtype=[("RA", "f8"), ("DEC", "f8"), ("TILEID", "i4")])
        tx["RA"] = tiles.ra
        tx["DEC"] = tiles.dec
        tx["TILEID"] = tiles.id
        tiles = tx

    # Load potential GFA targets and GFA locations
    targets = fitsio.read(gfafile)
    gfa = desimodel.focalplane.gfa.GFALocations(scale=2)

    # Pre-filter what GFA targets cover what tiles with some buffer.
    # find_points_in_tiles returns a list of lists;
    # convert to dictionary of lists keyed by tileid
    log.info("Finding overlap of {} GFA targets on {} tiles".format(
        len(targets), len(tiles)))
    gfa_tile_indices = dict()
    ii = desimodel.footprint.find_points_in_tiles(
        tiles, targets["RA"], targets["DEC"], radius=1.8)
    for i, tileid in enumerate(tiles["TILEID"]):
        gfa_tile_indices[tileid] = ii[i]

    gfa_targets = list()

    log.info("Generating GFA targets tables")
    for telra, teldec, tileid in zip(tiles["RA"], tiles["DEC"],
                                     tiles["TILEID"]):
        tmp = gfa.targets_on_gfa(telra, teldec,
                                 targets[gfa_tile_indices[tileid]])
        t = Table(tmp)

        # Rename some columns for downstream clarity and consistency
        for oldname, newname in [
                ("TYPE", "MORPHTYPE"),
                ("RA", "TARGET_RA"),
                ("DEC", "TARGET_DEC"),
                ("RA_IVAR", "TARGET_RA_IVAR"),
                ("DEC_IVAR", "TARGET_DEC_IVAR")]:
            if oldname in t.colnames:
                t.rename_column(oldname, newname)

        # Select which targets are good for ETC / GUIDE / FOCUS
        # 0 == good

        flag = np.zeros(len(t), dtype="i2")

        #- Not PSF-like
        isPSF = (t["MORPHTYPE"] == "PSF ") | (t["MORPHTYPE"] == "GPSF") | (t["MORPHTYPE"] == "PSF")
        flag[~isPSF] |= 2**0

        #- Not Isolated
        if len(tmp) > 1:
            notIsolated = ~isolated(tmp['RA'], tmp['DEC'])
            flag[notIsolated] |= 2**1

        #- Questionable astrometry / proper motion
        tych = (0 < t['REF_ID'])
        tych &= ( t['REF_ID'] < 1e10)
        flag[tych] |= 2**2

        #- Too faint
        faint = t['GAIA_PHOT_G_MEAN_MAG'] > faintlim
        flag[faint] |= 2**3

        # AR not passing the Gaia AEN criterion (PM correction done for AEN targets only)
        g = t["GAIA_PHOT_G_MEAN_MAG"]
        aen = t["GAIA_ASTROMETRIC_EXCESS_NOISE"]
        isaen = np.logical_or(
            (g <= 19.0) * (aen < 10.0 ** 0.5),
            (g >= 19.0) * (aen < 10.0 ** (0.5 + 0.2 * (g - 19.0))),
        )
        flag[~isaen] |= 2**4

        if len(flag)-np.count_nonzero(flag) == 0:
            log.error("ERROR: no good GFA targets for "
                      "ETC/GUIDE/FOCUS on tile {}".format(tileid))

        t["ETC_FLAG"] = flag
        t["GUIDE_FLAG"] = flag
        t["FOCUS_FLAG"] = flag

        gfa_targets.append(t)

    tm.stop()
    tm.report("  Identifying GFA targets")

    return gfa_targets


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""

:copyright:
    Simon Stähler (mail@simonstaehler.com), 2020
:license:
    None
"""
from taup_distance.taup_distance import get_dist
from obspy.taup import TauPyModel
import numpy as np

model_list = ('iasp91', './taup_tmp/model_example.npz')
for model_file in model_list:
    print('Test with %s' % model_file)
    model = TauPyModel(model_file)
    tSmP = 220.
    dist = get_dist(model=model, tSmP=tSmP, depth=50.)
    arrivals = model.get_travel_times(source_depth_in_km=50., distance_in_degree=dist,
                                      phase_list=('P', 'S'))
    has_P = False
    has_S = False
    for arr in arrivals:
        if arr.name == 'P' and not has_P:
            tP = arr.time
            has_P = True
        if arr.name == 'S' and not has_S:
            tS = arr.time
            has_S = True

    np.testing.assert_almost_equal(actual = tS - tP, desired = tSmP, decimal=3)
    print(dist)