import re
import warnings
from collections import OrderedDict
from operator import attrgetter
from urllib.parse import urljoin
from weakref import WeakKeyDictionary

from django.core.exceptions import PermissionDenied
from django.core.validators import (
    DecimalValidator,
    EmailValidator,
    MaxLengthValidator,
    MaxValueValidator,
    MinLengthValidator,
    MinValueValidator,
    RegexValidator,
    URLValidator,
)
from django.db import models
from django.http import Http404
from django.utils.encoding import force_str, smart_str
from rest_framework import exceptions, renderers, serializers
from rest_framework.compat import uritemplate
from rest_framework.fields import empty, Field
from rest_framework.request import clone_request
from rest_framework.schemas.generators import EndpointEnumerator, get_pk_name
from rest_framework.schemas.utils import is_list_view, get_pk_description
from rest_framework.settings import api_settings
from rest_framework.utils import formatting
from rest_framework.views import APIView

class ServerSwagger:
    def __init__(self, url, description):
        self.url = url
        self.description = description

class _UnvalidatedField(Field):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.allow_blank = True
        self.allow_null = True

    def to_internal_value(self, data):
        return data

    def to_representation(self, value):
        return value

class SortedPathSchemaGenerator:
    endpoint_inspector_cls = EndpointEnumerator

    # 'pk' isn't great as an externally exposed name for an identifier,
    # so by default we prefer to use the actual model field name for schemas.
    # Set by 'SCHEMA_COERCE_PATH_PK'.
    coerce_path_pk = None

    def __init__(
            self,
            title=None,
            url=None,
            description=None,
            patterns=None,
            urlconf=None,
            version=None,
    ):
        if url and not url.endswith("/"):
            url += "/"

        self.coerce_path_pk = api_settings.SCHEMA_COERCE_PATH_PK

        self.patterns = patterns
        self.urlconf = urlconf
        self.title = title
        self.description = description
        self.version = version
        self.url = url
        self.endpoints = None

    def _initialise_endpoints(self):
        if self.endpoints is None:
            inspector = self.endpoint_inspector_cls(self.patterns, self.urlconf)
            self.endpoints = inspector.get_api_endpoints()

    def _get_paths_and_endpoints(self, request):
        """
        Generate (path, method, view) given (path, method, callback) for paths.
        """
        paths = []
        view_endpoints = []
        for path, method, callback in self.endpoints:
            view = self.create_view(callback, method, request)
            path = self.coerce_path(path, method, view)
            if view:
                paths.append(path)
                view_endpoints.append((path, method, view))
        return paths, view_endpoints

    def _user_has_perm(self, user, endpoint_view, method):
        view_perms = endpoint_view.get_permissions()
        is_allowed = []
        for view_perm in view_perms:
            perms = view_perm.get_required_permissions(method, endpoint_view.get_serializer().Meta.model)
            is_allowed.append(all(user.has_perm(perm) if perm else True for perm in perms))
        return all(is_allowed)

    def create_view(self, callback, method, request=None):
        """
        Given a callback, return an actual view instance.
        """
        view = callback.cls(**getattr(callback, "initkwargs", {}))
        view.args = ()
        view.kwargs = {}
        view.format_kwarg = None
        view.request = None
        view.action_map = getattr(callback, "actions", None)

        actions = getattr(callback, "actions", None)
        if actions is not None:
            if method == "OPTIONS":
                view.action = "metadata"
            else:
                view.action = actions.get(method.lower())

        if request is not None:
            view.request = clone_request(request, method)
        is_allowed = True
        if not request.user.is_superuser:
            is_allowed = self._user_has_perm(request.user, view, method)
        if is_allowed:
            return view

    def coerce_path(self, path, method, view):
        """
        Coerce {pk} path arguments into the name of the model field,
        where possible. This is cleaner for an external representation.
        (Ie. "this is an identifier", not "this is a database primary key")
        """
        if not self.coerce_path_pk or "{pk}" not in path:
            return path
        model = getattr(getattr(view, "queryset", None), "model", None)
        if model:
            field_name = get_pk_name(model)
        else:
            field_name = "id"
        return path.replace("{pk}", "{%s}" % field_name)

    def has_view_permissions(self, path, method, view):
        """
        Return `True` if the incoming request has the correct view permissions.
        """
        if view.request is None:
            return True

        try:
            view.check_permissions(view.request)
        except (exceptions.APIException, Http404, PermissionDenied):
            return False
        return True

    def get_info(self):
        # Title and version are required by openapi specification 3.x
        info = {"title": self.title or "", "version": self.version or ""}

        if self.description is not None:
            info["description"] = self.description

        return info

    def get_paths(self, request=None):
        result = {}

        self._initialise_endpoints()

        paths, view_endpoints = self._get_paths_and_endpoints(request)

        # Only generate the path prefix for paths that will be included
        if not paths:
            return None

        for path, method, view in view_endpoints:
            if not self.has_view_permissions(path, method, view):
                continue
            operation = view.schema.get_operation(path, method)
            # Normalise path for any provided mount url.
            if path.startswith("/"):
                path = path[1:]
            path = urljoin(self.url or "/", path)

            result.setdefault(path, {})
            result[path][method.lower()] = operation
        return result

    def get_schema(self, request=None, public=False, servers: [ServerSwagger] = None):
        """
        Generate a OpenAPI schema.
        """

        paths = self.get_paths(None if public else request)
        if not paths:
            return None

        schema = {
            "openapi": "3.0.2",
            "info": self.get_info(),
            "components": {
                "securitySchemes": {
                    "ApiKeyAuth": {
                        "type": "apiKey",
                        "in": "header",
                        "name": "Authorization",
                        "value": "Token ",
                    }
                }
            },

            "paths": dict(OrderedDict(sorted(paths.items(), key=lambda t: t[0]))),
        }
        if servers is not None:
            servers = []
            for e in servers:
                servers.append({"url": e.url, "description": e.description})
            schema['servers'] = servers
        if schema is None:
            return
        return schema

class CustomAutoSchema:
    request_media_types = []
    response_media_types = []

    method_mapping = {
        "get": "Retrieve",
        "post": "Create",
        "put": "Update",
        "patch": "PartialUpdate",
        "delete": "Destroy",
    }

    header_regex = re.compile("^[a-zA-Z][0-9A-Za-z_]*:")

    def __init__(self):
        self.instance_schemas = WeakKeyDictionary()

    def __get__(self, instance, owner):
        """
        Enables `ViewInspector` as a Python _Descriptor_.

        This is how `view.schema` knows about `view`.

        `__get__` is called when the descriptor is accessed on the owner.
        (That will be when view.schema is called in our case.)

        `owner` is always the owner class. (An APIView, or subclass for us.)
        `instance` is the view instance or `None` if accessed from the class,
        rather than an instance.

        See: https://docs.python.org/3/howto/descriptor.html for info on
        descriptor usage.
        """
        if instance in self.instance_schemas:
            return self.instance_schemas[instance]

        self.view = instance
        return self

    def __set__(self, instance, other):
        self.instance_schemas[instance] = other
        if other is not None:
            other.view = instance

    @property
    def view(self):
        """View property."""
        assert self._view is not None, (
            "Schema generation REQUIRES a view instance. (Hint: you accessed "
            "`schema` from the view class rather than an instance.)"
        )
        return self._view

    @view.setter
    def view(self, value):
        self._view = value

    @view.deleter
    def view(self):
        self._view = None

    def get_description(self, path, method):
        """
        Determine a path description.

        This will be based on the method docstring if one exists,
        or else the class docstring.
        """
        view = self.view

        method_name = getattr(view, "action", method.lower())
        method_docstring = getattr(view, method_name, None).__doc__
        if method_docstring:
            # An explicit docstring on the method or action.
            return self._get_description_section(
                view, method.lower(), formatting.dedent(smart_str(method_docstring))
            )
        else:
            return self._get_description_section(
                view,
                getattr(view, "action", method.lower()),
                view.get_view_description(),
            )

    def _get_description_section(self, view, header, description):
        lines = [line for line in description.splitlines()]
        current_section = ""
        sections = {"": ""}

        for line in lines:
            if self.header_regex.match(line):
                current_section, separator, lead = line.partition(":")
                sections[current_section] = lead.strip()
            else:
                sections[current_section] += "\n" + line

        # TODO: SCHEMA_COERCE_METHOD_NAMES appears here and in `SchemaGenerator.get_keys`
        coerce_method_names = api_settings.SCHEMA_COERCE_METHOD_NAMES
        if header in sections:
            return sections[header].strip()
        if header in coerce_method_names:
            if coerce_method_names[header] in sections:
                return sections[coerce_method_names[header]].strip()
        return sections[""].strip()

    def get_operation(self, path, method):
        operation = {
            "operationId": self._get_operation_id(path, method),
            "description": self.get_description(path, method),
        }

        tags = self.get_tags(path, method)
        if tags:
            operation["tags"] = tags

        if self.is_deprecated(path, method):
            operation["deprecated"] = True

        parameters = []
        parameters += self._get_path_parameters(path)
        parameters += self._get_pagination_parameters(path, method)
        parameters += self._get_filter_parameters(path, method)

        operation["parameters"] = parameters

        request_body = self._get_request_body(path, method)
        if request_body:
            operation["requestBody"] = request_body
        operation["responses"] = self._get_responses(path, method)

        return operation

    def _get_operation_id(self, path, method):
        """
        Compute an operation ID from the model, serializer or view name.
        """
        method_name = getattr(self.view, "action", method.lower())
        if is_list_view(path, method, self.view):
            action = "list"
        elif method_name not in self.method_mapping:
            action = method_name
        else:
            action = self.method_mapping[method.lower()]

        # Try to deduce the ID from the view's model
        model = getattr(getattr(self.view, "queryset", None), "model", None)
        if model is not None:
            name = model.__name__

        # Try with the serializer class name
        elif hasattr(self.view, "get_serializer_class"):
            name = self.view.get_serializer_class().__name__
            if name.endswith("Serializer"):
                name = name[:-10]

        # Fallback to the view name
        else:
            name = self.view.__class__.__name__
            if name.endswith("APIView"):
                name = name[:-7]
            elif name.endswith("View"):
                name = name[:-4]

            # Due to camel-casing of classes and `action` being lowercase, apply title in order to find if action truly
            # comes at the end of the name
            if name.endswith(
                    action.title()
            ):  # ListView, UpdateAPIView, ThingDelete ...
                name = name[: -len(action)]

        if action == "list" and not name.endswith(
                "s"
        ):  # listThings instead of listThing
            name += "s"

        return action + name

    def get_tags(self, path, method):
        """
        Compute the tags.
        """
        try:
            return self.view.tags
        except AttributeError:
            return

    def is_deprecated(self, path, method):
        """
        Is deprecated.
        """
        try:
            return self.view.deprecated
        except AttributeError:
            return False

    def _get_path_parameters(self, path):
        """
        Return a list of parameters from templated path variables.
        """
        assert (
            uritemplate
        ), "`uritemplate` must be installed for OpenAPI schema support."

        model = getattr(getattr(self.view, "queryset", None), "model", None)
        parameters = []

        for variable in uritemplate.variables(path):
            description = ""
            if model is not None:  # TODO: test this.
                # Attempt to infer a field description if possible.
                try:
                    model_field = model._meta.get_field(variable)
                except Exception:
                    model_field = None

                if model_field is not None and model_field.help_text:
                    description = force_str(model_field.help_text)
                elif model_field is not None and model_field.primary_key:
                    description = get_pk_description(model, model_field)

            parameter = {
                "name": variable,
                "in": "path",
                "required": True,
                "description": description,
                "schema": {"type": "string", },  # TODO: integer, pattern, ...
            }
            parameters.append(parameter)

        return parameters

    def _get_filter_parameters(self, path, method):
        if not self._allows_filters(path, method):
            return []
        parameters = []
        for filter_backend in self.view.filter_backends:
            parameters += filter_backend().get_schema_operation_parameters(self.view)
        return parameters

    def _allows_filters(self, path, method):
        """
        Determine whether to include filter Fields in schema.

        Default implementation looks for ModelViewSet or GenericAPIView
        actions/methods that cause filtering on the default implementation.
        """
        if getattr(self.view, "filter_backends", None) is None:
            return False
        if hasattr(self.view, "action"):
            return self.view.action in [
                "list",
                "retrieve",
                "update",
                "partial_update",
                "destroy",
            ]
        return method.lower() in ["get", "put", "patch", "delete"]

    def _get_pagination_parameters(self, path, method):
        view = self.view

        if not is_list_view(path, method, view):
            return []

        paginator = self._get_paginator()
        if not paginator:
            return []

        return paginator.get_schema_operation_parameters(view)

    def _map_field(self, field):

        # Nested Serializers, `many` or not.
        if isinstance(field, serializers.ListSerializer):
            return {"type": "array", "items": self._map_serializer(field.child)}
        if isinstance(field, serializers.Serializer):
            data = self._map_serializer(field)
            data["type"] = "object"
            return data

        # Related fields.
        if isinstance(field, serializers.ManyRelatedField):
            return {"type": "array", "items": self._map_field(field.child_relation)}
        if isinstance(field, serializers.PrimaryKeyRelatedField):
            model = getattr(field.queryset, "model", None)
            if model is not None:
                model_field = model._meta.pk
                if isinstance(model_field, models.AutoField):
                    return {"type": "integer"}

        # ChoiceFields (single and multiple).
        # Q:
        # - Is 'type' required?
        # - can we determine the TYPE of a choicefield?
        if isinstance(field, serializers.MultipleChoiceField):
            return {
                "type": "array",
                "items": {"enum": list(field.choices)},
            }

        if isinstance(field, serializers.ChoiceField):
            return {
                "enum": list(field.choices),
            }

        # ListField.
        if isinstance(field, serializers.ListField):
            mapping = {
                "type": "array",
                "items": {},
            }
            if not isinstance(field.child, _UnvalidatedField):
                map_field = self._map_field(field.child)
                items = {"type": map_field.get("type")}
                if "format" in map_field:
                    items["format"] = map_field.get("format")
                mapping["items"] = items
            return mapping

        # DateField and DateTimeField type is string
        if isinstance(field, serializers.DateField):
            return {
                "type": "string",
                "format": "date",
            }

        if isinstance(field, serializers.DateTimeField):
            return {
                "type": "string",
                "format": "date-time",
            }

        # "Formats such as "email", "uuid", and so on, MAY be used even though undefined by this specification."
        # see: https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.0.2.md#data-types
        # see also: https://swagger.io/docs/specification/data-models/data-types/#string
        if isinstance(field, serializers.EmailField):
            return {"type": "string", "format": "email"}

        if isinstance(field, serializers.URLField):
            return {"type": "string", "format": "uri"}

        if isinstance(field, serializers.UUIDField):
            return {"type": "string", "format": "uuid"}

        if isinstance(field, serializers.IPAddressField):
            content = {
                "type": "string",
            }
            if field.protocol != "both":
                content["format"] = field.protocol
            return content

        # DecimalField has multipleOf based on decimal_places
        if isinstance(field, serializers.DecimalField):
            content = {"type": "number"}
            if field.decimal_places:
                content["multipleOf"] = float(
                    "." + (field.decimal_places - 1) * "0" + "1"
                )
            if field.max_whole_digits:
                content["maximum"] = int(field.max_whole_digits * "9") + 1
                content["minimum"] = -content["maximum"]
            self._map_min_max(field, content)
            return content

        if isinstance(field, serializers.FloatField):
            content = {"type": "number"}
            self._map_min_max(field, content)
            return content

        if isinstance(field, serializers.IntegerField):
            content = {"type": "integer"}
            self._map_min_max(field, content)
            # 2147483647 is max for int32_size, so we use int64 for format
            if (
                    int(content.get("maximum", 0)) > 2147483647
                    or int(content.get("minimum", 0)) > 2147483647
            ):
                content["format"] = "int64"
            return content

        if isinstance(field, serializers.FileField):
            return {"type": "string", "format": "binary"}

        # Simplest cases, default to 'string' type:
        FIELD_CLASS_SCHEMA_TYPE = {
            serializers.BooleanField: "boolean",
            serializers.JSONField: "object",
            serializers.DictField: "object",
            serializers.HStoreField: "object",
        }
        return {"type": FIELD_CLASS_SCHEMA_TYPE.get(field.__class__, "string")}

    def _map_min_max(self, field, content):
        if field.max_value:
            content["maximum"] = field.max_value
        if field.min_value:
            content["minimum"] = field.min_value

    def _map_serializer(self, serializer):
        # Assuming we have a valid serializer instance.
        # TODO:
        #   - field is Nested or List serializer.
        #   - Handle read_only/write_only for request/response differences.
        #       - could do this with readOnly/writeOnly and then filter dict.
        required = []
        properties = {}

        for field in serializer.fields.values():
            if isinstance(field, serializers.HiddenField):
                continue

            if field.required:
                required.append(field.field_name)

            schema = self._map_field(field)
            if field.read_only:
                schema["readOnly"] = True
            if field.write_only:
                schema["writeOnly"] = True
            if field.allow_null:
                schema["nullable"] = True
            if field.default and field.default != empty:  # why don't they use None?!
                schema["default"] = field.default
            if field.help_text:
                schema["description"] = str(field.help_text)
            self._map_field_validators(field, schema)

            properties[field.field_name] = schema

        result = {"properties": properties}
        if required:
            result["required"] = required

        return result

    def _map_field_validators(self, field, schema):
        """
        map field validators
        """
        for v in field.validators:
            # "Formats such as "email", "uuid", and so on, MAY be used even though undefined by this specification."
            # https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.0.2.md#data-types
            if isinstance(v, EmailValidator):
                schema["format"] = "email"
            if isinstance(v, URLValidator):
                schema["format"] = "uri"
            if isinstance(v, RegexValidator):
                schema["pattern"] = v.regex.pattern
            elif isinstance(v, MaxLengthValidator):
                attr_name = "maxLength"
                if isinstance(field, serializers.ListField):
                    attr_name = "maxItems"
                schema[attr_name] = v.limit_value
            elif isinstance(v, MinLengthValidator):
                attr_name = "minLength"
                if isinstance(field, serializers.ListField):
                    attr_name = "minItems"
                schema[attr_name] = v.limit_value
            elif isinstance(v, MaxValueValidator):
                schema["maximum"] = v.limit_value
            elif isinstance(v, MinValueValidator):
                schema["minimum"] = v.limit_value
            elif isinstance(v, DecimalValidator):
                if v.decimal_places:
                    schema["multipleOf"] = float(
                        "." + (v.decimal_places - 1) * "0" + "1"
                    )
                if v.max_digits:
                    digits = v.max_digits
                    if v.decimal_places is not None and v.decimal_places > 0:
                        digits -= v.decimal_places
                    schema["maximum"] = int(digits * "9") + 1
                    schema["minimum"] = -schema["maximum"]

    def _get_paginator(self):
        pagination_class = getattr(self.view, "pagination_class", None)
        if pagination_class:
            return pagination_class()
        return None

    def map_parsers(self, path, method):
        return list(map(attrgetter("media_type"), self.view.parser_classes))

    def map_renderers(self, path, method):
        media_types = []
        for renderer in self.view.renderer_classes:
            # BrowsableAPIRenderer not relevant to OpenAPI spec
            if renderer == renderers.BrowsableAPIRenderer:
                continue
            media_types.append(renderer.media_type)
        return media_types

    def _get_serializer(self, method, path):
        view = self.view

        if not hasattr(view, "get_serializer"):
            return None

        try:
            return view.get_serializer()
        except exceptions.APIException:
            warnings.warn(
                "{}.get_serializer() raised an exception during "
                "schema generation. Serializer fields will not be "
                "generated for {} {}.".format(view.__class__.__name__, method, path)
            )
            return None

    def _get_request_body(self, path, method):
        if method not in ("PUT", "PATCH", "POST"):
            return {}

        self.request_media_types = self.map_parsers(path, method)

        serializer = self._get_serializer(path, method)

        if not isinstance(serializer, serializers.Serializer):
            return {}

        content = self._map_serializer(serializer)
        # No required fields for PATCH
        if method == "PATCH":
            content.pop("required", None)
        # No read_only fields for request.
        for name, schema in content["properties"].copy().items():
            if "readOnly" in schema:
                del content["properties"][name]

        return {"content": {ct: {"schema": content} for ct in self.request_media_types}}

    def _get_responses(self, path, method):
        # TODO: Handle multiple codes and pagination classes.
        if method == "DELETE":
            return {"204": {"description": ""}}

        self.response_media_types = self.map_renderers(path, method)

        item_schema = {}
        serializer = self._get_serializer(path, method)

        if isinstance(serializer, serializers.Serializer):
            item_schema = self._map_serializer(serializer)
            # No write_only fields for response.
            for name, schema in item_schema["properties"].copy().items():
                if "writeOnly" in schema:
                    del item_schema["properties"][name]
                    if "required" in item_schema:
                        item_schema["required"] = [
                            f for f in item_schema["required"] if f != name
                        ]

        if is_list_view(path, method, self.view):
            response_schema = {
                "type": "array",
                "items": item_schema,
            }
            paginator = self._get_paginator()
            if paginator:
                response_schema = paginator.get_paginated_response_schema(
                    response_schema
                )
        else:
            response_schema = item_schema

        return {
            "200": {
                "content": {
                    ct: {"schema": response_schema} for ct in self.response_media_types
                },
                # description is a mandatory property,
                # https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.0.2.md#responseObject
                # TODO: put something meaningful into it
                "description": "",
            }
        }

class AdvanceApiView(APIView):
    schema = CustomAutoSchema()


import torch
import io
from PIL import Image
from torchvision import models,transforms

m = models.resnet18()
num_ftrs = m.fc.in_features
m.fc = torch.nn.Linear(num_ftrs, 1890)

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
m.load_state_dict(torch.load('model_state_1210_new.pth', map_location=torch.device('cpu')))
m = m.to(device)
m.eval()

# image -> tensor
def transform_image(image_bytes):
    transform = transforms.Compose([
                                    transforms.Resize(224),
                                    transforms.ToTensor(),
                                    transforms.Normalize(
                                        mean = [0.485, 0.456, 0.406],
                                        std = [0.229, 0.224, 0.225]
                                    )])

    image = Image.open(io.BytesIO(image_bytes))
    return transform(image).unsqueeze(0)

# predict
def get_pred(image_tensor):

    f = open('drug_name_1210.txt',encoding='utf-8', errors='ignore')
    a = f.read()
    a_list = a.split(',')
    b = list(a_list)
    f.close()

    image_tensor = image_tensor.to(device)
    outputs = m(image_tensor)

    _, test_indices = torch.sort(outputs, descending = True)
    ls=[0,0,0,0,0]
    p=[0,0,0,0,0]
    a = 0
    for i in test_indices[0][:5]:
        ls[a] = outputs[0][i].item()
        a+=1

    k = 0
    for i in ls:
        p[k] = (i-min(ls)+1)/(max(ls)-min(ls)+2)
        k+=1
    p = torch.Tensor(p)

    i = 0
    c = {}
    for idx in test_indices[0][:5]:
        c[b[idx]] = round(p[i].item()*100,2)
        i += 1
    return list(c.keys()), list(c.values())

########## old one ##########
# def get_pred(image_tensor):

#     f = open('drug_name_1203.txt',encoding='utf-8', errors='ignore')
#     a = f.read()
#     a_list = a.split(',')
#     b = list(a_list)
#     f.close()

#     image_tensor = image_tensor.to(device)
#     outputs = m(image_tensor)
#     percentage = torch.nn.functional.softmax(outputs, dim = 1)[0] * 100

#     _, test_indices = torch.sort(outputs, descending = True)

#     c = {}
#     for idx in test_indices[0][:5]:
#         c[b[idx.item()]] = round(percentage[idx].item(),2)

#     return list(c.keys()), list(c.values())


import numpy as np
import cv2
import sys
import time

#Initialization of Aruco type for robotic french cup is the Aruco 4*4
ARUCO_DICT = {
	"DICT_4X4_50": cv2.aruco.DICT_4X4_50,
	"DICT_4X4_100": cv2.aruco.DICT_4X4_100,
	"DICT_4X4_250": cv2.aruco.DICT_4X4_250,
	"DICT_4X4_1000": cv2.aruco.DICT_4X4_1000,
	"DICT_5X5_50": cv2.aruco.DICT_5X5_50,
	"DICT_5X5_100": cv2.aruco.DICT_5X5_100,
	"DICT_5X5_250": cv2.aruco.DICT_5X5_250,
	"DICT_5X5_1000": cv2.aruco.DICT_5X5_1000,
	"DICT_6X6_50": cv2.aruco.DICT_6X6_50,
	"DICT_6X6_100": cv2.aruco.DICT_6X6_100,
	"DICT_6X6_250": cv2.aruco.DICT_6X6_250,
	"DICT_6X6_1000": cv2.aruco.DICT_6X6_1000,
	"DICT_7X7_50": cv2.aruco.DICT_7X7_50,
	"DICT_7X7_100": cv2.aruco.DICT_7X7_100,
	"DICT_7X7_250": cv2.aruco.DICT_7X7_250,
	"DICT_7X7_1000": cv2.aruco.DICT_7X7_1000,
	"DICT_ARUCO_ORIGINAL": cv2.aruco.DICT_ARUCO_ORIGINAL,
	"DICT_APRILTAG_16h5": cv2.aruco.DICT_APRILTAG_16h5,
	"DICT_APRILTAG_25h9": cv2.aruco.DICT_APRILTAG_25h9,
	"DICT_APRILTAG_36h10": cv2.aruco.DICT_APRILTAG_36h10,
	"DICT_APRILTAG_36h11": cv2.aruco.DICT_APRILTAG_36h11
}

def aruco_display(corners, ids, rejected, image):

    #if an aruco 4*4 tag is detected
	if len(corners) > 0:

		ids = ids.flatten()

		for (markerCorner, markerID) in zip(corners, ids):

			corners = markerCorner.reshape((4, 2))
			(topLeft, topRight, bottomRight, bottomLeft) = corners

			topRight = (int(topRight[0]), int(topRight[1]))
			bottomRight = (int(bottomRight[0]), int(bottomRight[1]))
			bottomLeft = (int(bottomLeft[0]), int(bottomLeft[1]))
			topLeft = (int(topLeft[0]), int(topLeft[1]))

			cv2.line(image, topLeft, topRight, (0, 255, 0), 2)
			cv2.line(image, topRight, bottomRight, (0, 255, 0), 2)
			cv2.line(image, bottomRight, bottomLeft, (0, 255, 0), 2)
			cv2.line(image, bottomLeft, topLeft, (0, 255, 0), 2)

			cX = int((topLeft[0] + bottomRight[0]) / 2.0)
			cY = int((topLeft[1] + bottomRight[1]) / 2.0)
			cv2.circle(image, (cX, cY), 4, (0, 0, 255), -1)

			cv2.putText(image, str(markerID),(topLeft[0], topLeft[1] - 10), cv2.FONT_HERSHEY_SIMPLEX,
				0.5, (0, 255, 0), 2)
			print("[Inference] ArUco marker ID: {}".format(markerID))

	return image

def pose_estimation(frame, aruco_dict_type, matrix_coefficients, distortion_coefficients):

    #transform the image into a grayscale image
    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

    #Getting the fictionaris corresponding of our aruco markers
    cv2.aruco_dict = cv2.aruco.Dictionary_get(aruco_dict_type)

    #create a set of parameters for a good estimation if you don't specify parameters
    #it takes the default ones that remain good for common use
    parameters = cv2.aruco.DetectorParameters_create()

    #Function that detects the Markers
    corners, ids, rejected_img_points = cv2.aruco.detectMarkers(gray, cv2.aruco_dict,parameters=parameters,
        cameraMatrix=matrix_coefficients,
        distCoeff=distortion_coefficients)

    R_list=[]
    #if the detection detects at least one marker then we go into the if
    if len(corners) > 0:
        for i in range(0, len(ids)):

            '''rvec: A 3D rotation vector that describes the rotation between the marker and the camera.
            tvec: A 3D translation vector that describes the position of the marker relative to the camera.'''
            rvec, tvec, markerPoints = cv2.aruco.estimatePoseSingleMarkers(corners[i], 0.03, matrix_coefficients,
                                                                       distortion_coefficients)

            # Camera position relative to marker [x,y,z]
            print("distance from the aruco marker", tvec[0][0])

            #Euclidian distance
            distance = np.linalg.norm(tvec)

            #Convertion Rotation vector to a rotation matrix
            R, _ = cv2.Rodrigues(rvec)

            #Inversion of the matrix for having the camera position and rotation relative to marker
            R_inv = np.linalg.inv(R)

            #print("rotation vector", np.degrees(rvec))
            t_cam_marker = np.dot(-R_inv, np.transpose(tvec[0][0]))

            print("Camera position relative to marker: ", t_cam_marker)
            print("Camera orientation relative to marker (in degrees): ", np.degrees(cv2.Rodrigues(R_inv)[0]))

            #Display onto the frame all the information that we know
            cv2.putText(frame,f"Distance: {t_cam_marker:.2f} meters",(10, frame.shape[0]-20), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 1, cv2.LINE_AA)

            cv2.putText(frame,f"rotation: {np.degrees(cv2.Rodrigues(R_inv)[0]):.2f} degree",(10, frame.shape[0]-40), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 1, cv2.LINE_AA)

            cv2.aruco.drawDetectedMarkers(frame, corners)

            cv2.aruco.drawAxis(frame, matrix_coefficients, distortion_coefficients, rvec, tvec, 0.01)

    return frame

# def Frame_display(Id):

#       print(0)

#Define the Aruco type we want to detect
aruco_type = "DICT_4X4_100"

#Get that Aruco in the dictionary of the Aruco
arucoDict = cv2.aruco.Dictionary_get(ARUCO_DICT[aruco_type])

arucoParams = cv2.aruco.DetectorParameters_create()

#Settings the intrinsic parameter of the camera that we got with the calibration algortihm (chessboard)
intrinsic_camera = np.array(((951.19442957, 0, 655.18037018),(0,951.21008316, 381.97715139),(0,0,1)))
distortion = np.array((8.88842561e-04 , 3.25521835e-01  ,1.37760593e-02  ,1.45864253e-03,-2.15412814e+00))

#Opening the socket for the camera if it's not 2 try like 0 or 1
cap = cv2.VideoCapture(1)

#Setting up the width and height for the camera frame
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 1280)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 720)

#log out "begin.."
print("[INFO] begining video capture...")
print("[INFO] press 'q' to quit")

# while the stream is one
while cap.isOpened():

    print("[INFO] begining video capture...")

    #ret is for checking if the reading have succeed and image is the actual frame
    ret, img = cap.read()

    output = pose_estimation(img, ARUCO_DICT[aruco_type], intrinsic_camera, distortion)

    cv2.imshow('Estimated Pose', output)

    key = cv2.waitKey(1) & 0xFF
    if key == ord('q'):
        break

cap.release()
cv2.destroyAllWindows()


import os
from zipfile import ZipFile

import h5py
from openseapy.ml.autoclassifier_tools import MimicHDFFile
from tensorflow.keras.models import Model, load_model

# todo: autoclassifier example
from tensorflow.python.keras.saving.hdf5_format import save_model_to_hdf5, load_model_from_hdf5

class AutoClassifier(Model):
    """
    Autoclassifier class

    This class acts like a tensorflow Model object. The autoclassfier exists of three single tensorflow models,
    encoder, decoder  and classifier. This three models can be designed independently, but output dimension of the
    encoder have to be equal to the input dimensions of the decoder and classifier and the output dimension of the
    decoder have to be equal to the input dimension of the encoder.

    For training the Autoclassifier object can act as an autoencoder or autoclassifier. The mode can be chosen with the
    methods:
       - activate_autoclassifier_mode()
       - activate_autoencoder_mode()
    Then the return of a call of the Autoclassifier object corresponds to the chosen mode.
    """
    def __init__(self, encoder, decoder, classifier, mode='autoclassifier'):
        """
        Parameters
        ----------
        encoder: model
            Encoder model.
        decoder: model or None
            Decoder model. If None, no model will be saved and the mode autoencoder will not be work.
        classifier: model or None
            Classifier model. If None, no model will be saved and the mode autoclassifier will not be work.
        mode: str, optional
            Mode of the object. Default 'autoclassifier'.
        """
        super(AutoClassifier, self).__init__()

        self._mode = None

        self.encoder = encoder
        self.decoder = decoder
        self.classifier = classifier

        if mode == 'autoclassifier':
            self.activate_autoclassifier_mode()
        elif mode == 'autoencoder':
            self.activate_autoencoder_mode()
        else:
            raise ValueError('Mode can be only "autoencoder" or "autoclassifier" not "{}"'.format(mode))

    @property
    def mode(self):
        """
        Returns
        -------
        current mode: str
        """
        return self._mode

    def activate_autoclassifier_mode(self):
        """
        Activate autoclassifier mode.
        """
        self._mode = 'autoclassifier'

    def activate_autoencoder_mode(self):
        """
        Activate autoencoder mode.
        """
        self._mode = 'autoencoder'

    def call(self, x):
        """
        Call function of the object.

        Parameters
        ----------
        x: input data

        Returns
        -------
        Evaluated model based on mode.
        """
        if self.mode == 'autoclassifier':
            return self.autoclassify(x)
        elif self.mode == 'autoencoder':
            return self.autoencode(x)
        else:
            raise ValueError('Mode can be only "autoencoder" or "autoclassifier" not "{}"'.format(self.mode))

    def autoencode(self, x):
        """
        Evaluate autencoder.

        Parameters
        ----------
        x: input data

        Returns
        -------
        evaluated model
        """
        encoded = self.encode(x)
        decoded = self.decode(encoded)

        return decoded

    def autoclassify(self, x):
        """
        Evaluate autoclassifer.

        Parameters
        ----------
        x: input data

        Returns
        -------
        evaluated model
        """
        encoded = self.encode(x)
        classified = self.classify(encoded)

        return classified

    def encode(self, x):
        """
        Encode data.

        Parameters
        ----------
        x: input data

        Returns
        -------
        encoded data
        """
        if self.encoder is not None:
            return self.encoder(x)
        else:
            raise RuntimeError('Encoder is not defined!')

    def decode(self, x):
        """
        Decode data.

        Parameters
        ----------
        x: input data

        Returns
        -------
        decoded data
        """
        if self.decoder is not None:
            return self.decoder(x)
        else:
            raise RuntimeError('Decoder is not defined!')

    def classify(self, x):
        """
        Classify encoded data

        Parameters
        ----------
        x: input data

        Returns
        -------
        classified data
        """
        if self.classifier is not None:
            return self.classifier(x)
        else:
            raise RuntimeError('Classifier is not defined!')

    def save(self, filepath, overwrite=False, *args, **kwargs):
        """
        Save Autoclassifier object as hdf file.

        Parameters
        ----------
        filepath: String, pathlib.Path object or h5py.File
            path of the file to be saved
        overwrite: bool, optional
            Should an existing file be overwritten? Default True.
        """
        if not isinstance(filepath, h5py.File):
            if not overwrite and os.path.isfile(filepath):
                raise FileExistsError('File exists already. If you want to overwrite, set "overwrite=True".')

            f = h5py.File(filepath, mode='w')
            opened_new_file = True
        else:
            f = filepath
            opened_new_file = False

        try:
            if self.encoder is not None:
                if 'encoder' in list(f.keys()):
                    del f['encoder']
                f_encoder = f.create_group('encoder')

                save_model_to_hdf5(self.encoder, MimicHDFFile(f_encoder), *args, **kwargs)

            if self.decoder is not None:
                if 'decoder' in list(f.keys()):
                    del f['decoder']
                f_decoder = f.create_group('decoder')

                save_model_to_hdf5(self.decoder, MimicHDFFile(f_decoder), *args, **kwargs)

            if self.classifier is not None:
                if 'classifier' in list(f.keys()):
                    del f['classifier']
                f_classifier = f.create_group('classifier')

                save_model_to_hdf5(self.classifier, MimicHDFFile(f_classifier), *args, **kwargs)

            f.flush()
        finally:
            if opened_new_file:
                f.close()

    @classmethod
    def load(cls, filepath, *args, **kwargs):
        """
        Load Autocalssifier object from hdf file.

        Parameters
        ----------
        filepath: String, pathlib.Path object or h5py.File
            path of the file to be loaded

        Returns
        -------
        AutoClassifier object
        """
        opened_new_file = not isinstance(filepath, h5py.File)
        if opened_new_file:
            f = h5py.File(filepath, mode='r')
        else:
            f = filepath

        try:
            if 'encoder' in list(f.keys()):
                encoder = load_model_from_hdf5(MimicHDFFile(f['encoder']), *args, **kwargs)
            else:
                encoder = None

            if 'decoder' in list(f.keys()):
                decoder = load_model_from_hdf5(MimicHDFFile(f['decoder']), *args, **kwargs)
            else:
                decoder = None

            if 'classifier' in list(f.keys()):
                classifier = load_model_from_hdf5(MimicHDFFile(f['classifier']), *args, **kwargs)
            else:
                classifier = None

        finally:
            if opened_new_file:
                f.close()

        return cls(encoder=encoder, decoder=decoder, classifier=classifier)


from flask import Flask, render_template, redirect, url_for, request, jsonify

# import cv2 as cv2
# import numpy as np
# import urllib.request
# from PIL import Image
# import io
# from scipy.io import wavfile

# from pygame import mixer
from werkzeug.utils import secure_filename
import os
import whisper
import torch
from datetime import datetime

#wav2vec
from transformers.file_utils import cached_path, hf_bucket_url
import zipfile
from transformers import Wav2Vec2Processor, Wav2Vec2ForCTC
import soundfile as sf
import kenlm
from pyctcdecode import Alphabet, BeamSearchDecoderCTC, LanguageModel
import librosa

# from odoo.http import request
app = Flask(__name__, template_folder='templates')

# model = whisper.load_model("tiny")
model = whisper.load_model("base")
state_dict = torch.load("checkpoint-epoch=0013.ckpt", map_location="cpu")['state_dict']
# change all key of state_dict to remove "model."
new_state_dict = {k.replace("model.", ""): v for k, v in state_dict.items()}
model.load_state_dict(new_state_dict)
print(f"Model whisper base loaded")

cache_dir = './cache/'
processor_w2v = Wav2Vec2Processor.from_pretrained("nguyenvulebinh/wav2vec2-base-vietnamese-250h", cache_dir=cache_dir)
model_w2v = Wav2Vec2ForCTC.from_pretrained("nguyenvulebinh/wav2vec2-base-vietnamese-250h", cache_dir=cache_dir)
# lm_file = hf_bucket_url("nguyenvulebinh/wav2vec2-base-vietnamese-250h", filename='vi_lm_4grams.bin.zip')
# lm_file = cached_path(lm_file,cache_dir=cache_dir)
# with zipfile.ZipFile(lm_file, 'r') as zip_ref:
#     zip_ref.extractall(cache_dir)
lm_file = cache_dir + 'vi_lm_4grams.bin'

def get_decoder_ngram_model(tokenizer, ngram_lm_path):
    vocab_dict = tokenizer.get_vocab()
    sort_vocab = sorted((value, key) for (key, value) in vocab_dict.items())
    vocab = [x[1] for x in sort_vocab][:-2]
    vocab_list = vocab
    # convert ctc blank character representation
    vocab_list[tokenizer.pad_token_id] = ""
    # replace special characters
    vocab_list[tokenizer.unk_token_id] = ""
    # vocab_list[tokenizer.bos_token_id] = ""
    # vocab_list[tokenizer.eos_token_id] = ""
    # convert space character representation
    vocab_list[tokenizer.word_delimiter_token_id] = " "
    # specify ctc blank char index, since conventially it is the last entry of the logit matrix
    alphabet = Alphabet.build_alphabet(vocab_list, ctc_token_idx=tokenizer.pad_token_id)
    lm_model = kenlm.Model(ngram_lm_path)
    decoder = BeamSearchDecoderCTC(alphabet,
                                   language_model=LanguageModel(lm_model))
    return decoder

ngram_lm_model = get_decoder_ngram_model(processor_w2v.tokenizer, lm_file)
print("Huggingface model loaded")

@app.route('/')
def home():
   return render_template('home.html')

@app.route('/transcribe-whisper', methods=['POST'])
def transcribe():
   files = request.files
   file = files.get('file')

   file_name = secure_filename(str(datetime.now())) + ".wav"
   file_path = os.path.join("audio_folder", file_name)
   file.save(file_path)

   audio = whisper.load_audio(file_path)
   audio = whisper.pad_or_trim(audio)
   print("Audio loaded and trimmed")

   mel = whisper.log_mel_spectrogram(audio).to(model.device)
   # options = whisper.DecodingOptions(fp16 = False, withlm=F, beam_size=1,
   #      patience=1.0, lm_path="../../dataset_tokenized_3gram.binary", lm_alpha=0.75, lm_beta=0.0,
   #      without_timestamps=True, language="vi")
   options = whisper.DecodingOptions(fp16 = False, language="vi", without_timestamps=True)
   print("Model decoding...")
   result = whisper.decode(model, mel, options)

   return jsonify(result.text)

@app.route('/transcribe-w2v', methods=['POST'])
def transcribe_w2v():
   files = request.files
   file = files.get('file')

   file_name = secure_filename(str(datetime.now())) + ".wav"
   file_path = os.path.join("audio_folder", file_name)
   file.save(file_path)

   speech, sr = librosa.load(file_path)
   speech = librosa.resample(speech, orig_sr=sr, target_sr=16000)
   input_values = processor_w2v(
      speech,
      sampling_rate=16000,
      return_tensors="pt").input_values
   logits = model_w2v(input_values).logits[0]
   pred_ids = torch.argmax(logits, dim=-1)

   print("Model decoding...")
   beam_search_output = ngram_lm_model.decode(logits.cpu().detach().numpy(), beam_width=500)
   return jsonify(beam_search_output)

if __name__ == '__main__':
   app.run(debug=True)


from django.conf import settings
from django.urls import reverse
from django.core.files.base import ContentFile
from django.core.files import File

import tempfile
import os
import zipfile

import sarge
import toml
import magic
from pathlib import PurePath

from .models import Media

class BadgeRenderHelper:
    def __init__(self, badge_instance=None, badge_template=None, data=None):
        self.badge_instance = badge_instance

        if self.badge_instance:
            self.badge_template = badge_instance.template
        else:
            self.badge_template = badge_template

        if badge_instance:
            self.data = badge_instance.data
        else:
            self.data = data

    @property
    def config(self):
        return toml.load(PurePath(self.badge_template.repository, "config.toml"))

    def render_config(self, template_filename):
        for tf in self.config["files"]:
            if template_filename == tf["filename"]:
                return tf

    def _render(self, template_filename):
        render_config = self.render_config(template_filename)
        if self.badge_instance:
            url = reverse(
                "main:instance-filename",
                kwargs={
                    "slug": self.badge_template.slug,
                    "pk": self.badge_instance.pk,
                    "filename": template_filename,
                },
            )
        else:
            url = reverse(
                "main:preview-filename",
                kwargs={
                    "slug": self.badge_template.slug,
                    "filename": template_filename,
                },
            )

        file_format = render_config.get("format", "png")
        if file_format == "pdf":
            f = tempfile.NamedTemporaryFile(delete=False)
            tmppdf = f.name
            f.close()

            cmd = (
                "node {chrome_pdf_bin} "
                "--paper-width 8.27 --paper-height 11.69"
                "--no-margins --landscape --include-background --url {url} --pdf {tmppdf}".format(
                    chrome_pdf_bin=settings.CHROME_PDF_BIN,
                    tmppdf=tmppdf,
                    url="{}{}".format(settings.RENDER_PREFIX_URL, url),
                )
            )
            sarge.run(cmd, stdout=sarge.Capture())

            f = open(tmppdf, "rb")
            image = f.read()
            f.close()
            os.unlink(f.name)

        else:
            cmd = (
                "{capture_bin} {url} --width={width} --height={height} "
                "--type={type} --element={element} --no-default-background".format(
                    width=render_config.get("screen_width", 1000),
                    height=render_config.get("screen_height", 1000),
                    type=file_format,
                    element=sarge.shell_quote(
                        render_config.get("element", ".screenshot")
                    ),
                    url="{}{}".format(settings.RENDER_PREFIX_URL, url),
                    capture_bin=settings.CAPTURE_BIN,
                )
            )
            p = sarge.run(cmd, stdout=sarge.Capture())

            image = p.stdout.bytes

        mime = magic.from_buffer(image, mime=True)
        return image, mime

    def get_rendered_file(self, *, template_filename=None, materialize=False):
        if materialize:
            render_config = self.render_config(template_filename)
            file_format = render_config.get("format", "png")

            file_stem = PurePath(template_filename).stem
            media_filename = "{}.{}".format(file_stem, file_format)

            image, mime = self._render(template_filename=template_filename)
            media = Media(
                instance=self.badge_instance,
                kind="media",
                original_filename=media_filename.replace("single-", ""),
            )

            media.file.save(media_filename, content=ContentFile(image))
            media.save()
            return media

        else:
            return self._render(template_filename=template_filename)

    def build_archive(self):
        Media.objects.filter(instance=self.badge_instance).delete()

        rendered_files = []
        for tf in self.config["files"]:
            media = self.get_rendered_file(
                template_filename=tf["filename"], materialize=True
            )
            rendered_files.append(media)

        f = tempfile.NamedTemporaryFile(delete=False)
        zipf_name = f.name
        f.close()

        archive_name = self.badge_instance.external

        zipf = zipfile.ZipFile(zipf_name, "w", zipfile.ZIP_DEFLATED)
        for tf in rendered_files:
            arcname = tf.original_filename
            zipf.write(tf.file.path, arcname="{}/{}".format(archive_name, arcname))
        zipf.close()

        archive = Media(instance=self.badge_instance, kind="archive")
        archive.file.save(
            name=self.badge_instance.external + ".zip",
            content=File(open(zipf_name, "rb")),
            save=True,
        )
        archive.save()


# -*- coding: utf-8 -*-
import requests
import re
from config import *
from mysql import MySQL

# 实例化MySQL类
mysql = MySQL()

def get_page_html(url):
    """
        获取首页HTML源代码
    """

    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36'
    }
    response = requests.get(url, headers=headers)

    if response.status_code == 200:
        return response.text
    return None

def getTypeURL(html):
    """
        提取成都招聘热门职位的URL
    """
    # 提取成都招聘热门职位

    # 正则
    print('-'*100)
    print('-----获取类别-----')
    pattern1 = re.compile('
([\s\S]*?)
', re.S)
    types_html = re.findall(pattern1, html)

    # 提取详细分类URLxx
    pattern2 = re.compile('(.*?)', re.S)
    items = re.findall(pattern2, str(types_html))

    #print(items)
    # 添加到MySQL
    for item in items:
        # 遇到其他职位则跳过
        print()
        if item[1].find('其他') >= 0:
            continue

        print('得到类别：%s' % item[1])
        # 将类别名和url封装到字典
        data =  {
            'url': BASE_URL + item[0],
            'typename': item[1]
        }
        # 插入数据库
        mysql.insert('types_url', data)

def getJobsURL(types_url):
    """
        查询数据库获得类别的URL，根据类别URL获取职位的URL
    """
    print('='*100)

    for row in types_url:
        print('---------- 获取 %s 类工作 ----------' % (row[2]))
        # 获取分类HTML
        type_html = get_page_html(row[1])
        # 获取工作URL
        getJobURL(type_html)

        current_html = type_html
        # 获取类别的下一页，循环次数为NEXT_PAGE_NUM(在config.py-默认=99)
        for i in range(NEXT_PAGE_NUM):
            #print(current_html)
            current_html = getNextPage(current_html)    # 返回值为当前页的下一页作为下次循环的当前页
            if current_html == None:
                break

def getJobURL(type_html):
    """
        根据类别页获取工作URL，并加入MySQL
    """
    print('获取job地址中...')
    # 正则提取工作URL
([\s\S]*?)

    pattern1 = re.compile('
[\s\S]*?
([\s\S]*?)<div[\s\S]*?
[\s\S]*?
', re.S)
    jobs_html = re.findall(pattern1, type_html)

    # 提取每个工作的URL ([\s\S]*?)
    pattern2 = re.compile('href="http://[\s\S]*?.ganji.com([\s\S]*?)"[\s\S]*?post_url[\s\S]*?>([\s\S]*?)<', re.S)
    items = re.findall(pattern2, str(jobs_html))

    if items == None:
        return None
    else:
        # 添加到MySQL
        for item in items:

            # 去除垃圾URL，如公司URL
            other_url = None
            other_url = re.search(r'.*?gongsi.*?', item[0], re.M|re.I)

            # 没有匹配到垃圾URL则跳过本次循环
            if other_url != None:
                continue

            # 封装到字典
            data = {
                'url': BASE_URL + item[0],
                'jobname': item[1].strip()  # 去除首尾空白
            }
            # 加入数据库
            mysql.insert('jobs_url', data)

def getNextPage(current_html):
    """
        获取该类别的下一页
    """
    if current_html == None:
        return None
    # 提取分页ul
([\s\S]*?)

    pattern1 = re.compile('
([\s\S]*?)
', re.S)
    url_html = re.findall(pattern1, current_html)

    # 该类别的分页  href="(.*?)"
    pattern2 = re.compile('href="(.*?)"', re.S)

    # 下页URL
    i = 0
    next_page_url = None
    for url in re.findall(pattern2, str(url_html)):
        next_page_url = str(url)

    # 获取工作URL
    if next_page_url != None:

        next_page_url = BASE_URL + next_page_url
        print('---------- 获取子页 %s ----------' % (next_page_url))

        next_page_html = get_page_html(next_page_url)
        getJobURL(next_page_html)
        # 返回下一页的HTML
        return next_page_html

def getJobInfo(jobs_url):
    """
        根据数据库得到的job URL得到相关信息，并存入MySQL
    """
    print('$'*100)
    print('获取工作详细信息中...')
    flag = 0 # 已存储数量
    test_flag = 0 # 验证提醒(假设连续3次null，则提示验证，某些工作可能会遗漏爬取，但数量极少)

    for url in jobs_url:

        print(url[1])
        html = str(get_page_html(url[1]))

        #html = get_page_html('http://cd.ganji.com/zpshichangyingxiao/2759567089x.htm')
        if html == None:
            continue
        # 职位
        jobname = re.search(r'
[\s\S]*?
([\s\S]*?)
([\s\S]*?)
', html, re.M|re.I)
        if jobname == None:
            jobname = 'null'
        else:
            jobname = jobname.group(1)

        # 工资
        salary = re.search(r'
[\s\S]*?([\s\S]*?)[\s\S]*?([\s\S]*?)[\s\S]*?
', html, re.M|re.I)
        if salary == None:
            salary = 'null'
        else:
            salary = salary.group(1) + salary.group(2)

        # 公司名称
        company = re.search(r'
[\s\S]*?
[\s\S]*?([\s\S]*?)[\s\S]*?
[\s\S]*?
', html, re.M|re.I)
        if company == None:
            company = 'null'
        else:
            company = company.group(1)

        # 工作地点
        location = re.search(r'
[\s\S]*?
([\s\S]*?)<[\s\S]*?
[\s\S]*?
', html, re.M|re.I)
        if location == None:
            location = 'null'
        else:
            location = location.group(1).strip().replace(' ', '')

        # 验证判断
        if jobname == 'null' and salary == 'null' and company == 'null' and location == 'null':
            test_flag += 1
            if test_flag == 3:
                print("^"*20)
                print("需要验证（如果无需验证直接回车）...")
                print("验证后回车继续...", end='')
                input()
                print("/n^"*20)
                test_flag = 0
            continue
        # 工作信息字典
        data = {
            'name': jobname,
            'salary': salary,
            'company': company,
            'location': location
        }
        # 向数据库插入data 中的数据
        mysql.insert('jobs_info', data)

        # 打印
        flag = flag + 1
        if (flag % 100) == 0:
            print('='*20, end='')
            print(' 已存储: %s ' % (flag), end='')
            print('='*20)

def main():
    """
        主函数
    """

    # 1.获取首页HTML源代码
    html = get_page_html(BASE_URL + '/zhaopin')
    if html == None:
        print('获取首页失败!')
        exit(0)
    #print(html)

    # 2.提取HTML代码，获得成都招聘热门职位的URL，并加入MySQL
    getTypeURL(html)

    # 3.从数据库获取类别的URL，提取每个类别下的职位URL
    types_url = mysql.select('types_url', None)
    getJobsURL(types_url)

    # 4.从数据库获取职位的URL，得到HTML获取相关信息并加入MySQL
    jobs_url = mysql.select('jobs_url', None)
    getJobInfo(jobs_url)

if __name__ == '__main__':
    main()


import os
import re
import json
from get_commands import Retrieve
from post_results import Deliver
from register_task import Task

def format_response(status_code, result, message, log, **kwargs):
    response = {'outcome': result}
    if message:
        response['message'] = message
    if kwargs:
        for k, v in kwargs.items():
            if v:
                response[k] = v
    if log:
        log['response'] = response
        print(log)
    return {'statusCode': status_code, 'body': json.dumps(response)}

def lambda_handler(event, context):
    region = re.search('arn:aws:lambda:([^:]+):.*', context.invoked_function_arn).group(1)
    deployment_name = os.environ['DEPLOYMENT_NAME']
    results_queue_expiration = int(os.environ['RESULTS_QUEUE_EXPIRATION'])
    enable_task_results_logging = os.environ['ENABLE_TASK_RESULTS_LOGGING']
    log = {'event': event}
    results = None
    detail = None

    user_id = event['requestContext']['authorizer']['user_id']
    data = json.loads(event['body'])
    try:
        command = data['command']
    except:
        return format_response(400, 'failed', 'invalid request', log)

    if command not in ['register_task', 'get_commands', 'post_results']:
        return format_response(400, 'failed', f'{command} is not a valid command', log)

    if 'detail' in data:
        detail = data['detail']

    if 'results' in data:
        results = data['results']

    if command == 'register_task':
        if not detail:
            return format_response(400, 'failed', 'missing detail', log)
        else:
            t = Task(region, deployment_name, user_id, detail, log)
            response = t.registration()
            return response

    if command == 'get_commands':
        if not detail:
            return format_response(400, 'failed', 'missing detail', log)
        else:
            r = Retrieve(region, deployment_name, user_id, detail, log)
            response = r.retrieve_commands()
            return response

    if command == 'post_results':
        if not results:
            return format_response(400, 'failed', 'missing results', log)
        else:
            d = Deliver(region, deployment_name, results_queue_expiration, enable_task_results_logging, user_id, results, log)
            response = d.deliver_result()
            return response


from io import StringIO, TextIOWrapper
from zipfile import ZipFile

from django.core.management import call_command
from django.core.management.base import CommandError

import pytest
from requests import ConnectionError, HTTPError

from ...routes.models import Place
from ...routes.models.place import PlaceTuple, PlaceType
from ...routes.tests.factories import PlaceFactory
from ..geonames import PLACE_DATA_URL as GEO_DATA_URL
from ..geonames import (
    PLACE_TYPE_URL,
    get_geonames_remote_file,
    import_places_from_geonames,
    update_place_types_from_geonames,
)
from ..swissnames3d import PLACE_DATA_URL as SWISS_DATA_URL
from ..swissnames3d import (
    get_swissnames3d_remote_file,
    import_places_from_swissnames3d,
    unzip_swissnames3d_remote_file,
)
from ..utils import download_zip_file, get_csv_line_count, save_places_from_generator

#########
# utils #
#########

def test_get_csv_line_count(open_file):
    csv_file = open_file("swissnames3d_LV03_test.csv")
    assert get_csv_line_count(csv_file, header=True) == 202
    assert get_csv_line_count(csv_file, header=False) == 203
    assert csv_file.tell() == 0

def test_get_csv_line_count_empty_file(open_file):
    csv_file = open_file("text.txt")
    assert get_csv_line_count(csv_file, header=True) == 0
    assert get_csv_line_count(csv_file, header=False) == 0
    assert csv_file.tell() == 0

def test_download_zip_file(mock_zip_response):
    url = GEO_DATA_URL.format(scope="LI")
    mock_zip_response(url, "geonames_LI.zip", content_length="13091")
    assert isinstance(download_zip_file(url), ZipFile)

def test_download_zip_file_not_found(mock_html_not_found):
    url = GEO_DATA_URL.format(scope="ZZ")
    mock_html_not_found(url)
    with pytest.raises(HTTPError, match=f"404 Client Error: Not Found for url: {url}"):
        download_zip_file(url)

@pytest.mark.django_db
def test_save_places_from_generator():
    existing_places = PlaceFactory.create_batch(5, data_source="geonames")
    new_places = PlaceFactory.build_batch(20, data_source="geonames")
    places = existing_places + new_places

    data = (
        PlaceTuple(
            name=place.name,
            place_type=place.place_type.code,
            latitude=place.geom.y,
            longitude=place.geom.x,
            altitude=place.altitude,
            country=place.country,
            source_id=place.source_id,
            data_source=place.data_source,
            srid=4326,
        )
        for place in places
    )
    msg = "Created 20 new places and updated 5 places. "
    assert save_places_from_generator(data, count=20, source_info="test_source") == msg
    assert Place.objects.count() == 25

@pytest.mark.django_db
def test_save_places_from_generator_empty():
    places = []
    data = (place for place in places)
    msg = "Created 0 new places and updated 0 places. "
    assert save_places_from_generator(data, count=0, source_info="test_source") == msg
    assert Place.objects.count() == 0

@pytest.mark.django_db
def test_save_places_from_generator_bad_place_type(capsys):

    place = PlaceFactory.build(data_source="geonames")
    place.place_type.code = "BADCODE"
    data = (
        PlaceTuple(
            name=place.name,
            place_type=place.place_type.code,
            latitude=place.geom.y,
            longitude=place.geom.x,
            altitude=place.altitude,
            country=place.country,
            source_id=place.source_id,
            data_source=place.data_source,
            srid=4326,
        )
        for place in [place]
    )
    msg = "Created 0 new places and updated 0 places. "
    assert save_places_from_generator(data, count=1, source_info="test_source") == msg
    captured = capsys.readouterr()
    assert "Place type code: BADCODE does not exist.\n" in captured.out
    assert Place.objects.count() == 0

@pytest.mark.django_db
def test_save_places_from_generator_bad_country_type(capsys):
    place = PlaceFactory.build(data_source="geonames")
    data = (
        PlaceTuple(
            name=place.name,
            place_type=place.place_type.code,
            latitude=place.geom.y,
            longitude=place.geom.x,
            altitude=place.altitude,
            country="ZZ",
            source_id=place.source_id,
            data_source=place.data_source,
            srid=4326,
        )
        for place in [place]
    )
    msg = "Created 0 new places and updated 0 places. "
    assert save_places_from_generator(data, count=1, source_info="test_source") == msg
    captured = capsys.readouterr()
    assert "Country code: ZZ could not be found.\n" in captured.out
    assert Place.objects.count() == 0

###############
# geonames.py #
###############

def test_get_geonames_remote_file(mock_zip_response):
    url = GEO_DATA_URL.format(scope="LI")
    mock_zip_response(url, "geonames_LI.zip", content_length="13091")
    assert isinstance(get_geonames_remote_file(scope="LI"), TextIOWrapper)

def test_get_geonames_remote_file_not_found(mock_html_not_found):
    url = GEO_DATA_URL.format(scope="ZZ")
    mock_html_not_found(url)
    with pytest.raises(HTTPError):
        get_geonames_remote_file(scope="ZZ")

def test_get_geonames_remote_file_connection_error(mock_connection_error):
    url = GEO_DATA_URL.format(scope="XX")
    mock_connection_error(url)
    with pytest.raises(ConnectionError):
        get_geonames_remote_file(scope="XX")

@pytest.mark.django_db
def test_import_places_from_geonames_local_file(open_file):
    file = open_file("geonames_LI.txt")
    msg = "Created 200 new places and updated 0 places. "
    assert import_places_from_geonames(file=file) == msg

def test_import_places_from_geonames_not_found(mock_html_not_found):
    scope = "ZZ"
    url = GEO_DATA_URL.format(scope=scope)
    mock_html_not_found(url)
    msg = f"File for {scope} could not be downloaded from geonames.org: "
    assert msg in import_places_from_geonames("ZZ")

@pytest.mark.django_db
def test_update_place_types_from_geonames(mock_html_response):
    url = PLACE_TYPE_URL
    mock_html_response(url, "geonames_codes.html")
    PlaceType.objects.all().delete()
    update_place_types_from_geonames()
    assert PlaceType.objects.count() == 681

##################
# swissnames3d.py #
##################

def test_get_swissnames3d_remote_file(mock_zip_response):
    url = SWISS_DATA_URL
    mock_zip_response(url, "swissnames3d_data.zip", content_length="70171")
    assert isinstance(get_swissnames3d_remote_file(), TextIOWrapper)

def test_unzip_swissnames3d_remote_file(open_file):
    zip_file = ZipFile(open_file("swissnames3d_data.zip", binary=True))
    text_file = unzip_swissnames3d_remote_file(zip_file)
    assert isinstance(text_file, TextIOWrapper)
    assert text_file.name == "swissNAMES3D_LV95/csv_LV95_LN02/swissNAMES3D_PKT.csv"

@pytest.mark.django_db
def test_import_places_from_swissnames3d(open_file):
    file = open_file("swissnames3d_LV03_test.csv")
    msg = "Created 146 new places and updated 46 places. "
    assert import_places_from_swissnames3d(projection="LV03", file=file) == msg

##################################
# import_geonames_places command #
##################################

@pytest.mark.django_db
def test_import_geonames_places(mock_zip_response):
    scope = "LI"
    url = GEO_DATA_URL.format(scope=scope)
    mock_zip_response(url, "geonames_LI.zip", content_length="13091")

    out = StringIO()
    call_command("import_geonames_places", scope, stdout=out)

    msg = "Created 200 new places and updated 0 places. "
    assert msg in out.getvalue()
    assert Place.objects.filter(data_source="geonames").count() == 200

@pytest.mark.django_db
def test_import_geonames_places_multiple(
    mock_zip_response, mock_html_not_found, mock_connection_error, current_dir_path
):
    mock_zip_response(GEO_DATA_URL.format(scope="LI"), "geonames_LI.zip", content_length="13091")
    mock_html_not_found(GEO_DATA_URL.format(scope="ZZ"))
    mock_connection_error(GEO_DATA_URL.format(scope="XX"))
    file_path = current_dir_path / "data/geonames_LI.txt"

    out = StringIO()
    call_command(
        "import_geonames_places",
        "LI",
        "ZZ",
        "XX",
        "-f",
        file_path.as_posix(),
        stdout=out,
    )

    msgs = [
        "Created 200 new places and updated 0 places. ",
        "Created 0 new places and updated 200 places. ",
        "File for ZZ could not be downloaded from geonames.org: ",
        f'Error connecting to {GEO_DATA_URL.format(scope="XX")}. ',
    ]
    assert all([msg in out.getvalue() for msg in msgs])

def test_import_geonames_places_no_param(mock_zip_response):
    with pytest.raises(CommandError):
        call_command("import_geonames_places")

######################################
# import_geonames_places command #
######################################

@pytest.mark.django_db
def test_import_swissnames3d_places(mock_zip_response):
    url = SWISS_DATA_URL
    mock_zip_response(url, "swissnames3d_data.zip", content_length="70171")

    out = StringIO()
    call_command("import_swissnames3d_places", stdout=out)

    msg = "Created 146 new places and updated 46 places. "
    assert msg in out.getvalue()
    assert Place.objects.filter(data_source="swissnames3d").count() == 146

@pytest.mark.django_db
def test_import_swissnames3d_places_projection(mock_zip_response):
    url = SWISS_DATA_URL
    mock_zip_response(url, "swissnames3d_data.zip", content_length="70171")

    out = StringIO()
    call_command("import_swissnames3d_places", "-p", "LV03", stdout=out)

    msg = "Created 146 new places and updated 46 places. "
    assert msg in out.getvalue()
    assert Place.objects.filter(data_source="swissnames3d").count() == 146

@pytest.mark.django_db
def test_import_swissnames3d_places_file(current_dir_path):
    file_path = current_dir_path / "data/swissnames3d_LV95_test.csv"
    out = StringIO()
    call_command("import_swissnames3d_places", "-f", file_path.as_posix(), stdout=out)

    msg = "Created 146 new places and updated 46 places. "
    assert msg in out.getvalue()
    assert Place.objects.filter(data_source="swissnames3d").count() == 146

def test_import_swissnames3d_places_connection_error(mock_connection_error):
    mock_connection_error(SWISS_DATA_URL)
    out = StringIO()
    call_command("import_swissnames3d_places", stdout=out)

    msg = f"Error connecting to {SWISS_DATA_URL}. "
    assert msg in out.getvalue()

def test_import_swissnames3d_places_not_found(mock_html_not_found):
    mock_html_not_found(SWISS_DATA_URL)
    out = StringIO()
    call_command("import_swissnames3d_places", stdout=out)

    msg = f"Error downloading {SWISS_DATA_URL}: "
    assert msg in out.getvalue()


# Example 1
import numpy as np

X = np.array([[-1, -1], [-2, -1], [1, 1], [2, 1]])
y = np.array([1, 1, 2, 2])

from sklearn.svm import SVC

# clf = SVC(gamma='auto')

clf = SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,
          decision_function_shape='ovr', degree=3, gamma='auto', kernel='linear',
          max_iter=-1, probability=False, random_state=None, shrinking=True,
          tol=0.001, verbose=False)  # 可以根据前面介绍的参数，做出相应改变观察结果变化

clf.fit(X, y)
print(clf.predict([[-0.8, -1]]))

print(clf.support_vectors_)
print(clf.dual_coef_, clf.coef_, clf.intercept_)


import skimage.io as io
import numpy as np
import matplotlib.pyplot as plt
import skimage.util.noise as noise
import skimage.util as util
import scipy.ndimage as ndi
from numpy.fft import fftshift, fft2, ifft2, ifftshift
import skimage.exposure as ex

# Read the input image
img = io.imread('5.3.01.tiff')

# Add salt and pepper noise to the image
sp = noise.random_noise(img, mode='s&p', amount=0.3)

# Add periodic noise
r, c = img.shape
x, y = np.mgrid[0:r, 0:c].astype('float32')
p = np.sin(x / 3 + y / 3) + 1.0
gp = (2 * util.img_as_float(img) + p / 2) / 3

# Display images with salt and pepper noise and periodic noise
plt.subplot(1, 2, 1)
plt.title('Image with salt and pepper noise')
plt.imshow(sp, cmap='gray')
plt.subplot(1, 2, 2)
plt.title('Image with Periodic Noise')
plt.imshow(gp, cmap='gray')
plt.show()

# Apply median filtering to remove salt and pepper noise
img_med = ndi.median_filter(sp, 3)

# Apply the band-reject filter to the image with periodic noise
gf=fftshift(fft2(gp))
mg_spectrum=np.abs(gf)
temp= ex.rescale_intensity(abs(gf),out_range=(0,1))
gf2=util.img_as_ubyte(temp)
gf2[512,512]=0  #center of dft
i,j=np.where(gf2==gf2.max())
z=np.sqrt((x-512)**2+(y-512)**2)
k=1
d=np.sqrt(5832)
br=(z<np.floor(d-k))|(z>np.ceil(d+k))
gfr=gf*br
filtered_gp = np.real(ifft2(ifftshift(gfr)))

# Display the images with removed salt and pepper noise and periodic noise
plt.subplot(1, 2, 1)
plt.title('Removing salt and pepper noise')
plt.imshow(img_med, cmap='gray')
plt.subplot(1, 2, 2)
plt.title('Removing Periodic Noise')
plt.imshow(filtered_gp, cmap='gray')
plt.show()


"""Calculates P values from a given variants file and baseline.

Takes in a varscan SNP file and a baseline file
and in turn calculates P-values for each variant.
"""
import logging
import os
import sys
import numpy as np
import pandas as pd
from scipy import stats
import vcf

log = logging.getLogger(__name__)

def build_parser(parser):
    parser.add_argument('sample_variants',
                        help='Variants file for a sample')
    parser.add_argument('error_baseline',
                        help='An error profile baseline file to analyze against')
    parser.add_argument('output_file',
                        help='Output file to write to')
    parser.add_argument('--full_output', action='store_true',
                        help="Output full merged data from error profile and variant file as csv")

def action(args):
    ''' Takes in arguments for sample_variants, error_baseline,
    output_file and calculates P-values for variants.'''
    sample_variants = args.sample_variants
    error_baseline = args.error_baseline
    output_file = args.output_file
    full_output = args.full_output

    # Check that required input files exist
    if not os.path.isfile(sample_variants):
        print("Sample variants file not found. Exiting.")
        sys.exit()
    if not os.path.isfile(error_baseline):
        print("Error baseline file not found. Exiting.")
        sys.exit()

    # build pandas dataframe for sample from VCF
    columns = ['position_base','chromosome','position','var_base','ref_base','var_freq_pct','var_freq_flt','var_pval']
    vcf_reader = vcf.Reader(open(sample_variants))
    row_list = []
    for record in vcf_reader:
        call_data = record.samples[0].data
        # Need following data: Chromosome, Position, Var Base, Ref Base, Var Freq
        row_dict = {}
        row_dict['chromosome'] = record.CHROM
        row_dict['position'] = record.POS
        if len(record.ALT) == 1:
            row_dict['var_base'] = record.ALT[0]
        else:
            # May need to support this for indels/large deletions in future versions
            print("Multiple variant bases...not supported - passing")
            continue
        row_dict['ref_base'] = record.REF
        row_dict['var_freq_pct'] = call_data.FREQ
        row_dict['var_freq_flt'] = percent_to_float(call_data.FREQ)
        row_dict['var_pval'] = call_data.PVAL
        row_dict['position_base'] = str(row_dict['chromosome']) + ':' + str(row_dict['position']) + ':' + str(row_dict['var_base'])
        row_list.append(row_dict)
    sample_df = pd.DataFrame(row_list, columns=columns)

    # create another dataframe from the error baseline
    dtype={'chrom':str,'position':int,'base':str,'mean':float,'std':float,'alpha':float,'beta':float}

    # Read error_baseline file in from csv file
    error_df = pd.read_csv(error_baseline, dtype=dtype)
    error_df['position_base'] = error_df.apply(get_error_position_base_key,axis=1)
    error_df.set_index('position_base')

    # do inner join with sample_df on the common key position_base (ie chrom:position:base)
    merged_df = sample_df.merge(error_df, on='position_base',how='inner')

    # uw_dec_p_value calculated using beta distribution parameters
    merged_df['uw_dec_p_value'] = merged_df.apply(calculate_p_value,axis=1)

    # used in full output only
    merged_df['pdf'] = merged_df.apply(calculate_pdf,axis=1)
    merged_df['cdf'] = merged_df.apply(calculate_cdf,axis=1)

    # drop rows with null results
    final_df = merged_df[merged_df['uw_dec_p_value'].isnull()==False]

    # Write output file
    if full_output:
        merged_df_to_full_output(final_df, output_file)
    else:
        merged_df_to_munge_ready_output(final_df, output_file)

def get_sample_position_base_key(row):
    ''' Creates a position key consisting of the chromosome, position, and base'''
    return str(row['Chrom']) + ":" + str(row['Position']) + ":" + str(row['Var'])

def get_error_position_base_key(row):
    ''' Creates a position key consisting of the chromosome, position, and base'''
    return str(row['chrom']) + ":" + str(row['position']) + ":" + str(row['base'])

def calculate_pdf(row):
    ''' Calculates the probability density function given beta values.'''
    if row['var_freq_flt'] and row['alpha'] and row['beta']:
        var_freq_flt = row['var_freq_flt']
        if var_freq_flt == 1.0:
            var_freq_flt = var_freq_flt - 0.0001 #0.9999
        return stats.beta.pdf(var_freq_flt, row['alpha'], row['beta'])
    else:
        return None

def calculate_cdf(row):
    ''' Calculates the cumulative density function given beta distribution parameters.'''
    if row['var_freq_flt'] and row['alpha'] and row['beta']:
        var_freq_flt = row['var_freq_flt']
        if var_freq_flt == 1.0:
            var_freq_flt = var_freq_flt - 0.0001 #0.9999
        return stats.beta.cdf(var_freq_flt, row['alpha'], row['beta'])
    else:
        return None

def calculate_p_value(row):
    """ Calculates the cdf and inverts it."""
    if row['var_freq_flt'] and row['alpha'] and row['beta']:
        var_freq_flt = row['var_freq_flt']
        if var_freq_flt == 1.0:
            var_freq_flt = var_freq_flt - 0.0001 #0.9999
        return (1 - stats.beta.cdf(var_freq_flt, row['alpha'], row['beta']))
    else:
        return None

def percent_to_float(val):
    '''turns a percentage to a float.'''
    return float(val.strip('%'))/100

def merged_df_to_full_output(merged_df, output_file):
    '''output full csv'''
    merged_df.to_csv(output_file)

def merged_df_to_munge_ready_output(merged_df, output_file):
    ''' Just need to output:
    generic\tp.value\tchr\tstart\tstop\tref_base\tvar_base\n
    '''
    out_f = open(output_file, 'w')
    # loop through dataframe, write row to file
    for index, row in merged_df.iterrows():
        line = "generic\t" + str(row['uw_dec_p_value']) + "\t" + str(row['chrom']) + "\t" + str(row['position_x']) + "\t" + str(row['position_x']) + "\t" + str(row['ref_base']) + "\t" + str(row['var_base']) + "\n"
        out_f.write(line)
    out_f.close()


import math
import torch
import torch.distributed as dist
from torch.utils.data.sampler import Sampler
from torch._six import int_classes as _int_classes

class BatchSampler(Sampler):
    r"""Wraps another sampler to yield a mini-batch of indices.

    Args:
        sampler (Sampler): Base sampler.
        batch_size (int): Size of mini-batch.
        drop_last (bool): If ``True``, the sampler will drop the last batch if
            its size would be less than ``batch_size``

    Example:
        >>> list(BatchSampler(SequentialSampler(range(10)), batch_size=3, drop_last=False))
        [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]]
        >>> list(BatchSampler(SequentialSampler(range(10)), batch_size=3, drop_last=True))
        [[0, 1, 2], [3, 4, 5], [6, 7, 8]]
    """

    def __init__(self, sampler, batch_size, drop_last):
        if not isinstance(sampler, Sampler):
            raise ValueError("sampler should be an instance of "
                             "torch.utils.data.Sampler, but got sampler={}"
                             .format(sampler))
        if not isinstance(batch_size, _int_classes) or isinstance(batch_size, bool) or \
                batch_size <= 0:
            raise ValueError("batch_size should be a positive integer value, "
                             "but got batch_size={}".format(batch_size))
        if not isinstance(drop_last, bool):
            raise ValueError("drop_last should be a boolean value, but got "
                             "drop_last={}".format(drop_last))
        self.sampler = sampler
        self.batch_size = batch_size
        self.drop_last = drop_last

    def __iter__(self):
        batch = []
        for idx in self.sampler:
            batch.append(idx)
            if len(batch) == self.batch_size:
                yield batch
                batch = []
        if len(batch) > 0 and not self.drop_last:
            yield batch

    def __len__(self):
        if self.drop_last:
            return len(self.sampler) // self.batch_size
        else:
            return (len(self.sampler) + self.batch_size - 1) // self.batch_size

class IterationBasedBatchSampler(BatchSampler):
    """
    Wraps a BatchSampler, resampling from it until
    a specified number of iterations have been sampled
    """

    def __init__(self, batch_sampler, num_iterations, start_iter=0):
        self.batch_sampler = batch_sampler
        self.num_iterations = num_iterations
        self.start_iter = start_iter

    def __iter__(self):
        iteration = self.start_iter
        while iteration <= self.num_iterations:
            # if the underlying sampler has a set_epoch method, like
            # DistributedSampler, used for making each process see
            # a different split of the dataset, then set it
            if hasattr(self.batch_sampler.sampler, "set_epoch"):
                self.batch_sampler.sampler.set_epoch(iteration)
            for batch in self.batch_sampler:
                iteration += 1
                if iteration > self.num_iterations:
                    break
                yield batch

    def __len__(self):
        return self.num_iterations

class DistributedSampler(Sampler):
    """Sampler that restricts data loading to a subset of the dataset.
    It is especially useful in conjunction with
    :class:`torch.nn.parallel.DistributedDataParallel`. In such case, each
    process can pass a DistributedSampler instance as a DataLoader sampler,
    and load a subset of the original dataset that is exclusive to it.
    .. note::
        Dataset is assumed to be of constant size.
    Arguments:
        dataset: Dataset used for sampling.
        num_replicas (optional): Number of processes participating in
            distributed training.
        rank (optional): Rank of the current process within num_replicas.
    """

    def __init__(self, dataset, num_replicas=None, rank=None, shuffle=True,
            length_divisible=1):
        if num_replicas is None:
            if not dist.is_available():
                raise RuntimeError("Requires distributed package to be available")
            from src.tools.common import get_mpi_size
            num_replicas = get_mpi_size()
        if rank is None:
            if not dist.is_available():
                raise RuntimeError("Requires distributed package to be available")
            from src.tools.common import get_mpi_rank
            rank = get_mpi_rank()
        self.dataset = dataset
        self.num_replicas = num_replicas
        self.rank = rank
        self.epoch = 0
        self.num_samples = int(math.ceil(len(self.dataset) * 1.0 / self.num_replicas))
        if length_divisible > 1:
            import logging
            logging.info('before making divisible = {}'.format(self.num_samples))
            self.num_samples = ((self.num_samples + length_divisible - 1) //
                    length_divisible) * length_divisible
            logging.info('adjust to = {}'.format(self.num_samples))
        self.total_size = self.num_samples * self.num_replicas
        self.shuffle = shuffle

    def __iter__(self):
        if self.shuffle:
            # deterministically shuffle based on epoch
            g = torch.Generator()
            g.manual_seed(self.epoch)
            indices = torch.randperm(len(self.dataset), generator=g).tolist()
        else:
            indices = torch.arange(len(self.dataset)).tolist()

        # add extra samples to make it evenly divisible
        assert (self.total_size - len(indices)) <= len(indices), 'not implemented'
        indices += indices[: (self.total_size - len(indices))]
        assert len(indices) == self.total_size

        # subsample
        offset = self.num_samples * self.rank
        indices = indices[offset : offset + self.num_samples]
        assert len(indices) == self.num_samples

        return iter(indices)

    def __len__(self):
        return self.num_samples

    def set_epoch(self, epoch):
        self.epoch = epoch


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# coding: utf-8
import sys
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

# index_col=0  将第0列作为行索引
from sklearn.model_selection import cross_val_score

train_df = pd.read_csv('data/train.csv', index_col=0)
test_df = pd.read_csv('data/test.csv', index_col=0)
print(train_df.head())

prices = pd.DataFrame({'price': train_df['SalePrice'], 'log(price+1)': np.log1p(train_df['SalePrice'])})
prices.hist()  # 画图 看一下标签是否平滑
plt.show()

y_train = np.log1p(train_df.pop('SalePrice'))  # 将原来的标签删除 剩下log(price+1)列的数据

# y_train = train_df.pop('SalePrice')

all_df = pd.concat((train_df, test_df), axis=0)  # 将train_df, test_df合并
# 有些数据的取值只有四五，或者可数个。这类数据我们转为one_hot编码

# 发现MSSubClass值应该是分类值
print(all_df['MSSubClass'].dtypes)  # int64
all_df['MSSubClass'] = all_df['MSSubClass'].astype(str)
print(all_df['MSSubClass'].value_counts())

all_dummy_df = pd.get_dummies(all_df)
# 我们用均值填充
mean_cols = all_dummy_df.mean()
all_dummy_df = all_dummy_df.fillna(mean_cols)

# 再检查一下是否有缺失值
print(all_dummy_df.isnull().sum().sum())  # 0
#先找出数字型数据
numeric_cols = all_df.columns[all_df.dtypes != 'object']
print(numeric_cols)

# 对其标准化
numeric_col_mean = all_dummy_df.loc[:, numeric_cols].mean()
numeric_col_std = all_dummy_df.loc[:, numeric_cols].std()
all_dummy_df.loc[:, numeric_cols] = (all_dummy_df.loc[:, numeric_cols]-numeric_col_mean) / numeric_col_std

# 将合并的数据此时进行拆分  分为训练数据和测试数据
dummy_train_df = all_dummy_df.loc[train_df.index]
dummy_test_df = all_dummy_df.loc[test_df.index]

X_train = dummy_train_df.values
# X_test = dummy_test_df.values

from xgboost import XGBRegressor

clf = XGBRegressor(max_depth=5,n_estimators=100)
test_score = np.sqrt(-cross_val_score(clf, X_train, y_train, cv=10, scoring='neg_mean_squared_error'))
print (np.mean(test_score))

# 用sklearn自带的cross validation方法来测试模型
# params = [1, 2, 3, 4, 5, 6]
# test_scores = []
# for param in params:
#     clf = XGBRegressor(max_depth=param)
#     test_score = np.sqrt(-cross_val_score(clf, X_train, y_train, cv=10, scoring='neg_mean_squared_error'))
#     test_scores.append(np.mean(test_score))
# plt.plot(params, test_scores)
# plt.title("max_depth vs CV Error")
# plt.show()
# print(test_scores)
# print(test_scores.mean())
from tools import  DrawTools
clf.fit(X_train, y_train)
DrawTools.feature_importance(clf, dummy_train_df.columns)
plt.show()


import numpy as np
import matplotlib.pyplot as plt
import multiprocessing as mp

def calc_spike_times(d_params):

    nNeurons = d_params['nNeurons']

    outputFN = "dir_output/output_I40E40_{p[extdrivefreq]}freq_{p[ampSin]}ampSin_0.03gei_0.3gie_1.0gii_randomei_{p[trial]}trial_1initcond_{p[phase]}pibysix_noreplace.txt".format(p=d_params)
    for neuron in range(nNeurons):
        d_params['neuron'] = neuron
        outfile = open("spike_times/spike_times_{p[freq]}freq_{p[trial]}trial_{p[neuron]}neuron.txt".format(p=d_params),"w")
        v = np.loadtxt(FN,comments='time',delimiter=',')
        t = v[::10,0]
        v = v[::10,neuron+1]

        preT = v[:-1]-v[1:]
        postT = v[1:]-v[:-1]

        when_spiked = []

        #for j in range(numEs):
        for i in range(len(v[1:-1])):
            if (preT[i] < 0) and (postT[i+1] < 0) and (v[i+1]>-40) and (t[i+1] >= 1500.0 ) and (t[i+1] < 9000.0):
                outfile.write(str(t[i+1]))
                outfile.write(" ")
        outfile.write("\n")
        outfile.close()

pool = mp.Pool()
pibysix = (np.pi/6.0)

l_freqNpulse = []
l_freqNpulse += [[freq,freq] for freq in [8.0]]
l_freqNpulse += [[0.0,freq] for freq in [8.0]]

l_dparams = []

for freq,extdrivefreq in l_freqNpulse:
    if freq != 0.0:
        ampSin = 0.04
        phase =  4.0
    else:
        ampSin = 0.0
        phase = 0.0
    for trial in range(10):
        for init_cond in [1]:
            l_dparams.append({'nNeurons':str(40),'freq':str(freq),'ampSin':str(ampSin),'trial':str(trial),'init_cond':str(init_cond),'extdrivefreq':str(extdrivefreq),'phase':str(phase)})

pool.map(calc_spike_times,l_params)
end_t = time.time()
print("It took ", (end_t - st_t )/60.0 ," minutes to run the entire set of simulations!!")


import json
from unittest import TestCase

from service import DataManager

class DataManagerTest(TestCase):

    def setUp(self):
        self.data = DataManager()

    def test_insert_record(self):
        result = self.data.update_record(1,
                                         first_name="Test",
                                         last_name="Test",
                                         company="Test",
                                         phone_1="Test"
                                         )
        self.assertTrue(result)

        expected_data = {
            "last_name": "Test",
            "first_name": "Test",
            "middle_name": "Иванович",
            "company": "Test",
            "phone_1": "Test",
            "phone_2": "+79129998877"
        }

        with open("data.json", "r") as fp:
            data = json.load(fp)

        self.assertEqual(expected_data, data[0])


import numpy
import pickle

pkl_file = open('data_l.pkl', 'rb')

data1 = pickle.load(pkl_file)

bestresults = numpy.array(list(map(lambda x: x[1], data1)))

import matplotlib.pyplot as plt

fig = plt.figure()
# plt.yscale('log')

labels = [0.75, 0.375, 0.25, 0.75/4, 0.75/5, 0.75/6, 0.75/7, 0.75/8, 0.75/9, 0.075]

z = []
for i in range(10):
    seq = bestresults[i: 101: 10]
    cmap = plt.get_cmap('jet')
    colors = [cmap(i) for i in numpy.linspace(0, 1, 10)]

    q = []

    for j in range(10):
        q.append(seq[j].flatten())
        '''if 0 == j:
            plt.plot(seq[j].flatten(), color=colors[i], label='T(initial)=' + str(labels[i]))
        else:
            plt.plot(seq[j].flatten(), color=colors[i])
    '''
    qm = []
    qe = []
    offsets = [999]
    for offs in offsets:
        qv = list(map(lambda x: x[offs], q))
        qm.append(numpy.mean(qv))
        qe.append(numpy.std(qv))
    z.append((qm[0]))
    plt.errorbar([labels[i]], qm, numpy.sqrt(qe))

plt.plot(labels, z, 'k')

plt.xlabel('t')
plt.ylabel('E')
#plt.legend()
fig.show()

input("exit>")


"""
@file   image_embeddings.py
@author Nianchen Deng, Shanghai AI Lab
@brief  Learnable image embeddings
"""

__all__ = [
    'ImageEmbeddings'
]

import torch
import torch.nn as nn
from nr3d_lib.config import ConfigDict

from nr3d_lib.utils import torch_dtype
from nr3d_lib.fmt import log

from app.resources import Scene, SceneNode
from app.models.base import AssetAssignment, AssetModelMixin

class ImageEmbeddings(AssetModelMixin, nn.ModuleDict):
    assigned_to = AssetAssignment.SCENE
    def __init__(
        self,
        dims: int,
        ego_node_id: str=None, ego_class_name: str="Camera",
        weight_init: str="normal", weight_init_std: float=1.0,
        dtype=torch.float, device=torch.device('cuda')) -> None:
        super().__init__()

        self.dims = dims
        self.ego_node_id = ego_node_id
        self.ego_class_name = ego_class_name
        self.weight_init = weight_init
        self.weight_init_std = weight_init_std
        self.dtype = torch_dtype(dtype)
        self.device = device

    def populate(self, scene: Scene = None, obj: SceneNode = None, config: ConfigDict = None,
                 dtype=torch.float, device=torch.device('cuda'), **kwargs):
        self.scene = scene
        if self.ego_node_id is not None:
            ego_node_list = [self.scene.all_nodes[self.ego_node_id]]
        elif self.ego_class_name is not None:
            ego_node_list = self.scene.all_nodes_by_class_name[self.ego_class_name]
        else:
            raise RuntimeError(
                f"Invalid combination of arguments ego_node_id={self.ego_node_id}, "
                f"ego_class_name={self.ego_class_name}")

        self.exposures = nn.ModuleDict()
        n_frames = len(self.scene)
        for ego_node in ego_node_list:
            embedding_weight = torch.empty(n_frames, self.dims, dtype=self.dtype)
            if self.weight_init == "uniform":
                embedding_weight.uniform_(-self.weight_init_std, self.weight_init_std)
            elif self.weight_init == "normal":
                embedding_weight.normal_(0., self.weight_init_std)
            elif self.weight_init == "zero":
                embedding_weight.zero_()
            else:
                raise ValueError("Unknown weight initial method")
            self[ego_node.id] = nn.Embedding(n_frames, self.dims, dtype=self.dtype, _weight=embedding_weight, device=self.device)
        log.info(f"{self.scene.id} create image embeddings for {[node.id for node in ego_node_list]}")

    @classmethod
    def compute_model_id(cls, scene: Scene = None, obj: SceneNode = None, class_name: str = None) -> str:
        return f"{cls.__name__}#{scene.id}"


import yfinance as yf
import pymysql
import pandas as pd
import os
from yahoo_fin import stock_info as si
db = pymysql.connect("localhost", "stockuser", "123456", "stock_advisor")
cursor = db.cursor()
cursor.execute("SELECT symbol FROM symbols WHERE active=1")
symbols=cursor.fetchall()

def main():
    print('Starting market profits checking module')

    SUMM()

def SUMM():
    open('data/out2_tmp.csv', 'w').close()
    open('data/out3_tmp.csv', 'w').close()
    open('data/out4_tmp.csv', 'w').close()
    for symbol in symbols: #Loop trough the stock summary
          market=(symbol[0])
          #print (symbol)
          #market = yf.Ticker(symbol)
          print (market, summ_serf(market))
          if summ_serf(market)!=0:
              #print (market, summ_serf(market), count(market))
              f= open('data/out2_tmp.csv', 'a')
              print (str(market), file=f)
              f1=open('data/out3_tmp.csv', 'a')
              print (summ_serf(market), file=f1)
              f2 = open('data/out4_tmp.csv', 'a')
              print (count(market), file=f2)
    os.rename('data/out2_tmp.csv', 'data/out2.csv')
    os.rename('data/out3_tmp.csv', 'data/out3.csv')
    os.rename('data/out4_tmp.csv', 'data/out4.csv')

def summ_serf(marketname):
    db = pymysql.connect("localhost", "stockuser", "123456", "stock_advisor")
    cursor = db.cursor()
    market=marketname
    cursor.execute("SELECT SUM(percent_serf) FROM orders where active=0 and market= '%s'" % market)
    r = cursor.fetchall()
    for row in r:
        if row[0] is not None:
            return row[0]

    return 0

def count(marketname):
    db = pymysql.connect("localhost", "stockuser", "123456", "stock_advisor")
    cursor = db.cursor()
    market=marketname
    cursor.execute("SELECT COUNT(*) FROM orders where active=0 and market= '%s'" % market)
    r = cursor.fetchall()
    for row in r:
        if row[0] is not None:
            return row[0]

    return 0

if __name__ == "__main__":
    main()