import csv
from django.db.models import Max
from pyzbar.pyzbar import decode
from PIL import Image
from .models import ListOptions, List, fieldsPerformsProperties, Fields, Campaigns, t, msk
from entry.models import Register, Data, Blocks, Edit
import pytesseract
from csvmodule import UnicodeReader, UnicodeWriter
from DataEntry_v2.settings import MEDIA_ROOT, STATIC_ROOT
import os, re, shutil, imghdr
from glob import glob
from django.utils.translation import gettext as _
import magic

def getEncoding(fname):

    encoding = magic.from_file(fname)
    encoding2 = magic.from_file(fname, mime=True)
    if (encoding2=="text/plain" or encoding2=="text/csv"):
        return True
    else:
        return False

def adjustImageSize(filename, newfile):
    img=Image.open(filename)
    width=float(1200)
    origheight=float(img.size[1])
    origwidth=float(img.size[0])
    height=(origheight/origwidth)*width
    size=(int(width), int(height))
    img=img.convert("L")
    img=img.resize(size, Image.BILINEAR)
    if os.path.isfile(newfile): os.unlink(newfile)
    img.save(newfile)
    return

def loadImgs(fname, extractfolder, destfolder, blockid, idfields):

    isok, error = extract(fname, extractfolder)
    if isok:
        curdir = os.getcwd()
        os.chdir(extractfolder)
        block=Blocks.objects.filter(id=blockid).first()
        fields=Fields.objects.filter(id__in=idfields).order_by("fieldNum")
        regs=Register.objects.filter(block=block).order_by("regnum")
        for reg in regs:
            datas=Data.objects.filter(includes=reg)
            edits=Edit.objects.filter(register=reg)
            edits.delete()
            datas.delete()
        regs.delete()
        n=0
        for dp, dn, fn in os.walk(extractfolder):
            for nfile in [os.path.join(dp, f) for f in fn]:
                if not os.path.isfile(nfile):
                    continue
                filetype = imghdr.what(nfile)
                if filetype in ["jpeg", "gif", "png", "bmp", "tiff"]:
                    destfile="%08d.jpg" % n
                    adjustImageSize(nfile, os.path.join(destfolder, destfile))
                    register = Register.objects.create(block=block, image=destfile, regnum=n)
                    register.save()
                    for field in fields:
                        data = Data.objects.create(includes=register, field=field)
                        data.save()
                    n+=1
        os.chdir(curdir)

    return n

def loadList(fname, id):

    olist=List.objects.get(id=id)
    csv_reader = UnicodeReader(fname, delimiter=str(u':').encode('utf-8'))
    line_count = 0
    for row in csv_reader:
        listoption = ListOptions.objects.create(list=olist, code=row[0], value=row[1])
        listoption.save()
        line_count += 1

    return line_count

def createPathIfNotExists(destfolder):
    curdir=os.getcwd()
    try:
        os.chdir(destfolder)
    except:
        os.makedirs(destfolder)
    finally:
        os.chdir(curdir)
        shutil.rmtree(destfolder)
        os.makedirs(destfolder)

    os.chdir(curdir)

def extract(filename, destfolder):
    compressions={"\.zip$":"unzip -uo %s", "\.tar$":"tar xvf %s", "\.tgz$":"tar zxvf %s", "\.tar\.gz$":"tar zxvf %s","\.tar\.bz2$":"tar jxvf %s","\.rar$":"unrar e -o+ %s","\.7z$":"7z x -aoa %s"}

    if re.search("[^\d\w\.\_\-]+", filename.split("/")[-1]):
        return (False, _("Incorrect Filename. Allowed Chars: '0-9a-z._-A-Z'"))

    curdir=os.getcwd()
    createPathIfNotExists(destfolder)
    os.chdir(destfolder)

    for ext, cmd in compressions.items():
        if re.search(ext, filename.split("/")[-1], re.IGNORECASE):
            if os.system(cmd % filename)!=0:
                os.chdir(curdir)
                return (False, _("Error: processing compressed file: %s") % filename)
            else:
                os.chdir(curdir)
                return (True, "")
    return (False, _("Error: file type not supported."))

def cropImage(image, imgPos):
    pos = map(int, imgPos.split(","))
    img = Image.open(image).crop(pos)
    img.save("/tmp/%s.jpg" % imgPos)
    return img

def readBarCode(image, lang="", imgPos=""):
    img = cropImage(image, imgPos)
    barcode=decode(img)
    if len(barcode)>0:
        return barcode[0].data.strip()
    else:
        return ""

def getBlock(idcampaign, blocknum):
    if blocknum is None:
        blocknum=Blocks.objects.filter(campaign__id=int(idcampaign)).aggregate(Max('number'))['number__max']+1
    block=Blocks.objects.filter(campaign__id=int(idcampaign)).filter(number=int(blocknum))
    if len(block)==0:
        campaign=Campaigns.objects.get(id=int(idcampaign))
        block=Blocks.objects.create(campaign=campaign, number=int(blocknum))
        block.save()
    else:
        block=block.first()
    return block

def OCRImage(image, lang, imgPos):
    img=cropImage(image, imgPos)
    text=pytesseract.image_to_string(img, lang=lang).strip()
    return text

def getCSVCount(fname):
    lines=open(fname, "r+").readlines()
    numRegs=len(lines)
    return numRegs

def loadData(fname, idcampaign, blocknum):
    totregs=getCSVCount(fname)
    if blocknum==0:
        blocknums=Blocks.objects.filter(campaign__id=int(idcampaign)).order_by('number')
        if blocknums.exists():
            blocknum=blocknums.last().number+1
        else:
            blocknum=1
    populateDatabase(idcampaign, blocknum, totregs=totregs)
    return loadDataFields(fname, blocknum, idcampaign)

def loadDataFields(fname, blocknum, idcampaign):
    campaign=Campaigns.objects.get(id=int(idcampaign))
    fields = Fields.objects.filter(campaign=campaign).order_by("fieldNum")
    registers = Register.objects.filter(block__campaign=campaign).filter(block__number=int(blocknum))

    numRegs=0
    csv_reader = UnicodeReader(fname, delimiter=str(u':').encode('utf-8'))
    line_count = 0
    for register in registers:
        row=csv_reader.next()
        # populate with csv data
        fldn=0
        for field in fields:
            data=Data.objects.get(includes=register, field=field)
            if field.type=="str":
                dt=row[fldn] if len(row[fldn])<=field.length else row[fldn][:field.length]
            elif field.type=="bool":
                dt=bool(row[fldn])
            elif field.type=="int":
                dt=int(row[fldn])
            elif field.type=="dec":
                dt=float(row[fldn])
            else:
                dt=row[fldn]
            data.contents=dt
            data.save()
            fldn+=1
        numRegs += 1

    return numRegs

def putMask(dtype, dvalue, dlen):

    if dtype=="str":
        tp=u"%% %ds" % dlen
        dvalue=dvalue
    if dtype=="int":
        tp=u"%%0%dd" % dlen
        dvalue=int(dvalue)
    if dtype=="dec":
        tp=u"%%0%d.03f" % 4 if dlen<4 else dlen-3
        dvalue=float(dvalue)

    return tp % dvalue

def exportData(fname, idcampaign, nblock):
    fields = Fields.objects.filter(campaign__id=int(idcampaign))
    if int(nblock)==0:
        registers = Register.objects.filter(block__campaign__id=int(idcampaign)).order_by("block__number", "regnum")
    else:
        registers = Register.objects.filter(block__number=nblock).filter(block__campaign__id=int(idcampaign)).order_by("block__number", "regnum")

    numRegs=0
    if os.path.isfile(fname):
        os.unlink(fname)
    csv_writer = UnicodeWriter(fname, delimiter=str(u':').encode('utf-8'))
    for register in registers:
        row=[]
        # populate with csv data
        fldn=0
        for field in fields:
            cprops=[ prop.property.identifier for prop in fieldsPerformsProperties.objects.filter(field=field, property__identifier__startswith="C")]
            data=Data.objects.get(includes=register, field=field)
            if len(cprops)>0:
                ldata = ListOptions.objects.filter(id=data.contents.strip())[0]
                if cprops[0] in ["C1", "C2"]:
                    row.append(ldata.code)
                elif cprops[0] in ["C3", "C4"]:
                    row.append(ldata.value)
                else:
                    row.append(data.contents)
            else:
                row.append(data.contents)
            row[-1]=putMask(field.type, row[-1], field.length)
            fldn+=1
        csv_writer.writerow(row)
        numRegs += 1

    return numRegs

def populateDatabase(idcampaign, blocknum, totregs=None, imgfile=None):
    campaign=Campaigns.objects.get(id=int(idcampaign))
    fields = Fields.objects.filter(campaign=campaign)
    block=getBlock(idcampaign ,blocknum)
    registers = Register.objects.filter(block=block)

    fpaths=[]
    if imgfile is not None:
        extpath=os.path.join(MEDIA_ROOT, "images", "extract", "%04d" % int(idcampaign), "%04d" % int(blocknum))
        destpath=os.path.join(MEDIA_ROOT, "images", "entry", "%04d" % int(idcampaign), "%04d" % int(blocknum))
        createPathIfNotExists(extpath)
        createPathIfNotExists(destpath)
        ok=extract(imgfile, extpath)
        if ok[0]:
            n=0
            for dp, dn, fn in os.walk(extpath):
                for nfile in [os.path.join(dp, f) for f in fn]:
                    if not os.path.isfile(nfile):
                        continue
                    filetype = imghdr.what(nfile)
                    if filetype in ["jpeg", "gif", "png", "bmp", "tiff"]:
                        destfile="%08d.jpg" % n
                        dfl=os.path.join(destpath, destfile)
                        adjustImageSize(nfile, dfl)
                        fpaths.append(dfl)
                        n+=1

    numRegs = 0
    if campaign.imagePath.strip() == "" and totregs is not None:
        for register in registers:
            datas=Data.objects.filter(includes=register)
            edits=Edit.objects.filter(register=register)
            edits.delete()
            datas.delete()
        registers.delete()
        for numRegs in xrange(int(totregs)):
            register=Register.objects.create(regnum=numRegs, block=block)
            register.save()
            for field in fields:
                data = Data.objects.create(includes=register, field=field)
                data.save()

            numRegs += 1
    elif len(fpaths)==0:
        for register in registers:
            for data in datas:
                data.delete()
            for field in fields:
                data = Data.objects.create(includes=register, field=field)
                data.save()
            Edit.objects.filter(register=register).delete()
            numRegs += 1
    else:
        if len(registers):
            for register in registers:
                datas=Data.objects.filter(includes=register)
                datas.delete()
                Edit.objects.filter(register=register).delete()
            registers.delete()
        numRegs=0
        for fpath in fpaths:
            imgname=fpath.split("/")[-1]
            register=Register.objects.create(regnum=numRegs, image=imgname, block=block)
            register.save()
            for field in fields:
                data = Data.objects.create(includes=register, field=field)
                data.save()
            numRegs+=1

    return numRegs

def scanProjectImages(idcampaign, blocknum, propertyid="O1", func=OCRImage, lang="eng"):

    # OCR || BARCODE records

    block=getBlock(idcampaign ,blocknum)
    ocrfields=fieldsPerformsProperties.objects.filter(field__campaign__id=int(idcampaign), property__identifier=propertyid)
    ocrnum=0
    if len(ocrfields)>0:
        imagePath=ocrfields[0].field.campaign.imagePath+"/%04d/" % block.number
        registers=Register.objects.filter(block=block)
        for register in registers:
            for ocrfield in ocrfields:
                data=Data.objects.get(includes=register, field=ocrfield.field)
                text=func(os.path.join(imagePath, register.image), lang, ocrfield.field.imgPos)
                data.contents=text[:255] if len(text)>254 else text
                data.save()
                ocrnum+=1

    return ocrnum

def ocrProjectImages(idcampaign, lang):
    ocrnum=0
    for block in Blocks.objects.filter(campaign__id=int(idcampaign)).order_by("number"):
        ocrnum+=scanProjectImages(idcampaign, block.number, "O1", OCRImage, lang)
    return ocrnum

def barcodeProjectImages(idcampaign):
    ocrnum=0
    for block in Blocks.objects.filter(campaign__id=int(idcampaign)).order_by("number"):
        ocrnum+=scanProjectImages(idcampaign, block.number, "O2", readBarCode)
    return ocrnum


import http
from datetime import datetime

from flask import Blueprint, request, jsonify
from flask_jwt_extended import jwt_required, get_jwt_identity

from src.constants.permissions import WRITE_ACCESS, READ_ACCESS, ADMIN_ACCESS
from src.database.general_info.ads import Ads
from src.database.database_basis import db
from src.database.general_info.adgroups import Adgroups
from src.database.general_info.campaigns import Campaigns
from src.services.useful_funcs import EndpointManagingHelper
from src.services.validator import Validator

campaigns = Blueprint("campaigns", __name__, url_prefix="/api/v1/campaign")
validator = Validator()
helper = EndpointManagingHelper()

@campaigns.post("/add")
@jwt_required()
def add_campaign():
    schema = (
        ("campaign_name", "STRING", True),
        ("total_budget", "INTEGER", False),
        ("daily_budget", "INTEGER", False),
        ("status", "STRING", True),
    )
    enum_values = (("status", ("ENABLED", "DISABLED")),)
    unique_fields = ("campaign_name",)
    parameters = (
        "campaign_name",
        "total_budget",
        "daily_budget",
        "status",
    )
    try:
        validator.validate_types(schema, request.json)
        validator.validate_values(enum_values, request.json)
        validator.validate_unique(unique_fields, request.json, Campaigns)
        validator.validate_names(parameters, request.json)
    except Exception as error:
        return jsonify({"error": error.args}), http.HTTPStatus.BAD_REQUEST
    user = helper.get_user(get_jwt_identity())

    if not helper.check_user_permissions(user.advertiser_id, WRITE_ACCESS, "campaigns"):
        return jsonify({"message": "Access denied"}), http.HTTPStatus.UNAUTHORIZED

    campaign = Campaigns(
        advertiser_id=user.advertiser_id,
        campaign_name=request.json["campaign_name"],
        total_budget=request.json.get("total_budget", 0),
        daily_budget=request.json.get("daily_budget", 0),
        creation_date=datetime.utcnow(),
        status=request.json["status"],
    )
    db.session.add(campaign)
    db.session.commit()
    return (
        jsonify(
            {
                "message": "Campaign created",
                "campaign": {
                    "campaign_id": campaign.campaign_id,
                    "campaign_name": campaign.campaign_name,
                },
            }
        ),
        http.HTTPStatus.CREATED,
    )

@campaigns.get("/get")
@jwt_required()
def get_campaign():
    schema = (("campaign_id", "INTEGER", True),)
    params = ("campaign_id",)
    try:
        validator.validate_types(schema, request.json)
        validator.validate_names(params, request.json)
    except Exception as error:
        return jsonify({"error": error.args}), http.HTTPStatus.BAD_REQUEST
    advertiser_id = helper.get_user(get_jwt_identity()).advertiser_id

    if not helper.check_user_permissions(advertiser_id, READ_ACCESS, "campaigns"):
        return jsonify({"message": "Access denied"}), http.HTTPStatus.UNAUTHORIZED

    campaign = Campaigns.query.filter_by(
        campaign_id=request.json["campaign_id"], advertiser_id=advertiser_id
    ).first()
    if campaign:
        return (
            jsonify(
                {
                    "campaign_id": campaign.campaign_id,
                    "campaign_name": campaign.campaign_name,
                    "total_budget": campaign.total_budget,
                    "daily_budget": campaign.daily_budget,
                    "creation_date": campaign.creation_date,
                    "last_update_date": campaign.last_update_date,
                    "deletion_date": campaign.deletion_date,
                    "status": campaign.status,
                }
            ),
            http.HTTPStatus.OK,
        )
    else:
        return (
            jsonify(
                {
                    "error": f"There's no campaign with campaign_id {request.json['campaign_id']}"
                }
            ),
            http.HTTPStatus.BAD_REQUEST,
        )

@campaigns.get("/list")
@jwt_required()
def get_all_campaigns():
    advertiser_id = helper.get_user(get_jwt_identity()).advertiser_id

    if not helper.check_user_permissions(advertiser_id, READ_ACCESS, "campaigns"):
        return jsonify({"message": "Access denied"}), http.HTTPStatus.UNAUTHORIZED

    campaigns = Campaigns.query.filter_by(advertiser_id=advertiser_id)
    if campaigns.first():
        return (
            jsonify(
                [
                    {
                        "campaign_id": campaign.campaign_id,
                        "campaign_name": campaign.campaign_name,
                        "total_budget": campaign.total_budget,
                        "daily_budget": campaign.daily_budget,
                        "creation_date": campaign.creation_date,
                        "last_update_date": campaign.last_update_date,
                        "deletion_date": campaign.deletion_date,
                        "status": campaign.status,
                    }
                    for campaign in campaigns
                    if campaign.status != "DELETED"
                ]
            ),
            http.HTTPStatus.OK,
        )
    return jsonify(
        {"error": f"There are no campaigns for advertiser if {advertiser_id}."}
    )

@campaigns.post("/update")
@jwt_required()
def update_campaign():
    schema = (
        ("campaign_id", "INTEGER", True),
        ("total_budget", "INTEGER", False),
        ("daily_budget", "INTEGER", False),
        ("status", "STRING", False),
    )
    enum_values = (("status", ("ENABLED", "DISABLED")),)
    parameters = (
        "campaign_id",
        "total_budget",
        "daily_budget",
        "status",
    )
    immutable = ("campaign_id",)
    try:
        validator.validate_types(schema, request.json)
        validator.validate_values(enum_values, request.json)
        validator.validate_names(parameters, request.json)
    except Exception as error:
        return jsonify({"error": error.args}), http.HTTPStatus.BAD_REQUEST
    advertiser_id = helper.get_user(get_jwt_identity()).advertiser_id

    if not helper.check_user_permissions(advertiser_id, WRITE_ACCESS, "campaigns"):
        return jsonify({"message": "Access denied"}), http.HTTPStatus.UNAUTHORIZED

    campaign = Campaigns.query.filter_by(
        campaign_id=request.json["campaign_id"], advertiser_id=advertiser_id
    ).first()
    if not campaign:
        return (
            jsonify(
                {
                    "error": f"There's no campaign with campaign_id {request.json['campaign_id']}"
                }
            ),
            http.HTTPStatus.BAD_REQUEST,
        )
    if campaign.status == "DELETED":
        return jsonify(
            {"error": f"Campaign with {request.json['campaign_id']} id was deleted."}
        )
    for parameter in request.json:
        if parameter not in immutable:
            setattr(campaign, parameter, request.json[parameter])
    campaign.last_update_date = datetime.utcnow()
    db.session.commit()
    return (
        jsonify(
            {
                "message": "Campaign updated",
                "campaign": {
                    "campaign_id": campaign.campaign_id,
                    "campaign_name": campaign.campaign_name,
                },
            }
        ),
        http.HTTPStatus.OK,
    )

def delete_ads(adgroup_id):
    ads = Ads.query.filter_by(adgroup_id=adgroup_id)
    for ad in ads:
        ad.status = "DELETED"
        if not ad.deletion_date:
            ad.deletion_date = datetime.utcnow()
    db.session.commit()

def delete_adgroups(campaign_id):
    adgroups = Adgroups.query.filter_by(campaign_id=campaign_id)
    for adgroup in adgroups:
        adgroup.status = "DELETED"
        if not adgroup.deletion_date:
            adgroup.deletion_date = datetime.utcnow()
        delete_ads(adgroup.adgroup_id)
    db.session.commit()

@campaigns.post("/delete")
@jwt_required()
def delete_campaign():
    schema = (("campaign_id", "INTEGER", True),)
    parameters = ("campaign_id",)
    try:
        validator.validate_types(schema, request.json)
        validator.validate_names(parameters, request.json)
    except Exception as error:
        return jsonify({"error": error.args}), http.HTTPStatus.BAD_REQUEST
    advertiser_id = helper.get_user(get_jwt_identity()).advertiser_id

    if not helper.check_user_permissions(advertiser_id, ADMIN_ACCESS, "campaigns"):
        return jsonify({"message": "Access denied"}), http.HTTPStatus.UNAUTHORIZED

    campaign = Campaigns.query.filter_by(
        campaign_id=request.json["campaign_id"], advertiser_id=advertiser_id
    ).first()
    if not campaign:
        return (
            jsonify(
                {
                    "error": f"There's no campaign with campaign_id {request.json['campaign_id']}"
                }
            ),
            http.HTTPStatus.BAD_REQUEST,
        )
    campaign.deletion_date = datetime.utcnow()
    campaign.status = "DELETED"
    helper.lazy_delete(Campaigns)
    delete_adgroups(request.json["campaign_id"])
    return (
        jsonify(
            {
                "message": "Campaign deleted",
                "campaign": {"campaign_id": campaign.campaign_id},
            }
        ),
        http.HTTPStatus.OK,
    )

@campaigns.post("/undelete")
@jwt_required()
def undelete_campaign():
    schema = (("campaign_id", "INTEGER", True),)
    parameters = ("campaign_id",)
    try:
        validator.validate_types(schema, request.json)
        validator.validate_names(parameters, request.json)
    except Exception as error:
        return jsonify({"error": error.args}), http.HTTPStatus.BAD_REQUEST
    advertiser_id = helper.get_user(get_jwt_identity()).advertiser_id

    if not helper.check_user_permissions(advertiser_id, ADMIN_ACCESS, "campaigns"):
        return jsonify({"message": "Access denied"}), http.HTTPStatus.UNAUTHORIZED

    campaign = Campaigns.query.filter_by(
        advertiser_id=advertiser_id,
        campaign_id=request.json["campaign_id"],
        status="DELETED",
    ).first()
    if not campaign:
        return (
            jsonify(
                {
                    "error": f"There's no deleted campaign with id {request.json['campaign_id']}"
                }
            ),
            http.HTTPStatus.BAD_REQUEST,
        )
    campaign.deletion_date = None
    campaign.status = "ENABLED"
    db.session.commit()
    return (
        jsonify(
            {
                "message": "Campaign undeleted",
                "campaign": {"campaign_id": campaign.campaign_id},
            }
        ),
        http.HTTPStatus.OK,
    )


from datetime import datetime
import json
import logging
import os
import pytz
import re

from pydnp3 import opendnp3

DEFAULT_POINT_TOPIC = 'dnp3/point'
#DEFAULT_OUTSTATION_STATUS_TOPIC = 'mesa/outstation_status'
#DEFAULT_LOCAL_IP = "0.0.0.0"
DEFAULT_PORT = 20000

# PointDefinition.fcodes values:
DIRECT_OPERATE = 'direct_operate'       # This is actually DIRECT OPERATE / RESPONSE
SELECT = 'select'                       # This is actually SELECT / RESPONSE
OPERATE = 'operate'                     # This is actually OPERATE / RESPONSE

# Some PointDefinition.point_type values:
POINT_TYPE_ANALOG_INPUT = 'Analog Input'
POINT_TYPE_ANALOG_OUTPUT = 'Analog Output'
POINT_TYPE_BINARY_INPUT = 'Binary Input'
POINT_TYPE_BINARY_OUTPUT = 'Binary Output'

# Default event group and variation for each of these types, in case they weren't spec'd for a point in the data file.
EVENT_DEFAULTS_BY_POINT_TYPE = {
    POINT_TYPE_ANALOG_INPUT: {"group": 32, "variation": 3},
    POINT_TYPE_ANALOG_OUTPUT: {"group": 42, "variation": 3},
    POINT_TYPE_BINARY_INPUT: {"group": 2, "variation": 1},
    POINT_TYPE_BINARY_OUTPUT: {"group": 11, "variation": 1}
}

EVENT_CLASSES = {
    1: opendnp3.PointClass.Class1,
    2: opendnp3.PointClass.Class2,
    3: opendnp3.PointClass.Class3
}

GROUP_AND_VARIATIONS = {
    '1.1': opendnp3.StaticBinaryVariation.Group1Var1,
    '1.2': opendnp3.StaticBinaryVariation.Group1Var2,
    '2.1': opendnp3.EventBinaryVariation.Group2Var1,
    '2.2': opendnp3.EventBinaryVariation.Group2Var2,
    '2.3': opendnp3.EventBinaryVariation.Group2Var3,
    '3.2': opendnp3.StaticDoubleBinaryVariation.Group3Var2,
    '4.1': opendnp3.EventDoubleBinaryVariation.Group4Var1,
    '4.2': opendnp3.EventDoubleBinaryVariation.Group4Var2,
    '4.3': opendnp3.EventDoubleBinaryVariation.Group4Var3,
    '10.2': opendnp3.StaticBinaryOutputStatusVariation.Group10Var2,
    '11.1': opendnp3.EventBinaryOutputStatusVariation.Group11Var1,
    '11.2': opendnp3.EventBinaryOutputStatusVariation.Group11Var2,
    '20.1': opendnp3.StaticCounterVariation.Group20Var1,
    '20.2': opendnp3.StaticCounterVariation.Group20Var2,
    '20.5': opendnp3.StaticCounterVariation.Group20Var5,
    '20.6': opendnp3.StaticCounterVariation.Group20Var6,
    '21.1': opendnp3.StaticFrozenCounterVariation.Group21Var1,
    '21.2': opendnp3.StaticFrozenCounterVariation.Group21Var2,
    '21.5': opendnp3.StaticFrozenCounterVariation.Group21Var5,
    '21.6': opendnp3.StaticFrozenCounterVariation.Group21Var6,
    '21.9': opendnp3.StaticFrozenCounterVariation.Group21Var9,
    '21.10': opendnp3.StaticFrozenCounterVariation.Group21Var10,
    '22.1': opendnp3.EventCounterVariation.Group22Var1,
    '22.2': opendnp3.EventCounterVariation.Group22Var2,
    '22.5': opendnp3.EventCounterVariation.Group22Var5,
    '22.6': opendnp3.EventCounterVariation.Group22Var6,
    '23.1': opendnp3.EventFrozenCounterVariation.Group23Var1,
    '23.2': opendnp3.EventFrozenCounterVariation.Group23Var2,
    '23.5': opendnp3.EventFrozenCounterVariation.Group23Var5,
    '23.6': opendnp3.EventFrozenCounterVariation.Group23Var6,
    '30.1': opendnp3.StaticAnalogVariation.Group30Var1,
    '30.2': opendnp3.StaticAnalogVariation.Group30Var2,
    '30.3': opendnp3.StaticAnalogVariation.Group30Var3,
    '30.4': opendnp3.StaticAnalogVariation.Group30Var4,
    '30.5': opendnp3.StaticAnalogVariation.Group30Var5,
    '30.6': opendnp3.StaticAnalogVariation.Group30Var6,
    '32.1': opendnp3.EventAnalogVariation.Group32Var1,
    '32.2': opendnp3.EventAnalogVariation.Group32Var2,
    '32.3': opendnp3.EventAnalogVariation.Group32Var3,
    '32.4': opendnp3.EventAnalogVariation.Group32Var4,
    '32.5': opendnp3.EventAnalogVariation.Group32Var5,
    '32.6': opendnp3.EventAnalogVariation.Group32Var6,
    '32.7': opendnp3.EventAnalogVariation.Group32Var7,
    '32.8': opendnp3.EventAnalogVariation.Group32Var8,
    '40.1': opendnp3.StaticAnalogOutputStatusVariation.Group40Var1,
    '40.2': opendnp3.StaticAnalogOutputStatusVariation.Group40Var2,
    '40.3': opendnp3.StaticAnalogOutputStatusVariation.Group40Var3,
    '40.4': opendnp3.StaticAnalogOutputStatusVariation.Group40Var4,
    '42.1': opendnp3.EventAnalogOutputStatusVariation.Group42Var1,
    '42.2': opendnp3.EventAnalogOutputStatusVariation.Group42Var2,
    '42.3': opendnp3.EventAnalogOutputStatusVariation.Group42Var3,
    '42.4': opendnp3.EventAnalogOutputStatusVariation.Group42Var4,
    '42.5': opendnp3.EventAnalogOutputStatusVariation.Group42Var5,
    '42.6': opendnp3.EventAnalogOutputStatusVariation.Group42Var6,
    '42.7': opendnp3.EventAnalogOutputStatusVariation.Group42Var7,
    '42.8': opendnp3.EventAnalogOutputStatusVariation.Group42Var8,
    '50.4': opendnp3.StaticTimeAndIntervalVariation.Group50Var4,
    '121.1': opendnp3.StaticSecurityStatVariation.Group121Var1,
    '122.1': opendnp3.EventSecurityStatVariation.Group122Var1,
    '122.2': opendnp3.EventSecurityStatVariation.Group122Var2
}

POINT_TYPES_BY_GROUP = {
    # Single-Bit Binary: See DNP3 spec, Section A.2-A.5 and Table 11-17
    1: POINT_TYPE_BINARY_INPUT,         # Binary Input (static): Reporting the present value of a single-bit binary object
    2: POINT_TYPE_BINARY_INPUT,         # Binary Input Event: Reporting single-bit binary input events and flag bit changes
    # Double-Bit Binary: See DNP3 spec, Section A.4 and Table 11-15
    3: 'Double Bit Binary',             # Double-Bit Binary Input (static): Reporting present state value
    4: 'Double Bit Binary',             # Double-Bit Binary Input Event: Reporting double-bit binary input events and flag bit changes
    # Binary Output: See DNP3 spec, Section A.6-A.9 and Table 11-12
    10: POINT_TYPE_BINARY_OUTPUT,       # Binary Output (static): Reporting the present output status
    11: POINT_TYPE_BINARY_OUTPUT,       # Binary Output Event: Reporting changes to the output status or flag bits
    12: POINT_TYPE_BINARY_OUTPUT,       # Binary Output Command: Issuing control commands
    13: POINT_TYPE_BINARY_OUTPUT,       # Binary Output Command Event: Reporting control command was issued regardless of its source
    # Counter: See DNP3 spec, Section A.10-A.13 and Table 11-13
    20: 'Counter',                      # Counter: Reporting the count value
    21: 'Counter',                      # Frozen Counter: Reporting the frozen count value or changed flag bits
    22: 'Counter',                      # Counter Event: Reporting counter events
    23: 'Counter',                      # Frozen Counter Event: Reporting frozen counter events
    # Analog Input: See DNP3 spec, Section A.14-A.18 and Table 11-9
    30: POINT_TYPE_ANALOG_INPUT,        # Analog Input (static): Reporting the present value
    31: POINT_TYPE_ANALOG_INPUT,        # Frozen Analog Input (static): Reporting the frozen value
    32: POINT_TYPE_ANALOG_INPUT,        # Analog Input Event: Reporting analog input events or changes to the flag bits
    33: POINT_TYPE_ANALOG_INPUT,        # Frozen Analog Input Event: Reporting frozen analog input events
    34: POINT_TYPE_ANALOG_INPUT,        # Analog Input Reporting Deadband (static): Reading and writing analog deadband values
    # Analog Output: See DNP3 spec, Section A.19-A.22 and Table 11-10
    40: POINT_TYPE_ANALOG_OUTPUT,       # Analog Output Status (static): Reporting present value of analog outputs
    41: POINT_TYPE_ANALOG_OUTPUT,       # Analog Output (command): Controlling analog output values
    42: POINT_TYPE_ANALOG_OUTPUT,       # Analog Output Event: Reporting changes to the analog output or flag bits
    43: POINT_TYPE_ANALOG_OUTPUT,       # Analog Output Command Event: Reporting output points being commanded from any source
    # Time and Date: See DNP3 spec, Section A.23-A.25
    50: 'Time And Date',
    51: 'Time And Date',                # Time and Date Common Time-of-Occurrence
    52: 'Time And Date',                # Time Delay
    # Class Objects: See DNP3 spec, Section A.26
    60: 'Class Objects',
    # File-Control: See DNP3 spec, Section A.27
    70: 'File-Control',
    # Information Objects: See DNP3 spec, Section A.28-A.31
    80: 'Internal Indications',
    81: 'Device Storage',
    82: 'Device Profile',
    83: 'Data Set Registration',
    # Data Set Objects: See DNP3 spec, Section A.32-A.35
    85: 'Data Set Prototype',
    86: 'Data Set Descriptor',
    87: 'Data Set',
    88: 'Data Set Event',
    # Application & Status of Operation Information Objects: See DNP3 spec, Section A.36-A.37
    90: 'Application',
    91: 'Status of Requested Operation',
    # Floating-Point (Obsolete): See DNP3 spec, Section A.38
    100: 'Floating-Point',
    # Numeric Static Objects: See DNP3 spec, Section A.39-A.40
    101: 'BCD',                         # Device-dependent values in Binary-Coded Decimal form (Table 11-11)
    102: 'Unsigned Integer',
    # Octet String: See DNP3 spec, Section A.41-A.42 and Table 11-16
    110: 'Octet String',                # To convey the present value
    111: 'Octet String',                # Reporting an octet string event
    # Virtual Terminal: See DNP3 spec, Section A.43-A.44 and Table 11-18
    112: 'Virtual Terminal',            # Conveying data to the command interpreter at the outstation
    113: 'Virtual Terminal',            # Conveying data from the command interpreter at the outstation
    # Security: See DNP3 spec, Section A.45
    120: 'Authentication',
    # Security Statistic: See DNP3 spec, Section A.46-A.47 and Table 11-20
    121: 'Security Statistic',          # Reporting the current value of the statistics
    122: 'Security Statistic'           # Reporting changes to the statistics
}

_log = logging.getLogger(__name__)

class DNP3Exception(Exception):
    """Raise exceptions that are specific to the DNP3 agent. No special exception behavior is needed at this time."""
    pass

class PointDefinitions(object):
    """In-memory repository of PointDefinitions."""

    def __init__(self, point_definitions_path=None):
        self._points = {}
        self._point_variation_dict = {}
        self._point_name_dict = {}
        if point_definitions_path:
            file_path = os.path.expandvars(os.path.expanduser(point_definitions_path))
            self.load_points_from_json_file(file_path)

    def __getitem__(self, name):
        """Return the PointDefinition associated with this name. Must be unique."""
        return self.get_point_named(name)

    def load_points_from_json_file(self, point_definitions_path):
        """Load and cache a dictionary of PointDefinitions, indexed by point_type and point index."""
        if point_definitions_path:
            try:
                file_path = os.path.expandvars(os.path.expanduser(point_definitions_path))
                _log.debug('Loading DNP3 point definitions from {}.'.format(file_path))
                with open(file_path, 'r') as f:
                    # Filter comments out of the file's contents before loading it as JSON.
                    self.load_points(json.loads(self.strip_comments(f.read())))
            except Exception as err:
                raise ValueError('Problem parsing {}. Error={}'.format(point_definitions_path, err))
        else:
            _log.debug('No point_definitions_path specified, loading no points')

    def strip_comments(self, raw_string):
        """
            Return a string with comments stripped.

            Both JavaScript-style comments (//... and /*...*/) and hash (#...) comments are removed.
            Thanks to VOLTTRON volttron/platform/agent/utils.py/strip_comments() for this logic.
        """
        def _repl(match):
            return match.group(1) or ''

        _comment_re = re.compile(r'((["\'])(?:\\?.)*?\2)|(/\*.*?\*/)|((?:#|//).*?(?=\n|$))', re.MULTILINE | re.DOTALL)
        return _comment_re.sub(_repl, raw_string)

    def load_points(self, point_definitions_json):
        """Load and cache a dictionary of PointDefinitions, indexed by point_type and point index."""
        try:
            self._points = {}           # If they're already loaded, force a reload.
            for element in point_definitions_json:
                # Load a PointDefinition (or subclass) from JSON, and add it to the dictionary of points.
                # If the point defines an array, load additional definitions for each interior point in the array.
                try:
                    if element.get('type', None) != 'array':
                        point_def = PointDefinition(element)
                        self.index_point(point_def)
                    else:
                        point_def = ArrayHeadPointDefinition(element)
                        self.index_point(point_def)
                        # Load a separate ArrayPointDefinition for each interior point in the array.
                        for pt in point_def.create_array_point_definitions(element):
                            self.index_point(pt)
                except ValueError as err:
                    raise DNP3Exception('Validation error for point with json: {}: {}'.format(element, err))
        except Exception as err:
            raise ValueError('Problem parsing PointDefinitions. Error={}'.format(err))
        _log.debug('Loaded {} PointDefinitions'.format(len(self.all_points())))

    def index_point(self, point_def):
        """Add a PointDefinition to the dictionary of points."""
        point_type_dict = self._points.setdefault(point_def.point_type, {})
        if point_def.index in point_type_dict:
            error_message = 'Discarding DNP3 duplicate {0} (conflicting {1})'
            raise DNP3Exception(error_message.format(point_def, point_type_dict[point_def.index]))
        else:
            point_type_dict[point_def.index] = point_def

    def _points_dictionary(self):
        """Return a (cached) dictionary of PointDefinitions, indexed by point_type and point index."""
        return self._points

    def for_group_and_index(self, group, index):
        return self._points_dictionary().get(PointDefinition.point_type_for_group(group), {}).get(index, None)

    def point_value_for_command(self, command_type, command, index, op_type):
        """
            A DNP3 Select or Operate was received from the master. Create and return a PointValue for its data.

        :param command_type: Either 'Select' or 'Operate'.
        :param command: A ControlRelayOutputBlock or else a wrapped data value (AnalogOutputInt16, etc.).
        :param index: DNP3 index of the payload's data definition.
        :param op_type: An OperateType, or None if command_type == 'Select'.
        :return: An instance of PointValue
        """
        function_code = command.functionCode if type(command) == opendnp3.ControlRelayOutputBlock else None
        point_type = POINT_TYPE_BINARY_OUTPUT if function_code else POINT_TYPE_ANALOG_OUTPUT
        point_def = self.for_point_type_and_index(point_type, index)
        # print(point_def)
        if not point_def:
            raise DNP3Exception('No DNP3 PointDefinition found for point type {0} and index {1}'.format(point_type,
                                                                                index))
        point_value = PointValue(command_type,
                                 function_code,
                                 command.value if not function_code else None,
                                 point_def,
                                 index,
                                 op_type)
        _log.debug('Received DNP3 {}'.format(point_value))
        return point_value

    def for_point_type_and_index(self, point_type, index):
        """
            Return a PointDefinition for a given data type and index.

        @param point_type: A point type (string).
        @param index: Unique integer index of the PointDefinition to be looked up.
        @return: A PointDefinition.
        """
        return self._points_dictionary().get(point_type, {}).get(index, None)

    def _points_by_variation(self):
        """Return a (cached) dictionary of PointDefinitions, indexed by group, variation and index."""
        if not self._point_variation_dict:
            for point_type, inner_dict in self._points_dictionary().items():
                for index, point_def in inner_dict.items():
                    if self._point_variation_dict.get(point_def.group, None):
                        self._point_variation_dict[point_def.group] = {}
                    if self._point_variation_dict[point_def.group].get(point_def.variation, None):
                        self._point_variation_dict[point_def.group][point_def.variation] = {}
                    self._point_variation_dict[point_def.group][point_def.variation][index] = point_def
        return self._point_variation_dict

    def point_for_variation_and_index(self, group, variation, index):
        """Return a PointDefinition for a given group, variation and index."""
        return self._points_by_variation().get(group, {}).get(variation, {}).get(index, None)

    def points_by_name(self):
        """Return a (cached) dictionary of PointDefinition lists, indexed by name."""
        if not self._point_name_dict:
            for point_type, inner_dict in self._points_dictionary().items():
                for index, point_def in inner_dict.items():
                    point_name = point_def.name
                    if point_name not in self._point_name_dict:
                        self._point_name_dict[point_name] = []
                    self._point_name_dict[point_name].append(point_def)
        return self._point_name_dict

    def point_named(self, name, index=None):
        """
            Return the PointDefinition with the indicated name and (optionally) index.

        :param name: (string) The point's name.
        :param index: (integer) An optional index value. If supplied, search for an array point at this DNP3 index.
        :return A PointDefinition, or None if no match.
        """
        point_def_list = self.points_by_name().get(name, None)
        if point_def_list is None:
            return None                     # No points with that name

        if index is not None:
            # Return the PointDefinition with a matching index.
            for pt in point_def_list:
                if pt.index == index:
                    return pt
            return None

        # In multi-element lists, give preference to the ArrayHeadPointDefinition.
        for pt in point_def_list:
            if pt.is_array_head_point:
                return pt
        return point_def_list[0]

    def get_point_named(self, name, index=None):
        """
            Return the PointDefinition with the indicated name and (optionally) index.
            Raise an exception if none found.

        :param name: (string) The point's name.
        :param index: (integer) An optional index value. If supplied, search for an array point at this DNP3 index.
        :return A PointDefinition.
        """
        point_def = self.point_named(name, index=index)
        if point_def is None:
            if index is not None:
                raise DNP3Exception('No point named {} with index {}'.format(name, index))
            else:
                raise DNP3Exception('No point named {}'.format(name))
        return point_def

    def all_points(self):
        """Return a flat list of all PointDefinitions."""
        point_list = []
        for inner_dict in self._points_dictionary().values():
            point_list.extend(inner_dict.values())
        return point_list

    def points_by_mrid(self):
        """Return a (cached) dictionary of PointDefinition lists, indexed by name."""
        if not self._point_mrid_dict:
            for point_type, inner_dict in self._points_dictionary().items():
                for point_def in inner_dict.items():
                    print(point_def)
                    point_mrid = point_def.measurement_id
                    # if point_mrid not in self._point_mrid_dict:
                    #     self._point_mrid_dict[point_mrid] = []
                    self._point_mrid_dict[point_mrid].append(point_def)
        return self._point_mrid_dict

    def all_point_names(self):
        return self.points_by_name().keys()

    def all_point_mrid(self):
        return self.points_by_mrid().keys()

class BasePointDefinition(object):
    """Abstract superclass for PointDefinition data holders."""

    def __init__(self, element_def):
        """Initialize an instance of the PointDefinition from a dictionary of point attributes."""
        self.name = str(element_def.get('name', ''))
        self.data_type = element_def.get('data_type', None)
        self.group = element_def.get('group', None)
        self.variation = element_def.get('variation', None)
        self.index = element_def.get('index', None)
        self.description = element_def.get('description', '')
        self.measurement_id = element_def.get('measurement_id', None)
        self.measurement_type = element_def.get('measurement_type', None)
        self.magnitude = element_def.get('magnitude', None)
        self.value = element_def.get('value', None)
        self.attribute = element_def.get('attribute', None)
        #Old, unused data
        self.scaling_multiplier = element_def.get('scaling_multiplier', 1)
        self.units = element_def.get('units', '')
        self.event_class = element_def.get('event_class', 2)
        self.event_group = element_def.get('event_group', None)
        self.event_variation = element_def.get('event_variation', None)
        self.selector_block_start = element_def.get('selector_block_start', None)
        self.selector_block_end = element_def.get('selector_block_end', None)
        self.save_on_write = element_def.get('save_on_write', None)
        self.type = element_def.get('type', None)

    @property
    def is_array_point(self):
        return False

    @property
    def is_array_head_point(self):
        return False

    @property
    def is_array(self):
        return self.is_array_point or self.is_array_head_point

    def validate_point(self):
        """A PointDefinition has been created. Perform a variety of validations on it."""
        if self.type is not None and self.type not in ['array', 'selector_block']:
            raise ValueError('Invalid type for {}: {}'.format(self.name, self.type))
        if self.group is None:
            raise ValueError('Missing group for {}'.format(self.name))
        if self.variation is None:
            raise ValueError('Missing variation for {}'.format(self.name))
        if self.index is None:
            raise ValueError('Missing index for {}'.format(self.name))

        # Use intelligent defaults for event_group and event_variation based on data type
        if self.event_group is None:
            if self.point_type in EVENT_DEFAULTS_BY_POINT_TYPE:
                self.event_group = EVENT_DEFAULTS_BY_POINT_TYPE[self.point_type]["group"]
            else:
                raise ValueError('Missing event group for {}'.format(self.name))
        if self.event_variation is None:
            if self.point_type in EVENT_DEFAULTS_BY_POINT_TYPE:
                self.event_variation = EVENT_DEFAULTS_BY_POINT_TYPE[self.point_type]["variation"]
            else:
                raise ValueError('Missing event variation for {}'.format(self.name))

        if self.is_selector_block:
            if self.selector_block_start is None:
                raise ValueError('Missing selector_block_end for block named {}'.format(self.name))
            if self.selector_block_end is None:
                raise ValueError('Missing selector_block_end for block named {}'.format(self.name))
            if self.selector_block_start > self.selector_block_end:
                raise ValueError('Selector block end index < start index for block named {}'.format(self.name))
        else:
            if self.selector_block_start is not None:
                raise ValueError('selector_block_start defined for non-selector-block point {}'.format(self.name))
            if self.selector_block_end is not None:
                raise ValueError('selector_block_end defined for non-selector-block point {}'.format(self.name))

    def as_json(self):
        """Return a json description of the PointDefinition."""
        point_json = {
            "name": self.name,
            "group": self.group,
            "variation": self.variation,
            "index": self.index,
            "scaling_multiplier": self.scaling_multiplier,
            "event_class": self.event_class
        }
        if self.type is not None:
            point_json["type"] = self.type
        if self.description is not None:
            point_json["description"] = self.description
        if self.units is not None:
            point_json["units"] = self.units
        if self.event_group is not None:
            point_json["event_group"] = self.event_group
        if self.event_variation is not None:
            point_json["event_variation"] = self.event_variation
        if self.selector_block_start is not None:
            point_json["selector_block_start"] = self.selector_block_start
        if self.selector_block_end is not None:
            point_json["selector_block_end"] = self.selector_block_end
        if self.save_on_write is not None:
            point_json["save_on_write"] = self.save_on_write
        return point_json

    def __str__(self):
        """Return a string description of the PointDefinition."""
        try:
            return '{0} {1} ({2}, index={3}, type={4})'.format(self.__class__.__name__,
                                                               self.name,
                                                               self.group_and_variation,
                                                               self.index,
                                                               self.point_type)
        except UnicodeEncodeError as err:
            _log.error('Unable to convert point definition to string, err = {}'.format(err))
            return ''

    @property
    def group_and_variation(self):
        """Return a string representation of the PointDefinition's group and variation."""
        return '{0}.{1}'.format(self.group, self.variation)

    @property
    def event_group_and_variation(self):
        """Return a string representation of the PointDefinition's event group and event variation."""
        return '{0}.{1}'.format(self.event_group, self.event_variation)

    @property
    def point_type(self):
        """Return the PointDefinition's point type, derived from its group (indexing is within point type)."""
        return self.point_type_for_group(self.group)

    @property
    def is_input(self):
        """Return True if the PointDefinition is a Binary or Analog input point (i.e., sent by the Outstation)."""
        return self.point_type in [POINT_TYPE_ANALOG_INPUT, POINT_TYPE_BINARY_INPUT]

    @property
    def is_output(self):
        """Return True if the PointDefinition is a Binary or Analog output point (i.e., sent by the Master)."""
        return self.point_type in [POINT_TYPE_ANALOG_OUTPUT, POINT_TYPE_BINARY_OUTPUT]

    @property
    def is_selector_block(self):
        return self.type == 'selector_block'

    @property
    def eclass(self):
        """Return the PointDefinition's event class, or the default (2) if no event class was defined for the point."""
        return EVENT_CLASSES.get(self.event_class, 2)

    @property
    def svariation(self):
        """Return the PointDefinition's group-and-variation enumerated type."""
        return GROUP_AND_VARIATIONS.get(self.group_and_variation)

    @property
    def evariation(self):
        """Return the PointDefinition's event group-and-variation enumerated type."""
        return GROUP_AND_VARIATIONS.get(self.event_group_and_variation)

    @classmethod
    def point_type_for_group(cls, group):
        """Return the point type for a group value."""
        ptype = POINT_TYPES_BY_GROUP.get(group, None)
        if ptype is None:
            _log.error('No DNP3 point type found for group {}'.format(group))
        return ptype

class PointDefinition(BasePointDefinition):
    """Data holder for an OpenDNP3 data element."""

    def __init__(self, element_def):
        """Initialize an instance of the PointDefinition from a dictionary of point attributes."""
        super(PointDefinition, self).__init__(element_def)
        self.validate_point()

    def validate_point(self):
        """A PointDefinition has been created. Perform a variety of validations on it."""
        super(PointDefinition, self).validate_point()
        if self.type is not None and self.type != 'selector_block':
            raise ValueError('Invalid type for {}: {}'.format(self.name, self.type))

class ArrayHeadPointDefinition(BasePointDefinition):
    """Data holder for an OpenDNP3 data element that is the head point in an array."""

    def __init__(self, json_element):
        """
            Initialize an ArrayPointDefinition instance.
            An ArrayPointDefinition defines an interior point (not the head point) in an array.

        :param json_element: A JSON dictionary of point attributes.
        """
        super(ArrayHeadPointDefinition, self).__init__(json_element)
        self.array_points = json_element.get('array_points', None)
        self.array_times_repeated = json_element.get('array_times_repeated', None)
        self.array_point_definitions = []         # Holds all ArrayPointDefinitions belonging to this array.
        self.validate_point()

    def validate_point(self):
        """An ArrayHeadPointDefinition has been created. Perform a variety of validations on it."""
        super(ArrayHeadPointDefinition, self).validate_point()
        if self.type != 'array':
            raise ValueError('Invalid type {} for array named {}'.format(self.type, self.name))
        if self.array_points is None:
            raise ValueError('Missing array_points for array named {}'.format(self.name))
        if self.array_times_repeated is None:
            raise ValueError('Missing array_times_repeated for array named {}'.format(self.name))

    @property
    def is_array_point(self):
        return True

    @property
    def is_array_head_point(self):
        return True

    def as_json(self):
        """Return a json description of the ArrayHeadPointDefinition."""
        point_json = super(ArrayHeadPointDefinition, self).as_json()
        # array_points has been excluded because it's not a simple data type. Is it needed in the json?
        # if self.array_points is not None:
        #     point_json["array_points"] = self.array_points
        if self.array_times_repeated is not None:
            point_json["array_times_repeated"] = self.array_times_repeated
        return point_json

    @property
    def array_last_index(self):
        """Calculate and return the array's last index value."""
        if self.is_array_head_point:
            return self.index + self.array_times_repeated * len(self.array_points) - 1
        else:
            return None

    def create_array_point_definitions(self, element):
        """Create a separate ArrayPointDefinition for each interior point in the array."""
        for row_number in range(self.array_times_repeated):
            for column_number, pt in enumerate(self.array_points):
                # The ArrayHeadPointDefinition is already defined -- don't create a redundant definition.
                if row_number > 0 or column_number > 0:
                    array_pt_def = ArrayPointDefinition(element, self, row_number, column_number, pt['name'])
                    self.array_point_definitions.append(array_pt_def)
        return self.array_point_definitions

class ArrayPointDefinition(BasePointDefinition):
    """Data holder for an OpenDNP3 data element that is interior to an array."""

    def __init__(self, json_element, base_point_def, row, column, array_element_name):
        """
            Initialize an ArrayPointDefinition instance.
            An ArrayPointDefinition defines an interior point (not the head point) in an array.

        :param json_element: A JSON dictionary of point attributes.
        :param base_point_def: The PointDefinition of the head point in the array.
        :param row: The point's row number in the array.
        :param column: The point's column number in the array.
        :param array_element_name: The point's column name in the array.
        """
        super(ArrayPointDefinition, self).__init__(json_element)
        self.base_point_def = base_point_def
        self.row = row
        self.column = column
        self.index = self.base_point_def.index + row * len(self.base_point_def.array_points) + column
        self.array_element_name = array_element_name
        self.validate_point()

    def validate_point(self):
        """An ArrayPointDefinition has been created. Perform a variety of validations on it."""
        super(ArrayPointDefinition, self).validate_point()
        if self.type != 'array':
            raise ValueError('Invalid type {} for array named {}'.format(self.type, self.name))
        if self.base_point_def is None:
            raise ValueError('Missing base point definition for array point named {}'.format(self.name))
        if self.row is None:
            raise ValueError('Missing row number for array point named {}'.format(self.name))
        if self.column is None:
            raise ValueError('Missing column number for array point named {}'.format(self.name))
        if self.index is None:
            raise ValueError('Missing index value for array point named {}'.format(self.name))
        if self.array_element_name is None:
            raise ValueError('Missing array element name for array point named {}'.format(self.name))

    @property
    def is_array_point(self):
        return True

    @property
    def is_array_head_point(self):
        return False

    def as_json(self):
        """Return a json description of the ArrayPointDefinition."""
        point_json = super(ArrayPointDefinition, self).as_json()
        if self.row is not None:
            point_json["row"] = self.row
        if self.row is not None:
            point_json["column"] = self.column
        if self.row is not None:
            point_json["array_element_name"] = self.array_element_name
        return point_json

class PointValue(object):
    """Data holder for a point value (DNP3 measurement or command) received by an outstation."""

    def __init__(self, command_type, function_code, value, point_def, index, op_type):
        """Initialize an instance of the PointValue."""
        # Don't rely on VOLTTRON utils in this code, which may run outside of VOLTTRON
        # self.when_received = utils.get_aware_utc_now()
        self.when_received = pytz.UTC.localize(datetime.utcnow())
        self.command_type = command_type
        self.function_code = function_code
        self.value = value
        self.point_def = point_def
        self.index = index
        self.op_type = op_type

    def __str__(self):
        """Return a string description of the PointValue."""
        str_desc = 'Point value {0} ({1}, {2}.{3}, {4})'
        return str_desc.format(self.value or self.function_code,
                               self.name,
                               self.point_def.group_and_variation,
                               self.index,
                               self.command_type)

    @property
    def name(self):
        """Return the name of the PointDefinition."""
        return self.point_def.name

    def unwrapped_value(self):
        """Unwrap the point's value, returning the sample data type (e.g. an integer, binary, etc. instance)."""
        if self.value is None:
            # For binary commands, send True if function_code is LATCH_ON, False otherwise
            return self.function_code == opendnp3.ControlCode.LATCH_ON
        else:
            return self.value

class PointArray(object):
    """Data holder for  Array."""

    def __init__(self, point_def):
        """
            The "points" variable is a dictionary of dictionaries:
                0: {
                    0: PointValue,
                    1: PointValue
                },
                1: {
                    0: PointValue,
                    1: PointValue
                }
                (etc.)
            It's stored as dictionaries indexed by index numbers, not as lists,
            because there's no guarantee that array elements will arrive in order.

        :param point_def: The PointDefinition of the array's head point.
        """
        _log.debug('New Array {} starting at {} with bounds ({}, {})'.format(point_def.name,
                                                                             point_def.index,
                                                                             point_def.index,
                                                                             point_def.array_last_index))
        self.point_def = point_def
        self.points = {}

    def __str__(self):
        return 'Array, points = {}'.format(self.points)

    def as_json(self):
        """
            Return a JSON representation of the PointArray:

                [
                    {name1: val1a, name2: val2a, ...},
                    {name1: val1b, name2: val2b, ...},
                    ...
                ]
        """
        names = [d['name'] for d in self.point_def.array_points]
        json_array = []
        for pt_dict_key in sorted(self.points):
            pt_dict = self.points[pt_dict_key]
            json_array.append({name: (pt_dict[i].value if i in pt_dict else None) for i, name in enumerate(names)})
        return json_array

    def contains_index(self, index):
        """Answer whether this Array contains the point index."""
        return self.point_def.index <= index <= self.point_def.array_last_index

    def add_point_value(self, point_value):
        """Add point_value to the Array's "points" dictionary."""
        point_def = point_value.point_def
        row = 0 if point_def.is_array_head_point else point_def.row
        col = 0 if point_def.is_array_head_point else point_def.column
        if row not in self.points:
            self.points[row] = {}
        self.points[row][col] = point_value


# -*- coding: utf-8 -*-
"""
Created on Fri Nov 16 10:24:51 2018

@author: Alexandre
"""

############################################################################
from pyro.dynamic  import integrator
from pyro.control  import linear
############################################################################

# Double integrator
si = integrator.SimpleIntegrator()
di = integrator.DoubleIntegrator()
ti = integrator.TripleIntegrator()

# Controller
ctl      = linear.ProportionnalSingleVariableController()
ctl.gain = 2

# New cl-dynamic
clsi = ctl + si
clsi.plot_phase_plane_trajectory([10],10,0,0)
clsi.sim.plot('xu')

cldi = ctl + di
cldi.plot_phase_plane_trajectory([10,0],10,0,1)
cldi.sim.plot('xu')

clti = ctl + ti

clti.plot_phase_plane_trajectory_3d([0.1,0,0],10)
clti.sim.plot('xu')


"""
Grab stashed trips parquet file.
Add in service hours, narrow it down to 1 trip per route,
and subset it to parallel routes only.

Create a df that has the selected trip per route
with all the stop info (and stop sequence) attached.
"""
import datetime
import dask.dataframe as dd
import dask_geopandas as dg
import geopandas as gpd
import os
import pandas as pd

os.environ["CALITP_BQ_MAX_BYTES"] = str(130_000_000_000)

from shared_utils import gtfs_utils, rt_dates, rt_utils
from bus_service_utils import utils as bus_utils
from calitp_data_analysis import geography_utils, utils

ANALYSIS_DATE = rt_dates.PMAC["Q2_2022"]
COMPILED_CACHED = f"{rt_utils.GCS_FILE_PATH}compiled_cached_views/"

def grab_service_hours(selected_date: str,
                       valid_trip_keys: list) -> pd.DataFrame:
    daily_service_hours = gtfs_utils.get_trips(
        selected_date = selected_date,
        itp_id_list = None,
        # Keep more columns, route_id, shape_id, direction_id so the metrolink fix
        # can be propagated
        trip_cols = ["calitp_itp_id", "service_date", "trip_key",
                     "route_id", "shape_id", "direction_id", "service_hours"],
        get_df = True,
        custom_filtering = {"trip_key": valid_trip_keys}
    )

    daily_service_hours.to_parquet(
        f"{bus_utils.GCS_FILE_PATH}service_hours_{selected_date}.parquet")

def merge_trips_with_service_hours(selected_date: str)-> pd.DataFrame:

    trips = dd.read_parquet(
        f"{COMPILED_CACHED}trips_{selected_date}.parquet")

    daily_service_hours = pd.read_parquet(
        f"{bus_utils.GCS_FILE_PATH}service_hours_{selected_date}.parquet")

    df = dd.merge(
        trips,
        daily_service_hours[["trip_key", "service_hours"]],
        on = "trip_key",
        how = "inner",
    ).compute()

    df = df.drop(columns = ["calitp_extracted_at", "calitp_deleted_at"])

    return df

def select_one_trip(df: pd.DataFrame) -> pd.DataFrame:

    # Across trip_ids, for the same route_id, there are differing max_stop_sequence
    # Can't use max_stop_sequence to find longest route_length
    # Use service hours instead to find faster trip during free-flowing traffic
    group_cols = ["calitp_itp_id", "route_id"]

    # Select a trip that closest to 25th percentile (lower means faster!)
    # This groupby ruins the index, throws an error, so just merge in as separate df
    quantile = (df.groupby(group_cols)["service_hours"]
                .quantile(0.25).reset_index()
                .rename(columns = {"service_hours": "p25"})
               )

    df = pd.merge(df, quantile,
                  on = group_cols,
                  how = "inner",
                  validate = "m:1"
            )

    # Select trip that is closest to 25th percentile (min_diff)
    df["difference"] = abs(df.service_hours - df.p25)
    df["min_diff"] = df.groupby(group_cols)["difference"].transform("min")

    df['faster_trip'] = df.apply(
        lambda x: 1 if x.difference == x.min_diff else 0, axis=1)

    # If there are multiple trips selected for a route, do a sort/drop duplicates
    df2 = (df[df.faster_trip==1]
           .sort_values(group_cols + ["trip_id"], ascending=[True, True, True])
           .drop_duplicates(subset=group_cols)
           .drop(columns = ["faster_trip", "difference", "min_diff", "p25"])
           .reset_index(drop=True)
          )

    return df2

def grab_stops_for_trip_selected(trip_df: dd.DataFrame,
                                 selected_date: str) -> gpd.GeoDataFrame:

    stop_times = dd.read_parquet(
        f"{COMPILED_CACHED}st_{selected_date}.parquet")

    stop_times_for_trip = dd.merge(
        trip_df,
        stop_times[["trip_key", "stop_id", "stop_sequence"]],
        on = "trip_key",
        how = "inner"
    )

    # Add in stop geom
    stops = dg.read_parquet(
        f"{COMPILED_CACHED}stops_{selected_date}.parquet"
    )

    stop_times_with_geom = dd.merge(
        stops[["calitp_itp_id", "stop_id", "stop_name", "geometry"]].drop_duplicates(),
        stop_times_for_trip,
        on = ["calitp_itp_id", "stop_id"],
        how = "inner"
    ).to_crs(geography_utils.WGS84)

    stop_times_with_geom2 = (stop_times_with_geom.drop(
        columns = ["calitp_extracted_at", "calitp_deleted_at"])
        .sort_values(["calitp_itp_id", "route_id", "trip_id", "stop_sequence"])
        .reset_index(drop=True)
    ).compute()

    return stop_times_with_geom2

if __name__ == "__main__":

    trips = dd.read_parquet(
        f"{COMPILED_CACHED}trips_{ANALYSIS_DATE}.parquet")

    valid_trip_keys = trips.trip_key.unique().compute().tolist()
    #grab_service_hours(ANALYSIS_DATE, valid_trip_keys)

    df = merge_trips_with_service_hours(ANALYSIS_DATE)

    one_trip = select_one_trip(df)

    trips_with_stops = grab_stops_for_trip_selected(one_trip, ANALYSIS_DATE)

    utils.geoparquet_gcs_export(
        trips_with_stops,
        bus_utils.GCS_FILE_PATH,
        f"trips_with_stops_{ANALYSIS_DATE}"
    )


from __future__ import absolute_import, print_function

import os
from os.path import dirname, join

import jinja2
from flask import Flask, cli, current_app
from flask_babelex import Babel
from invenio_access import InvenioAccess
from invenio_accounts import InvenioAccounts
from invenio_accounts.views import blueprint as accounts_blueprint
from invenio_admin import InvenioAdmin
from invenio_assets import InvenioAssets
from invenio_db import InvenioDB, db
from invenio_files_rest import InvenioFilesREST
from invenio_files_rest.models import Location
from invenio_i18n import InvenioI18N
from invenio_indexer import InvenioIndexer
from invenio_indexer.api import RecordIndexer
from invenio_jsonschemas import InvenioJSONSchemas
from invenio_oauth2server import InvenioOAuth2Server, InvenioOAuth2ServerREST
from invenio_oauth2server.views import server_blueprint, settings_blueprint
from invenio_pidstore import InvenioPIDStore
from invenio_records import InvenioRecords
from invenio_records_rest import InvenioRecordsREST
from invenio_records_rest.facets import terms_filter
from invenio_records_rest.utils import PIDConverter
from invenio_records_ui import InvenioRecordsUI
from invenio_rest import InvenioREST
from invenio_search import InvenioSearch
from invenio_search_ui import InvenioSearchUI
from invenio_search_ui.bundles import js
from invenio_theme import InvenioTheme

from invenio_deposit import InvenioDeposit, InvenioDepositREST

# Create Flask application
app = Flask(__name__)

app.config.update(
    CELERY_ALWAYS_EAGER=True,
    CELERY_CACHE_BACKEND='memory',
    CELERY_EAGER_PROPAGATES_EXCEPTIONS=True,
    CELERY_RESULT_BACKEND='cache',
    JSONSCHEMAS_HOST='localhost:5000',
    JSONSCHEMAS_URL_SCHEME='http',
    REST_ENABLE_CORS=True,
    SECRET_KEY='changeme',
    SQLALCHEMY_TRACK_MODIFICATIONS=True,
    DEPOSIT_SEARCH_API='/deposits',
    RECORDS_REST_FACETS=dict(
        deposits=dict(
            aggs=dict(
                status=dict(terms=dict(
                    field='_deposit.status'
                )),
            ),
            post_filters=dict(
                status=terms_filter(
                    '_deposit.status'
                ),
            )
        )
    ),
    RECORDS_UI_DEFAULT_PERMISSION_FACTORY=None,
    SQLALCHEMY_DATABASE_URI=os.getenv('SQLALCHEMY_DATABASE_URI',
                                      'sqlite:///app.db'),
    SEARCH_UI_SEARCH_API='/deposits',
    SEARCH_UI_JSTEMPLATE_RESULTS='templates/app/deposit.html',
    DATADIR=join(dirname(__file__), 'data/upload'),
    OAUTH2SERVER_CACHE_TYPE='simple',
    OAUTHLIB_INSECURE_TRANSPORT=True,
)

Babel(app)

# Set jinja loader to first grab templates from the app's folder.
app.jinja_loader = jinja2.ChoiceLoader([
    jinja2.FileSystemLoader(join(dirname(__file__), "templates")),
    app.jinja_loader
])

app.url_map.converters['pid'] = PIDConverter

InvenioDB(app)
InvenioI18N(app)
InvenioTheme(app)
InvenioJSONSchemas(app)
InvenioAccounts(app)
InvenioAccess(app)
InvenioRecords(app)
InvenioRecordsUI(app)

InvenioRecordsREST(app)
InvenioDeposit(app)
InvenioDepositREST(app)

search = InvenioSearch(app)
search.register_mappings('deposits', 'invenio_deposit.mappings')

InvenioSearchUI(app)
InvenioREST(app)
InvenioIndexer(app)
InvenioPIDStore(app)
InvenioAdmin(app)
InvenioOAuth2Server(app)
InvenioOAuth2ServerREST(app)

InvenioFilesREST(app)

assets = InvenioAssets(app)
assets.env.register('invenio_search_ui_search_js', js)

app.register_blueprint(accounts_blueprint)

app.register_blueprint(settings_blueprint)
app.register_blueprint(server_blueprint)

@app.cli.group()
def fixtures():
    """Command for working with test data."""

@fixtures.command()
@cli.with_appcontext
def records():
    """Load records."""
    from flask_login import login_user, logout_user
    from invenio_accounts.models import User
    from invenio_deposit.api import Deposit

    user = User.query.one()

    with current_app.test_request_context():
        login_user(user)
        with db.session.begin_nested():
            Deposit.create({'title': 'Test'})
        logout_user()
        db.session.commit()

@fixtures.command()
@cli.with_appcontext
def location():
    """Load default location."""
    d = current_app.config['DATADIR']
    with db.session.begin_nested():
        Location.query.delete()
        loc = Location(name='local', uri=d, default=True)
        db.session.add(loc)
    db.session.commit()


import pygame
import random

pygame.init()

color_white = (255, 255, 255)
color_yellow = (255, 255, 102)
color_black = (0, 0, 0)
color_red = (213, 50, 80)
color_green = (100, 200, 50)
color_blue = (173, 216, 230)

display_width = 600
display_height = 400

display = pygame.display.set_mode((display_width, display_height))
pygame.display.set_caption('Snek')

clock = pygame.time.Clock()

snake_particle = 10
snake_speed = 15

font_style = pygame.font.SysFont("bahnschrift", 25)
score_font = pygame.font.SysFont("arial", 35)

def show_score(score):
    value = score_font.render("Score: " + str(score), True, color_yellow)
    display.blit(value, [0, 0])

def snake(snake_particle, snake_array):
    for x in snake_array:
        pygame.draw.rect(display, color_black, [
                         x[0], x[1], snake_particle, snake_particle])

def message(msg, color, x, y):
    mesg = font_style.render(msg, True, color)
    display.blit(mesg, [x, y])

def gameLoop():
    game_closed = False
    game_lost = False

    x1 = display_width / 2
    y1 = display_height / 2

    x1_change = 0
    y1_change = 0

    # create snake with length 3
    snake_array = []
    snake_length = 3
    for x in range(snake_length - 1, 0, -1):
        snake_array.append([x1, y1 + x * snake_particle])

    food_posX = round(random.randrange(
        0, display_width - snake_particle) / 10.0) * 10.0
    food_posY = round(random.randrange(
        0, display_height - snake_particle) / 10.0) * 10.0

    while not game_closed:
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                game_closed = True

            elif event.type == pygame.KEYDOWN:
                if game_lost == False:
                    if event.key == pygame.K_LEFT:
                        if x1 - snake_particle != snake_array[-2][0]:
                            x1_change = -snake_particle
                            y1_change = 0
                    elif event.key == pygame.K_RIGHT:
                        if x1 + snake_particle != snake_array[-2][0]:
                            x1_change = snake_particle
                            y1_change = 0
                    elif event.key == pygame.K_UP:
                        if y1 - snake_particle != snake_array[-2][1]:
                            x1_change = 0
                            y1_change = -snake_particle
                    elif event.key == pygame.K_DOWN:
                        if y1 + snake_particle != snake_array[-2][1]:
                            x1_change = 0
                            y1_change = snake_particle

                else:
                    if event.type == pygame.KEYDOWN:
                        if event.key == pygame.K_q or event.key == pygame.K_ESCAPE:
                            game_closed = True
                            game_lost = False
                        if event.key == pygame.K_r:
                            gameLoop()

        if game_lost == True:
            display.fill(color_blue)
            message("LMAO NOOB",
                    color_red, display_width * 0.15, display_height * 0.35)
            message("R - Replay || Q / ESC - Quit", color_red,
                    display_width * 0.15, display_height * 0.50)
            show_score(snake_length - 3)

        else:
            if x1 >= display_width or x1 < 0 or y1 >= display_height or y1 < 0:
                game_lost = True

            x1 += x1_change
            y1 += y1_change

            snake_Head = []
            snake_Head.append(x1)
            snake_Head.append(y1)
            if snake_Head != snake_array[-1]:
                snake_array.append(snake_Head)
                if len(snake_array) > snake_length:
                    del snake_array[0]

            for x in snake_array[:-1]:
                if x == snake_Head:
                    game_lost = True

            if game_lost == False and x1 == food_posX and y1 == food_posY:
                food_posX = round(random.randrange(
                    0, display_width - snake_particle) / 10.0) * 10.0
                food_posY = round(random.randrange(
                    0, display_height - snake_particle) / 10.0) * 10.0
                snake_length += 1

            display.fill(color_blue)
            pygame.draw.rect(display, color_green, [
                             food_posX, food_posY, snake_particle, snake_particle])

            snake(snake_particle, snake_array)
            show_score(snake_length - 3)

        pygame.display.update()
        clock.tick(snake_speed)

    pygame.quit()
    quit()

gameLoop()


# -*- coding: utf-8 -*-
import unittest
import datetime

from helpers import *

class ChangeProfile(unittest.TestCase):

    def test_change_profile(self):
        serverid = xmlrpclib.Server(server_https(), transport=SpecialTransport())
        #file = open('rsid.txt')
        rsid = check_rsid()

        # Генерим пол 100% отличный от того, что сейчас в аккаунте
        method_caller = serverid.__getattr__('Rambler::Id::get_profile')
        response = method_caller({'rsid': rsid})
        genderlist = 'mf'
        try:
            initialgender = str(response['profile']['gender'])
            if initialgender == 'm':
                gender = 'f'
            elif initialgender == 'f':
                gender = 'm'
        except KeyError:
            gender = random.choice(genderlist)

        # Словарь рандомных значений:
        random_dict = {'rsid': rsid,  'firstname' : randomstring(7),  'backup_email' : randomstring(4) + '@' + randomstring(4)+'.ru',
            'gender' : str(gender), 'birthday' : randomint(), 'lastname' : randomstring(10),
                       'geoid': randomint()}

        # Сгенеренную рандомную юникс-дату приводим к человеческому формату для дальнейшего сравнения полученной
        # человеческой даты с этим говном:
        date = datetime.datetime.fromtimestamp(random_dict['birthday'])
        date = str(date.strftime('%Y-%m-%d'))

        # Меняем инфу об аккаунте:
        method_caller = serverid.__getattr__('Rambler::Id::change_profile')
        change = method_caller(random_dict)

        # Удаляем рсид и меняем дату рождения на человеческую.
        random_dict['birthday'] = date
        del random_dict['rsid']

        # Получаем измененную инфу
        method_caller = serverid.__getattr__('Rambler::Id::get_profile')
        response = method_caller({'rsid': rsid})
        changed_profile = response['profile']

        # Тут происходит сравнение полученных значений от апишки с той инфой, что мы отсылали:
        for key in random_dict:
            try:
                assert random_dict[key] == changed_profile[key]
            except AssertionError:
                raise AssertionError(str(key) + ' is not changed')

if __name__ == "__main__":
    unittest.main()


import base64
import json
import logging
import numpy as np
from flask import Flask, jsonify, request, render_template
from six.moves import http_client
from drugner import DrugNER

import drugner

app = Flask(__name__)

# Load model
nerModel = DrugNER('drug', '/models/drug/')

PLACEHOLDER_TEXT = '''Beta-adrenergic agonist medicines may produce significant hypokalemia in some patients, possibly through intracellular shunting, which has the potential to produce adverse cardiovascular effects. The decrease in serum potassium is usually transient, not requiring supplementation. Beta-agonist medications may produce transient hyperglycemia in some patients. In clinical trials evaluating BREO ELLIPTA in subjects with COPD or asthma, there was no evidence of a treatment effect on serum glucose or potassium.'''

PLACEHOLDER_RESPONSE ={'entities': [{'text': 'may', 'start_char': 34, 'end_char': 37, 'label': 'Factor'}, {'text': 'significant', 'start_char': 46, 'end_char': 57, 'label': 'Severity'}, {'text': 'hypokalemia', 'start_char': 58, 'end_char': 69, 'label': 'AdverseReaction'}, {'text': 'decrease in serum potassium', 'start_char': 200, 'end_char': 227, 'label': 'AdverseReaction'}, {'text': 'Beta-agonist medications', 'start_char': 281, 'end_char': 305, 'label': 'DrugClass'}, {'text': 'may', 'start_char': 306, 'end_char': 309, 'label': 'Factor'}, {'text': 'transient', 'start_char': 318, 'end_char': 327, 'label': 'Severity'}, {'text': 'hyperglycemia', 'start_char': 328, 'end_char': 341, 'label': 'AdverseReaction'}, {'text': 'potassium', 'start_char': 500, 'end_char': 509, 'label': 'AdverseReaction'}]}

def _get_entities_from_text(request):

    if request.method == 'POST':
        input_text = request.form.get('text')
        if input_text == PLACEHOLDER_TEXT:
            entities = PLACEHOLDER_RESPONSE
            text = input_text
        else:
            entities = nerModel.evaluate(input_text)
            text = input_text
    elif request.method == 'GET':
        entities = None
        text = PLACEHOLDER_TEXT

    return entities, text

def _annotate_text(annotations, text):

    locs = np.empty((len(annotations), 2), dtype=np.int16)
    for i, annotation in enumerate(annotations):
        locs[i, :] = (annotation['start_char'], annotation['end_char'])
    locs = np.sort(locs, axis=0)

    annotations.sort(key=lambda x: x['start_char'])

    def build_annotation_block(text, label):
        if label is not None:
            text_class = label.lower().replace(" ", "")
        else:
            text_class = None
        return {'text': text, 'label': label, 'class': text_class}

    # initialize
    annotated_text = []
    text_block = ''
    annotating = False
    annotated_loc = 0
    for i, char in enumerate(text):
        if locs.shape[0] > 0  and i >= locs[annotated_loc, 0] and i <= locs[annotated_loc, 1]:
            # if we should continue annotating add the character
            if i == locs[annotated_loc, 1]:
                # finished label, add text block
                block = build_annotation_block(text_block, annotations[annotated_loc]['label'])
                annotated_text.append(block)
                annotating = False
                text_block = ''
                text_block += char
                if (annotated_loc < locs.shape[0] - 1):
                    annotated_loc += 1
            elif i == locs[annotated_loc, 0]:
                # starting label, add text block
                block = build_annotation_block(text_block, None)
                annotated_text.append(block)
                text_block = ''
                text_block += char
            else:
                annotating = True
                text_block += char
        else:
            text_block += char
            if i == len(text) - 1:
                block = build_annotation_block(text_block, None)
                annotated_text.append(block)

    return annotated_text

def _base64_decode(encoded_str):
    # Add paddings manually if necessary.
    num_missed_paddings = 4 - len(encoded_str) % 4
    if num_missed_paddings != 4:
        encoded_str += b'=' * num_missed_paddings
    return base64.b64decode(encoded_str).decode('utf-8')

@app.route('/', methods=['POST', 'GET'])
def index():

    """Identify FDA adverse events from text"""
    # TODO: change variable scops in conditionals
    entities, text = _get_entities_from_text(request)

    annotated_text = None
    if entities is not None:
        annotated_text = _annotate_text(entities['entities'], text)

    return render_template('index.html',
                           input=text,
                           annotated_text=annotated_text)

@app.route('/ner/drug', methods=['POST'])
def ner():
    """Identify FDA adverse events from text"""
    text = request.form.get('text')

    def get_response(text):
        if (text == PLACEHOLDER_TEXT):
            return PLACEHOLDER_RESPONSE
        else:
            return nerModel.evaluate(text)

    response = get_response(text)

    return render_template('index.html', entities=response, input=text)

@app.route('/ner/drug.json', methods=['POST'])
def nerJson():
    """Identify FDA adverse events from text"""
    text = request.get_json().get('text', '')
    response = nerModel.evaluate(text)
    return jsonify(response)

@app.errorhandler(http_client.INTERNAL_SERVER_ERROR)
def unexpected_error(e):
    """Handle exceptions by returning swagger-compliant json."""
    logging.exception('An error occured while processing the request.')
    response = jsonify({
        'code': http_client.INTERNAL_SERVER_ERROR,
        'message': 'Exception: {}'.format(e)})
    response.status_code = http_client.INTERNAL_SERVER_ERROR
    return response

if __name__ == '__main__':
    # This is used when running locally. Gunicorn is used to run the
    # application on Google App Engine. See entrypoint in app.yaml.
    app.run(host='0.0.0.0', port=8080, debug=True)


'''
change the other format to coco format and save the info to a json file
@author:cvhadessun 2019-12-13
'''
import os
import json
import sys
import glob
import cv2
from linemod_tools import Linemod,LabelInfo
import datetime
class CocoFormat(object):
    '''
    transform the coco dataset format
    '''
    def __init__(self,dataset_name,path):
        #path: dict{"model_path":[...],"data_path":[...]}

        self.licenses=[{"name":dataset_name,"id":1,"url":"219.223.170.43"}]
        self.info= {"author":"cvhadessun"}
        self.categories=[]
        self.images=[]
        self.annoatations=[]
        self.path=path
        self.ann_id=0
        date=datetime.datetime.now()
        self.date=str(date.year)+'-'+str(date.month)+'-'+str(date.day)
        # self.ycb=ycb_v

    def getimg(self,path,img_name,id):
        #append image to images and get the w and h of the image
        path_img=path
        # print(path_img)
        img=cv2.imread(path_img)
        h,w=img.shape[:-1]
        dict_img= {"height": h,
              "flickr_url": "ycb-video",
              "license": 1,
              "id":id,
              "width": w,
              "date_captured":self.date,
              "file_name": img_name}
        self.images.append(dict_img)

    def getann_per_img(self,id,label_info):
        #save the label of one image to json file
        # id:the image id
        # label_info:the annotation info of one image

        num_objects=label_info.num_objects
        category_ids=label_info.category_ids
        #from the label_info get the annotation and save to annotations
        for i in range(num_objects):
            #initial
            dict_ann={}
            #get one annotation of one object
            pose=label_info.label['{}'.format(category_ids[i])]['pose']
            model=label_info.label['{}'.format(category_ids[i])]['model']
            segmentation=label_info.label['{}'.format(category_ids[i])]['segmentation']
            keypoints=label_info.label['{}'.format(category_ids[i])]['keypoints']
            category_id=category_ids[i]
            area=label_info.label['{}'.format(category_ids[i])]['area']
            bbox=label_info.label['{}'.format(category_ids[i])]['bbox']
            #append to annotations
            dict_ann["area"]=area
            dict_ann["bbox"]=bbox
            dict_ann["category_id"]=category_id
            dict_ann["keypoints"]=keypoints
            dict_ann["iscrowd"]=0
            dict_ann["num_keypoints"]=len(keypoints)//3
            dict_ann["image_id"]=id
            dict_ann["id"]=self.ann_id+i+1
            # print(dict_ann["id"])
            dict_ann["segmentation"]=segmentation
            # dict_ann['full_mask']=full_polygon

            self.annoatations.append(dict_ann)
        self.ann_id+=1+i
        # print(self.ann_id)

    def categories_content(self,dataset_ob):
        #to save the categories content for all object
        num_category=dataset_ob.categories_info["num_object"]
        _categories=[]
        for i in range(num_category):
            dict_cate={"supercategory": "object",
                "keypoints": ["1", "2", "3", "4",
                            "5", "6", "7", "8",],
                "skeleton": [[1, 2], [1, 3], [2, 4], [1, 5], [5, 7], [7, 8],
                            [3, 4], [5, 6], [2, 6], [6, 8], [8, 4]]
                        }
            dict_cate["id"]=i+1
            dict_cate["name"]=dataset_ob.categories_info["name"][i]
            _categories.append(dict_cate)
        self.categories=_categories

    def save_json(self,save_path):
        #save data as a .json file

        json_dict={
            "licenses":self.licenses,
            "categories":self.categories,
            "images":self.images,
            "annotations":self.annoatations,
            "info":self.info

        }
        # json_dict={
        #     "licenses":self.licenses,
        #     "categories":self.categories,
        #     "images":self.images,
        #     "info":self.info

        # }

        json_path=os.path.join(save_path,'kpts_coco.json')
        with open (json_path,'w') as fp:
            json.dump(json_dict,fp)
            print("Done")
            fp.close()

def main(root_path,class_name):
    #generate the coco format label json file

    '''
    :root_path:the path of root path of dataset
    :dataset_name: the path of object name,eg:cat,...

    '''
    #initialize
    dataset=Linemod(root_path,class_name)
    class_ob_path=os.path.join(root_path,class_name)
    img_path=os.path.join(os.path.join(root_path,class_name),'data')
    # categories_info=dataset.categories_info
    img_list=dataset.img_names #the image names make of img_list
    ob_coco=CocoFormat('linemod',img_path)
    ob_coco.categories_content(dataset)
    #process all image
    for i,per_name in enumerate(img_list):
        id=i
        # print(per_name)
        # print(ob_coco.ann_id)
        label=LabelInfo(class_ob_path,per_name)
        ob_coco.getimg(os.path.join(class_ob_path,'data'),per_name,id)
        ob_coco.getann_per_img(id,label)
        # print(ob_coco.ann_id)

    ob_coco.save_json(class_ob_path)

    return True

# root_path='./../data'
# class_name='cat'
# flag=main(root_path,class_name)


# Import socket module
import socket
import json

def sendMsg(socketconn, data):
	"""
	"""
	serialized_data = json.dumps(data)
	encoded_data = serialized_data.encode()
	socketconn.send(encoded_data)

# Create a socket object
s = socket.socket()

# Define the port on which you want to connect
port = 12346

# connect to the server on local computer
try:
	s.connect(('127.0.0.1', port))
	# receive data from the server
	print( s.recv(1024).decode() )
	msg = {"type" : "msgtype", "data" : "msgdata"*1000}
	# s.send(msg)
	s.send(json.dumps(msg).encode())
	# close the connection
	s.close()
except Exception as e:
	print("Connection not possible")
	raise e


"""
Common functionalities for schemas, can include classes, functions.
"""

from marshmallow import fields, validate
from webargs.flaskparser import use_args

from app import ma
from app.resources.accounts import constants as ACCT

default_exclude = ('created_date', 'modified_date', 'row_id', )

# user profile details
user_profile_fields = ['first_name', 'last_name', 'designation',
                       'profile_photo_url', 'profile_thumbnail_url']
# account fields
account_fields = ['row_id', 'account_name', 'account_type']
# user details that will be passed while populating user relation
user_fields = ['row_id', ]
user_fields += ['profile.' + fld for fld in user_profile_fields] + [
    'profile.links']
user_fields += ['account.' + fld for fld in account_fields]

# corporate access:
# event fields which will pass while populating event relation
ca_event_fields = ['row_id', 'title', 'event_type_id', 'event_sub_type_id',
                   'invite_logo_filename', 'invite_banner_filename',
                   'started_at', 'ended_at', 'collaborators', 'creator']
# webinar:
# webinar fields which will pass while populating webinar relation
webinar_fields = ['row_id', 'title', 'invite_logo_filename',
                  'invite_banner_filename', 'started_at', 'ended_at']
# webcast:
# webcast fields which will pass while populating webcast relation
webcast_fields = ['row_id', 'title', 'invite_logo_filename',
                  'invite_banner_filename', 'started_at', 'ended_at']

#e_meeting
#e_meeting fields which will pass while populating e_meeting relation
e_meeting_fields = ['row_id','meeting_subject','meeting_agenda','meeting_datetime']

# ca open meeting:
# ca open meeting fields which will pass while populating ca open
# meeting relation
ca_open_meeting_fields = ['row_id', 'title', 'event_type_id', 'creator',
                          'event_sub_type_id', 'started_at', 'ended_at']

def use_args_with(schema_cls, schema_kwargs=None, **kwargs):
    """
    Small helper to reduce boilerplate from: (unused)
    https://webargs.readthedocs.io/en/latest/advanced.html#reducing-boilerplate
    """
    schema_kwargs = schema_kwargs or {}

    def factory(request):
        strict = schema_kwargs.pop('strict', True)
        # Filter based on 'fields' query parameter
        only = request.args.get('pfields', schema_kwargs.pop('only', None))
        # Respect partial updates for PATCH requests
        partial = request.method == 'PATCH'
        # Add current request to the schema's context
        # and ensure we're always using strict mode
        return schema_cls(
            only=only, partial=partial, strict=strict,
            context={'request': request}, **schema_kwargs
        )
    return use_args(factory, **kwargs)

class BaseReadArgsSchema(ma.Schema):
    """
    A base schema for reading filters, pagination, sort from args using
    webargs.
    """
    page = fields.Integer(load_only=True, missing=1)
    per_page = fields.Integer(load_only=True, missing=10)
    sort_by = fields.List(fields.String(), load_only=True, missing=['row_id'])
    sort = fields.String(validate=validate.OneOf(['asc', 'dsc']),
                         missing='asc')
    # specific fields for return
    pfields = fields.List(fields.String(), load_only=True)
    # operator
    operator = fields.String(validate=validate.OneOf(['and', 'or']),
                             missing='and')
    # dates
    created_date_from = fields.DateTime(load_only=True)
    created_date_to = fields.DateTime(load_only=True)
    apply_domain_filter = fields.Boolean(load_only=True, missing=True)

    _db_exclude_fields = [
        'page', 'per_page', 'sort_by', 'sort', 'pfields', 'operator',
        'created_date_from', 'created_date_to', ]

class BaseCommonSchema(ma.Schema):
    """
    Schema for Common of every api
    """
    launch = fields.Boolean(load_only=True)
    order = fields.Boolean(load_only=True)
    token = fields.String(load_only=True)