import itertools
import operator
import types
from typing import Any, List, Optional, Tuple, Type

import torch
from executorch.exir.dialects.edge._ops import EdgeOpOverload
from executorch.exir.error import ExportError, ExportErrorType
from executorch.exir.lowered_backend_module import LoweredBackendModule
from executorch.exir.verification.arg_validator import (
    EdgeOpArgValidator,
    RunHigherOrderOperatorError,
)

from torch._export.verifier import SpecViolationError, Verifier
from torch._ops import OpOverload
from torch._subclasses import FakeTensor
from torch.export.exported_program import ExportedProgram
from torch.fx import GraphModule

ALLOWED_META_KEYS = {"spec", "stack_trace"}

def _check_tensors_are_contiguous(gm: GraphModule) -> None:
    # Tensors be of contiguous format
    for name, param in itertools.chain(gm.named_parameters(), gm.named_buffers()):
        if isinstance(param, torch.Tensor):
            if not param.is_contiguous():
                raise SpecViolationError(
                    f"Tensors in Aten dialect must be contiguous, {name} is not contiguous"
                )

class EXIRATenDialectVerifierBase(Verifier):
    dialect = "OLD_EXIR_ATEN_DISABLED"

    def allowed_getattr_types(self) -> Tuple[Type[Any], ...]:
        return (
            torch.fx.GraphModule,
            LoweredBackendModule,
            torch.Tensor,
            torch.ScriptObject,
        )

    def allowed_op_types(self):
        return super().allowed_op_types() + (torch._ops.OpOverloadPacket,)

    def __call__(self, *args, **kwargs):
        if hasattr(self, "_check_graph_module"):
            return self._check_graph_module(*args, **kwargs)
        elif hasattr(self, "check_valid"):
            return self.check_valid(*args, **kwargs)
        else:
            raise RuntimeError("")

class EXIRATenDialectVerifier(EXIRATenDialectVerifierBase):
    dialect = "OLD_EXIR_ATEN"

    def check_valid_op(self, op):
        if isinstance(op, OpOverload):
            # TODO These special ops should be removable easily.
            if op.namespace in (
                "quantized_decomposed",
                "boltnn_nimble",
                "nimble",
                "quantized",
            ) or op in (
                torch.ops.aten.mkldnn_rnn_layer.default,
                torch.ops.aten._upsample_bilinear2d_aa.default,
                torch.ops.aten.quantize_per_tensor.default,
                torch.ops.aten.dequantize.self,
                torch.ops.aten.max.default,
            ):
                return
            if torch.Tag.core not in op.tags and torch.Tag.view_copy not in op.tags:
                # NOTE(qihan): whether view_copy operators are marked as canonical is still under
                #            discussion.
                raise SpecViolationError(
                    f"Operator {op.__module__}.{op.__name__} is not Aten Canonical."
                )

def get_aten_verifier(enable: bool = True):
    return EXIRATenDialectVerifier if enable else EXIRATenDialectVerifierBase

def _get_inputs(graph_module: GraphModule) -> List[Optional[FakeTensor]]:
    def extract_input(node: torch.fx.Node) -> Optional[FakeTensor]:
        if "val" in node.meta:
            return node.meta["val"]

        if len(node.users) == 0:
            return None

        # TODO(ycao): `val` should always exist after we enable shape environment
        # serialization and deserialization.
        raise ExportError(
            ExportErrorType.VIOLATION_OF_SPEC,
            f"Cannot construct an input for graph module: {graph_module}.",
        )

    return [
        extract_input(node)
        for node in graph_module.graph.nodes
        if node.op == "placeholder"
    ]

def _check_tensor_args_matching_op_allowed_dtype(gm: GraphModule) -> None:
    validator = EdgeOpArgValidator(gm)
    inputs = _get_inputs(gm)
    try:
        validator.run(*inputs)
    except RunHigherOrderOperatorError:
        # NB: ignore higher order operator in the graph.
        # If we lower a graph module to delegate and then compose it with some other graph module, retrace it,
        # if we also turn on edge ops and validator (_check_ir_validity=True), we will run
        # into RunHigherOrderOperatorError. The only thing we can do right now is to ignore this error, since
        # by definition it's still a valid Edge dialect. This is not ideal because it ignores possible invalidity
        # later in the graph.
        return

    if validator.violating_ops:
        raise SpecViolationError(
            f"These operators are taking Tensor inputs with mismatched dtypes: {validator.violating_ops}"
        )

def EXIREdgeDialectVerifier(  # noqa: C901
    check_edge_ops: bool = True,
    enable: bool = True,
    class_only: bool = False,
):
    class _EXIREdgeDialectVerifier(Verifier):
        dialect = "EDGE"

        def __init__(self) -> None:
            self.check_edge_ops = check_edge_ops

            if self.check_edge_ops:
                self.check_valid_op = self.check_valid_edge_op
            else:
                self.check_valid_op = self.check_valid_aten_op

        def allowed_getattr_types(self) -> Tuple[Type[Any], ...]:
            return (
                torch.fx.GraphModule,
                LoweredBackendModule,
                torch.Tensor,
                torch.ScriptObject,
            )

        def allowed_op_types(self):
            return super().allowed_op_types() + (EdgeOpOverload, types.FunctionType)

        def check_valid_edge_op(self, op):
            if not enable:
                return
            if op in [operator.getitem, torch.ops.aten.sym_size.int]:
                return

            if isinstance(op, OpOverload) and not isinstance(op, EdgeOpOverload):
                raise SpecViolationError(
                    "Operator {}.{} is not an Edge operator.".format(
                        op.__module__, op.__name__
                    )
                )
            if isinstance(op, EdgeOpOverload):
                self.check_valid_aten_op(op._op)
            if isinstance(op, types.FunctionType):
                assert op.__name__ in ("alloc",)

        def check_valid_aten_op(self, op) -> None:
            if isinstance(op, OpOverload):
                if (
                    torch.Tag.core not in op.tags  # type: ignore[attr-defined]
                    and torch.Tag.view_copy not in op.tags  # type: ignore[attr-defined]
                ):
                    # NOTE(qihan): whether view_copy operators are marked as canonical is still under
                    #            discussion.
                    raise SpecViolationError(
                        "Operator {}.{} is not Aten Canonical.".format(
                            op.__module__, op.__name__
                        )
                    )

        def check_additional(self, gm: GraphModule) -> None:
            if not enable:
                return
            if self.check_edge_ops:
                _check_tensors_are_contiguous(gm)
                _check_tensor_args_matching_op_allowed_dtype(gm)

        def is_valid(self, gm: GraphModule) -> bool:
            try:
                self(gm)
                return True
            except SpecViolationError:
                return False

        def __call__(self, ep_or_gm):
            if not enable:
                return
            gm = ep_or_gm
            if isinstance(gm, ExportedProgram):
                gm = ep_or_gm.graph_module
            if hasattr(self, "_check_graph_module"):
                return self._check_graph_module(gm)
            elif hasattr(self, "check_valid"):
                return self.check_valid(gm)
            else:
                raise RuntimeError("")

    ret = _EXIREdgeDialectVerifier
    if not class_only:
        ret = ret()
    return ret

EXIREdgeDialectVerifier()


# ---------------
# ParamCopy - Substance 3D Designer plugin
# (c) 2019-2022 Eyosido Software SARL
# ---------------

import os, weakref
from functools import partial

from PySide2.QtCore import QObject
from PySide2.QtWidgets import QToolBar

import sd
from sd.context import Context
from sd.api.sdapplication import SDApplication
from sd.api.sduimgr import SDUIMgr

from paramcopy.pccore import pclog
from paramcopy.pccore.pcdata import PCData

class PCGraphCustomToolbarMgr(QObject):
    """
    Handles a single custom toolbar per graph view used by a single external component
    """
    def __init__(self, callback, toolbarIcon):
        super().__init__()
        self.sdApp = sd.getContext().getSDApplication()
        self.sdUiMgr = self.sdApp.getQtForPythonUIMgr()
        self.toolbars = {} # key: graphViewId, value: weak reference on created toolbar (so we don't prevent Qt to delete the toolbars)
        self.callback = partial(callback) # callback must create/setup a single QToolBar object and return it.
        self.toolbarIcon = toolbarIcon
        self.registerGraphViewCreated()

    # --- Public
    def cleanup(self):
        self.removeAllToolbars()
        if self.graphViewCreatedCbId:
            self.sdUiMgr.unregisterCallback(self.graphViewCreatedCbId)

    # --- Private
    def registerGraphViewCreated(self):
        self.graphViewCreatedCbId = self.sdUiMgr.registerGraphViewCreatedCallback( partial(self.onGraphViewCreated, uiMgr=self.sdUiMgr))

    def removeAllToolbars(self):
        for toolbarRef in self.toolbars.values():
            weakref.proxy(toolbarRef).deleteLater()
        self.toolbars = {}

    def onGraphViewCreated(self, graphViewId, uiMgr):
        if not self.toolbars.get(graphViewId):
            toolbar = self.callback()   # let user create and setup the QToolBar
            toolbar.destroyed.connect(partial(self.onToolbarDestroyed, graphViewId=graphViewId))
            self.toolbars[graphViewId] = toolbar
            self.sdUiMgr.addToolbarToGraphView(graphViewId, toolbar, icon = self.toolbarIcon, tooltip = toolbar.toolTip())

    def onToolbarDestroyed(self, graphViewId):
        # self.sender() is not the toolbar object, so we need to look-up by graphViewId
        if self.toolbars.get(graphViewId):
            del self.toolbars[graphViewId]


import sys,os,subprocess,string,re
from threading import Timer
import time,socket,json
from os import path

base_dir = path.dirname(path.abspath(sys.path[0]))
print(base_dir)
sys.path.append(base_dir)

class HfsSetup():
    def __init__(self,hver='',plants='',adict={}):
        self._run_code = True
        self._plants = plants
        self._hfs_version = hver
        self._servers = ['10.60.96.203']
        self._server = self._servers[0]
        self._Kapps = ["houdinifx.exe","hython.exe"]
        self._folders = ["temp"]
        self.Makerdir()
        self._temp = "%s/temp"%base_dir
        self._code_base_path = adict['codebase'] # C:/script/CG/Houdini
        self._B_plugin_path = adict['plugingase']  # B:/plugins/houdini
        self._D_houdini_path = adict['hfsbase']  # D:/plugins/houdini

        self._info_return = []

    def getcustip(self):
        self._hostname = socket.gethostname()
        ## print(self._hostname)
        _ip_end = re.findall(r"\d+",self._hostname)
        _ip_a = _ip_end[0]
        z_num = 0
        for i in range(0,len(_ip_a)):
            if not _ip_a[i]=="0":
                z_num=i
                break
        _ip_a =_ip_a[z_num:]
        _ipList = socket.gethostbyname_ex(self._hostname)
        self._this_ip = ''
        for elm in _ipList[-1]:
            _ip_ss = elm.split(".")
            if _ip_a in _ip_ss:
                self._this_ip = elm
                break

    def Makerdir(self):
        for elm in self._folders:
            _elm = "%s/%s"%(base_dir,elm)
            if not os.path.exists(_elm):
                os.makedirs(_elm)

    def ConfigApp_win(self):
        self._CopyHfs = True
        self._info_return.append("Server tyr to chang to: %s"%self._server)
        ## copy files
        # 7z tools
        localpath = r"D:/plugins/tools/7z"
        z7 = "D:/plugins/tools/7z/7z.exe"
        fromdir = self._B_plugin_path.replace("/plugins/houdini","") + r"/tools/7-Zip"
        h_source = os.path.abspath("%s/apps/win/%s.7z"%(self._B_plugin_path,self._hfs_version))
        h_amd = os.path.abspath("%s/%s.7z"%(self._D_houdini_path,self._hfs_version))
        if os.path.exists(h_amd):
            self._CopyHfs = False

        ## set hfs server
        version_base = self._hfs_version[:2]
        py_path = os.path.abspath("%s/function/HoudiniLibs/hfsserver.py %s"%(self._code_base_path,base_dir))
        _lic_info = {"hver":version_base,"server":self._server}
        _lic_info_f = "%s/temp/lic_info.json"%base_dir
        with open(_lic_info_f,"w")as f:
            json.dump(_lic_info,f)
            f.close()
        server_cmds = py_path

        ## unzip houdini
        cmd_un7z = z7 + " x -y -aos "
        cmd_un7z += "%s/%s.7z"%(self._D_houdini_path,self._hfs_version) # D:/plugins/houdini
        cmd_un7z += " -o%s" % ("%s/%s"%(self._D_houdini_path,self._hfs_version))

        ## subprocess
        # creat the handl
        setserver_log = open(r'%s/Server_info.txt'%self._temp,'wt')
        copy7ztool_log = open(r'%s/Copy_ziptool.txt'%self._temp,'wt')
        Uzi_Houdini_log = open(r'%s/Uzip_Houdini.txt'%self._temp,'wt')

        set_server = subprocess.Popen(server_cmds,stdout=setserver_log,shell=True)
        copy7ztool = subprocess.Popen("robocopy /S /NDL /NFL %s %s %s" % (fromdir, localpath, "*"),stdout=copy7ztool_log,shell=True)
        if self._CopyHfs:
            copyhoudini = subprocess.Popen("copy %s %s" % (h_source, h_amd),shell=True)
            copyhoudini.wait()
        copy7ztool.wait()

        UzipHoudini = subprocess.Popen(cmd_un7z,stdout=Uzi_Houdini_log,shell=True)
        _s_result = set_server.wait()
        if not _s_result:
            # print("License server changed to: %s"%self._server)
            self._info_return.append("License server changed to: %s"%self._server)
        UzipHoudini.wait()

        # finish,close the handl
        setserver_log.close()
        copy7ztool_log.close()
        Uzi_Houdini_log.close()
        _h_result = UzipHoudini.returncode

        ## os.remove(h_amd)
        if not _h_result:
            print("Houdini setup finished. ")

    def KillApps(self):
        for app in self._Kapps:
            if self._plants == "win":
                cmds = r'c:\windows\system32\cmd.exe /c c:\windows\system32\TASKKILL.exe /F /IM %s'%app
            elif self._plants == "Linux":
                cmds = ''
            subprocess.call(cmds,shell=True)

    def Extued(self):
        # print("Try to kill the houdinifx and hython before houdini app setup")
        self._info_return.append("Try to kill the houdinifx and hython before houdini app setup")
        try:
            self.KillApps()
        except:
            pass
        if self._run_code and self._plants=="win":
            self.ConfigApp_win()
        elif self._run_code and self._plants=="Linux":
            self.ConfigApp_Linux()

def main(version='',plants='win',adict={}):

    s_time = time.time()
    app = HfsSetup(version,plants,adict)
    app.Extued()
    n_time = time.time()
    print("Times for apps setup: %d s"%(n_time-s_time))
    app._info_return.append("Times for apps setup: %d s"%(n_time-s_time))

    return app._info_return

if __name__ == '__main__':
    main('160557','win')

'''
    _info_file = "%s/temp/app_info.json"%base_dir
    _plugin_info = {}
    if os.path.exists(_info_file):
        with open(_info_file,"r")as f:
            _plugin_info = json.load(f)
            f.close()
'''


from django.db import models
from core.models import Service, PlCoreBase, Slice, Instance, Tenant, TenantWithContainer, Node, Image, User, Flavor, Subscriber
from core.models.plcorebase import StrippedCharField
import os
from django.db import models, transaction
from django.forms.models import model_to_dict
from django.db.models import Q
from operator import itemgetter, attrgetter, methodcaller
import traceback
from xos.exceptions import *
from core.models import SlicePrivilege, SitePrivilege
from sets import Set
from urlparse import urlparse

CEILOMETER_KIND = "ceilometer"

class CeilometerService(Service):
    KIND = CEILOMETER_KIND

    class Meta:
        app_label = "ceilometer"
        verbose_name = "Ceilometer Service"
        proxy = True

    @property
    def ceilometer_pub_sub_url(self):
        return self.get_attribute("ceilometer_pub_sub_url", None)

    @ceilometer_pub_sub_url.setter
    def ceilometer_pub_sub_url(self, value):
        self.set_attribute("ceilometer_pub_sub_url", value)

class MonitoringChannel(TenantWithContainer):   # aka 'CeilometerTenant'
    class Meta:
        proxy = True

    KIND = CEILOMETER_KIND
    LOOK_FOR_IMAGES=[ #"trusty-server-multi-nic-docker", # CloudLab
                      "ceilometer-trusty-server-multi-nic",
                      #"trusty-server-multi-nic",
                    ]

    sync_attributes = ("private_ip", "private_mac",
                       "ceilometer_ip", "ceilometer_mac",
                       "nat_ip", "nat_mac", "ceilometer_port",)

    default_attributes = {}
    def __init__(self, *args, **kwargs):
        ceilometer_services = CeilometerService.get_service_objects().all()
        if ceilometer_services:
            self._meta.get_field("provider_service").default = ceilometer_services[0].id
        super(MonitoringChannel, self).__init__(*args, **kwargs)
        self.set_attribute("use_same_instance_for_multiple_tenants", True)

    def can_update(self, user):
        #Allow creation of this model instances for non-admin users also
        return True

    def save(self, *args, **kwargs):
        if not self.creator:
            if not getattr(self, "caller", None):
                # caller must be set when creating a monitoring channel since it creates a slice
                raise XOSProgrammingError("MonitoringChannel's self.caller was not set")
            self.creator = self.caller
            if not self.creator:
                raise XOSProgrammingError("MonitoringChannel's self.creator was not set")

        if self.pk is None:
            #Allow only one monitoring channel per user
            channel_count = sum ( [1 for channel in MonitoringChannel.objects.filter(kind=CEILOMETER_KIND) if (channel.creator == self.creator)] )
            if channel_count > 0:
                raise XOSValidationError("Already %s channels exist for user Can only create max 1 MonitoringChannel instance per user" % str(channel_count))

        super(MonitoringChannel, self).save(*args, **kwargs)
        model_policy_monitoring_channel(self.pk)

    def delete(self, *args, **kwargs):
        self.cleanup_container()
        super(MonitoringChannel, self).delete(*args, **kwargs)

    @property
    def addresses(self):
        if (not self.id) or (not self.instance):
            return {}

        addresses = {}
        for ns in self.instance.ports.all():
            if "private" in ns.network.name.lower():
                addresses["private"] = (ns.ip, ns.mac)
            elif ("nat" in ns.network.name.lower()) or ("management" in ns.network.name.lower()):
                addresses["nat"] = (ns.ip, ns.mac)
            #TODO: Do we need this client_access_network. Revisit in VTN context
            #elif "ceilometer_client_access" in ns.network.labels.lower():
            #    addresses["ceilometer"] = (ns.ip, ns.mac)
        return addresses

    @property
    def nat_ip(self):
        return self.addresses.get("nat", (None, None))[0]

    @property
    def nat_mac(self):
        return self.addresses.get("nat", (None, None))[1]

    @property
    def private_ip(self):
        return self.addresses.get("nat", (None, None))[0]

    @property
    def private_mac(self):
        return self.addresses.get("nat", (None, None))[1]

    @property
    def ceilometer_ip(self):
        return self.addresses.get("ceilometer", (None, None))[0]

    @property
    def ceilometer_mac(self):
        return self.addresses.get("ceilometer", (None, None))[1]

    @property
    def site_tenant_list(self):
        tenant_ids = Set()
        for sp in SitePrivilege.objects.filter(user=self.creator):
            site = sp.site
            for cs in site.controllersite.all():
               if cs.tenant_id:
                   tenant_ids.add(cs.tenant_id)
        return tenant_ids

    @property
    def slice_tenant_list(self):
        tenant_ids = Set()
        for sp in SlicePrivilege.objects.filter(user=self.creator):
            slice = sp.slice
            for cs in slice.controllerslices.all():
               if cs.tenant_id:
                   tenant_ids.add(cs.tenant_id)
        for slice in Slice.objects.filter(creator=self.creator):
            for cs in slice.controllerslices.all():
                if cs.tenant_id:
                    tenant_ids.add(cs.tenant_id)
        if self.creator.is_admin:
            #TODO: Ceilometer publishes the SDN meters without associating to any tenant IDs.
            #For now, ceilometer code is changed to pusblish all such meters with tenant
            #id as "default_admin_tenant". Here add that default tenant as authroized tenant_id
            #for all admin users.
            tenant_ids.add("default_admin_tenant")
        return tenant_ids

    @property
    def tenant_list(self):
        return self.slice_tenant_list | self.site_tenant_list

    @property
    def tenant_list_str(self):
        return ", ".join(self.tenant_list)

    @property
    def ceilometer_port(self):
        # TODO: Find a better logic to choose unique ceilometer port number for each instance
        if not self.id:
            return None
        return 8888+self.id

    @property
    def ceilometer_url(self):
        if not self.private_ip:
            return None
        return "http://" + self.private_ip + ":" + str(self.ceilometer_port) + "/"

def model_policy_monitoring_channel(pk):
    # TODO: this should be made in to a real model_policy
    with transaction.atomic():
        mc = MonitoringChannel.objects.select_for_update().filter(pk=pk)
        if not mc:
            return
        mc = mc[0]
        mc.manage_container()

SFLOW_KIND = "sflow"
SFLOW_PORT = 6343
SFLOW_API_PORT = 33333

class SFlowService(Service):
    KIND = SFLOW_KIND

    class Meta:
        app_label = "ceilometer"
        verbose_name = "sFlow Collection Service"
        proxy = True

    default_attributes = {"sflow_port": SFLOW_PORT, "sflow_api_port": SFLOW_API_PORT}

    sync_attributes = ("sflow_port", "sflow_api_port",)

    @property
    def sflow_port(self):
        return self.get_attribute("sflow_port", self.default_attributes["sflow_port"])

    @sflow_port.setter
    def sflow_port(self, value):
        self.set_attribute("sflow_port", value)

    @property
    def sflow_api_port(self):
        return self.get_attribute("sflow_api_port", self.default_attributes["sflow_api_port"])

    @sflow_api_port.setter
    def sflow_api_port(self, value):
        self.set_attribute("sflow_api_port", value)

    def get_instance(self):
        if self.slices.exists():
            slice = self.slices.all()[0]
            if slice.instances.exists():
                return slice.instances.all()[0]

        return None

    @property
    def sflow_api_url(self):
        if not self.get_instance():
            return None
        return "http://" + self.get_instance().get_ssh_ip() + ":" + str(self.sflow_api_port) + "/"

class SFlowTenant(Tenant):
    class Meta:
        proxy = True

    KIND = SFLOW_KIND

    sync_attributes = ("listening_endpoint", )

    default_attributes = {}
    def __init__(self, *args, **kwargs):
        sflow_services = SFlowService.get_service_objects().all()
        if sflow_services:
            self._meta.get_field("provider_service").default = sflow_services[0].id
        super(SFlowTenant, self).__init__(*args, **kwargs)

    @property
    def creator(self):
        from core.models import User
        if getattr(self, "cached_creator", None):
            return self.cached_creator
        creator_id=self.get_attribute("creator_id")
        if not creator_id:
            return None
        users=User.objects.filter(id=creator_id)
        if not users:
            return None
        user=users[0]
        self.cached_creator = users[0]
        return user

    @creator.setter
    def creator(self, value):
        if value:
            value = value.id
        if (value != self.get_attribute("creator_id", None)):
            self.cached_creator=None
        self.set_attribute("creator_id", value)

    @property
    def listening_endpoint(self):
        return self.get_attribute("listening_endpoint", None)

    @listening_endpoint.setter
    def listening_endpoint(self, value):
        if urlparse(value).scheme != 'udp':
            raise XOSProgrammingError("SFlowTenant: Only UDP listening endpoint URLs are accepted...valid syntax is: udp://ip:port")
        self.set_attribute("listening_endpoint", value)

    def save(self, *args, **kwargs):
        if not self.creator:
            if not getattr(self, "caller", None):
                # caller must be set when creating a SFlow tenant since it creates a slice
                raise XOSProgrammingError("SFlowTenant's self.caller was not set")
            self.creator = self.caller
            if not self.creator:
                raise XOSProgrammingError("SFlowTenant's self.creator was not set")

        if not self.listening_endpoint:
            raise XOSProgrammingError("SFlowTenant's self.listening_endpoint was not set")

        if self.pk is None:
            #Allow only one sflow channel per user and listening_endpoint
            channel_count = sum ( [1 for channel in SFlowTenant.objects.filter(kind=SFLOW_KIND) if ((channel.creator == self.creator) and (channel.listening_endpoint == self.listening_endpoint))] )
            if channel_count > 0:
                raise XOSValidationError("Already %s sflow channels exist for user Can only create max 1 tenant per user and listening endpoint" % str(channel_count))

        super(SFlowTenant, self).save(*args, **kwargs)

    def delete(self, *args, **kwargs):
        super(MonitoringChannel, self).delete(*args, **kwargs)

    @property
    def authorized_resource_list(self):
        return ['all']

    @property
    def authorized_resource_list_str(self):
        return ", ".join(self.authorized_resource_list)


import socket

TCP_IP = '127.0.0.1'
TCP_PORT = 5005
BUFFER_SIZE = 1024

s = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
s.connect((TCP_IP,TCP_PORT))
message ="Player"
s.send(message.encode())

while 1:
  data = s.recv(1024).decode()
  print("received:",data)
  s.send(input().encode())


from django.http import *
from forms import UploadForm
from django import template
from django.template.loader import get_template
from django.template import Context, RequestContext
from django.utils.decorators import method_decorator
from django.shortcuts import render_to_response
from django.contrib.auth import authenticate, login, logout
from django.contrib.auth.decorators import login_required
from django.views.generic.base import TemplateView, View
from django.views.decorators.csrf import csrf_exempt
from django.contrib.sessions.models import Session
from django.contrib.auth.models import User, Group, Permission
from models import *
from django.db import models
from django.db.models import Count, Min, Sum, Max, Avg
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.utils import unittest
from django.db import connection, transaction
import logging
import hashlib
from google.appengine.api import files

try:
    files.gs
except AttributeError:
    import gs
    files.gs = gs

PERPAGE=50

def checkadminlogin_dispatch(f):
    def wrap(request, *args, **kwargs):
        if 'IsLogin' in request.session and request.session['IsLogin'] and 'Staff' in request.session and request.session['Staff'].username !="":
            staff_list = Admins.objects.filter(username = request.session['Staff_username'], pass_field = hashlib.md5(request.session['Staff_password']).hexdigest())
            if staff_list:
                request.session['IsLogin'] = True
                request.session['Staff'] = staff_list[0]
                success = True
            else:
                return HttpResponseRedirect('/logout')
            logging.info('Fetch Started::  %s', staff_list[0])
        else:
            return HttpResponseRedirect('/logout')
        return f(request, *args, **kwargs)
    return wrap

class CsrfExemptMixin(object):
    @method_decorator(csrf_exempt)
    def dispatch(self, request, *args, **kwargs):
        return super(CsrfExemptMixin, self).dispatch(request, *args, **kwargs)

class LoginRequiredMixin(object):
    @method_decorator(checkadminlogin_dispatch)
    def dispatch(self,request, *args, **kwargs):
        return super(LoginRequiredMixin, self).dispatch(request, *args, **kwargs)

@csrf_exempt
def render_template(request, template, data=None):
    errs =""
    if request.method == 'GET' and 'err' in request.GET:
        data.update({'errs':request.GET['err']})

    response = render_to_response(template, data,
                              context_instance=RequestContext(request))
    return response

class CMSClass(LoginRequiredMixin, TemplateView):

    def get(self, request, *args, **kwargs):
        count = Extrapages.objects.count()
        if request.GET['page'] == "":
            page_num = 1
        else:
            page_num = request.GET['page']
        page_num = int(page_num)
        offset = page_num * 100
        allpages = Extrapages.objects.all()[offset-100:offset]
        content = {'page_title': "Summary",
                   'allpages':allpages,
                   'count':count,
                   'page_num':page_num,
                   }
        return render_template(request, "cms_pages.htm", content)

class CMSEditClass(LoginRequiredMixin, TemplateView):

    def get(self, request, *args, **kwargs):

        pageid = request.GET['pageid']
        allpages = Extrapages.objects.get(id=pageid)
        content = {'page_title': "Summary",
                   'allpages':allpages,

                   }
        return render_template(request, "cms_pages_edit.htm", content)

class EmailViewClass(LoginRequiredMixin, TemplateView):

    def get(self, request, *args, **kwargs):
        count = Emails.objects.count()
        if request.GET['page'] == "":
            page_num = 1
        else:
            page_num = request.GET['page']
        page_num = int(page_num)
        offset = page_num * 100
        allpages = Emails.objects.all()[offset-100:offset]
        content = {'page_title': "Admin :: Email List",
                   'allpages':allpages,
                   'count':count,
                   'page_num':page_num,
                   }
        return render_template(request, "email_pages.htm", content)

class EmailEditClass(LoginRequiredMixin, TemplateView):

    def get(self, request, *args, **kwargs):
        pageid = request.GET['id']
        allpages = Emails.objects.get(id=pageid)
        content = {'page_title': "Admin::Email Edit",
                   'allpages':allpages,
                   }
        return render_template(request, "email_pages_edit.htm", content)

class CMSAddFormClass(LoginRequiredMixin, TemplateView):

    def get(self, request, *args, **kwargs):
        content = {'page_title': "Summary",}
        return render_template(request, "cms_pages_add.htm", content)

class TitlesContentClass(LoginRequiredMixin, TemplateView):

    def get(self, request, *args, **kwargs):
        count = Html.objects.count()
        if request.GET['page'] == "":
            page_num = 1
        else:
            page_num = request.GET['page']
        page_num = int(page_num)
        offset = page_num * 100
        allpages = Html.objects.all()[offset-100:offset]
        content = {'page_title': "Summary",
                   'allpages':allpages,
                   'count':count,
                   'page_num':page_num,
                   }
        return render_template(request, "titles_content.htm", content)

class ProductWishListClass(LoginRequiredMixin, TemplateView):

    def get(self, request, *args, **kwargs):
        if request.GET['page'] == "":
            page_num = 1
        else:
            #pages = count/100
            page_num = request.GET['page']
        page_num = int(page_num)
        offset = page_num * 100
        #allitems = ProductWaitinglist.objects.annotate(dcount=Count('catalogid')).values('catalogid',
        #                                                                                'current_stock',
        #                                                                                'products__catalogid').all()[offset-100:offset]
        allitems = ProductWaitinglist.objects.raw('select count(*) as dcount,product_waitinglist.catalogid,products.id,name,current_stock from product_waitinglist,products where product_waitinglist.catalogid=products.catalogid group by catalogid')[offset-100:offset]
        count = ProductWaitinglist.objects.values('catalogid').annotate(dcount=Count('catalogid')).count()
        #return HttpResponse(allitems)
        content = {'page_title': "Summary",
                   'allitems':allitems,
                   'count':count,
                   'page_num':page_num,
                   }
        return render_template(request, "products_wish_list.htm", content)

class ProductWishViewClass(LoginRequiredMixin, TemplateView):

    def get(self, request, *args, **kwargs):
        if request.GET['page'] == "":
            page_num = 1
        else:
            page_num = request.GET['page']
        page_num = int(page_num)
        offset = page_num * 100
        itemid = request.GET['itemid']
        allitems = ProductWaitinglist.objects.filter(catalogid=itemid).all()[offset-100:offset]

        count = ProductWaitinglist.objects.filter(catalogid=itemid).all().count()
        #return HttpResponse(allitems)
        content = {'page_title': "Summary",
                   'allitems':allitems,
                   'count':count,
                   'page_num':page_num,
                   'itemid':itemid,
                   }
        return render_template(request, "products_wish_list_view_list.htm", content)

class ReviewAllClass(LoginRequiredMixin, TemplateView):

    def get(self, request, *args, **kwargs):
        if request.GET['page'] == "":
            page_num = 1
        else:
            #pages = count/100
            page_num = request.GET['page']
        page_num = int(page_num)
        offset = page_num * 100

        allitems = ProductReview.objects.raw('select count(*) as dcount,product_review.catalogid,products.id,name,thumbnail from product_review, products where product_review.catalogid=products.catalogid group by catalogid')[offset-100:offset]
        count = ProductReview.objects.values('catalogid').annotate(dcount=Count('catalogid')).count()
        #return HttpResponse(allitems)
        content = {'page_title': "Summary",
                   'allitems':allitems,
                   'count':count,
                   'page_num':page_num,
                   }
        return render_template(request, "products_7_reviews.htm", content)

class ProductsReviewsViewClass(LoginRequiredMixin, TemplateView):

    def get(self, request, *args, **kwargs):

        itemid = request.GET['itemid']
        allitems = ProductReview.objects.filter(catalogid=itemid).all()

        count = ProductReview.objects.filter(catalogid=itemid).all().count()
        #return HttpResponse(allitems)
        content = {'page_title': "Summary",
                   'allitems':allitems,
                   'count':count,
                   'itemid':itemid,
                   }
        return render_template(request, "products_review_view_list.htm", content)

class ProductsReviewEditFormClass(LoginRequiredMixin,TemplateView):
    def get(self, request, *args, **kwargs):
        itemid = request.GET['itemid']
        allitems = ProductReview.objects.get(id=itemid)
        content = {'page_title': "Summary",
                   'allitems':allitems,
                   #'count':count,
                   #'page_num':page_num,
                   'itemid':itemid,
                   }
        return render_template(request, "products_7_reviews_edit_2_edit.htm", content)

class ApanelViewClass(LoginRequiredMixin, TemplateView):
    def get(self, request, *args, **kwargs):
        content = {'page_title': "Profile",}
        return render_template(request, "home-page-admin.htm", content)

class CustomersViewClass(LoginRequiredMixin,TemplateView):
    def get(self, request, *args, **kwargs):
        count = customers.objects.count()
        if request.GET['page'] == "":
            page_num = 1
        else:
            page_num = request.GET['page']
        page_num = int(page_num)
        offset = page_num * 100
        content = {'page_title': "Profile",
                   'customers':customers.objects.all()[offset-100:offset],
                   'count':count,
                   'page_num':page_num,
                   }
        return render_template(request, "customers.htm", content)

class CRMViewClass(LoginRequiredMixin,TemplateView):
    def get(self, request, *args, **kwargs):

        if request.GET['page'] == "":
            page_num = 1
        else:
            page_num = request.GET['page']
        if 'status' in request.GET and request.GET['status'] != "":
            status = request.GET['status']
        else:
            status = 1
        count = Crm.objects.filter(status=status).count()
        page_num = int(page_num)
        offset = page_num * 100
        content = {'page_title': "Profile",
                   'allitems':Crm.objects.all().filter(status=status)[offset-100:offset],
                   'count':count,
                   'page_num':page_num,
                   }
        return render_template(request, "crm.htm", content)

class CRMEditViewClass(LoginRequiredMixin,TemplateView):
    def get(self, request, *args, **kwargs):
        crmid = request.GET['id']
        allitems = Crm.objects.get(id=crmid)
        categories = ProductCategory.objects.all()
        content = {'page_title': "Profile",
                   'allitems':allitems,
                   'manufacturers':Manufacturer.objects.all(),
                   'categories': categories,}
        return render_template(request, "crm_edit.htm", content)

class StaffViewClass(LoginRequiredMixin,TemplateView):
    def get(self, request, *args, **kwargs):
        content = {'page_title': "Site Staff",
                   'customers':Admins.objects.all()[:100],
                   'count':Admins.objects.count(),}
        return render_template(request, "admins.htm", content)

class CategoryViewClass(LoginRequiredMixin,TemplateView):
    def get(self, request, *args, **kwargs):
        count = Category.objects.count()
        if request.GET['page'] == "":
            page_num = 1

        else:
            #pages = count/100
            page_num = request.GET['page']
        page_num = int(page_num)
        offset = page_num * 100
        content = {'page_title': "Profile",
                   'customers':Category.objects.all()[offset-100:offset],
                   'count':count,
                   'page_num':page_num,}
        return render_template(request, "categories.htm", content)

class CustomerAddFormClass(LoginRequiredMixin, TemplateView):
    def get(self, request, *args, **kwargs):
        content = {'title': "Add Customer",}
        return render_template(request, "customer_add.htm", content)

class CustomerInfoClass(LoginRequiredMixin,TemplateView):
    #summary = Customers.objects.all()
    def get(self, request, *args, **kwargs):
        cid = request.GET['id']
        customer = customers.objects.get(contactid=cid)
        customeremail= customer.email
        customerrewards = CustomerRewards.objects.filter(contactid=cid).all()
        totalrewards = CustomerRewards.objects.filter(contactid=cid).aggregate(Sum('points'))
        #customers_promocode = SwfCustomerCreditsLog.objects.values_list('customers_promocode', flat=True)
        #customers_promocode = customers_promocode['customers_promocode']
        #storerewards = SwfCustomerCreditsLog.objects.filter(customers_email_address=customeremail)
        storerewards = SwfCustomerCreditsLog.objects.raw('select *,swf_customer_credits_log.id as sid from swf_customer_credits_log , promotions where customers_promocode = coupon AND customers_email_address="'+customeremail+'" AND customers_promocode != ""')
        fulldata = list(storerewards)
        try:
            wish_id = WshWishlist.objects.get(customerid=cid)
            wishitems = WsiWishlistitems.objects.filter(wsh_id=wish_id.wsh_id)
        except Exception as e:
            wishitems = ""

        content = {'page_title': "Customers Info",
                   'customer': customer,
                   'customerorders':Orders.objects.filter(ocustomerid=cid).all(),
                   'wishlists':wishitems,
                   'customerrewards':customerrewards,
                   'totalrewards':totalrewards,
                   'storerewards':fulldata,
                   }
                   #'count':Admins.objects.count(),}
        return render_template(request, "customers_info.htm", content)

class ProductsViewClass(LoginRequiredMixin,TemplateView):
    def get(self, request, *args, **kwargs):
        count = Products.objects.count()
        if request.GET['page'] == "":
            page_num = 1
        else:
            #pages = count/100
            page_num = request.GET['page']
        page_num = int(page_num)
        offset = page_num * 100
        content = {'page_title': "Profile",
                   'allitems':Products.objects.all()[offset-100:offset],
                   'count':count,
                   'page_num':page_num,}
        return render_template(request, "products.htm", content)

class ProductViewClass(LoginRequiredMixin,TemplateView):
    def get(self, request, *args, **kwargs):
        pid = request.GET['pid']
        allitems = Products.objects.get(catalogid=pid)
        categories = ProductCategory.objects.all().filter(catalogid=pid)
        content = {'page_title': "Profile",
                   'allitems':allitems,
                   'manufacturers':Manufacturer.objects.all(),
                   'categories': categories,}
        return render_template(request, "productedit.htm", content)

class ProductRelatedClass(LoginRequiredMixin,TemplateView):
    def get(self, request, *args, **kwargs):
        pid = request.GET['pid']
        allitems = Products.objects.get(catalogid=pid)
        categories = ProductCategory.objects.all().filter(catalogid=pid)
        content = {'page_title': "Profile",
                   'allitems':allitems,
                   'manufacturers':Manufacturer.objects.all(),
                   'categories': categories,}
        return render_template(request, "productrelated.htm", content)

class ProductsImagesViewClass(LoginRequiredMixin,TemplateView):
    def get(self, request, *args, **kwargs):
        pid = request.GET['pid']
        allitems = Products.objects.get(catalogid=pid)
        categories = ProductCategory.objects.all().filter(catalogid=pid)
        content = {'page_title': "Profile",
                   'allitems':allitems,
                   'manufacturers':Manufacturer.objects.all(),
                   'categories': categories,}
        return render_template(request, "images_products.htm", content)

class ApanelViewOrdersClass(LoginRequiredMixin,TemplateView):

    def get(self, request, *args, **kwargs):
        order_status = request.GET['order_status']
        if order_status < 1:
            order_status = 1
        else:
            order_status = order_status
        count = Orders.objects.filter(order_status=order_status).count()
        if request.GET['page'] == "":
            page_num = 1
        else:
            #pages = count/100
            page_num = request.GET['page']
        page_num = int(page_num)
        offset = page_num * 100
        allitems = Orders.objects.all().filter(order_status=order_status)[offset-100:offset]
        order_status_links = OrderStatus.objects.all().filter(visible='1')
        #crm_messages=CrmMessages.objects.select_related(crmid__orderid='8623')

        #return HttpResponse(crm_messages)
        content = {'page_title': "Orders",
                   'allitems':allitems,
                   'count':count,
                   'page_num':page_num,
                   'order_status':order_status,
                   'order_links':order_status_links,}
        return render_template(request, "vieworders.htm", content)

class ApanelViewOrdersStatusClass(LoginRequiredMixin,TemplateView):
    def get(self, request, *args, **kwargs):
        allitems = OrderStatus.objects.all()
        content = {'page_title': "Orders Status",
                   'allitems':allitems,
                   'order_links':OrderStatus.objects.all().filter(visible='1'),}
        return render_template(request, "orders_status.htm", content)

class OrderPageClass(LoginRequiredMixin,TemplateView):
    def get(self, request, *args, **kwargs):
        oid = request.GET['oid']
        order_status_links = OrderStatus.objects.all().filter(visible='1')
        allitems = Orders.objects.get(orderid=oid)
        try:
            transactions = Transactions.objects.get(orderid=oid)
            amount = transactions.amount
            totalamt = Oitems.objects.filter(orderid=oid).aggregate(Sum('unitprice'))
            totalamt = totalamt['unitprice__sum']
        except Exception as e:
            transactions = ""
            totalamt = 0
            amount = 0
        alloiitems = Oitems.objects.all().filter(orderid=oid)
        finaltotal = (totalamt + int(allitems.oshipcost)) - allitems.coupondiscount
        balance = finaltotal - amount
        content = {'page_title': "Orders Status",
                   'allitems':allitems,
                   'alloiitems':alloiitems,
                   'order_links':order_status_links,
                   'totalamt':totalamt,
                   'finaltotal':finaltotal,
                   'paidamt':finaltotal,
                   'transactions':transactions,
                   'balance':balance,
                   }
        return render_template(request, "orderpage.htm", content)

class AddAdminsFormClass(LoginRequiredMixin,TemplateView):
    def get(self, request, *args, **kwargs):
        allitems = Admins.objects.all()
        if "mode" in request.GET:
            mode = request.GET['mode']
        else:
            mode = ""
            allitems = ""
        if "id" in request.GET:
            allitems = Admins.objects.get(id=request.GET['id'])
        else:
             allitems = ""
        content = {'page_title': "Add User",
                   'allitems':allitems,
                   'mode':mode,}
        return render_template(request, "admins_add.htm", content)

class RmaPagesClass(LoginRequiredMixin,TemplateView):
    def get(self, request, *args, **kwargs):
        count = Rma.objects.count()
        if request.GET['page'] == "":
            page_num = 1
        else:
            #pages = count/100
            page_num = request.GET['page']
        page_num = int(page_num)
        offset = page_num * 100
        allitems = Rma.objects.all()[offset-100:offset]
        content = {'page_title': "Orders Status",
                   'allitems':allitems,
                   'count':count,}
        return render_template(request, "rma_pages.htm", content)

class RmaViewClass(LoginRequiredMixin,TemplateView):
    def get(self, request, *args, **kwargs):
        rmaid=request.GET['rmaid']
        allitems = Rma.objects.get(idrma=rmaid)
        content = {'page_title': "View RMA",
                   'allitems':allitems,}
        return render_template(request, "rmaview.htm", content)

class ShippingManagerViewClass(LoginRequiredMixin,TemplateView):
    def get(self, request, *args, **kwargs):
        if "mode" in request.GET:
            mode = request.GET['mode']
        else:
            mode = ""
        allitems = ShippingCategory.objects.all()
        content = {'page_title': "Admin: Shipping Manager View",
                   'allitems':allitems,
                   'mode':mode,}
        return render_template(request, "adminshippingmanager.htm", content)

class TaxManagerViewClass(LoginRequiredMixin,TemplateView):
    def get(self, request, *args, **kwargs):
        if "mode" in request.GET:
            mode = request.GET['mode']
        else:
            mode = ""
        allitems = Tax.objects.all()
        content = {'page_title': "Admin: Tax Manager View",
                   'allitems':allitems,
                   'mode':mode,}
        return render_template(request, "taxmanager.htm", content)

class GiftCertificatesViewClass(LoginRequiredMixin,TemplateView):
    def get(self, request, *args, **kwargs):
        count = GiftCertificates.objects.all().count()
        if request.GET['page'] == "":
            page_num = 1
        else:
            #pages = count/100
            page_num = request.GET['page']
        page_num = int(page_num)
        offset = page_num * 100
        allitems = GiftCertificates.objects.all()[offset-100:offset]
        content = {'page_title': "Admin: Gift Certificate View",
                   'allitems':allitems,
                   'page_num':page_num,
                   'count':count,
                   'order_links':OrderStatus.objects.all().filter(visible='1'),}
        return render_template(request, "giftcertificate_pages.htm", content)

class EditGiftCertificateClass(LoginRequiredMixin,TemplateView):
    def get(self, request, *args, **kwargs):
        giftid=request.GET['id']
        allitems = GiftCertificates.objects.get(id=giftid)
        total = allitems.certificate_amount + allitems.certificate_expenses
        content = {'page_title': "Admin :: Edit Gift Certificate",
                   'allitems':allitems,
                   'order_links':OrderStatus.objects.all().filter(visible='1'),
                   'total':total}
        return render_template(request, "edit_giftcertificate.htm", content)

class ProductArticleViewClass(LoginRequiredMixin,TemplateView):
    def get(self, request, *args, **kwargs):
        pid = request.GET['pid']
        prod = Products.objects.get(catalogid=pid)
        allitems = ProductArticle.objects.all().filter(catalogid=pid)
        count = allitems.count()
        content = {'page_title': "Admin: Product Articles",
                   'allitems':allitems,
                   'prod':prod,
                   'count':count,
                   }
        return render_template(request, "product_articles.htm", content)

class ProductArticleEditViewClass(LoginRequiredMixin, TemplateView):
    def get(self, request, *args, **kwargs):
        pid = request.GET['id']
        allpages = ProductArticle.objects.get(id=pid)
        content = {'page_title': "Admin :: Edit Article",
                   'allpages':allpages,}
        return render_template(request, "product_article_edit.htm", content)

class ProductArticleAddFormClass(LoginRequiredMixin, TemplateView):
    def get(self, request, *args, **kwargs):
        pid = request.GET['pid']
        content = {'page_title': "Admin :: Add Article",
                   'pid':pid,}
        return render_template(request, "product_article_add.htm", content)

class ProductReviewsViewClass(LoginRequiredMixin,TemplateView):
    def get(self, request, *args, **kwargs):
        pid = request.GET['pid']
        prod = Products.objects.get(catalogid=pid)
        allitems = ProductReview.objects.filter(catalogid=pid).all()
        count = allitems.count()
        content = {'page_title': "Admin: Product Articles",
                   'allitems':allitems,
                   'prod':prod,
                   'count':count,
                   }
        return render_template(request, "product_reviews.htm", content)

class ProductOptionEditViewClass(LoginRequiredMixin, TemplateView):
    def get(self, request, *args, **kwargs):
        pid = request.GET['pid']
        allpages = Products.objects.get(catalogid=pid)
        content = {'page_title': "Admin :: Edit Options",
                   'allpages':allpages,
                   'prod':pid,}
        return render_template(request, "product_options_edit.htm", content)

class BannersViewClass(LoginRequiredMixin, TemplateView):
    def get(self, request, *args, **kwargs):
        allpages = SiteBanners.objects.all()
        content = {'page_title': "Admin :: Banner Managements",
                   'allitems':allpages,}
        return render_template(request, "viewbanners.htm", content)

class BannerEditViewClass(LoginRequiredMixin, TemplateView):
    def get(self, request, *args, **kwargs):
        bid = request.GET['bid']
        filename = "/gs/swf_product_images/banner/banner5.png"
        allpages = SiteBanners.objects.get(id=bid)
        content = {'page_title': "Admin :: Edit banner",
                   'allpages':allpages,
                   'bannerpath':filename,}
        return render_template(request, "editbanner.htm", content)

class BannersAddFormClass(LoginRequiredMixin, TemplateView):
    def get(self, request, *args, **kwargs):
        content = {'page_title': "Admin :: Add Banner Managements",}
        return render_template(request, "addbanner.htm", content)

class GCSfilesClass(LoginRequiredMixin, TemplateView):
    def get(self, request, *args, **kwargs):
        content = {'page_title': "Admin :: Add Banner Managements",}
        file_list = files.listdir('/gs/swf_product_images')
        for file_name in file_list:
          if not file_name.__contains__('$folder$'):
            self.response.write('<a href="https://storage.cloud.google.com/%s">%s<a><br>' %(file_name[4:], file_name[4:]))
        #return render_template(request, "gcsfiles.htm", content)

class CouponsViewClass(LoginRequiredMixin,TemplateView):
    def get(self, request, *args, **kwargs):
        count = Promotions.objects.count()
        if "page" in request.GET and request.GET['page'] != "":
            page_num = request.GET['page']
        else:
            page_num = 1
            #pages = count/100

        page_num = int(page_num)
        offset = page_num * 100
        allitems = Promotions.objects.all()[offset-100:offset]
        content = {'page_title': "Orders Status",
                   'allitems':allitems,
                   'count':count,}
        return render_template(request, "viewcoupons.htm", content)


"""
Created on Wed Feb 24 12:34:17 2021

@author: Narmin Ghaffari Laleh
"""

##############################################################################

from dataGenerator.dataSetGenerator_ClamMil import Generic_MIL_Dataset
import utils.utils as utils
from extractFeatures import ExtractFeatures
from utils.core_utils import Train_MIL_CLAM
from utils.data_utils import ConcatCohorts_Classic
from eval.eval import CalculatePatientWiseAUC, CalculateTotalROC, MergeResultCSV
from sklearn.model_selection import StratifiedKFold
import numpy as np
import os
import pandas as pd
import random
from sklearn import preprocessing
import torch

##############################################################################

def CLAM_MIL_Training(args):

    targetLabels = args.target_labels
    args.feat_dir = args.feat_dir[0]

    for targetLabel in targetLabels:
        for repeat in range(args.repeatExperiment):

            args.target_label = targetLabel
            random.seed(args.seed)
            args.projectFolder = utils.CreateProjectFolder(args.project_name, args.adressExp, targetLabel, args.model_name, repeat+1)
            print(args.projectFolder)
            if os.path.exists(args.projectFolder):
                continue
            else:
                os.mkdir(args.projectFolder)

            args.result_dir = os.path.join(args.projectFolder, 'RESULTS')
            os.makedirs(args.result_dir, exist_ok = True)
            args.split_dir = os.path.join(args.projectFolder, 'SPLITS')
            os.makedirs(args.split_dir, exist_ok = True)

            reportFile  = open(os.path.join(args.projectFolder,'Report.txt'), 'a', encoding="utf-8")
            reportFile.write('-' * 30 + '\n')
            reportFile.write(str(args))
            reportFile.write('-' * 30 + '\n')
            if args.extractFeature:
                imgs = os.listdir(args.datadir_train[0])
                imgs = [os.path.join(args.datadir_train[0], i) for i in imgs]
                ExtractFeatures(data_dir = imgs, feat_dir = args.feat_dir, batch_size = args.batch_size, target_patch_size = -1, filterData = True)
            print('\nLOAD THE DATASET FOR TRAINING...\n')
            patientsList, labelsList, args.csvFile = ConcatCohorts_Classic(imagesPath = args.datadir_train,
                                                                          cliniTablePath = args.clini_dir, slideTablePath = args.slide_dir,
                                                                          label = targetLabel, minNumberOfTiles = args.minNumBlocks,
                                                                          outputPath = args.projectFolder, reportFile = reportFile, csvName = args.csv_name,
                                                                          patientNumber = args.numPatientToUse)

            yTrueLabel = utils.CheckForTargetType(labelsList)
            le = preprocessing.LabelEncoder()
            yTrue = le.fit_transform(yTrueLabel)
            args.num_classes = len(set(yTrue))
            args.target_labelDict = dict(zip(le.classes_, range(len(le.classes_))))
            utils.Summarize(args, list(yTrue), reportFile)
            print('\nLoad the DataSet...')
            dataset = Generic_MIL_Dataset(csv_path = args.csvFile,
                                          data_dir = args.feat_dir,
                                          shuffle = False,
                                          seed = args.seed,
                                          print_info = True,
                                          label_dict  = args.target_labelDict,
                                          patient_strat = True,
                                          label_col = args.target_label,
                                          ignore = [],
                                          reportFile = reportFile)
            if len(patientsList) < 20:
                continue

            if args.train_full:
                print('-' * 30)
                print('IT IS A FULL TRAINING FOR ' + targetLabel + '!')
                train_data = pd.DataFrame(list(zip(patientsList, yTrue, yTrueLabel)), columns = ['PATIENT', 'yTrue', 'yTrueLabel'])
                if args.early_stopping:
                    val_data = train_data.groupby('yTrue', group_keys = False).apply(lambda x: x.sample(frac = 0.1))
                    train_data = train_data[~train_data['PATIENT'].isin(list(val_data['PATIENT']))]
                    train_data.reset_index(inplace = True, drop = True)
                    val_data.reset_index(inplace = True, drop = True)
                    df = pd.DataFrame({'train': pd.Series(train_data['PATIENT']), 'test': pd.Series([]), 'val' : pd.Series(val_data['PATIENT'])})
                    df.to_csv(os.path.join(args.split_dir, 'TrainSplit.csv'), index = False)
                    train_dataset, val_dataset, test_dataset = dataset.Return_splits(from_id = False, csv_path = os.path.join(args.split_dir, 'TrainSplit.csv'))
                else:
                    df = pd.DataFrame({'train': pd.Series(train_data['PATIENT']), 'test': pd.Series([]), 'val' : pd.Series([])})
                    df.to_csv(os.path.join(args.split_dir, 'TrainValSplit.csv'), index = False)
                    train_dataset, val_dataset, test_dataset = dataset.Return_splits(from_id = False, csv_path = os.path.join(args.split_dir, 'TrainValSplit.csv'))

                datasets = (train_dataset, val_dataset, test_dataset)
                model, _, _  = Train_MIL_CLAM(datasets = datasets, fold = 'FULL', args = args, trainFull = True)
                torch.save(model.state_dict(), os.path.join(args.projectFolder, 'RESULTS', 'finalModel'))
                print()
                print('-' * 30)
                reportFile.close()
            else:
                print('IT IS A ' + str(args.k) + 'FOLD CROSS VALIDATION TRAINING FOR ' + targetLabel + '!')
                patientID = np.array(patientsList)
                yTrue = np.array(yTrue)
                yTrueLabel = np.array(yTrueLabel)

                folds = args.k
                kf = StratifiedKFold(n_splits = folds, random_state = args.seed, shuffle = True)
                kf.get_n_splits(patientID, yTrue)

                foldcounter = 1

                for train_index, test_index in kf.split(patientID, yTrue):

                    testPatients = patientID[test_index]
                    trainPatients = patientID[train_index]
                    testyTrue = yTrue[test_index]
                    trainyTrue = yTrue[train_index]
                    testyTrueLabel = yTrueLabel[test_index]
                    trainyTrueLabel = yTrueLabel[train_index]

                    print('GENERATE NEW TILES...\n')
                    print('FOR TRAIN SET...\n')
                    train_data = pd.DataFrame(list(zip(trainPatients, trainyTrue, trainyTrueLabel)), columns = ['PATIENT', 'yTrue', 'yTrueLabel'])
                    print('FOR VALIDATION SET...\n')
                    val_data = train_data.groupby('yTrue', group_keys = False).apply(lambda x: x.sample(frac = 0.1))
                    train_data = train_data[~train_data['PATIENT'].isin(list(val_data['PATIENT']))]
                    print('FOR TEST SET...\n')
                    test_data = pd.DataFrame(list(zip(testPatients, testyTrue, testyTrueLabel)), columns = ['PATIENT', 'yTrue', 'yTrueLabel'])
                    train_data.reset_index(inplace = True, drop = True)
                    test_data.reset_index(inplace = True, drop = True)
                    val_data.reset_index(inplace = True, drop = True)

                    print('-' * 30)
                    print("K FOLD VALIDATION STEP => {}".format(foldcounter))
                    print('-' * 30)

                    df = pd.DataFrame({'train': pd.Series(train_data['PATIENT']), 'test': pd.Series(test_data['PATIENT']), 'val' : pd.Series(val_data['PATIENT'])})
                    df.to_csv(os.path.join(args.split_dir, 'TrainTestValSplit_{}.csv'.format(foldcounter)), index = False)
                    train_dataset, val_dataset, test_dataset = dataset.Return_splits(from_id = False, csv_path = os.path.join(args.split_dir, 'TrainTestValSplit_{}.csv'.format(foldcounter)))
                    datasets = (train_dataset, val_dataset, test_dataset)

                    model, results, test_auc  = Train_MIL_CLAM(datasets = datasets, fold = foldcounter, args = args, trainFull = False)
                    reportFile.write('AUC calculated by CLAM' + '\n')
                    reportFile.write(str(test_auc) + '\n')
                    reportFile.write('-' * 30 + '\n')

                    patients = []
                    filaNames = []
                    yTrue_test = []
                    yTrueLabe_test = []
                    probs = {}

                    for i_temp in range(args.num_classes):
                        key = utils.get_key_from_value(args.target_labelDict, i_temp)
                        probs[key] = []

                    for item in list(results.keys()):
                        temp = results[item]
                        patients.append(temp['PATIENT'])
                        filaNames.append(temp['FILENAME'])
                        yTrue_test.append(temp['label'])
                        yTrueLabe_test.append(utils.get_key_from_value(args.target_labelDict, temp['label']))

                        for key in list(args.target_labelDict.keys()):
                            probs[key].append(temp['prob'][0][utils.get_value_from_key(args.target_labelDict, key)])

                    probs = pd.DataFrame.from_dict(probs)
                    df = pd.DataFrame(list(zip(patients, filaNames, yTrue_test, yTrueLabe_test)), columns =['PATIENT', 'FILENAME', 'yTrue', 'yTrueLabel'])
                    df = pd.concat([df, probs], axis = 1)
                    testResultsPath = os.path.join(args.result_dir, 'TEST_RESULT_SLIDE_BASED_FOLD_' + str(foldcounter) + '.csv')
                    df.to_csv(testResultsPath, index = False)
                    CalculatePatientWiseAUC(resultCSVPath = testResultsPath, args = args, foldcounter = foldcounter , clamMil = True, reportFile = reportFile)
                    reportFile.write('-' * 30 + '\n')
                    foldcounter +=  1

                patientScoreFiles = []
                slideScoreFiles = []
                for i in range(args.k):
                    patientScoreFiles.append('TEST_RESULT_PATIENT_BASED_FOLD_' + str(i + 1) + '.csv')
                    slideScoreFiles.append('TEST_RESULT_SLIDE_BASED_FOLD_' + str(i + 1) + '.csv')

                CalculateTotalROC(resultsPath = args.result_dir, results = patientScoreFiles, target_labelDict =  args.target_labelDict, reportFile = reportFile)
                reportFile.write('-' * 30 + '\n')
                MergeResultCSV(args.result_dir, slideScoreFiles, milClam = True)
                reportFile.close()

##############################################################################


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import cm as cm
import matplotlib.lines as mlines

df= pd.read_csv('/home/nishchay/Documents/Arcon/Day7/winequality-red.csv')
X1=df.iloc[:,11].values
Y1=df.iloc[:,0].values
Y2=df.iloc[:,1].values

fig = plt.figure()
ax1 = fig.add_subplot(111)
cmap = cm.get_cmap('jet', 20)
cax = ax1.imshow(df.corr(), interpolation="nearest", cmap=cmap)
ax1.grid(True)
plt.title('Wine Quality Correlation')
labels=['fixed acidity', 'volatile acidity', 'citric acid', 'residual sugar','chlorides', 'free sulfur dioxide','asdf']
ax1.set_xticklabels(labels,fontsize=6)
ax1.set_yticklabels(labels,fontsize=6)
fig.colorbar(cax, ticks=[- .6,- .5,- .4,- .3,- .2,- .1,0,.1,.2,.3,.4,.5,.6,.7,.8,.9,1])
plt.show()
#################################################################################
col_labels = df.columns[1:]
corMat2 = df.corr().values[::-1]
fig, axes = plt.subplots(nrows=1,ncols=1)
ax0 = axes
ax0.set_xticks(np.linspace(.5,12.5,11))
ax0.set_xticklabels(col_labels,rotation=45)

ax0.set_yticks(np.linspace(.5,12.5,11))
ax0.set_yticklabels(col_labels[::-1],rotation=45)
#ax0.set_yticklabels(col_labels,rotation=45)

#visualize correlations using heatmap
cax = ax1.imshow(df.corr(), interpolation="nearest", cmap=cmap)
cmap = cm.get_cmap('jet', 20)
fig.colorbar(cax, ticks=[- .6,- .5,- .4,- .3,- .2,- .1,0,.1,.2,.3,.4,.5,.6,.7,.8,.9,1])
plt.pcolor(corMat2,cmap='jet')
plt.show()

############################################################################

plt.plot(Y1,X1,'r--',Y2,X1,'bs')
plt.xlabel('Wine Quality')
plt.ylabel('fixed acidity')
red_line = mlines.Line2D(Y1,X1,color='red',marker='_',markersize=10,label='Fixed Acidity')
blue_line=mlines.Line2D(Y2,X1,color='blue',marker='|',markersize=10,label='Volatile Acidity')
plt.legend(handles=[red_line,blue_line])
plt.show()

labels = 'Python', 'C++', 'Ruby', 'Java'
sizes = [215, 130, 245, 210]
colors = ['gold', 'yellowgreen', 'lightcoral', 'lightskyblue']
explode = (0.1, 0, 0, 0)  # explode 1st slice

plt.pie(sizes, explode=explode, labels=labels, colors=colors,
        autopct='%1.1f%%', shadow=True, startangle=140)

plt.axis('equal')
plt.show()


'''
https://leetcode.com/explore/challenge/card/february-leetcoding-challenge-2021/584/week-1-february-1st-february-7th/3630/
'''
# Definition for a binary tree node.
# class TreeNode:
#     def __init__(self, val=0, left=None, right=None):
#         self.val = val
#         self.left = left
#         self.right = right
class Solution:
    def rightSideView(self, root: TreeNode) -> List[int]:
        if not root:
            return None
        result = []
        self.dfs(root, result, 0)

        return result

    def dfs(self, node, trace, depth):
        if depth == len(trace):
            trace.append(node.val)
        if node.right:
            self.dfs(node.right, trace, depth + 1)
        if node.left:
            self.dfs(node.left, trace, depth + 1)


#!/bin/python
import xml.etree.ElementTree as ET
import sys

tree = ET.parse(sys.argv[1])
root = tree.getroot()
'''
print root.find('deckname').text
main = root.find('./zone')
for c in main.findall(path='card'):
    print c.get('number')+c.get('name')
'''
for c in root[2]:
    print(c.get('number') + ' ' + c.get('name'))

for c in root[3]:
    print('SB: '+c.get('number') + ' ' + c.get('name'))


import pytest
import numpy as np
import os
from netZooPy import dragon

def test_dragon():
    #1. test1
    print('Start Dragon run ...')
    n = 1000
    p1 = 500
    p2 = 100
    X1, X2, Theta, _ = dragon.simulate_dragon_data(eta11=0.005, eta12=0.005, eta22=0.05,
                                            p1=100, p2=500, epsilon=[0.1,0.1],
                                            n=n, seed=123)
    os.system('curl -O https://netzoo.s3.us-east-2.amazonaws.com/netZooPy/unittest_datasets/dragonx1.npy')
    os.system('curl -O https://netzoo.s3.us-east-2.amazonaws.com/netZooPy/unittest_datasets/dragonx2.npy')
    X1=np.load('dragonx1.npy')
    X2=np.load('dragonx2.npy')
    lambdas, lambdas_landscape = dragon.estimate_penalty_parameters_dragon(X1, X2)
    lambdasSingle=tuple([int(10*x)/10 for x in lambdas]) # 3 digit precision
    alamb=lambdas_landscape[1,1]
    assert(lambdasSingle == (0.9, 0.9))
    assert((alamb < 398.7*1.002) & (alamb > 398.7*0.998)) #0.2% of error
    assert(int(X1[1,1]*1000)/1000 ==0.880)
    assert(int(X2[1,1]*1000)/1000 ==0.664)

    #2. test2
    r = dragon.get_partial_correlation_dragon(X1, X2, lambdas)
    adj_p_vals, p_vals = dragon.estimate_p_values_dragon(r, n, p1, p2, lambdas)
    p_valstest=int(p_vals[2,1]*100)/100
    adj_p_valstest=int(adj_p_vals[2,1]*10)/10 # 3 digit precision
    assert(int(np.max(r)*100)/100 == 0.03)
    assert(int(r[1,2]*100000)/100000 == 0.00012)
    assert(p_valstest == 0.96)
    assert(adj_p_valstest == 0.9)

    #3. test monte carlo p-values
    p1 = 3
    p2 = 4
    n = 10
    lam = [0,0] # no shrinkage
    test11 = np.array([[1,1/2.,-1/4.],
                   [1/2.,1,1/8.],
                   [1/4.,1/8.,1]])
    test12 = np.array([[-3/4.,1/2.,1/4.,0],
                  [-3/4.,1/2.,1/4.,0],
                  [-3/4.,1/2.,1/4.,0]])

    test21 = np.transpose(test12)

    test22 = np.array([[1,-3/4.,1/2.,-1/4.],
                  [-3/4.,1,1/8.,1/16.],
                  [1/2.,1/8.,1,1/32.],
                  [-1/4.,1/16.,1/32.,1]])
    test_mc_mat = np.identity(p1+p2)
    test_mc_mat[0:3,0:3] = test11
    test_mc_mat[0:3,3:7] = test12
    test_mc_mat[3:7,0:3] = test21
    test_mc_mat[3:7,3:7] = test22

    dragon_p_mc = dragon.dragon.estimate_p_values_mc(test_mc_mat,n,p1,p2,lam,seed=412)
    ground_truth_mc_p = np.array([[0,0,1/3.,0,1/12.,7/12.,1],
                                 [0,0,2/3.,0,1/12.,7/12.,1],
                                 [1/3.,2/3.,0,0,1/12,7/12.,1],
                                 [0,0,0,0,0,0,5/6.],
                                 [1/12.,1/12.,1/12.,0,0,5/6.,5/6.],
                                 [7/12.,7/12.,7/12.,0,5/6.,0,5/6.],
                                 [1,1,1,5/6.,5/6.,5/6.,0]])
    assert(np.array_equal(dragon_p_mc,ground_truth_mc_p))
    return()

def test_remove_zero_variance_preds():
    layer1 = np.array([[1,2,3],
                   [1,5,6],
                   [1,4,9],
                   [1,10,11]])
    layer2 = np.array([[1,2,3],
                   [2,5,6],
                   [3,4,9],
                   [4,10,11]])
    layer1_manual_complete = np.array([[2,3],
                   [5,6],
                   [4,9],
                   [10,11]])
    layer1_complete = dragon.dragon.remove_zero_variance_preds(layer1)
    layer2_complete = dragon.dragon.remove_zero_variance_preds(layer2)
    assert(np.array_equal(layer1_complete, layer1_manual_complete))
    assert(np.array_equal(layer2_complete, layer2))
    return()

def test_zero_variance_exception_estimate_penalty_parameters_dragon():
    layer1 = np.array([[1,2,3],
                   [1,5,6],
                   [1,4,9],
                   [1,10,11]])
    layer2 = np.array([[1,2,3],
                   [2,5,6],
                   [3,4,9],
                   [4,10,11]])
    with pytest.raises(Exception) as exc:
        dragon.dragon.estimate_penalty_parameters_dragon(X1 = layer1, X2 = layer2)

    assert(str(exc.value) == "[netZooPy.dragon.dragon.estimate_penalty_parameters_dragon] Found variables with zero variance. These must be removed before use of DRAGON. Consider use of `dragon.dragon.remove_zero_variance_preds`.")
    return()

def test_zero_variance_exception_get_shrunken_covariance_dragon():
    layer1 = np.array([[1,2,3],
                   [1,5,6],
                   [1,4,9],
                   [1,10,11]])
    layer2 = np.array([[1,2,3],
                   [2,5,6],
                   [3,4,9],
                   [4,10,11]])

    with pytest.raises(Exception) as exc:
        dragon.dragon.get_shrunken_covariance_dragon(X1 = layer1, X2 = layer2, lambdas = [0.5,0.5])
    assert(str(exc.value) == "[netZooPy.dragon.dragon.get_shrunken_covariance_dragon] Found variables with zero variance. These must be removed before use of DRAGON. Consider use of `dragon.dragon.remove_zero_variance_preds`.")
    return()

def test_singularity_exception():
    layer1 = np.array([[1,2,3],
                   [2,5,6]])
    layer2 = np.array([[1,2,3],
                   [2,5,6]])
    with pytest.raises(Exception) as exc:
        dragon.dragon.get_shrunken_covariance_dragon(X1 = layer1, X2 = layer2, lambdas=[0,0]) # no shrinkage
    assert(str(exc.value) == "[dragon.dragon.get_shrunken_covariance_dragon] Sigma is not invertible for the input values of lambda. Make sure that you are using `estimate_penalty_parameters_dragon` to select lambda. You may have variables with very small variance or highly collinear variables in your data. Consider removing such variables.")
    return()


import numpy as np
def Poisson1D( v, L ):
    # Solve 1-d Poisson equation:
    #     d^u / dx^2 = v   for  0 <= x <= L
    # using spectral method
    J = len(v)
    # Fourier transform source term
    v_tilde = np.fft.fft(v)
    # vector of wave numbers
    k = (2*np.pi/L)*np.concatenate((np.linspace(0,J/2-1,J/2),np.linspace(-J/2,-1,J/2)))
    k[0] = 1
    # Calculate Fourier transform of u
    u_tilde = np.divide(-v_tilde,np.power(k,2))
    # Inverse Fourier transform to obtain u
    u = np.real(np.fft.ifft(u_tilde))
    # Specify arbitrary constant by forcing corner u = 0;
    u = u - u[0]
    return u


from __future__ import absolute_import
from __future__ import print_function

import os
import sys
import optparse
import collections

if 'SUMO_HOME' in os.environ:
    tools = os.path.join(os.environ['SUMO_HOME'], 'tools')
    sys.path.append(tools)
    import sumolib  # noqa
else:
    sys.exit("please declare environment variable 'SUMO_HOME'")

def get_options(args=None):
    optParser = optparse.OptionParser()
    optParser.add_option("-n", "--net-file", dest="netfile",
                         help="define the net file (mandatory)")
    optParser.add_option("-o", "--output-file", dest="outfile",
                         default="tlsAdaptation.add.xml", help="define the output filename")
    optParser.add_option("-r", "--route-files", dest="routefiles",
                         help="define the route file seperated by comma(mandatory)")
    optParser.add_option("-b", "--begin", dest="begin", type="int",
                         default=0, help="begin time of the optimization period with unit second")
    optParser.add_option("-y", "--yellow-time", dest="yellowtime", type="int",
                         default=4, help="yellow time")
    optParser.add_option("-a", "--all-red", dest="allred", type="int",
                         default=0, help="all-red time")
    optParser.add_option("-l", "--lost-time", dest="losttime", type="int",
                         default=4, help="lost time for start-up and clearance in each phase")
    optParser.add_option("-g", "--min-green", dest="mingreen", type="int",
                         default=4, help=" minimal green time when there is no traffic volume")
    optParser.add_option("--green-filter-time", dest="greenFilter", type="int", default=0,
                         help="when computing critical flows, do not count phases with a green time below INT")
    optParser.add_option("-c", "--min-cycle", dest="mincycle", type="int",
                         default=20, help="minimal cycle length")
    optParser.add_option("-C", "--max-cycle", dest="maxcycle", type="int",
                         default=120, help="maximal cycle length")
    optParser.add_option("-e", "--existing-cycle", dest="existcycle", action="store_true",
                         default=False, help="use the existing cycle length")
    optParser.add_option("--write-critical-flows", dest="write_critical_flows", action="store_true",
                         default=False, help="print critical flows for each tls and phase")
    optParser.add_option("-p", "--program", dest="program", default="a",
                         help="save new definitions with this program id")
    optParser.add_option("-H", "--saturation-headway", dest="satheadway", type="float", default=2,
                         help="saturation headway in seconds for calculating hourly saturation flows")
    optParser.add_option("-R", "--restrict-cyclelength", dest="restrict", action="store_true",
                         default=False, help="restrict the max. cycle length as the given one")
    optParser.add_option("-u", "--unified-cycle", dest="unicycle", action="store_true", default=False,
                         help=" use the calculated max cycle length as the cycle length for all intersections")
    optParser.add_option("-v", "--verbose", dest="verbose", action="store_true",
                         default=False, help="tell me what you are doing")
    (options, args) = optParser.parse_args(args=args)

    if not options.netfile or not options.routefiles:
        optParser.print_help()
        sys.exit()

    return options

def getFlows(net, routeFiles, tlsList, begin, verbose):
    tlsFlowsMap = {}
    end = begin + 3600
    for tls in tlsList:
        tlsFlowsMap[tls._id] = collections.defaultdict(lambda: collections.defaultdict(int))
    for file in routeFiles.split(','):
        if verbose:
            print("route file:%s" % file)
        for veh in sumolib.output.parse(file, 'vehicle'):
            if float(veh.depart) >= end:
                break
            if float(veh.depart) >= begin:
                edgeList = veh.route[0].edges.split()
                for tls in tlsList:
                    # c: [[inLane, outLane, linkNo],[],..]
                    for c in tls.getConnections():
                        inEdge = c[0].getEdge().getID()
                        outEdge = c[1].getEdge().getID()
                        if inEdge in edgeList:
                            beginIndex = edgeList.index(inEdge)
                            if beginIndex < len(edgeList) - 1 and edgeList[beginIndex + 1] == outEdge:
                                pce = 1.
                                if veh.type == "bicycle":
                                    pce = 0.2
                                elif veh.type in ["moped", "motorcycle"]:
                                    pce = 0.5
                                elif veh.type in ["truck", "trailer", "bus", "coach"]:
                                    pce = 3.5
                                tlsFlowsMap[tls._id][inEdge + " " + outEdge][c[2]] += pce

    # remove the doubled counts
    connFlowsMap = {}
    for t in tlsList:
        connFlowsMap[t.getID()] = {}
        for subRoute in tlsFlowsMap[t.getID()]:
            totalConns = len(tlsFlowsMap[t.getID()][subRoute])
            for conn in tlsFlowsMap[t.getID()][subRoute]:
                tlsFlowsMap[t.getID()][subRoute][conn] /= totalConns
                connFlowsMap[t.getID()][conn] = tlsFlowsMap[t.getID()][subRoute][conn]

        # remove the redundant connection flows
        connFlowsMap = removeRedundantFlows(t, connFlowsMap)

    return connFlowsMap

def getEffectiveTlsList(tlsList, connFlowsMap, verbose):
    effectiveTlsList = []
    for tl in tlsList:
        if len(tl.getPrograms()) == 0:
            continue
        valid = True
        for program in tl.getPrograms().values():
            for phase in program.getPhases():
                if len(phase.state) > len(tl.getConnections()):
                    print("Skipping TLS '%s' due to unused states (%s states, %s connections)" % (
                        tl.getID(), len(phase.state), len(tl.getConnections())))
                    valid = False
                    break
        if valid:
            for conn in connFlowsMap[tl.getID()]:
                if connFlowsMap[tl.getID()][conn] > 0:
                    effectiveTlsList.append(tl)
                    break
    return effectiveTlsList

def removeRedundantFlows(t, connFlowsMap):
    # if two or more intersections share the lane-lane connection indices together,
    # the redundant connection flows will set to zero.
    connsList = t.getConnections()
    connsList = sorted(connsList, key=lambda connsList: connsList[2])
    redundantConnsList = []
    identical = True
    for c1 in connsList:
        for c2 in connsList:
            if c1[2] != c2[2]:
                if c1[1]._edge == c2[0]._edge:
                    identical = identityCheck(c1[0]._edge, c2[0]._edge._incoming, identical)
                    if identical:
                        for toEdge in c2[0]._edge._outgoing:
                            for c in c2[0]._edge._outgoing[toEdge]:
                                if c._tlLink not in redundantConnsList:
                                    redundantConnsList.append(c._tlLink)
                    else:
                        for conn_1 in c1[0]._edge._outgoing[c2[0]._edge]:
                            if conn_1._direction == 's':
                                for toEdge in c2[0]._edge._outgoing:
                                    for conn_2 in c2[0]._edge._outgoing[toEdge]:
                                        if conn_2._tlLink not in redundantConnsList:
                                            redundantConnsList.append(conn_2._tlLink)
    for conn in redundantConnsList:
        if conn in connFlowsMap[t._id]:
            connFlowsMap[t._id][conn] = 0.
    return connFlowsMap

def identityCheck(e1, incomingLinks, identical):
    for i in incomingLinks:
        if i != e1:
            identical = False
            break
    return identical

def getLaneGroupFlows(tl, connFlowsMap, phases, greenFilter):
    connsList = tl.getConnections()
    groupFlowsMap = {}  # i(phase): duration, laneGroup1, laneGroup2, ...
    connsList = sorted(connsList, key=lambda connsList: connsList[2])

    # check if there are shared lane groups, i.e. some lane groups have only "g" (no "G")
    ownGreenConnsList = []
    for i, p in enumerate(phases):
        for j, control in enumerate(p.state):
            if control == "G" and j not in ownGreenConnsList:
                ownGreenConnsList.append(j)
    yellowRedTime = 0
    greenTime = 0
    currentLength = 0
    phaseLaneIndexMap = collections.defaultdict(list)
    for i, p in enumerate(phases):
        currentLength += p.duration
        if 'G' in p.state and 'y' not in p.state and p.duration >= greenFilter:
            greenTime += p.duration
            groupFlowsMap[i] = [p.duration]
            groupFlows = 0
            laneIndexList = []
            for j, control in enumerate(p.state):
                inEdge = connsList[j][0]._edge._id
                if j == 0:
                    exEdge = inEdge
                if (inEdge == exEdge and control == 'G') or (inEdge == exEdge and
                                                             control == 'g' and j not in ownGreenConnsList):
                    if j in connFlowsMap[tl._id]:
                        groupFlows += connFlowsMap[tl._id][j]
                    if connsList[j][0].getIndex() not in laneIndexList:
                        laneIndexList.append(connsList[j][0].getIndex())

                if exEdge != inEdge or j == len(p.state) - 1:
                    if laneIndexList:
                        phaseLaneIndexMap[i].append(laneIndexList)
                        groupFlowsMap[i].append(groupFlows)

                    laneIndexList = []
                    groupFlows = 0
                    if control == "G":
                        if j in connFlowsMap[tl._id]:
                            groupFlows = connFlowsMap[tl._id][j]
                            if connsList[j][0].getIndex() not in laneIndexList:
                                laneIndexList.append(connsList[j][0].getIndex())
                exEdge = inEdge
        elif 'G' not in p.state and 'g' in p.state and 'y' not in p.state and 'r' not in p.state:
            print("Check: only g for all connections:%s in phase %s" % (tl._id, i))
        elif ('G' not in p.state and 'g' not in p.state) or ('G' not in p.state and 'y' in p.state and 'r' in p.state):
            yellowRedTime += int(p.duration)
        if options.verbose and i in groupFlowsMap:
            print("phase: %s" % i)
            print("group flows: %s" % groupFlowsMap[i])
            print("The used lanes: %s" % phaseLaneIndexMap[i])
    if options.verbose:
        print("the current cycle length:%s sec" % currentLength)
    return groupFlowsMap, phaseLaneIndexMap, currentLength

def getMaxOptimizedCycle(groupFlowsMap, phaseLaneIndexMap, currentLength, cycleList, options):
    lostTime = len(groupFlowsMap) * options.losttime + options.allred
    satFlows = 3600. / options.satheadway
    # calculate the critical flow ratios and the respective sum
    criticalFlowRateMap = {}
    for i in groupFlowsMap:   # [duration. groupFlow1, groupFlow2...]
        criticalFlowRateMap[i] = 0.
        maxFlow = 0
        index = None
        if len(groupFlowsMap[i][1:]) > 0:
            for j, f in enumerate(groupFlowsMap[i][1:]):
                if f >= maxFlow:
                    maxFlow = f
                    index = j
            criticalFlowRateMap[i] = (maxFlow / float((len(phaseLaneIndexMap[i][index])))) / satFlows
        else:
            criticalFlowRateMap[i] = 0.
    sumCriticalFlows = sum(criticalFlowRateMap.values())

    if options.existcycle:
        optCycle = currentLength
    elif sumCriticalFlows >= 1.:
        optCycle = options.maxcycle
        if options.verbose:
            print("Warning: the sum of the critical flows >= 1:%s" % sumCriticalFlows)
    else:
        optCycle = int(round((1.5 * lostTime + 5.) / (1. - sumCriticalFlows)))

    if not options.existcycle and optCycle < options.mincycle:
        optCycle = options.mincycle
    elif not options.existcycle and optCycle > options.maxcycle:
        optCycle = options.maxcycle

    cycleList.append(optCycle)

    return cycleList

def optimizeGreenTime(tl, groupFlowsMap, phaseLaneIndexMap, currentLength, options):
    lostTime = len(groupFlowsMap) * options.losttime + options.allred
    satFlows = 3600. / options.satheadway
    # calculate the critical flow ratios and the respective sum
    criticalFlowRateMap = {}
    for i in groupFlowsMap:   # [duration. groupFlow1, groupFlow2...]
        criticalFlowRateMap[i] = 0.
        maxFlow = 0
        index = None
        if len(groupFlowsMap[i][1:]) > 0:
            for j, f in enumerate(groupFlowsMap[i][1:]):
                if f >= maxFlow:
                    maxFlow = f
                    index = j
            criticalFlowRateMap[i] = (maxFlow / float((len(phaseLaneIndexMap[i][index])))) / satFlows
        else:
            criticalFlowRateMap[i] = 0.
    sumCriticalFlows = sum(criticalFlowRateMap.values())
    if options.write_critical_flows:
        print(tl.getID(), criticalFlowRateMap)

    if options.existcycle:
        optCycle = currentLength
    elif sumCriticalFlows >= 1.:
        optCycle = options.maxcycle
        if options.verbose:
            print("Warning: the sum of the critical flows >= 1:%s" % sumCriticalFlows)
    else:
        optCycle = int(round((1.5 * lostTime + 5.) / (1. - sumCriticalFlows)))

    if not options.existcycle and optCycle < options.mincycle:
        optCycle = options.mincycle
    elif not options.existcycle and optCycle > options.maxcycle:
        optCycle = options.maxcycle

    # calculate the green time for each critical group
    effGreenTime = optCycle - lostTime
    totalLength = lostTime
    minGreenPhasesList = []
    adjustGreenTimes = 0
    totalGreenTimes = 0
    subtotalGreenTimes = 0
    for i in criticalFlowRateMap:
        groupFlowsMap[i][0] = effGreenTime * \
            (criticalFlowRateMap[i] / sum(criticalFlowRateMap.values())) - options.yellowtime + options.losttime
        groupFlowsMap[i][0] = int(round(groupFlowsMap[i][0]))
        totalGreenTimes += groupFlowsMap[i][0]
        if groupFlowsMap[i][0] < options.mingreen:
            groupFlowsMap[i][0] = options.mingreen
            minGreenPhasesList.append(i)
        else:
            subtotalGreenTimes += groupFlowsMap[i][0]
        totalLength += groupFlowsMap[i][0]

    # adjust the green times if minimal green times are applied for keeping the defined maximal cycle length.
    if minGreenPhasesList and totalLength > options.maxcycle and options.restrict:
        if options.verbose:
            print("Re-allocate the green splits!")
        adjustGreenTimes = totalGreenTimes - len(minGreenPhasesList) * options.mingreen
        for i in groupFlowsMap:
            if i not in minGreenPhasesList:
                groupFlowsMap[i][0] = int((groupFlowsMap[i][0] / float(subtotalGreenTimes)) * adjustGreenTimes)

    if options.verbose:
        totalLength = lostTime
        for i in groupFlowsMap:
            totalLength += groupFlowsMap[i][0]
            print("Green time for phase %s: %s" % (i, groupFlowsMap[i][0]))
        print("the optimal cycle length:%s" % totalLength)

    return groupFlowsMap

def main(options):
    net = sumolib.net.readNet(options.netfile, withPrograms=True, withPedestrianConnections=True)
    tlsList = net.getTrafficLights()

    if options.verbose:
        print("the total number of tls: %s" % len(tlsList))
    print("Begin time:%s" % options.begin)
    # get traffic flows for each connection at each TL
    connFlowsMap = getFlows(net, options.routefiles, tlsList, options.begin, options.verbose)

    # remove the tls where no traffic volumes exist
    effectiveTlsList = getEffectiveTlsList(tlsList, connFlowsMap, options.verbose)

    with open(options.outfile, 'w') as outf:
        outf.write('<?xml version="1.0" encoding="UTF-8"?>\n')
        outf.write('<additional>\n')

        if len(effectiveTlsList) > 0:
            if options.unicycle:
                cycleList = []
                if options.verbose:
                    print("Firstly only calculate the maximal optimized cycle length! ")
                for tl in effectiveTlsList:
                    if options.verbose:
                        print("tl-logic ID: %s" % tl._id)
                    programs = tl.getPrograms()
                    for pro in programs:
                        phases = programs[pro].getPhases()

                        # get the connection flows and group flows
                        groupFlowsMap, phaseLaneIndexMap, currentLength = getLaneGroupFlows(tl, connFlowsMap, phases, 0)

                        # only optimize the cycle length
                        cycleList = getMaxOptimizedCycle(groupFlowsMap, phaseLaneIndexMap,
                                                         currentLength, cycleList, options)

                options.maxcycle = max(cycleList)
                options.mincycle = max(cycleList)
                options.restrict = True
                if options.verbose:
                    print("The maximal optimized cycle length is %s." % max(cycleList))
                    print(" It will be used for calculating the green splits for all intersections.")

            # calculate the green splits; the optimal length will be also calculate if options.unicycle is set as false.
            for tl in effectiveTlsList:
                if options.verbose:
                    print("tl-logic ID: %s" % tl._id)
                programs = tl.getPrograms()
                for pro in programs:
                    phases = programs[pro].getPhases()

                    # get the connection flows and group flows
                    groupFlowsMap, phaseLaneIndexMap, currentLength = getLaneGroupFlows(
                        tl, connFlowsMap, phases, options.greenFilter)

                    # optimize the cycle length and calculate the respective green splits
                    groupFlowsMap = optimizeGreenTime(tl, groupFlowsMap, phaseLaneIndexMap, currentLength, options)

                # write output
                outf.write('    <tlLogic id="%s" type="%s" programID="%s" offset="%i">\n' %
                           (tl._id, programs[pro]._type, options.program, programs[pro]._offset))

                phases = programs[pro].getPhases()
                for i, p in enumerate(phases):
                    duration = p.duration
                    if i in groupFlowsMap:
                        duration = groupFlowsMap[i][0]
                    outf.write('        <phase duration="%s" state="%s"/>\n' % (duration, p.state))
                outf.write('    </tlLogic>\n')
        else:
            print("There are no flows at the given intersections. No green time optimization is done.")
        outf.write('</additional>\n')

if __name__ == "__main__":
    options = get_options(sys.argv)
    main(options)