from __future__ import absolute_import
from builtins import object
from builtins import range
from builtins import object

import json
import logging
import lxml.etree as et
import re
import sys
from gettext import gettext as _

import splunk.util
from splunk.models.view_escaping import forminput
from splunk.models.view_escaping.base import POST_SEARCH_MODE, SAVED_SEARCH_MODE, TEMPLATE_SEARCH_MODE
from splunk.models.view_escaping.base import DEFAULT_SEARCH_ID
from splunk.models.view_escaping.dashboard import SimpleDashboard
from splunk.models.view_escaping.drilldown import parseDrilldown
from splunk.models.view_escaping.drilldown import parseDrilldownAction
from splunk.models.view_escaping.panel import Panel
# I am leaving the * because refactoring may have un foreseen issues. the code uses `globals().items()` :(
from splunk.models.view_escaping.panelElement import *
from splunk.models.view_escaping.panelElement import normalizeBoolean, Event, BasePanel, Chart
from splunk.models.view_escaping.row import Row
from splunk.models.view_escaping.search import Search, createSearchFromSearchXml
from splunk.models.view_escaping.tokendeps import parseTokenDeps
from splunk.models.view_escaping.validation_helper import WarningsCollector

logger = logging.getLogger('splunk.models.view_escaping')

ID_PATTERN = re.compile(r'^[a-z]\w*$', re.IGNORECASE)

def getValidationMessages(dashboardNode, viewName=None, digest=None, sourceApp=None):
    logger.info('Validating dashboard XML for view=%s', viewName)
    collector = WarningsCollector(logger=logger)
    try:
        with collector:
            createDashboardFromXml(dashboardNode, viewName, digest, sourceApp)
    except:
        pass
    return collector.getMessages()

def createDashboardFromXml(dashboardNode, viewName=None, digest=None, sourceApp=None):
    """
    @param dashboardNode: lxml node representing a dashboard or form
    @param viewName: the name of the xml file
    @param digest: boolean
    @return:
    """

    logger.debug("Parsing view: simplexml=%s" % viewName)
    dashboard = SimpleDashboard(viewName=viewName, digest=digest, sourceApp=sourceApp)

    # Parse global options
    dashboard.label = dashboardNode.findtext('./label')
    dashboard.description = dashboardNode.findtext('./description')

    dashboard.customScript = dashboardNode.get('script', None)
    dashboard.customStylesheet = dashboardNode.get('stylesheet', None)
    dashboard.onUnloadCancelJobs = normalizeBoolean(
        dashboardNode.get('onunloadCancelJobs', dashboard.onUnloadCancelJobs))

    dashboard.matchTagName = dashboardNode.tag
    if dashboard.matchTagName not in ['form', 'dashboard']:
        raise Exception(_('Invalid root element'))

    if dashboardNode.tag == 'form':
        # update the fieldset
        fieldsetNode = dashboardNode.find('./fieldset')
        if fieldsetNode is not None:
            dashboard.autoRun = splunk.util.normalizeBoolean(
                fieldsetNode.get('autoRun', False))
            dashboard.submitButton = splunk.util.normalizeBoolean(
                fieldsetNode.get('submitButton', True))

            for item in fieldsetNode:
                logger.debug("Found new element in fieldset: form=%s item=%s" % (viewName, item.tag))
                if item.tag == 'html':
                    panelInstance = createPanelElementFromXml(item)
                    dashboard.fieldset.append(panelInstance)
                elif item.tag == 'input':
                    inputDefaults = dict()
                    if not dashboard.submitButton:
                        inputDefaults['searchWhenChanged'] = True
                    inputInstance = forminput.createInput(item.get('type'), inputDefaults)
                    inputInstance.fromXml(item, sourceApp)

                    dashboard.fieldset.append(inputInstance)

    # set core view attributes
    for k in dashboard.standardAttributeMap:
        v = dashboardNode.get(k)
        if v is not None:
            if k in dashboard.booleanAttributeKeys:
                v = splunk.util.normalizeBoolean(v)
            elif k in dashboard.integerAttributeKeys:
                try:
                    v = int(v)
                except:
                    msg = "Dashboard attribute %s should have an integer value. Value found: %s" % (k, v)
                    logger.error(msg)
                    raise Exception(msg)
            setattr(dashboard, k, v)

    # Legacy searchTemplate extraction
    search = createSearchFromXml(dashboardNode, id=DEFAULT_SEARCH_ID)
    if search:
        dashboard.searches.append(search)
    for searchNode in dashboardNode.findall('search'):
        search = createSearchFromSearchXml(searchNode)
        if search:
            logger.debug("Appending new search to view=%s" % viewName)
            dashboard.searches.append(search)

    for rowNode in dashboardNode.findall('row'):
        logger.debug("Appending new row to view=%s" % viewName)
        dashboard.rows.append(createRowFromXml(rowNode, sourceApp))

    normalizeIdentifiers(dashboard)

    def prepareEventSearch(searchObj):
        if searchObj is not None:
            if searchObj.searchMode == POST_SEARCH_MODE:
                prepareEventSearch(dashboard.get_search(searchObj.baseSearchId))
            else:
                searchObj.statusBuckets = 300

    for row in dashboard.rows:
        for panel in row.panels:
            for panelElement in panel.panelElements:
                if isinstance(panelElement, Event):
                    prepareEventSearch(panelElement.search)

    return dashboard

def verify_id(id, seen, type):
    if id is not None:
        if id in seen:
            raise Exception(_('Duplicate ID "%s" of %s' % (id, type)))
        if not ID_PATTERN.match(id):
            raise Exception(_(
                'ID "%s" for %s does not match pattern %s ' +
                '(ID has to begin with a letter and must not '
                'container characters other than letters, numbers and "_")') % (id, type, ID_PATTERN.pattern))
        seen.add(id)

class IdSeq(object):
    def __init__(self):
        self.val = 1

    def __next__(self):
        while True:
            yield self.val
            self.val += 1

def auto_id(pattern, seen, seq):
    id = None
    for seq_no in next(seq):
        id = pattern % seq_no
        if id not in seen:
            break
    seen.add(id)
    return id

def normalizeIdentifiers(dashboard):
    """
    Ensure all dashboard elements have an unique id

    @param dashboard: Dashboard object
    @return: no return value
    """
    seen = set()
    for row in dashboard.rows:
        verify_id(row.id, seen, "row")
    for panel in dashboard.all_panels():
        verify_id(panel.id, seen, "panel")
    for el in dashboard.all_elements():
        verify_id(el.id, seen, "panel element")
    for el in dashboard.all_fields():
        verify_id(el.id, seen, "input field")
    for (search, obj) in dashboard.all_searches():
        verify_id(search.id, seen, "search")

    globalSearch = None

    search_seq = IdSeq()
    row_seq = IdSeq()
    panel_seq = IdSeq()
    element_seq = IdSeq()
    input_seq = IdSeq()

    for (search, obj) in dashboard.all_searches():
        if search.id is None:
            search.id = globalSearch = auto_id("search%d", seen, search_seq)

    for row in dashboard.rows:
        if row.id is None:
            row.id = auto_id("row%d", seen, row_seq)
            row.idGenerated = True
    for panel in dashboard.all_panels():
        if panel.id is None:
            panel.id = auto_id("panel%d", seen, panel_seq)
            panel.idGenerated = True
    for el in dashboard.all_elements():
        if el.id is None:
            el.id = auto_id("element%d", seen, element_seq)
        if el.context is None:
            if getattr(el, 'searchMode', None):
                el.context = auto_id("search%d", seen, search_seq)
            elif globalSearch is not None:
                el.context = globalSearch
    for field in dashboard.all_fields():
        if field.id is None:
            field.id = auto_id("input%d", seen, input_seq)
        if getattr(field, 'context', None) is None:
            if getattr(field, 'search', None) or getattr(field, 'savedSearch', None):
                field.context = auto_id("search%d", seen, search_seq)

def createSearchFromXml(searchNode, id=None):
    """
    Parses a search from search, dashboard, panel element xml nodes
    @param searchNode: Lxml representing a form or dashboard element
    @param id: and optional id to force id to
    @return:
    """

    # define search mode XML node -> object property mappings
    searchModeNodeMap = [
        ('searchString', TEMPLATE_SEARCH_MODE),
        ('searchName', SAVED_SEARCH_MODE),
        ('searchTemplate', TEMPLATE_SEARCH_MODE),
        ('searchPostProcess', POST_SEARCH_MODE)
    ]

    opt = dict()
    for pair in searchModeNodeMap:
        if searchNode.find(pair[0]) is not None:
            opt['searchMode'] = pair[1]
            opt['searchCommand'] = (
                searchNode.findtext(pair[0])).replace("\n", " ").replace("\t", " ")
            break
    else:
        return False
    for node in ['earliestTime', 'latestTime']:
        nodeVal = searchNode.findtext(node)
        if nodeVal:
            opt[node] = nodeVal
    tokenDeps = parseTokenDeps(searchNode)
    if id:
        opt['id'] = id
    if tokenDeps:
        opt['tokenDeps'] = tokenDeps
    return Search(**opt)

def createRowFromXml(rowNode, sourceApp=None):
    """
    Parses a row xml node
    @param rowNode: Lxml representing a form or dashboard element
    @return:
    """
    logger.debug("parsing dashboard row node")
    if rowNode.get('grouping'):
        rowGroupings = list(map(
            int,
            rowNode.get('grouping').replace(' ', '').strip(',').split(',')))
        logger.debug("Found row grouping=%s" % rowGroupings)
    else:
        rowGroupings = None

    row = Row(rowGroupings)
    row.tokenDeps = parseTokenDeps(rowNode)
    row.id = rowNode.get('id')

    if len(rowNode) is 0:
        logger.warn('Dashboard row is empty (line %d)', rowNode.sourceline)

    else:
        hasPanels = False
        hasVisualizations = False
        for panelElementNode in rowNode:
            if panelElementNode.tag == "panel":
                if hasVisualizations:
                    raise Exception(_('Row, on line=%s, should not combine visualizations and panels. Panel, on line=%s') % (rowNode.sourceline, panelElementNode.sourceline))
                hasPanels = True
                if rowGroupings is not None:
                    raise Exception(_('Row on line=%s specifies row grouping but has <panel> children, which is not allowed') % rowNode.sourceline)
                row.panels.append(createPanelFromXML(panelElementNode, sourceApp))
            elif panelElementNode.tag == et.Comment:
                continue
            else:
                if hasPanels:
                    raise Exception(_('Row, on line=%s, should not combine visualizations and panels. Visualization, on line=%s') % (rowNode.sourceline, panelElementNode.sourceline))
                hasVisualizations = True
                try:
                    panelElement = createPanelElementFromXml(panelElementNode)
                    if panelElement:
                        row.appendPanelElement(panelElement)
                except NotImplementedError:
                    raise Exception(_('Row, on line=%s, contains unknown node=%s on line=%s.') % (rowNode.sourceline, panelElementNode.tag, panelElementNode.sourceline))
    return row

def createPanelFromXML(panelNode, sourceApp=None):
    panel = Panel(None)
    panel.tokenDeps = parseTokenDeps(panelNode)
    panel.id = panelNode.get('id', None)

    if len(panelNode) is 0:
        logger.warn('Dashboard panel is empty (line %d)', panelNode.sourceline)

    ref = panelNode.get('ref', None)
    if ref:
        panel.ref = ref
        panel.app = panelNode.get('app', sourceApp)
    else:
        panel.title = panelNode.findtext('title')
        for panelElementNode in panelNode:
            if panelElementNode.tag == 'input':
                inputInstance = forminput.createInput(
                    panelElementNode.get('type'), dict(searchWhenChanged=True))
                inputInstance.fromXml(panelElementNode, sourceApp)
                panel.fieldset.append(inputInstance)
            elif panelElementNode.tag == 'search':
                search = createSearchFromSearchXml(panelElementNode)
                if search:
                    panel.searches.append(search)
            elif panelElementNode.tag not in ('title', 'description'):
                panelElement = createPanelElementFromXml(panelElementNode)
                if panelElement:
                    panel.appendPanelElement(panelElement)
    return panel

def extractHtmlContent(node):
    if normalizeBoolean(node.attrib.get("encoded", False)):
        return node.text
    else:
        if len(node) == 0 and node.text is None:
            return ''
        else:
            parser = et.XMLParser(resolve_entities=False)
            clone = et.XML(et.tostring(node).strip(), parser)
            # Strip attributes on outer <html> node, so we can safely strip it once serialized
            clone.attrib.clear()
            html = et.tostring(clone)
            if sys.version_info > (3, 0) and isinstance(html, bytes):
                html = html.decode()
            if html.startswith('<html>') and html.endswith('</html>'):
                return html[6:-7].strip()
            else:
                return html

def _isPrimarySearchNode(node):
    type = node.get('type')
    return type == 'primary' or type is None

def _findPrimarySearchNode(panelElementNode):
    searchNodes = panelElementNode.findall('search')
    return next((sn for sn in searchNodes if _isPrimarySearchNode(sn)), None)

def createPanelElementFromXml(panelElementNode):
    def createHtmlPanelElementFromXml(panelElementNode, panelInstance):
        """
        Override default parser and just get the text
        """
        logger.debug("parsing html panel element")
        panelInstance.options['useTokens'] = normalizeBoolean(panelElementNode.attrib.get("tokens", True))

        src = panelElementNode.get('src')
        if src:
            panelInstance.options['serverSideInclude'] = src
            return

        panelInstance.options['rawcontent'] = extractHtmlContent(panelElementNode)
        panelInstance.tokenDeps = parseTokenDeps(panelElementNode)

    def createChartPanelElementFromXml(node, element):
        createDefaultPanelElementFromXml(node, element)
        selectionNode = node.find('selection')
        if selectionNode:
            element.selection = []
            for actionNode in [node for node in selectionNode if et.iselement(node) and isinstance(node.tag, str)]:
                action = parseDrilldownAction(actionNode)
                if action:
                    element.selection.append(action)

    def createTablePanelElementFromXml(panelElementNode, panelInstance):
        """
        Add a format tag to provide cell formatting
        (primarily for sparklines at the current time)
        Each format tag may have some option sub-tags
        Each option may contain list or option tags generating
        lists or dictionaries respectively

        Each format tag may specify a field to filter on
        (defaults to all fields) and a type to apply (eg 'sparkline')
        """
        logger.debug("parsing table panel element")

        def parseOption(node):
            """Extract nested options/lists"""
            listNodes = node.findall('list')
            optionNodes = node.findall('option')

            if listNodes and optionNodes:
                raise ValueError("Tag cannot contain both list and option subtags")
            elif listNodes:
                result = []
                for listNode in listNodes:
                    result.append(parseOption(listNode))
                return result
            elif optionNodes:
                result = {}
                for optionNode in optionNodes:
                    result[optionNode.get('name')] = parseOption(optionNode)
                return result
            else:
                return node.text

        createDefaultPanelElementFromXml(panelElementNode, panelInstance)
        fieldFormats = {}
        for formatNode in panelElementNode.findall('format'):
            logger.debug("Parsing format view node")
            field = formatNode.get('field', '*')
            formatType = formatNode.get('type', 'text')
            options = {}
            for optionNode in formatNode.findall('option'):
                options[optionNode.get('name')] = parseOption(optionNode)
            fieldFormats.setdefault(field, []).append({
                'type': formatType,
                'options': options
            })
        panelInstance.fieldFormats = fieldFormats

    def createVizPanelElementFromXml(panelElementNode, panelInstance):
        createDefaultPanelElementFromXml(panelElementNode, panelInstance)
        panelInstance.type = panelElementNode.get('type', None)

    def createDefaultPanelElementFromXml(panelElementNode, panelInstance):
        logger.debug("Parsing default panel element options")
        title = panelElementNode.findtext('title')
        if title is not None:
            setattr(panelInstance, 'title', title)

        optionTypeMap = {}
        if hasattr(panelInstance.__class__, 'optionTypeMap'):
            optionTypeMap = getattr(panelInstance.__class__, 'optionTypeMap')

        # option params get their own container
        for node in panelElementNode.findall('option'):
            optionName = node.get('name')
            optionValue = node.text
            if optionName in optionTypeMap:
                optionValue = optionTypeMap[optionName](optionValue)
            if isinstance(optionValue, str) and optionValue != None:
                optionValue = optionValue.strip()
            if optionName not in ['id', 'el', 'managerid', 'tokenDependencies', 'resizable']:
                panelInstance.options[optionName] = optionValue

        # handle different search modes
        if getattr(panelInstance.__class__, 'hasSearch'):
            # SPL-143552: make sure the primary search is found. Other types of search nodes will be ignored for now.
            primarySearchNode = _findPrimarySearchNode(panelElementNode)
            search = createSearchFromSearchXml(primarySearchNode) if primarySearchNode is not None else None
            if search:
                panelInstance.search = search
            else:
                search = createSearchFromXml(panelElementNode)
                if search:
                    panelInstance.search = search
                else:
                    # create a default search if there is not one
                    panelInstance.search = Search(searchMode=POST_SEARCH_MODE)

        # handle field lists
        if panelElementNode.find('fields') is not None:
            fields = panelElementNode.findtext('fields').strip()
            if len(fields) and fields[0] == '[' and fields[-1] == ']':
                panelInstance.searchFieldList = json.loads(fields)
            else:
                panelInstance.searchFieldList = splunk.util.stringToFieldList(fields)

        # extract simple XML drilldown params
        panelInstance.simpleDrilldown = parseDrilldown(panelElementNode.find('drilldown'))

        panelInstance.tokenDeps = parseTokenDeps(panelElementNode)

        # extract the contents of all top-level comment nodes
        for node in panelElementNode.xpath('./comment()'):
            panelInstance.comments.append(node.text)

        # extract the comments from inside the drilldown tag
        for node in panelElementNode.xpath('./drilldown/comment()'):
            panelInstance.drilldownComments.append(node.text)

    def createPanel(name):
        """
        Factory method for creating an appropriate panel object based upon the
        name.  Returns an instance of a BasePanel subclass, or throws a
        NotImplementedError if no suitable mapper is found.

        This method works by inspecting all objects that subclass BasePanel and
        attempting to match their matchTagName class attribute.
        """

        if not name:
            raise ValueError('Cannot create panel from nothing')

        for obj in globals().values():
            try:
                if issubclass(obj, BasePanel) and name == obj.matchTagName:
                    #  only Chart objects need to be instantiated
                    if obj is Chart:
                        return Chart()
                    else:
                        return obj()
            except:
                pass
        raise NotImplementedError(
            _('Cannot find object mapper for panel type: %s') % name)

    if not isinstance(panelElementNode.tag, splunk.util.string_type):
        return False
    panelType = panelElementNode.tag
    panelInstance = createPanel(panelType)

    id = panelElementNode.attrib.get('id')
    if id is not None:
        panelInstance.id = id

    logger.debug("found panel element type=%s" % panelType)
    if panelType == 'table':
        createTablePanelElementFromXml(panelElementNode, panelInstance)
    elif panelType == 'chart':
        createChartPanelElementFromXml(panelElementNode, panelInstance)
    elif panelType == 'html':
        createHtmlPanelElementFromXml(
            panelElementNode, panelInstance)
    elif panelType == 'viz':
        createVizPanelElementFromXml(panelElementNode, panelInstance)
    else:
        createDefaultPanelElementFromXml(panelElementNode, panelInstance)
    return panelInstance

if __name__ == '__main__':
    import unittest

    def getRowXml(args='', panels=1):
        nodes = ['<row %(args)s>']
        for i in range(0, panels):
            nodes.append('<single></single>')
        nodes.append('</row>')
        xml = ''.join(nodes)
        return xml % {'args': args}

    def getPanelElementXml(type="foo", options=None, args=None):
        options = options or (
            '<searchString> | metadata type="sources" | '
            'stats count</searchString>')
        args = args or ''
        xml = '''
            <%(type)s %(args)s>
                %(options)s
            </%(type)s>
        '''
        return xml % {'type': type, 'options': options, 'args': args}

    class CreatePanelElementTests(unittest.TestCase):

        def createPanel(self, type="foo", options=None, args=None):
            xml = getPanelElementXml(type, options, args)
            root = et.fromstring(xml)
            return createPanelElementFromXml(root)

        def testCreateUnknownPanel(self):
            with self.assertRaises(NotImplementedError):
                d = self.createPanel('foo')

        def testCreateAllowedPanels(self):
            for panelType in ['single', 'chart', 'table',
                              'html', 'map', 'event', 'list', 'viz']:
                d = self.createPanel(panelType)
                self.assertTrue(d.matchTagName == panelType)

        def testCreateHTMLPanel(self):
            d = self.createPanel(
                'html', args='src="/foo/bar"')
            self.assertTrue(d.options.get('serverSideInclude') == '/foo/bar')
            self.assertFalse(d.options.get('rawcontent'))
            d = self.createPanel(
                'html', options='<div>Test</div>', args='src="/foo/bar"')
            self.assertTrue(d.options.get('serverSideInclude') == '/foo/bar')
            self.assertFalse(d.options.get('rawcontent'))
            d = self.createPanel(
                'html', options='<div>Test</div>')
            self.assertFalse(d.options.get('serverSideInclude'))
            self.assertEqual(d.options.get('rawcontent'), '<div>Test</div>')

        def testCreateTablePanel(self):
            """tables only have special format fields"""
            d = self.createPanel('table', options='''
                <format>
                    <option name="fff">
                        <list>foo</list>
                        <list>bop</list>
                    </option>
                    <option name="bippity">
                        <option name="bippity">bop</option>
                    </option>
                </format>
                <format field="foobar" type="num">
                    <option name="fff">
                        <list>foo</list>
                        <list>bop</list>
                    </option>
                    <option>bar</option>
                </format>
                ''')
            self.assertTrue(d.fieldFormats == { # pylint: disable=E1103
                '*': [{'type': 'text',
                       'options': {'bippity': {'bippity': 'bop'},
                                   'fff': ['foo', 'bop']
                       }
                      }],
                'foobar': [{'type': 'num',
                            'options': {None: 'bar',
                                        'fff': ['foo', 'bop']
                            }
                           }]
            })
            #  formats can't mix options and lists.
            #  lists can contain options and options can contain list but
            #     neither can contain both.
            with self.assertRaises(ValueError):
                d = self.createPanel(
                    'table',
                    options='''
                        <format>
                            <option name="fff">
                                <list>foo</list>
                                <list>bop</list>
                                <option name="should">not work</option>
                            </option>
                            <option>bar</option>
                        </format>
                    ''')

        def testCreateChartPanelWithSelection(self):
            chart = self.createPanel('chart', options='''
                <selection>
                    <set token="foo">$start$</set>
                    <unset token="bar" />
                </selection>
                ''')

            self.assertIsNotNone(chart.selection)
            self.assertEqual(2, len(chart.selection))
            self.assertEqual("settoken", chart.selection[0].type)
            self.assertEqual("unsettoken", chart.selection[1].type)

        def testCreateCustomVizPanel(self):
            customViz = self.createPanel('viz', args='type="testapp.testviz"')
            self.assertTrue(customViz.type == 'testapp.testviz')

        def testCreateDefaultPanel(self):
            """All panels have some things in common"""
            #s test common nodes
            d = self.createPanel('single', options='''
                <title>Title1</title>
                <searchString>search string</searchString>
                <earliestTime>0</earliestTime>
                <latestTime>50</latestTime>
                <fields>foo bar baz</fields>
                ''')
            self.assertEqual(d.search.earliestTime, '0')
            self.assertEqual(d.search.latestTime, '50')
            self.assertEqual(d.title, 'Title1')
            self.assertEqual(d.searchFieldList, 'foo bar baz'.split())

            # should be able to accept one of the search modes
            d = self.createPanel('single', options='''
                <searchString>search string</searchString>
                ''')
            self.assertTrue(hasattr(d, 'search'),  'panel elements should have a search object')
            self.assertEqual(getattr(d.search, 'searchMode'), 'template')
            self.assertEqual(getattr(d.search, 'searchCommand'), 'search string')
            d = self.createPanel('single', options='''
                <searchName>search saved</searchName>
                <searchTemplate>search template</searchTemplate>
                <searchPostProcess>search postsearch</searchPostProcess>
                ''')
            self.assertEqual(getattr(d.search, 'searchMode'), 'saved')
            self.assertEqual(getattr(d.search, 'searchCommand'), 'search saved')
            d = self.createPanel('single', options='''
                <searchTemplate>search template</searchTemplate>
                ''')
            self.assertEqual(getattr(d.search, 'searchMode'), 'template')
            self.assertEqual(getattr(d.search, 'searchCommand'), 'search template')
            d = self.createPanel('single', options='''
                <searchPostProcess>search postsearch</searchPostProcess>
                ''')
            self.assertEqual(getattr(d.search, 'searchMode'), 'postsearch')
            self.assertEqual(getattr(d.search, 'searchCommand'), 'search postsearch')
            self.assertEqual(getattr(d.search, 'baseSearchId'), 'global')

            # Options should not override internal settings
            d = self.createPanel('chart', options='''
                <option name="id">xyz</option>
                <option name="managerid">abc</option>
                <option name="el">abc</option>
                <option name="tokenDependencies">abc</option>
                <option name="resizable">False</option>
                ''')

            for option in d.options:
                if option in ['id', 'el', 'managerid', 'tokenDependencies', 'resizable']:
                    self.assertTrue(False, 'Options should not override internal settings')

        def testCreateSavedSearch(self):
            """All panels have some things in common"""
            #s test common nodes
            d = self.createPanel('single', options='''
                <title>Title1</title>
                <search ref="search"/>
                ''')
            self.assertEqual(d.search.searchCommand, 'search')

        def testTokenDependencies(self):
            for panelType in ('table', 'chart', 'single', 'map', 'list', 'html'):
                panel = createPanelElementFromXml(et.fromstring('''
                    <%(type)s depends="$foo$">
                    </%(type)s>
                ''' % dict(type=panelType)))
                self.assertIsNotNone(panel)
                self.assertIsNotNone(panel.tokenDeps)
                self.assertEquals(panel.tokenDeps.depends, '$foo$')
                self.assertEquals(panel.tokenDeps.rejects, '')

                panel = createPanelElementFromXml(et.fromstring('''
                    <%(type)s rejects="$foo$">
                    </%(type)s>
                ''' % dict(type=panelType)))
                self.assertIsNotNone(panel)
                self.assertIsNotNone(panel.tokenDeps)
                self.assertEquals(panel.tokenDeps.rejects, '$foo$')
                self.assertEquals(panel.tokenDeps.depends, '')

                panel = createPanelElementFromXml(et.fromstring('''
                    <%(type)s id="foo" depends="$foo$" rejects="$bar$">
                    </%(type)s>
                ''' % dict(type=panelType)))
                self.assertIsNotNone(panel)
                self.assertIsNotNone(panel.tokenDeps)
                self.assertEquals(panel.tokenDeps.depends, '$foo$')
                self.assertEquals(panel.tokenDeps.rejects, '$bar$')

                panel = createPanelElementFromXml(et.fromstring('''
                    <%(type)s>
                    </%(type)s>
                ''' % dict(type=panelType)))
                self.assertIsNotNone(panel)
                self.assertIsNone(panel.tokenDeps)

                panel = createPanelElementFromXml(et.fromstring('''
                    <%(type)s depends="" rejects="">
                    </%(type)s>
                ''' % dict(type=panelType)))
                self.assertIsNotNone(panel)
                self.assertIsNone(panel.tokenDeps)

                dashboard = createDashboardFromXml(et.fromstring('''
                    <form>

                        <fieldset>
                            <input token="foobar" rejects="$foobar$" />
                        </fieldset>

                        <row depends="$foobar$">
                            <panel rejects="$foobar$">
                                <chart depends="$x$" rejects="$y$">

                                </chart>
                            </panel>
                        </row>
                    </form>
                '''))

                self.assertIsNotNone(dashboard.rows[0].tokenDeps)
                self.assertEquals(dashboard.rows[0].tokenDeps.depends, "$foobar$")
                self.assertEquals(dashboard.rows[0].tokenDeps.rejects, "")
                self.assertIsNotNone(dashboard.rows[0].panels[0].tokenDeps)
                self.assertEquals(dashboard.rows[0].panels[0].tokenDeps.depends, "")
                self.assertEquals(dashboard.rows[0].panels[0].tokenDeps.rejects, "$foobar$")
                self.assertIsNotNone(dashboard.rows[0].panels[0].tokenDeps)
                self.assertEquals(dashboard.rows[0].panels[0].tokenDeps.depends, "")
                self.assertEquals(dashboard.rows[0].panels[0].tokenDeps.rejects, "$foobar$")
                self.assertIsNotNone(dashboard.rows[0].panels[0].panelElements[0].tokenDeps)
                self.assertEquals(dashboard.rows[0].panels[0].panelElements[0].tokenDeps.depends, "$x$")
                self.assertEquals(dashboard.rows[0].panels[0].panelElements[0].tokenDeps.rejects, "$y$")
                self.assertIsNotNone(dashboard.fieldset[0].tokenDeps)
                self.assertEquals(dashboard.fieldset[0].tokenDeps.depends, "")
                self.assertEquals(dashboard.fieldset[0].tokenDeps.rejects, "$foobar$")
                dashboard = createDashboardFromXml(et.fromstring('''
                    <form>

                        <fieldset>
                            <input token="multiselect" type="multiselect" rejects="$foobar$">
                                <search ref="foo"/>
                            </input>
                        </fieldset>

                        <row depends="$foobar$">
                            <panel rejects="$foobar$">
                                <chart depends="$x$" rejects="$y$">

                                </chart>
                            </panel>
                        </row>
                    </form>
                '''))
                self.assertEquals(dashboard.fieldset[0].search.searchCommand, "foo")

        def testSimpleDrilldownPopulated(self):
            for panelType in ('table', 'chart', 'single', 'map', 'list'):
                xmlNode = et.fromstring('''
                    <%(type)s id="panel1">
                        <title>Panel 1</title>
                        <drilldown>
                            <set token="foobar">($click.value$)</set>
                        </drilldown>
                    </%(type)s>
                ''' % dict(type=panelType))
                panel = createPanelElementFromXml(xmlNode)
                self.assertIsNotNone(panel)
                self.assertEquals(len(panel.simpleDrilldown), 1)

    class _findPrimarySearchNodeTests(unittest.TestCase):
        def testAnnotationBeforePrimary(self):
            chart = et.fromstring('''
              <chart>
                <title>test_chart_annotation_pdf_printing</title>
                <search type="annotation">
                  <query>index=_internal (log_level="WARN" OR log_level="ERROR" OR log_level="INFO") | eval annotation_label = "Category is" | eval annotation_category = log_level | table _time annotation_label annotation_category</query>
                  <earliest>-1m@m</earliest>
                  <latest>now</latest>
                </search>
                <search>
                  <query>index=_internal | timechart count</query>
                  <earliest>-1m@m</earliest>
                  <latest>now</latest>
                </search>
                <option name="charting.annotation.categoryColors">{"ERROR":"0xFF0000","WARN":"0x0000FF", "INFO": "0x008000"}</option>
                <option name="charting.chart">area</option>
              </chart>
            ''')
            primary = _findPrimarySearchNode(chart)
            self.assertIsNone(primary.get('type'), 'primary search type')
            self.assertEqual(primary.findtext('query'), 'index=_internal | timechart count', 'primary search type')

        def test_isPrimarySearchNode(self):
            primaryNode = et.fromstring('''
                <search type="primary">
                  <query>index=_internal | timechart count</query>
                  <earliest>-1m@m</earliest>
                  <latest>now</latest>
                </search>
            ''')
            self.assertTrue(_isPrimarySearchNode(primaryNode))

            defaultNode = et.fromstring('''
                <search>
                  <query>index=_internal | timechart count</query>
                  <earliest>-1m@m</earliest>
                  <latest>now</latest>
                </search>
            ''')
            self.assertTrue(_isPrimarySearchNode(defaultNode))

            annotationNode = et.fromstring('''
                <search type="annotation">
                  <query>index=_internal (log_level="WARN" OR log_level="ERROR" OR log_level="INFO") | eval annotation_label = "Category is" | eval annotation_category = log_level | table _time annotation_label annotation_category</query>
                  <earliest>-1m@m</earliest>
                  <latest>now</latest>
                </search>
            ''')
            self.assertFalse(_isPrimarySearchNode(annotationNode))

    class CreatePanelTests(unittest.TestCase):
        def getPanel(self, xml='<panel></panel>'):
            root = et.fromstring(xml)
            return createPanelFromXML(root)

        def testEmptyPanel(self):
            self.assertTrue(isinstance(self.getPanel(), Panel), '')

        def testPanelId(self):
            panel = self.getPanel(xml='<panel id="foo"></panel>')
            self.assertTrue(isinstance(panel, Panel), 'Panel should exist')
            self.assertEqual(panel.id, "foo", 'Panel should have id=foo')

        def testPanelMultipleSearches(self):
            """Extract new search format and multiple searches"""
            #s test common nodes
            d = self.getPanel(xml='''
            <panel>
                <search id="panel_search">
                    <query>search string also</query>
                    <earliest>-15m</earliest>
                    <latest>now</latest>
                </search>
                <search base="panel_search">
                    <query>search postsearch</query>
                </search>
            </panel>
            ''')
            self.assertEqual(len(d.searches), 2)
            self.assertEqual(getattr(d.searches[0], 'searchMode'), 'template')
            self.assertEqual(getattr(d.searches[0], 'searchCommand'), 'search string also')
            self.assertEqual(getattr(d.searches[0], 'id'), 'panel_search')
            self.assertEqual(getattr(d.searches[0], 'latestTime'), 'now')
            self.assertEqual(getattr(d.searches[0], 'earliestTime'), '-15m')
            self.assertEqual(getattr(d.searches[1], 'searchMode'), 'postsearch')
            self.assertEqual(getattr(d.searches[1], 'searchCommand'), 'search postsearch')
            self.assertEqual(getattr(d.searches[1], 'baseSearchId'), 'panel_search')

        def testPanelSavedSearch(self):
            """Extract new search format saved search"""
            d = self.getPanel(xml='''
            <panel>
                <search ref="panel_search"/>
            </panel>
            ''')
            self.assertEqual(len(d.searches), 1)
            self.assertEqual(getattr(d.searches[0], 'searchMode'), 'saved')
            self.assertEqual(getattr(d.searches[0], 'searchCommand'), 'panel_search')

    class CreateRowTests(unittest.TestCase):
        def getRowLxml(self, args='', panels=1):
            xml = getRowXml(args, panels)
            root = et.fromstring(xml)
            return createRowFromXml(root)

        def testCreateRow(self):
            d = self.getRowLxml()
            self.assertTrue(d)
            self.assertEqual(len(d.panels), 1)
            self.assertEqual(
                d.panels[0].panelElements[0].matchTagName, 'single')

        def testRowGrouping(self):
            d = self.getRowLxml(args='grouping="2,1"', panels=3)
            self.assertTrue(d)
            self.assertEqual(len(d.panels), 2)
            self.assertEqual(len(d.panels[0].panelElements), 2)
            self.assertEqual(len(d.panels[1].panelElements), 1)

        def testRowId(self):
            d = self.getRowLxml(args='id="foo"')
            self.assertTrue(d)
            self.assertEqual(d.id, "foo", "Row Id should be foo")

        def testCreateRowWith3Panels(self):
            d = self.getRowLxml(args='', panels=3)
            self.assertTrue(d)
            self.assertEqual(len(d.panels), 3)

        def testCreateRowWithPanels(self):
            xml = '''
                    <row>
                        <panel>
                            <single/>
                        </panel>
                    </row>
                '''
            root = et.fromstring(xml)
            d =  createRowFromXml(root)
            self.assertTrue(d)
            self.assertEqual(len(d.panels), 1)
            self.assertEqual(
                d.panels[0].panelElements[0].matchTagName, 'single')

        def testRowGroupingWithPanels(self):
            xml = '''
                    <row>
                        <panel>
                            <single/>
                            <single/>
                        </panel>
                        <panel>
                            <single/>
                        </panel>
                    </row>
                '''
            root = et.fromstring(xml)
            d =  createRowFromXml(root)
            self.assertTrue(d)
            self.assertEqual(len(d.panels), 2)
            self.assertEqual(len(d.panels[0].panelElements), 2)
            self.assertEqual(len(d.panels[1].panelElements), 1)

        def testCreateRowWithComments(self):
            xml = '''
                    <row>
                        <!-- this better work -->
                        <panel/>
                    </row>
                '''
            root = et.fromstring(xml)
            row =  createRowFromXml(root)
            xml = '''
                    <row>
                        <!-- this better work -->
                        <chart/>
                    </row>
                '''
            root = et.fromstring(xml)
            row =  createRowFromXml(root)

        def testCreateRowWithTitleException(self):
            with self.assertRaises(Exception):
                xml = '''
                        <row>
                            <title/>
                            <panel/>
                        </row>
                    '''
                root = et.fromstring(xml)
                row =  createRowFromXml(root)
            with self.assertRaises(Exception):
                xml = '''
                        <row>
                            <title/>
                            <chart/>
                        </row>
                    '''
                root = et.fromstring(xml)
                row =  createRowFromXml(root)

    class CreateDashboardTests(unittest.TestCase):
        def getSimpleLxml(self, root='dashboard', rows=1, fieldset=''):
            nodes = []
            nodes.append('<%(root)s>')
            nodes.append(fieldset)
            for i in range(0, rows):
                nodes.append(getRowXml())
            nodes.append('</%(root)s>')
            xml = ''.join(nodes)
            xml = xml % {'root': root}
            root = et.fromstring(xml)
            return createDashboardFromXml(root)

        def testCreateDashboard(self):
            d = self.getSimpleLxml()
            self.assertTrue(d)
            self.assertEqual(d.matchTagName, 'dashboard')

        def testCreateForm(self):
            d = self.getSimpleLxml(root='form')
            self.assertTrue(d)
            self.assertEqual(d.matchTagName, 'form')

        def testCreateFormWithFieldset(self):
            fieldset = '''
            <fieldset>
                <input token="foo" searchWhenChanged="True"></input>
                <html></html>
                <shouldntShow></shouldntShow>
            </fieldset>
            '''
            d = self.getSimpleLxml(root='form', fieldset=fieldset)
            self.assertTrue(d)
            self.assertTrue(d.fieldset)
            self.assertEqual(len(d.fieldset), 2)
            self.assertEqual(d.fieldset[0].__class__.__name__, 'TextInput')
            self.assertEqual(d.fieldset[0].searchWhenChanged, True)
            self.assertEqual(d.fieldset[1].matchTagName, 'html')
            self.assertEqual(d.matchTagName, 'form')

        def testCreateUnsupportedRoot(self):
            with self.assertRaises(Exception):
                self.getSimpleLxml(root='notFormOrDashboard')

        def testValidationMessages(self):
            msgs = getValidationMessages(et.fromstring('<dashboard></dashboard>'))
            self.assertIsNotNone(msgs)
            self.assertEquals(len(msgs), 0)

            msgs = getValidationMessages(et.fromstring('''
                <dashboard>
                    <row>
                        <table>
                            <drilldown>
                                <set token="foo">...</set>
                                <condition field="bar"></condition>
                            </drilldown>
                        </table>
                    </row>
                </dashboard>
            '''))
            self.assertIsNotNone(msgs)
            self.assertGreater(len(msgs), 0)

            msgs = getValidationMessages(et.fromstring('''
                <dashboard>
                    <row>
                        <table>
                            <drilldown>
                                <set field="bar" token="foo">...</set>
                            </drilldown>
                        </table>
                    </row>
                </dashboard>
            '''))
            self.assertIsNotNone(msgs)
            self.assertGreater(len(msgs), 0)

            msgs = getValidationMessages(et.fromstring('''
                <dashboard>
                    <row>
                        <table>
                            <drilldown>
                                <link field="bar">...</link>
                                <link field="bar">...</link>
                            </drilldown>
                        </table>
                    </row>
                </dashboard>
            '''))
            self.assertIsNotNone(msgs)
            self.assertGreater(len(msgs), 0)

            msgs = getValidationMessages(et.fromstring('''
                <dashboard>
                    <row>
                    </row>
                </dashboard>
            '''))
            self.assertIsNotNone(msgs)
            self.assertGreater(len(msgs), 0)

            msgs = getValidationMessages(et.fromstring('<foobar></foobar>'))
            self.assertIsNotNone(msgs)
            self.assertGreater(len(msgs), 0)

    class ExtractHTMLTests(unittest.TestCase):
        def testExtractEncodedHTML(self):
            self.assertEqual(" foo ", extractHtmlContent(et.fromstring('<html encoded="1"><![CDATA[ foo ]]></html>')))

        def testExtractEmptyHTML(self):
            self.assertEqual('', extractHtmlContent(et.fromstring('<html></html>')))
            self.assertEqual('', extractHtmlContent(et.fromstring('<html/>')))
            self.assertEqual('', extractHtmlContent(et.fromstring('<html id="foobar" />')))

        def testExtractInlineHTML(self):
            self.assertEqual('<h1>test</h1><p>foo</p>', extractHtmlContent(et.fromstring('<html id="foobar"><h1>test</h1><p>foo</p></html>')))

    loader = unittest.TestLoader()
    unittest.TextTestRunner(verbosity=2).run(unittest.TestSuite([
        loader.loadTestsFromTestCase(CreatePanelElementTests),
        loader.loadTestsFromTestCase(_findPrimarySearchNodeTests),
        loader.loadTestsFromTestCase(CreateRowTests),
        loader.loadTestsFromTestCase(CreateDashboardTests),
        loader.loadTestsFromTestCase(CreatePanelTests),
        loader.loadTestsFromTestCase(ExtractHTMLTests)
    ]))


'''
Created on Feb 6, 2011

@author: jehan
'''

import pyexiv2
import mimetypes
from filemovers import mover
from filemovers.mover import Mover
import os, re
import datetime

class DumpToRenameMover(Mover):
    _config = None
    def __init__(self,config):
        self._config = config
        mimetypes.init()

    def handleFile(self,handledFilePath):
        print("handleFile", handledFilePath)

        # extract the date of the file
        imgDatetime = self.getImageDate(handledFilePath)

        # no date? Ignore the file.
        if None != imgDatetime:
            print("Handling",os.path.basename(handledFilePath),imgDatetime)
            dateFolderPath = self.makeFolder(imgDatetime, handledFilePath)
            mover.moveFile(handledFilePath,
                           os.path.join(dateFolderPath,os.path.basename(handledFilePath)))

        # do some trailing work
#        for root,folders,files in os.walk(self._config.dump_folder): #@UnusedVariable
#            for fold in folders:
#                if 0 == len(os.listdir(os.path.join(root,fold))):
#                    os.rmdir(os.path.join(root,fold))

    def makeFolder(self, imgDatetime, handledFilePath):
        print("making a new folder")
        dateFolders = mover.findDateFolders(self._config.rename_folder)

        if mover.mkDateFolderName(imgDatetime) in dateFolders:
            print("folder exists")
            dateFolderPath = dateFolders[mover.mkDateFolderName(imgDatetime)][1]
        else:
            print("creating new folder")
            # make dir if it does not exist
            dateFolderPath = os.path.join(self._config.rename_folder,
                                   mover.mkDateFolderName(imgDatetime))

            if self._config.use_original_folder_names:
                print("import original folder name")
                originalFolderName =  os.path.basename(os.path.dirname(handledFilePath))
                dateFolderPath += ' '+ originalFolderName
            print("make dir",dateFolderPath)

            os.mkdir( dateFolderPath )

            if self._config.make_fusioned_folder_names:
                print("try to merge folders")
                self.makeRangeFolders(imgDatetime, dateFolders)
            else:
                print("I won't try merging folders")

        return dateFolderPath

    def moveFile(self,dateFolderPath,handledFilePath):
        print("moving...")
        fileName = os.path.splitext(os.path.basename(handledFilePath))
        dest  = os.path.join(dateFolderPath,fileName[0]+fileName[1])
        i = 0
        while os.path.exists(dest):
            i+=1
            dest  = os.path.join(dateFolderPath,fileName[0]+'_'+str(i)+fileName[1])
            if 100 == i:
                print("Tried renaming the file up to "+fileName[0]+'_'+str(i)+fileName[1]+" without success.")
                print("Giving up since we might be in a loop. File not moved:",handledFilePath)
                return
        os.rename(handledFilePath, dest)
        print("moved")

    def getImageDate(self,handledFilePath,useFilesystem=True):
        print("Reading metadata:",handledFilePath)
        imgDatetime = None
        try:
            image = pyexiv2.ImageMetadata(handledFilePath)
            image.read()
            #print image.exifKeys()
            try:
                imgDatetime = image['Exif.Photo.DateTimeOriginal']
                #print "found DateTimeOriginal"
            except KeyError:
                imgDatetime = image['Exif.Image.DateTime']
                #print "found DateTime"
            imgDatetime = imgDatetime.value
            if type(imgDatetime) != datetime.datetime:
                if type(imgDatetime) == str:
                    print("pyexif returned a string rather than a datetime object")
                    dateParse = re.match(r"^(\d{4}):(\d{2}):(\d{2}).*",imgDatetime).groups()
                    imgDatetime = datetime.datetime(int(dateParse[0]),int(dateParse[1]),int(dateParse[2]));
                else:
                    print("pyexif returned nothing worthwhile")
                    raise Exception()
        except Exception as e:
            if type(e) == IOError or type(e) == KeyError:
                print("Probably no EXIF data :",e)
                if useFilesystem:
                    print("is it an image or a video?")
                    t = mimetypes.guess_type(handledFilePath)
                    print("Mime : ",t)
                    if type(t[0]) == type("") and (
                            t[0].startswith('video') or  t[0].startswith('image')
                            ):
                        print("It's an image or a video. Trying the last modif date")
                        imgDatetime = datetime.datetime.fromtimestamp(os.path.getmtime(handledFilePath))
                        print("got:", imgDatetime)
            else: raise
        print("date time is",imgDatetime)
        return imgDatetime

    def makeRangeFolders(self,imgDatetime,dateFolders):
        # append to a prevRange if dir exists for previous or next day
        # search for possible matches
        prevDay = imgDatetime-datetime.timedelta(days=1)
        #prev = os.path.join(self._config.rename_folder,mkDateFolderName(prevDay))

        nextDay = imgDatetime+datetime.timedelta(days=1)
        #next = os.path.join(self._config.rename_folder,mkDateFolderName(nextDay))

        rangeFolders = mover.findRangeFolders(self._config.rename_folder)

        prevRange = None

        if mover.mkDateFolderName(prevDay) in dateFolders:
            print("prev exists")
            if len(rangeFolders):
                for start,finish,rangeFoldCurrentPath in rangeFolders: #@UnusedVariable
                    if mover.sameDay(finish,prevDay):
                        print("prevRange with prev")
                        prevRange = rangeFoldCurrentPath
                        break

            if None != prevRange:
                newPrevRange = mover.mkRangeFolderName(start, imgDatetime,self._config.rename_folder)
                print("we have a prevRange. Rename from",prevRange,"to",newPrevRange)
                os.rename(prevRange,newPrevRange)
                prevRange = newPrevRange
                prevRangeStart = start
            else:
                prevRange = mover.mkRangeFolderName(prevDay,imgDatetime,self._config.rename_folder)
                print("No prevRange. We make it from ",prevDay,"and",imgDatetime,":",prevRange)
                os.mkdir(prevRange)
                prevRangeStart = prevDay

        nextRange = None
        if mover.mkDateFolderName(nextDay) in dateFolders:
            print("next exists")
            if len(rangeFolders):
                for start,finish,rangeFoldCurrentPath in rangeFolders: #@UnusedVariable
                    if mover.sameDay(start,nextDay):
                        print("nextRange with next")
                        nextRange = rangeFoldCurrentPath
                        break

            if None != nextRange:
                print("We have a nextRange",nextRange)
                if None == prevRange:
                    newNextRange = mover.mkRangeFolderName(imgDatetime,finish,self._config.rename_folder)
                    print("but no prev: rename nextRange from ",nextRange,"to",newNextRange)
                    os.rename(nextRange,newNextRange)
                else:
                    mergeRange = mover.mkRangeFolderName(prevRangeStart,finish,self._config.rename_folder)
                    print("with prev: We rename from ",prevRange,"to",mergeRange)
                    os.rename(prevRange,mergeRange)
                    print("we delete",nextRange)
                    os.rmdir(nextRange)
            else:
                if None == prevRange:
                    nextRange = mover.mkRangeFolderName(imgDatetime,nextDay,self._config.rename_folder)
                    print("No nextRange. We make it:",nextRange)
                    os.mkdir(nextRange)
                else:
                    print("No nextRange but we have a prevRange: we merge")
                    mergeRange = mover.mkRangeFolderName(prevRangeStart,nextDay,self._config.rename_folder)
                    print("with prev: We rename from ",prevRange,"to",mergeRange)
                    os.rename(prevRange,mergeRange)
        # link in prevRange now that we have one


#!/usr/bin/env python

# -*- coding: UTF-8 -*-
# @package
# @brief
#
# @version    $Revision: $
# @author     Sergey Green
# @note       (c) Kapsch TrafficCom
# @note       $Date:     $
# @note       $URL:      $
#
#
import sys
import string
import logging
import ldp
import ldp_datetime
import ldp_db

class LdpUpdate:
    def __init__(self):
        self.log = string.replace(sys.argv[0],".py",".log")
        self.ldp = ldp.Ldp()
        self.ldp_datetime = ldp_datetime.LdpDateTime()
        self.ldp_db = ldp_db.Ldp_Db()
        self.include= [
            "accountExpires",
            "autoReplyMessage",
            "badPasswordTime",
            "badPwdCount",
            "c",
            "cn",
            "co",
            "codePage",
            "company",
            "countryCode",
            "department",
            "description",
            "displayName",
            "extensionAttribute1",
            "facsimileTelephoneNumber",
            "givenName",
            "homePhone",
            "instanceType",
            "l",
            "lastLogonTimestamp",
            "lockoutTime",
            "logonCount",
            "mail",
            "mailNickname",
            "mDBOverHardQuotaLimit",
            "mDBOverQuotaLimit",
            "mDBStorageQuota",
            "mDBUseDefaults",
            "memberOf",
            "mobile",
            "name",
            "physicalDeliveryOfficeName",
            "postalCode",
            "primaryGroupID",
            "pwdLastSet",
            "sAMAccountName",
            "sAMAccountType",
            "scriptPath",
            "sn",
            "st",
            "streetAddress",
            "telephoneNumber",
            "userAccountControl",
            "whenChanged",
            "whenCreated",
            "wWWHomePage"
        ]

    def initLogging(self):
        LOG = self.log
        FORMAT = '%(asctime)s %(message)s'
        logging.basicConfig(filename=LOG,level=logging.INFO,format=FORMAT)
        logging.info("-")

    def cleanupTables(self):

        sql = "delete from USERS"
        self.ldp_db.writeValues(sql)

        sql = "delete from GROUPS"
        self.ldp_db.writeValues(sql)

    def updateLdp(self):
        a = self.ldp.getAll()
        sql = ""
        users_count  = 0
        groups_count = 0
        for user in a:
            users_keys   = "updated,"
            users_values = "now(),"

            for key,value in user[1].iteritems():
                for v in range(len(value)):
                    if key in self.include:
                        if key == "memberOf": # write to groups db
                            try:
                                sql = "insert into GROUPS (updated,sAMAccountName,memberOf) values (%s,'%s','%s')" % ("now()",user[1]["sAMAccountName"][0],self.extrValues("CN",value[v]))
                                self.ldp_db.writeValues(sql)
                                groups_count +=1
                            except:
                                break
                        else: # write to users db
                            if key in ["whenCreated","whenChanged"]:
                                users_keys += "%s," % key
                                users_values += "'%s'," % string.replace(str(self.ldp_datetime.convert_from_oz( value[v])),"'","")
                            elif key in ["lastLogonTimestamp","badPasswordTime","pwdLastSet","accountExpires"]:
                                if key == "accountExpires":
                                    if value[v]:
                                        users_keys += "%s," % key
                                        users_values += "'%s'," % string.replace(value[v],"'","")
                                else:
                                    users_keys += "%s," % key
                                    users_values += "'%s'," % string.replace(str(self.ldp_datetime.convert_from_timestamp( value[v])),"'","")
                            else:
                                users_keys += "%s," % key
                                users_values += "'%s'," % string.replace(value[v],"'","")

            if len(users_keys):
                sql = "insert into USERS (%s) values (%s)" % (users_keys[:-1],users_values[:-1])
                self.ldp_db.writeValues(sql)
                users_count +=1

        self.collectStats(users_count, groups_count)

    def extrValues(self,k,v):
        v = string.replace(v,"\\","")
        return string.split(string.split(string.split(v,",")[0],k)[1],"=")[1]

    def collectStats(self,u,g):
        logging.info('Updated Users                   = %d' % u)
        logging.info('Updated Groups                  = %d' % g)
        sql = 'select count( distinct memberOf ) from GROUPS'
        logging.info('Total Groups                    = %d' % self.ldp_db.getCounts(sql))
        sql = 'select count(*) from USERS where accountExpires = 9223372036854775807'
        logging.info('Active Employees                = %d' % self.ldp_db.getCounts(sql))
        sql = 'select count(*) from USERS where accountExpires = 9223372036854775807 and streetAddress like "%Europlatz%"'
        logging.info('Active Employees (Am Europlatz) = %d' % self.ldp_db.getCounts(sql))
        sql = 'select count(*) from USERS where accountExpires = 0'
        logging.info('Former Employees                = %d' % self.ldp_db.getCounts(sql))
        sql = 'select count(*) from USERS where accountExpires = 0 and streetAddress like "%Europlatz%"'
        logging.info('Former Employees (Am Europlatz) = %d' % self.ldp_db.getCounts(sql))
        logging.info("-")
        logging.info("-"*40)

def main():
    l = LdpUpdate()
    l.initLogging()
    l.cleanupTables()
    l.updateLdp()

if __name__ == '__main__':
    main()


import sys
import time
import logging
import argparse

import numpy as np
import cv2
import tensorflow as tf
import tensorflow.contrib.tensorrt as trt
import centroidtracker as ot
import imutils
import csv
import paho.mqtt.client as mqtt
from utils.camera import add_camera_args, Camera
from utils.od_utils import read_label_map, build_trt_pb, load_trt_pb, \
                           write_graph_tensorboard, detect
#from utils.visualization import BBoxVisualization
import math

import json

#Global Parameters -- Vincent
ALPHA = 0.5
FONT = cv2.FONT_HERSHEY_PLAIN
TEXT_SCALE = 1.0
TEXT_THICKNESS = 1
BLACK = (0, 0, 0)
WHITE = (255, 255, 255)
person_detection = False #Control Display status
hand_wash_status=0
ct = ot.CentroidTracker()
temp=[]
client = mqtt.Client()

#It should connect to MQTT server
#Jetson TX2(vincent) has been set as a MQTT server
#If publisher/subscriber on JetsonTX2(vincet) please connect to localhost
#Else please connect to  192.168.0.101 in the LAN
client.connect("ia.ic.polyu.edu.hk",1883,60)
#Global Parameters -- Vincent

# Constants
DEFAULT_MODEL = 'ssd_mobilenet_v1_coco'
DEFAULT_LABELMAP = 'third_party/models/research/object_detection/' \
                   'data/labelmap.pbtxt'
WINDOW_NAME = 'AI-Hygiene-Tracking[Room]'
BBOX_COLOR = (0, 255, 0)  # green

def on_connect(client, userdata, flags, rc):
    print("Connected with result code "+str(rc))
    client.subscribe("MDSSCC/AIHH/SENSOR")

def on_message(client, userdata, msg):
    global hand_wash_status

    x = msg.payload.decode()
    message_dict = json.loads(x)
    if(message_dict["status"] == "1"):
        hand_wash_status = 1
    else:
        hand_wash_status = 0
        print("message get!")

#Not using loop_forever since it is using the current thread
client.loop_start()
client.on_connect = on_connect
client.on_message = on_message

def parse_args():
    """Parse input arguments."""
    desc = ('This script captures and displays live camera video, '
            'and does real-time object detection with TF-TRT model '
            'on Jetson TX2/TX1/Nano')
    parser = argparse.ArgumentParser(description=desc)
    parser = add_camera_args(parser)
    parser.add_argument('--model', dest='model',
                        help='tf-trt object detecion model '
                        '[{}]'.format(DEFAULT_MODEL),
                        default=DEFAULT_MODEL, type=str)
    parser.add_argument('--build', dest='do_build',
                        help='re-build TRT pb file (instead of using'
                        'the previously built version)',
                        action='store_true')
    parser.add_argument('--tensorboard', dest='do_tensorboard',
                        help='write optimized graph summary to TensorBoard',
                        action='store_true')
    parser.add_argument('--labelmap', dest='labelmap_file',
                        help='[{}]'.format(DEFAULT_LABELMAP),
                        default=DEFAULT_LABELMAP, type=str)
    parser.add_argument('--num-classes', dest='num_classes',
                        help='(deprecated and not used) number of object '
                        'classes', type=int)
    parser.add_argument('--confidence', dest='conf_th',
                        help='confidence threshold [0.3]',
                        default=0.3, type=float)
    args = parser.parse_args()
    return args

def open_display_window(width, height):
    """Open the cv2 window for displaying images with bounding boxeses."""
    cv2.namedWindow(WINDOW_NAME, cv2.WINDOW_NORMAL)
    cv2.resizeWindow(WINDOW_NAME, width, height)
    cv2.moveWindow(WINDOW_NAME, 0, 0)
    cv2.setWindowTitle(WINDOW_NAME, 'AI-Hygiene-Tracker[ROOM]')

def draw_help_and_fps(img, fps):
    """Draw help message and fps number at top-left corner of the image."""
    help_text = "'Esc' to Quit, 'H' for FPS & Help, 'F' for Fullscreen"
    font = cv2.FONT_HERSHEY_PLAIN
    line = cv2.LINE_AA

    fps_text = 'FPS: {:.1f}'.format(fps)
    cv2.putText(img, help_text, (11, 20), font, 1.0, (32, 32, 32), 4, line)
    cv2.putText(img, help_text, (10, 20), font, 1.0, (240, 240, 240), 1, line)
    cv2.putText(img, fps_text, (11, 50), font, 1.0, (32, 32, 32), 4, line)
    cv2.putText(img, fps_text, (10, 50), font, 1.0, (240, 240, 240), 1, line)
    return img

def set_full_screen(full_scrn):
    """Set display window to full screen or not."""
    prop = cv2.WINDOW_FULLSCREEN if full_scrn else cv2.WINDOW_NORMAL
    cv2.setWindowProperty(WINDOW_NAME, cv2.WND_PROP_FULLSCREEN, prop)

#<<-----------------------------------------------------Vincent---------------------------------->>#
def gen_colors(num_colors):
    """Generate different colors.

    # Arguments
      num_colors: total number of colors/classes.

    # Output
      bgrs: a list of (B, G, R) tuples which correspond to each of
            the colors/classes.
    """
    import random
    import colorsys

    hsvs = [[float(x) / num_colors, 1., 0.7] for x in range(num_colors)]
    random.seed(1234)
    random.shuffle(hsvs)
    rgbs = list(map(lambda x: list(colorsys.hsv_to_rgb(*x)), hsvs))
    bgrs = [(int(rgb[2] * 255), int(rgb[1] * 255),  int(rgb[0] * 255))
            for rgb in rgbs]
    return bgrs

def draw_boxed_text(img, text, topleft, color):
    """Draw a transluent boxed text in white, overlayed on top of a
    colored patch surrounded by a black border. FONT, TEXT_SCALE,
    TEXT_THICKNESS and ALPHA values are constants (fixed) as defined
    on top.

    # Arguments
      img: the input image as a numpy array.
      text: the text to be drawn.
      topleft: XY coordinate of the topleft corner of the boxed text.
      color: color of the patch, i.e. background of the text.

    # Output
      img: note the original image is modified inplace.
    """
    assert img.dtype == np.uint8
    img_h, img_w, _ = img.shape
    if topleft[0] >= img_w or topleft[1] >= img_h:
        return
    margin = 3
    size = cv2.getTextSize(text, FONT, TEXT_SCALE, TEXT_THICKNESS)
    w = size[0][0] + margin * 2
    h = size[0][1] + margin * 2
    # the patch is used to draw boxed text
    patch = np.zeros((h, w, 3), dtype=np.uint8)
    patch[...] = color
    cv2.putText(patch, text, (margin+1, h-margin-2), FONT, TEXT_SCALE,
                WHITE, thickness=TEXT_THICKNESS, lineType=cv2.LINE_8)
    cv2.rectangle(patch, (0, 0), (w-1, h-1), BLACK, thickness=1)
    w = min(w, img_w - topleft[0])  # clip overlay at image boundary
    h = min(h, img_h - topleft[1])
    # Overlay the boxed text onto region of interest (roi) in img
    roi = img[topleft[1]:topleft[1]+h, topleft[0]:topleft[0]+w, :]
    cv2.addWeighted(patch[0:h, 0:w, :], ALPHA, roi, 1 - ALPHA, 0, roi)
    return img

class BBoxVisualization():
    """BBoxVisualization class implements nice drawing of boudning boxes.

    # Arguments
      cls_dict: a dictionary used to translate class id to its name.
    """

    def __init__(self, cls_dict):
        self.cls_dict = cls_dict
        self.colors = gen_colors(len(cls_dict))

    def draw_bboxes(self, img, box, conf, cls):
        """Draw detected bounding boxes on the original image."""
        global person_detection,rects,ct,temp,hand_wash_status
        print("================================================================================")
        for bb, cf, cl in zip(box, conf, cls):
            cl = int(cl)
            print("Confidence=",cf)
            if((cl == 1)and(cf>=0.1)): #Vincent:Only process "person" (label id of persion = 1) + confidence control
                y_min, x_min, y_max, x_max = bb[0], bb[1], bb[2], bb[3]
                mid_x = int((x_min+x_max)/2)
                mid_y = int((y_min+y_max)/2)
                if((mid_y>=130)and(mid_y<=721))and((mid_x>=160)and(mid_x<=697)):
                    temp.append(x_min)
                    temp.append(y_min)
                    temp.append(x_max)
                    temp.append(y_max)

                    color = self.colors[cl]

                #print("Boundary Box Coordinates:")
                #print("X-min:%d Y-min:%d X-max:%d Y-max:%d"%(x_min,y_min,x_max,y_max))
                #print("Buffer:",temp)
                #print("Hand-Wash?",hand_wash_status)
                    id_x = int((x_min+x_max)/2)
                    id_y = int((y_min+y_max)/2) +30
                    debug = "W="+str(hand_wash_status)
                    cv2.putText(img, debug ,(id_x, id_y+30), cv2.FONT_HERSHEY_PLAIN, 1, (0,255,0), 2, cv2.LINE_AA)
                    rects.append(temp)
                    temp=[]
                    cv2.rectangle(img, (x_min, y_min), (x_max, y_max), color, 2)
                    txt_loc = (max(x_min+2, 0), max(y_min+2, 0))
                    cls_name = self.cls_dict.get(cl, 'CLS{}'.format(cl))
                    txt = '{} {:.2f}'.format(cls_name, cf)
                    img = draw_boxed_text(img, txt, txt_loc, color)
                    person_detection = True
        #print("TrFackers=",rects)
        #if(person_detection == True):  #Vincent: Detection status display
            #cv2.putText(img, "Status:Person IN ROOM", (10, 90), cv2.FONT_HERSHEY_PLAIN, 1.5, (0,255,0), 2, cv2.LINE_AA)
        #else:
        #    cv2.putText(img, "Status:EMPTY", (10, 90), cv2.FONT_HERSHEY_PLAIN, 1.5, (0,0,255), 2, cv2.LINE_AA)
        person_detection = False
        return img,hand_wash_status
#<<-----------------------------------------------------Vincent---------------------------------->>#
def pixel_info(event,x,y,flag,param):
    if event == cv2.EVENT_LBUTTONDOWN:  #When Left is clicked in the mouse
        print("X:{} Y:{}".format(x,y))

def loop_and_detect(cam, tf_sess, conf_th, vis, od_type):
    """Loop, grab images from camera, and do object detection.

    # Arguments
      cam: the camera object (video source).
      tf_sess: TensorFlow/TensorRT session to run SSD object detection.
      conf_th: confidence/score threshold for object detection.
      vis: for visualization.
    """
    show_fps = True
    full_scrn = False
    fps = 0.0
    tic = time.time()
    global rects,ct,temp
    zone_x_bed = 0
    zone_y_bed = 0

    zone_x_clean = 0
    zone_y_clean = 0

    #Boundary boxes for RTSP (low resolution)
    zone_x_min_bed,zone_y_min_bed,zone_x_max_bed,zone_y_max_bed = 366,369,521,667
    zone_x_min_clean,zone_y_min_clean,zone_x_max_clean,zone_y_max_clean = 194,300,330,420
    zone_x_min_door,zone_y_min_door,zone_x_max_door,zone_y_max_door = 631,308,674,414
    zone_x_min_alchol,zone_y_min_alchol,zone_x_max_alchol,zone_y_max_alchol = 430,329,470,356

    distance_thres_bed = 165
    distance_thres_clean = 105
    distance_thres_alchol = 100

    counter_msg = 0
    fail_msg = 0
    pass_msg = 0
    global hand_wash_status,args,client
    hd = 0
    wash_delay = 0
    invalid_id = []
    invalid_id.append(999)
    enter,leave = False,False
    restart_flag = False #restart issue
    backup_label = None  #restart issue
    none_buff = 0 #restart issue
    previous_id = 999
    personal_status = []
    for i in range(0,1000):
        personal_status.append(0)
    #CSV Log File
    if(5>2):
        while True:
            #if cv2.getWindowProperty(WINDOW_NAME, 0) < 0:
                # Check to see if the user has closed the display window.
                # If yes, terminate the while loop.
            #    break
            if(restart_flag == True):
                cam = Camera(args)
                cam.open()
                cam.start()
                print("Camera is opened!")
                dummy_img = np.zeros((720, 1280, 3), dtype=np.uint8)
                _, _, _ = detect(dummy_img, tf_sess, conf_th=.3, od_type=od_type)
                print("Loading dummy image!")
                restart_flag = False

            rects = []
            img = cam.read()
            if img is not None:
                img = cv2.flip(img,0)
                #check mqtt status
                #client.on_message = on_message
                #client.loop_forever()
                box, conf, cls = detect(img, tf_sess, conf_th, od_type=od_type)
                img,hd = vis.draw_bboxes(img, box, conf, cls)
                cv2.rectangle(img,(zone_x_min_bed,zone_y_min_bed),(zone_x_max_bed,zone_y_max_bed),(255,102,255),2)
                zone_x_bed = int((zone_x_min_bed+zone_x_max_bed)/2.0)
                zone_y_bed = int((zone_y_min_bed+zone_y_max_bed)/2.0)
                cv2.circle(img, (zone_x_bed, zone_y_bed), 4, (255,102,255), -1)
                cv2.putText(img, "Patient", (zone_x_bed-40, zone_y_bed-20),cv2.FONT_HERSHEY_SIMPLEX, 0.5,(255,102,255), 1)
                cv2.rectangle(img, (zone_x_min_clean,zone_y_min_clean),(zone_x_max_clean,zone_y_max_clean),(255,255,51),2)
                zone_x_clean = int((zone_x_min_clean+zone_x_max_clean)/2.0)
                zone_y_clean = int((zone_y_min_clean+zone_y_max_clean)/2.0)
                cv2.circle(img, (zone_x_clean, zone_y_clean), 4, (255,255,51), -1)
                cv2.putText(img, "CLEANING ZONE", (zone_x_clean-110, zone_y_clean-20),cv2.FONT_HERSHEY_SIMPLEX, 0.5,(255,255,51), 1)
                cv2.rectangle(img, (zone_x_min_door,zone_y_min_door),(zone_x_max_door,zone_y_max_door),(127,0,255),2)
                zone_x_door = int((zone_x_min_door+zone_x_max_door)/2.0)
                zone_y_door = int((zone_y_min_door+zone_y_max_door)/2.0)
                cv2.circle(img, (zone_x_door, zone_y_door), 4, (127,0,255), -1)
                cv2.putText(img, "ENTRANCE", (zone_x_door-40, zone_y_door-20),cv2.FONT_HERSHEY_SIMPLEX, 0.5,(127,0,255), 1)
                cv2.rectangle(img, (zone_x_min_alchol,zone_y_min_alchol),(zone_x_max_alchol,zone_y_max_alchol),(255,255,51),2)
                zone_x_alchol = int((zone_x_min_alchol+zone_x_max_alchol)/2.0)
                zone_y_alchol = int((zone_y_min_alchol+zone_y_max_alchol)/2.0)
                cv2.circle(img, (zone_x_alchol, zone_y_alchol), 4, (255,255,51), -1)
                cv2.putText(img, "CLEANING ZONE", (zone_x_alchol-35, zone_y_alchol-20),cv2.FONT_HERSHEY_SIMPLEX, 0.5,(255,255,51), 1)

                #ROI
                cv2.rectangle(img, (160,130),(697,721),(0,255,255),2)

                distance_bed = 0
                distance_clean = 0
                #Detection Zone
                objects,valid_checker = ct.update(rects)
                flag = False
                for ((objectID, centroid),(objectID,valid)) in zip(objects.items(),valid_checker.items()):
		    # draw both the ID of the object and the centroid of the
		    # object on the output frame
                    #text_id = "ID {}".format(objectID)
                    text_id = "id = 0"
                    backup_label = str(objectID)
                    text = "staff"
                    cv2.putText(img, text, (centroid[0] - 10, centroid[1] - 10),cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
                    cv2.putText(img, text_id, (centroid[0] - 10, centroid[1] - 50),cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
                    cv2.circle(img, (centroid[0], centroid[1]), 4, (0, 255, 0), -1)
                    distance_bed = int(math.sqrt((centroid[0]-zone_x_bed)**2+(centroid[1]-zone_y_bed)**2))
                    distance_clean = int(math.sqrt((centroid[0]-zone_x_clean)**2+(centroid[1]-zone_y_clean)**2))
                    distance_clean_alchol = int(math.sqrt((centroid[0]-zone_x_alchol)**2+(centroid[1]-zone_y_alchol)**2))
                    enter = ct.display_enter_status(objectID)
                    leave = ct.display_leave_status(objectID)
                    flag = ct.display_hygiene(objectID)
                    if(distance_clean_alchol <= distance_thres_alchol):
                        cv2.line(img,(centroid[0], centroid[1]),(zone_x_alchol,zone_y_alchol),(0,0,255),1)
                    #    if(hand_wash_status == 1):
                    #        personal_status[objectID] = 1
                    #    ct.update_wash(True,objectID)

                    if(distance_bed <= distance_thres_bed):
                        personal_status[objectID] = 0
                        hand_wash_status = 0
                        cv2.line(img,(centroid[0], centroid[1]),(zone_x_bed,zone_y_bed),(0,255,0),1)
                        #Update Hygiene Status as the staff is originally cleaned
                        ct.update_hygiene(False,objectID)
                        """
                        #Never enter
                        if(enter == False):
                            ct.update_enter(True,objectID)
                            #If the staff did not wash hand and go to patient directly
                            hand_wash_flag = ct.display_wash(objectID)
                            if(hand_wash_flag == False):
                                ct.update_valid(False,objectID)
                                m = 0
                                match = True
                                #Check whether this ID is marked as fail or not (on the screen)
                                while(m<len(invalid_id)):
                                    if(objectID==invalid_id[m]):
                                        #ObjectID is found in the invalid bank
                                        match = True
                                    else:
                                        match = False
                                    m+=1
                                #If it is not in the bank,then mark it and fail counter + 1
                                if(match == False):
                                    fail_msg +=1
                                    invalid_id.append(objectID)
                        #Enter again with uncleaned => invalid
                        else:
                            if(flag == False):
                                #Re-enter the patient zone
                                if((enter == True)and(leave == True)):
                                    ct.update_valid(False,objectID)
                                    m = 0
                                    match = True
                                    #Check whether this ID is marked as fail or not (on the screen)
                                    while(m<len(invalid_id)):
                                        if(objectID==invalid_id[m]):
                                            #ObjectID is found in the invalid bank
                                            match = True
                                        else:
                                            match = False
                                        m+=1
                                    #If it is not in the bank,then mark it and fail counter + 1
                                    if(match == False):
                                        fail_msg +=1
                                        invalid_id.append(objectID)
                    else:
                        if(enter == True):
                            ct.update_leave(True,objectID)

                    """
                    if((distance_clean <= distance_thres_clean)or(distance_clean_alchol <= distance_thres_alchol)):
                        if(distance_clean <= distance_thres_clean):
                            cv2.line(img,(centroid[0], centroid[1]),(zone_x_clean,zone_y_clean),(51,255,255),1)
                        if(hand_wash_status == 1):
                            personal_status[objectID] = 1
                        #hand_wash_status = 1
                        #Update Hygiene Status
                        ct.update_hygiene(True,objectID)
                        #Reset IN/OUT Mechanism
                        ct.update_enter(False,objectID)
                        ct.update_leave(False,objectID)
                        ct.update_wash(True,objectID)
                    #Return hygiene status
                    flag = ct.display_hygiene(objectID)
                    #if(previous_id!=objectID):
                        #hand_wash_status = 0
                        #personal_status = 0
                    previous_id = objectID

                    with open('./path_analyzer/path_log.csv','a',newline='') as csv_log_file:
                        log_writer=csv.writer(csv_log_file)

                        log_writer.writerow([objectID,centroid[0],centroid[1],int(distance_bed),int(distance_clean),int(distance_clean_alchol),int(hand_wash_status),int(0)])
#log_writer.writerow([objectID,centroid[0],centroid[1],int(distance_bed),int(distance_clean),int(distance_clean_alchol),int(hand_wash_status),int(personal_status[objectID])])
                if(restart_flag == False):
                    if show_fps:
                        img = draw_help_and_fps(img, fps)
                    #img = img[200:721, 160:697]
                    cv2.imshow(WINDOW_NAME, img)
                    cv2.setMouseCallback(WINDOW_NAME,pixel_info)  #Receive mouse click on HSV_Picker
                    toc = time.time()
                    curr_fps = 1.0 / (toc - tic)
                    # calculate an exponentially decaying average of fps number
                    fps = curr_fps if fps == 0.0 else (fps*0.9 + curr_fps*0.1)
                    tic = toc

            else:
                print("None Image  --> None Buff = {}".format(none_buff))
                none_buff+=1
                if(none_buff == 1000):
                    print("[SYSTEM] VSTARCAMERA Restart")
                    cam.stop()  # terminate the sub-thread in camera
                    #tf_sess.close()
                    #tf.reset_default_graph()
                    #tf.contrib.keras.backend.clear_session()
                    cam.release()
                    restart_flag = True
                    none_buff = 0
                    #img = None
                    cv2.destroyAllWindows()

            hand_wash_status = 0
            if(restart_flag == False):
                key = cv2.waitKey(1)
                if key == 27:  # ESC key: quit program
                    break
                elif key == ord('H') or key == ord('h'):  # Toggle help/fps
                    show_fps = not show_fps
                elif key == ord('F') or key == ord('f'):  # Toggle fullscreen
                    full_scrn = not full_scrn
                    set_full_screen(full_scrn)

        #client.loop_start()
        #client.loop_forever()
        client.reconnect()

def main():
    logging.basicConfig(level=logging.INFO)
    logger = logging.getLogger(__name__)
    # Ask tensorflow logger not to propagate logs to parent (which causes
    # duplicated logging)
    logging.getLogger('tensorflow').propagate = False
    global args #restart issue
    args = parse_args()
    logger.info('called with args: %s' % args)

    # build the class (index/name) dictionary from labelmap file
    logger.info('reading label map')
    cls_dict = read_label_map(args.labelmap_file)

    pb_path = './data/{}_trt.pb'.format(args.model)
    log_path = './logs/{}_trt'.format(args.model)
    if args.do_build:
        logger.info('building TRT graph and saving to pb: %s' % pb_path)
        build_trt_pb(args.model, pb_path)

    logger.info('opening camera device/file')
    cam = Camera(args)
    cam.open()
    if not cam.is_opened:
        sys.exit('Failed to open camera!')

    logger.info('loading TRT graph from pb: %s' % pb_path)
    trt_graph = load_trt_pb(pb_path)

    logger.info('starting up TensorFlow session')
    tf_config = tf.ConfigProto()
    tf_config.gpu_options.allow_growth = True
    #tf_sess = tf.Session(config=tf_config, graph=trt_graph) -- Vincent
    #Solve : "unable to satfisfy explicit device /dev/CPU:0 -- Vincent
    tf_sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True,log_device_placement=True),graph=trt_graph)
    if args.do_tensorboard:
        logger.info('writing graph summary to TensorBoard')
        write_graph_tensorboard(tf_sess, log_path)

    logger.info('warming up the TRT graph with a dummy image')
    od_type = 'faster_rcnn' if 'faster_rcnn' in args.model else 'ssd'
    dummy_img = np.zeros((720, 1280, 3), dtype=np.uint8)
    _, _, _ = detect(dummy_img, tf_sess, conf_th=.3, od_type=od_type)

    cam.start()  # ask the camera to start grabbing images
    # grab image and do object detection (until stopped by user)
    logger.info('starting to loop and detect')
    vis = BBoxVisualization(cls_dict)
    open_display_window(cam.img_height, cam.img_width)
    #<<----------------------------------Vincent---------------------------------->>#
    print("[INFO] Version: HT-002")                        #Vincent
    print("[INFO]Screen Dimensions (H,W) =",cam.img_height,"x",cam.img_width)
    #Low resolution RTSP: 360 x 640
    #<<----------------------------------Vincent---------------------------------->>#
    result=loop_and_detect(cam, tf_sess, args.conf_th, vis, od_type=od_type)
    logger.info('cleaning up')
    cam.stop()  # terminate the sub-thread in camera
    tf_sess.close()
    cam.release()
    cv2.destroyAllWindows()

if __name__ == '__main__':
    main()


from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.externals import joblib
from sklearn.metrics import classification_report
from sklearn.svm import SVC
import pandas as pd
import numpy as np
from numpy.random import shuffle
import os
import sys
import argparse
class train_svm():

    def __init__(self, keypoints_path, npsave_path, command, model_path):

        self.keypoints_folder = keypoints_path
        self.command = command
        self.model_path = model_path
        files = os.listdir(self.keypoints_folder)
        first_time = True
        for filename in files:
            self.keypoints_path = self.keypoints_folder + filename + '/keypoints.txt'
            self.filename = filename
            self.npsave_path = self.keypoints_folder + filename + '/np_keypoints.txt'
            Feature, self.lenx, self.leny = self.feature_generate()
            Label = self.label_generate()
            if first_time == True:
                self.feature = Feature
                self.label = Label
                first_time = False
            else:
                self.feature = np.insert(self.feature, 0, Feature, axis=0)
                self.label = np.insert(self.label, 0, Label, axis=0)
            #print(self.feature[array])
            #print(type(self.feature))
        np.savetxt('/home/hts/hts2019/openpose/svm/feature.txt', self.feature, fmt = '%f')
        print('---loading feature done!---')
        print('feature sample size', np.shape(self.feature) )
        print()
            #self.label.append(self.label.generate())
        np.savetxt('/home/hts/hts2019/openpose/svm/label.txt', self.label, fmt = '%f')
        print('---loading label done!---')
        print('label size', np.shape(self.label))
        print()
        self.keyboard_control()

    def feature_generate(self):

        with open(self.keypoints_path, 'r') as file:
            my_data = file.readlines()
            count = 0
            array1 = np.zeros(33)
            array2 = np.zeros((1,33))
            for line in my_data:
                #print(line)
                count += 1
                if count == 34:
                    count = 0
                    #print('ok')
                    #print(array1)
                    array2 = np.insert(array2, 0, array1, axis=0)
                else:
                    array1[count-1] = line
                    #print(array1[count-1])
            #print(array2)
            array2 = np.delete(array2, -1, axis=0)
            #print(array2)
            axisx = np.shape(array2)[0]
            axisy = np.shape(array2)[1]
            #print(axisx, axisy)
            np.savetxt(self.npsave_path, array2, fmt = '%d')
            return array2, axisx, axisy

    def label_generate(self):

        array3 = np.empty((self.lenx, 1) )
        for index in range(self.lenx):
            if self.filename == 'a':
                array3[index] = 1
            if self.filename == 'b':
                array3[index] = 2
            if self.filename == 'c':
                array3[index] = 3
            if self.filename == 'd':
                array3[index] = 4
            if self.filename == 'e':
                array3[index] = 5
        #print(array3)
        return array3

    def svm_training(self):

        print('---start training svm---')
        #split dataset in two equal parts
        #print(np.shape(self.feature), np.shape(self.label))
        X_train, X_test, Y_train, Y_test = train_test_split(self.feature, self.label, test_size = 0.5, random_state = 0)
        np.savetxt('/home/hts/hts2019/openpose/svm/X_train.txt', X_train, fmt = '%f')
        np.savetxt('/home/hts/hts2019/openpose/svm/X_test.txt', X_test, fmt = '%f')
        np.savetxt('/home/hts/hts2019/openpose/svm/Y_train.txt', Y_train, fmt = '%d')
        np.savetxt('/home/hts/hts2019/openpose/svm/Y_test.txt', Y_test, fmt = '%d')
        print('---split data done!---')
        print()
        #set the parameters by cross-validation
        tuned_parameters = [{'kernel': ['rbf'], 'gamma': np.logspace(-1, 2, 4, endpoint = True),
                             'C': np.logspace(-1, 2, 4, endpoint = True)},
                             {'kernel': ['linear'],'C': np.logspace(-1, 2, 4, endpoint = True)},
                             {'kernel': ['poly'], 'C': np.logspace(-1, 2, 4, endpoint = True), 'degree': [1, 2, 3], 'gamma': np.logspace(-1, 2, 4, endpoint = True)}]
        scores = ['precision', 'recall']

        for score in scores:

            print ('tuning hyper-parameters for %s' % score)
            clf = GridSearchCV(SVC(decision_function_shape='ovr'), tuned_parameters, cv = 5, scoring = '%s_weighted' % score)
            print(np.shape(X_train), np.shape(Y_train.ravel()))
            clf.fit(X_train, Y_train.ravel())

        print("Best parameters set found on development set:")
        print()
        print(clf.best_params_)
        print()
        print("Grid scores on development set:")
        print()
        means = clf.cv_results_['mean_test_score']
        stds = clf.cv_results_['std_test_score']
        for mean, std, params in zip(means, stds, clf.cv_results_['params']):
            print("%0.3f (+/-%0.03f) for %r"
                % (mean, std * 2, params))
        # for params, mean_score, scores in clf.cv_results_:
        #     print("%0.3f (+/-%0.03f) for %r"
        #         % (mean_score, scores.std() * 2, params))
        print()

        print("Detailed classification report:")
        print()
        print("The model is trained on the full development set.")
        print("The scores are computed on the full evaluation set.")
        print()
        target_names = ['lying', 'lie on the side', 'sitting', 'standing']
        Y_true, Y_pred = Y_test.ravel(), clf.predict(X_test)
        np.savetxt('/home/hts/hts2019/openpose/svm/Y_true.txt', Y_true, fmt = '%d')
        np.savetxt('/home/hts/hts2019/openpose/svm/Y_pred.txt', Y_pred, fmt = '%d')
        print(classification_report(Y_true, Y_pred, target_names = target_names))
        print()

        print('SVM model saving ......')
        model_save_path = '/home/hts/hts2019/openpose/svm/train_madel.m'
        joblib.dump(clf, model_save_path)

        return

    def predict_pic(self):

        my_clf = joblib.load(self.model_path)
        my_pre = my_clf.predict(self.feature)
        target_names = ['lying', 'lie on the side', 'sitting', 'standing']
        #print(my_pre, target_names[my_pre])
        print(my_pre)

        return

    def predict_video(self):

        my_clf = joblib.load(self.model_path)
        print("Detailed classification report:")
        print()
        target_names = ['lying', 'lie on the side', 'sitting', 'standing']
        Y_true, Y_pred = self.label.ravel(), my_clf.predict(self.feature)
        np.savetxt('/home/hts/hts2019/openpose/svm/Y_true.txt', Y_true, fmt = '%d')
        np.savetxt('/home/hts/hts2019/openpose/svm/Y_pred.txt', Y_pred, fmt = '%d')
        print(classification_report(Y_true, Y_pred, target_names = target_names))

        return

    def keyboard_control(self):

        command = self.command
        try:
            if command == 'predict_pic':
                self.predict_pic()
            elif command == 'svm_training':
                self.svm_training()
            elif command == 'predict_video':
                self.predict_video()
            else:
                print("---Invalid Command!---")
        except Exception as e:
            print(e)

# read_data('/home/hts/hts2019/openpose/video_res/Res_keypoints/a/keypoints.txt')
if __name__ == "__main__":

    parser = argparse.ArgumentParser()
    parser.add_argument("--keypoints_path", type = str, default = None,
                        help = "path to the folder for the json to be processed (default: None)")
    parser.add_argument("--npsave_path", type = str, default = None,
                        help = "path to the folder for the results to be saved (default: None)")
    parser.add_argument("--command", type = str, default = 'svm_training',
                        help = "next command(default: 'svm_training')")
    parser.add_argument("--model_path", type = str, default = None,
                        help="path to the model to be loaded (default: None)")
    # parser.add_argument("--label", type=str, default=None,
    #                     help="label (default: None)")
    args = parser.parse_args()
    keypoints_path = args.keypoints_path
    npsave_path = args.npsave_path
    command = args.command
    model_path = args.model_path
    # label = args.label
    train_svm(keypoints_path, npsave_path, command, model_path)


from django.shortcuts import render
from .models import frb_name
# Create your views here.
from django.template import loader
from django.http import HttpResponse,Http404, HttpRequest
import json
from candidates.models import frb_name,add_info,Verification
from django.utils import timezone
def index(request):
    latest_question_list = frb_name.objects.order_by('-pub_date')[:100]
    template = loader.get_template('candidates/index.html')
    context = {
        'FRB_info': latest_question_list,
    }
    return HttpResponse(template.render(context, request))

def detail(request, question_id):
    try:
        question = frb_name.objects.get(pk=question_id)
    except question.DoesNotExist:
        raise Http404("Index out of range")
    verify=question.verification_set.order_by('-pub_date')[:1]
    return render(request, 'candidates/detail.html', {'question': question,'latest_verify':verify})

def info(request, question_id):
    try:
        question = frb_name.objects.get(pk=question_id)
    except frb_name.DoesNotExist:
        raise Http404("Index out of range")
    template = loader.get_template('candidates/info.html')
    context={'question':question}
    return HttpResponse(template.render(context, request))

def add_candidate(request):
    if 'application/json' in request.META['CONTENT_TYPE']:
        #print ('hi')
        data = json.loads(request.body)
        #frb = data.get('frb_name', None)
        print("received",data["pub_date"])
        # newfrb = {'name':'test',
        #     'sn':37.1,
        #     'dm':1342,
        #     'boxcar':1,
        #     'latency':400.,
        #     'pub_date':str(datetime.datetime.now(pytz.UTC))[:-6],
        #     "beam":0,
        #     "mjd":500,
        #     }
        newfrb=frb_name(frb_name=data["name"],
        pub_date=timezone.now(),
        beam=data["beam"],dm_value=data['dm'],sn_value=data['sn'],width_value=data['boxcar'],
        mjd=data['mjd'],latency_ms=data['latency'])#,latestverify=''
        #frb_name.objects.create()
        newfrb.save()
        return HttpResponse("Success")

def veto(request):
    if 'application/x-www-form-urlencoded' in request.META['CONTENT_TYPE']:
        data = json.loads(request.body)
        #frb = data.get('frb_name', None)
        print("received",data["pub_date"])
        """ newfrb = {'name':'test',
            'sn':37.1,
            'dm':1342,
            'boxcar':1,
            'latency':400.,
            'pub_date':str(datetime.datetime.now(pytz.UTC))[:-6],
            "beam":0,
            "mjd":500,
            }"""
        #frb_name.objects.create()
        return HttpResponse("vetoed")

#def slackresponses(request):