from fhirclient import client
import json
import requests
import dateutil.parser
import requests
import datetime
import pytz
import fhirclient.models.patient as p
import fhirclient.models.bundle as b
import fhirclient.models.procedure as prod
import fhirclient.models.observation as ob
import fhirclient.models.condition as con
import fhirclient.models.encounter as enc
import csv

settings = {
    'app_id': 'my_web_app',
    'api_base': 'https://api-v5-stu3.hspconsortium.org/DBMIBC/open/'
}
smart = client.FHIRClient(settings=settings)

pid = "cf-1522104153916"
patient = p.Patient.read(pid, smart.server)
print(patient)

bundle = requests.get('https://api-v5-stu3.hspconsortium.org/DBMIBC/open/Patient?birthdate>=2018-01-01')  # Not working
data = json.loads(bundle.text)
print(data)

CODES = {}
CODES['weight'] = {'system': "http://loinc.org", "code": "LP18015-5"}
CODES['height'] = {'system': "http://loinc.org", "code": "8302-2"}
CODES['birthWeigth'] = {'system': "http://loinc.org", "code": "8339-4"}
CODES['apgarScore'] = {'system': "http://loinc.org", "code": "9274-2"}

def getNextURL(links):
    """
    Helper function for getAllPatients, get the next url so we can loop to get all resources
    :param links:
    :return:
    """
    if type(links) == list:
        for link in links:
            if link.relation == 'next':
                return link.url
    return None

def getAllRecentBabies(server=smart.server, targetnumber=1000, bornInDays=500):
    bundle = b.Bundle.read_from('Patient', server)
    targetGap = datetime.timedelta(days=bornInDays)
    res = []
    while len(res) < targetnumber:
        if bundle is None or bundle.entry is None:
            return res
        for entry in bundle.entry:
            pt = entry.resource
            if pt.name is None:
                name = "Name None"
            else:
                name = pt.name[0].as_json()
            if pt.birthDate is None:
                dob = "DOB None"
            else:
                dob = pt.birthDate.date

            print(len(res), pt.id, name, dob)
            if dob == 'DOB None':
                continue

            if type(dob) == datetime.date:
                gap = datetime.date.today() - dob
                #stop =  > targetGap
            if type(dob) == datetime.datetime:
                gap =datetime.datetime.now(tz=pytz.utc) - dob  # Make now as a datetime timezone-aware,
                #stop =  > targetGap
            #print("stop is ", stop)

            if gap > targetGap:
                continue

            print("Add patient: ", name, dob)

            res.append(pt)

        nexturl = getNextURL(bundle.link)
        if nexturl is None:
            return res
        data = requests.get(nexturl)     # request from a URL, not by fhirclient
        if data is None or data.text is None:
            return res
        bundle = b.Bundle(jsondict=json.loads(data.text))

def getRecentBabies(bornInDays=10, server=smart.server):
    today = datetime.date.today()
    targetDay = today - datetime.timedelta(days=bornInDays)
    # default count per page is 10 and the maximum count is 50 to prevent system overload by the FHIR server.
    # If there are more than 50 babies born, special treatment is need to read page by page.
    search = p.Patient.where(struct={'birthdate': ">%s" % targetDay, '_count': '50', '_sort': '-birthdate'})
    patients = search.perform_resources(smart.server)
    print(len(patients))
    return patients

CODEFILE = "FHIR_resource_codes_1.txt"
def getCODETABLE(file=CODEFILE, page=''):
    codeTable = {}
    with open(file) as f:
        reader = csv.DictReader(f, delimiter='\t')
        if page == '':
            for row in reader:
                if row['page'] != '':
                    codeTable[row['name']] = row
        else:
            for row in reader:
                if row['page'] == page:
                    codeTable[row['name']] = row

    return codeTable
CODETABLE = getCODETABLE(CODEFILE)

def getMedicalInfo(resource, type, codes):
    r_code = resource.code.coding
    for code in codes:
        c = codes[code]
        if c['resource'] != type:
            continue
        #print("try: ", c)
        for rc in r_code:
            #print(rc.as_json())
            if rc.code == c['code'] and rc.system == c['system']:
                #print (c['name'], resource.as_json())
                return (c['name'], resource.as_json())
    return (None, None)

def getPatientMedical(pid, server=smart.server, codes=CODETABLE):
    medInfo = {}
    search = p.Patient.where(struct={'_id': pid})
    patients = search.perform_resources(server)
    if len(patients) > 1:
        print("Too many patients")
    if len(patients) < 1:
        print("no patient!")
        return (None, medInfo)
    pt = patients[0]
    search = ob.Observation.where(struct={'subject': pid})
    conditions = search.perform_resources(smart.server)
    #print(codes[''])
    for condition in conditions:
        name, res = getMedicalInfo(condition, 'obs', codes)
        if name != None:
           medInfo[name] = res
    search = con.Condition.where(struct={'subject': pid})
    conditions = search.perform_resources(smart.server)
    #print(codes[''])
    for condition in conditions:
        name, res = getMedicalInfo(condition, 'condition', codes)
        if name != None:
           medInfo[name] = res
    return pt, medInfo

def getMedInfoValue(medInfo, name):
    if name not in medInfo:
        return None, None
    info = medInfo[name]
    #print(info)
    if info['resourceType'] == 'Observation':
        try:
            value = info['valueQuantity']['value']
        except:
            value = None
        try:
            unit = info['valueQuantity']['unit']
        except:
            unit = None
        return value, unit
    else:
        return True, None

def getMotherID(pid):
    """
    From a child's pid, search its birth Encounter which is a part of the mother's encount,
    find the mother's pid
    :param pid:
    :return:
    """
    search = enc.Encounter.where(struct={'subject': pid})
    encounters = search.perform_resources(smart.server)
    motherEncounter = None
    try:
        for encounter in encounters:
            if encounter.reason[0].coding[0].code == '169836001' \
                    and encounter.reason[0].coding[0].system == 'http://snomed.info/sct' \
                    and encounter.partOf != None:
                # print(encounter.partOf.reference.split('/')[1])
                motherEncounter = encounter.partOf.reference.split('/')[1]
    except:
        motherEncounter = None
    if motherEncounter == None:
        return None
    search = enc.Encounter.where(struct={'_id': motherEncounter})
    encounters = search.perform_resources(smart.server)
    motherID = encounters[0].subject.reference.split('/')[1]
    return motherID

#today = datetime.date.today()
#print(today)
#yesterday = today - datetime.timedelta(days=30)
#print(yesterday)
#res = getRecentBabies()
#res2 = getAllRecentBabies()
#print(res[0].as_json())
#print(res2[0].as_json())

#newbabies = getAllRecentBabies(smart.server, 100, 10)

#for baby in newbabies:
#    print ("%s, %s" %(baby.name[0].given[0].encode('utf-8'), baby.name[0].family.encode('utf-8')))
#    print(baby.birthDate.date)


"""empty message

Revision ID: 072139cf913e
Revises: 4d0ebbee0261
Create Date: 2020-08-13 19:36:00.204569

"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql

# revision identifiers, used by Alembic.
revision = '072139cf913e'
down_revision = '4d0ebbee0261'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('paychecks',
    sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False),
    sa.Column('pay_date', sa.Date(), nullable=False),
    sa.Column('gross', sa.Float(precision=2), nullable=False),
    sa.Column('net', sa.Float(precision=2), nullable=False),
    sa.Column('user_email', sa.Text(), nullable=True),
    sa.ForeignKeyConstraint(['user_email'], ['users.email'], ),
    sa.PrimaryKeyConstraint('id')
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('paychecks')
    # ### end Alembic commands ###


import matplotlib.pyplot as plt
import numpy as np

#x = np.random.normal(size = 1000)
#plt.hist(x, density=True, bins=30) # density
#plt.ylabel('Probability')
#plt.show()

x = np.arange(2)
plt.title("Primary Key constraint v2 performance")
barlist = plt.bar(x, height= [150,18])
plt.xticks(x, ['success','invalid'])

plt.ylabel(ylabel = "Time (ms)")
plt.xlabel(xlabel = "Result of the check")

barlist[0].set_color('g')
barlist[1].set_color('r')
plt.show()


# Copyright (c) 2020  PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

from ...registry import RECOGNIZERS
from .base import BaseRecognizer
import paddle
from paddlevideo.utils import get_logger

logger = get_logger("paddlevideo")

@RECOGNIZERS.register()
class Recognizer2D(BaseRecognizer):
    """2D recognizer model framework."""

    def forward_train(self, imgs, labels, reduce_sum, **kwargs):
        """Define how the model is going to train, from input to output.
        """
        # As the num_segs is an attribute of dataset phase, and didn't pass to build_head phase, should obtain it from imgs(paddle.Tensor) now, then call self.head method.

        batches = imgs.shape[0]
        imgs = paddle.reshape(imgs, [-1]+imgs.shape[2:])
        num_segs = imgs.shape[0] // batches
        feature = self.extract_feature(imgs)
        cls_score = self.head(feature, num_segs)
        #labels = labels.squeeze()
        #XXX: unsqueeze label to [label] ?
        loss_metrics = self.head.loss(cls_score, labels, reduce_sum, **kwargs)
        return loss_metrics

    def forward_valid(self, imgs):
        """Define how the model is going to valid, from input to output."""
        #XXX add testing code.
        imgs = paddle.reshape(imgs, [-1]+imgs.shape[2:])
        num_segs = imgs.shape[0] // batches
        feature = self.extract_feature(imgs)
        cls_score = self.head(feature, num_segs)

        return cls_score


#!/usr/bin/env python
#-*- coding:utf-8 -*-

import os.path, collections, time
from PyQt4 import QtGui, QtCore, QtWebKit, QtNetwork

allowedExtensions = ['.html', '.css', '.js', '.xhtml', '.dhtml']

class CookieJar(QtNetwork.QNetworkCookieJar):
    def __init__(self, ctr):
        self.ctr = ctr
        self.init = False
        self.cookieStorage = []
        QtNetwork.QNetworkCookieJar.__init__(self)
        self.time = time.time()
        self.updateCookie()
        self.initCookie()
        self.init = True

    def initCookie(self):
        self.setAllCookies([QtNetwork.QNetworkCookie.parseCookies(c)[0] for c in self.cookieStorage])

    def readCookie(self):
        return [c.toRawForm().data().decode() for c in self.allCookies()]

    def saveCookie(self):
        self.cookieStorage = [c.toRawForm().data().decode() for c in self.allCookies()]

    def getSyncCookie(self):
        return self.ctr.getSyncCookie()

    def updateCookie(self):
        if time.time() - self.time < 5 and self.init is True:
            return

        syncCookie = self.getSyncCookie()
        selfCookieMap = {v.split(';')[0].split('=')[0]: v for v in
                         [c.toRawForm().data().decode() for c in self.allCookies()] if v is not None}

        selfCookieMap.update(syncCookie)
        self.cookieStorage = selfCookieMap.values()
        self.initCookie()
        self.time = time.time()

class NetworkAccessManager(QtNetwork.QNetworkAccessManager):
    def __init__(self, ctr, allowedExtensions, cache_size=100):
        self.ctr = ctr
        QtNetwork.QNetworkAccessManager.__init__(self)

        self.cookie = CookieJar(self)
        self.setCookieJar(self.cookie)

        # initialize the manager cache
        cache = QtNetwork.QNetworkDiskCache()
        #QDesktopServices.storageLocation(QDesktopServices.CacheLocation)
        cache.setCacheDirectory('webCache')

        # need to convert cache value to bytes
        cache.setMaximumCacheSize(cache_size * 1024 * 1024)
        self.setCache(cache)

        self.allowedExtensions = allowedExtensions

        self.connect(self, QtCore.SIGNAL('sslErrors(QNetworkReply*, const QList<QSslError>&)'), self.sslErrors)

    def sslErrors(self, reply, errors):
        '''
        requiredCert = reply.sslConfiguration().peerCertificate()
        defaultConfig = QtNetwork.QSslConfiguration.defaultConfiguration()
        caList = defaultConfig.caCertificates()
        caList.append(requiredCert)
        defaultConfig.setCaCertificates(caList)
        QtNetwork.QSslConfiguration.setDefaultConfiguration(defaultConfig)
        reply.setSslConfiguration(defaultConfig)'''
        reply.ignoreSslErrors()

    def createRequest(self, operation, request, data):
        self.cookie.updateCookie()

        if data is not None:
            print('******')
            print(data)

        #operation == self.GetOperation:
        request.setAttribute(QtNetwork.QNetworkRequest.CacheLoadControlAttribute, QtNetwork.QNetworkRequest.PreferCache)

        request = self.bookTicket(request)

        '''
        for i in (dir(request)):
            print(i)

        rawHeaderList = request.rawHeaderList()
        for i in rawHeaderList:
            print(request.rawHeader(i))
        '''

        reply = QtNetwork.QNetworkAccessManager.createRequest(self, operation, request, data)
        reply.error.connect(self.catchError)

        #reply.downloadProgress.connect(self.progress)
        reply.finished.connect(self.finished)

        return reply

    def finished(self):
        self.ctr.loadFinished()

    def bookTicket(self, request):

        if (request.url().toString().startswith(
                'https://dynamic.12306.cn/otsweb/order/querySingleAction.do?method=submutOrderRequest')
            or request.url().toString().startswith('http://127.0.0.1/env.php')):
            print('match')
            request.setRawHeader('Host', 'dynamic.12306.cn')
            request.setRawHeader('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8')

            request.setRawHeader('Referer', 'https://dynamic.12306.cn/otsweb/order/querySingleAction.do?method=init')
            request.setRawHeader('User-Agent', 'Mozilla/5.0 (MSIE 9.0 Windows NT 6.1 Trident/5.0)')
            request.setRawHeader('Connection', 'keep-alive')
            request.setRawHeader('Content-Type', 'application/x-www-form-urlencoded')
        '''
(Request-Line)	POST /otsweb/order/querySingleAction.do?method=submutOrderRequest HTTP/1.1
Host	dynamic.12306.cn
User-Agent	Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:21.0) Gecko/20100101 Firefox/21.0
Accept	text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
Accept-Language	zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3
Accept-Encoding	gzip, deflate
Referer	https://dynamic.12306.cn/otsweb/order/querySingleAction.do?method=init
Cookie	JSESSIONID=22FF47CF0FFEAC7FBD65DB984C7C836F; BIGipServerotsweb=2446590218.48160.0000
Connection	keep-alive
Content-Type	application/x-www-form-urlencoded
Content-Length	737

        '''
        return request

    def isForbidden(self, request):
        extensions = os.path.splitext(request.url().toString())

        if extensions is not None and extensions[1].lower() in self.allowedExtensions:
            return True
        else:
            return False

    def catchError(self, eid):
        if eid not in (301, ):
            print('Error:', eid, self.sender().url().toString())

    def readCookie(self):
        self.cookie.readCookie()

    def closeEvent(self):
        self.cookie.saveCookie()

    def getSyncCookie(self):
        return self.ctr.getSyncCookie()

class WebPage(QtWebKit.QWebPage):
    def __init__(self):
        QtWebKit.QWebPage.__init__(self)
        self.userAgent = 'Mozilla/5.0 (MSIE 9.0 Windows NT 6.1 Trident/5.0)'

    def userAgentForUrl(self, url):
        return self.userAgent

    def javaScriptAlert(self, frame, message):
        #pass
        print('Alert:', message)

    def javaScriptConsoleMessage(self, message, line_number, source_id):
        pass
        #print('Console:', message, line_number, source_id)

class WebView(QtWebKit.QWebView):
    _windows = set()

    def __init__(self, ctr, parent=None):

        '''
        connect(ui->webView,SIGNAL(linkClicked(QUrl)),this,SLOT(lick(QUrl)));//连接信号和槽
ui->webView->page()->setLinkDelegationPolicy(QWebPage::DelegateAllLinks);//当一个链接被激活
ui.webView->page()->mainFrame()->evaluateJavaScript("Test()");
        '''
        super(WebView, self).__init__(parent)
        self.ctr = ctr

        self.settings().setAttribute(QtWebKit.QWebSettings.DeveloperExtrasEnabled, True)
        self.settings().setAttribute(QtWebKit.QWebSettings.JavascriptEnabled, True)
        self.settings().setAttribute(QtWebKit.QWebSettings.JavascriptCanOpenWindows, True)
        self.settings().setAttribute(QtWebKit.QWebSettings.PluginsEnabled, True)

        webPage = WebPage()
        self.manager = NetworkAccessManager(self, allowedExtensions)
        webPage.setNetworkAccessManager(self.manager)
        self.setPage(webPage)
        self.connect(self, QtCore.SIGNAL('linkClicked(QUrl)'), lambda x: self.require(x))
        self.page().setLinkDelegationPolicy(QtWebKit.QWebPage.DelegateAllLinks)

        #self.require('http://127.0.0.1/env.php')
        #self.load(QtCore.QUrl('http://www.12306.cn/mormhweb/'))

    def loadFinished(self):
        self.page().mainFrame().evaluateJavaScript("test()");
        print('loadFinished')

    def closeEvent(self, event):
        self.manager.closeEvent()
        #print(self.manager.readCookie())
        self._removeWindow(self)
        event.accept()

    def createWindow(self, webWindowType):
        window = self.newWindow()
        if webWindowType == QtWebKit.QWebPage.WebModalDialog:
            window.setWindowModality(QtCore.Qt.ApplicationModal)
        window.show()
        return window

    def getSyncCookie(self):
        return self.ctr.getSyncCookie()

    def require(self, url):
        self.load(QtCore.QUrl(url))

    def post(self, data):
        if data is None or len(data) < 0:
            return
        formData = ''
        for i, v in data.items():
            if not i.lower() == 'url':
                formData += '<input type="text" name="%s" value="%s" />' % (i, v)

        form = '<form action="%s" method="post" name="postForm">%s<input type="submit" value="submit"></form>' % (
            data['url'], formData)
        #script = ''
        script = ''
        html = '%s%s' % (form, script)

        self.setHtml(html)

    def bookTicket(self, ticketData, condition):

        url = 'http://127.0.0.1/env.php'
        #url = 'https://dynamic.12306.cn/otsweb/order/querySingleAction.do?method=submutOrderRequest'

        data = ticketData.split("#")
        query = collections.OrderedDict()

        query['station_train_code'] = data[0]
        query['train_date'] = condition["orderRequest.train_date"]
        query['seattype_num'] = condition['seatTypeAndNum']
        query['from_station_telecode'] = data[4]
        query['to_station_telecode'] = data[5]
        query['include_student'] = condition['includeStudent']
        query['from_station_telecode_name'] = condition['orderRequest.from_station_name']
        query['to_station_telecode_name'] = condition['orderRequest.to_station_name']
        query['round_train_date'] = condition["roundTrainDate"]
        query['round_start_time_str'] = condition['roundStartTimeStr']
        query['single_round_type'] = condition['singleRoundType']
        query['train_pass_type'] = condition['trainPassType']
        query['train_class_arr'] = condition['trainClassArr']
        query['start_time_arr'] = condition['orderRequest.start_time_str']
        query['lishi'] = data[1]
        query['train_start_time'] = data[2]
        query['trainno4'] = data[3]
        query['arrive_time'] = data[6]
        query['from_station_name'] = data[7]
        query['to_station_name'] = data[8]
        query['from_station_no'] = data[9]
        query['to_station_no'] = data[10]
        query['ypInfoDetail'] = data[11]
        query['mmStr'] = data[12]
        query['locationCode'] = data[13]
        query['url'] = url

        self.post(query)

class SimpleBrowsers(QtGui.QDialog):
    def __init__(self, ctr, parent=None):
        super(SimpleBrowsers, self).__init__(parent)
        self.ctr = ctr
        self.webView = WebView(self)

        page = self.webView.page()
        #page.settings().setAttribute(QtWebKit.QWebSettings.JavascriptEnabled, False)
        page.settings().setAttribute(QtWebKit.QWebSettings.PluginsEnabled, True)
        page.settings().setAttribute(QtWebKit.QWebSettings.DeveloperExtrasEnabled, True)
        self.webInspector = QtWebKit.QWebInspector(self)
        self.webInspector.setPage(page)

        shortcut = QtGui.QShortcut(self)
        shortcut.setKey(QtCore.Qt.Key_F12)
        shortcut.activated.connect(self.toggleInspector)
        self.webInspector.setVisible(False)

        self.splitter = QtGui.QSplitter(self)
        self.splitter.setOrientation(QtCore.Qt.Vertical)

        self.verticalLayout = QtGui.QVBoxLayout()
        self.verticalLayout.addWidget(self.splitter)

        self.splitter.addWidget(self.webView)
        self.splitter.addWidget(self.webInspector)

        self.setLayout(self.verticalLayout)

    def closeEvent(self, event):
        self.webView.closeEvent(event)
        event.accept()

    def getSyncCookie(self):
        return self.ctr.getSyncCookie()

    def bookTicket(self, ticketData, condition):
        self.webView.bookTicket(ticketData, condition)

    def post(self, data):
        self.webView.post(data)

    def toggleInspector(self):
        self.webInspector.setVisible(not self.webInspector.isVisible())

class Ctr:
    def __init__(self):
        self.ticketData = 'K528#05:45#08:00#630000K5280C#GZQ#HYQ#13:45#广州#衡阳#01#05#1*****30124*****00081*****01023*****0028#A86124DD6525AE6502C2860A53F45662115F625406DEA2133AEA577F#Q6'
        self.condition = {'trainPassType': 'QB', 'orderRequest.to_station_name': '衡阳', 'orderRequest.trainCodeText': '',
                          'orderRequest.from_station_telecode': 'GZQ', 'roundStartTimeStr': '00:00--24:00',
                          'orderRequest.from_station_name': '广州', 'singleRoundType': 1,
                          'orderRequest.start_time_str': '00:00--24:00', 'trainClassArr': 'QB#D#Z#T#K#QT#',
                          'orderRequest.to_station_telecode': 'HYQ', 'roundTrainDate': '2013-06-27',
                          'orderRequest.train_date': '2013-06-27', 'seatTypeAndNum': '', 'includeStudent': '00'}

    '''
{'station_train_code':	'G101',
'train_date	:'2013-06-26',
'seattype_num':'',
'from_station_telecode'	:'VNP',
'to_station_telecode':	'AOH',
'include_student'	:'00',
'from_station_telecode_name':	'åäº¬',
'to_station_telecode_name'	:'ä¸æµ·',
'round_train_date'	:'2013-06-26',
'round_start_time_str'	:'00:00--24:00',
'single_round_type'	:'1',
'train_pass_type':'QB',
'train_class_arr':	'QB#D#Z#T#K#QT#',
'start_time_str':	'00:00--24:00',
'lishi'	:'05:23',
'train_start_time':	'07:00',
'trainno4'	:'240000G10102',
'arrive_time'	:'12:23',
'from_station_name' :	'åäº¬å',
'to_station_name'	:'ä¸æµ·è¹æ¡¥',
'from_station_no'	:'01',
'to_station_no'	:'08',
'ypInfoDetail'	:'O*****0216M*****01219*****0021',
'mmStr'	:'0C18275B4E6471D2D28186C83D976EDB16A862912EA6B33F9A4775D0',
'locationCode'	:'P3'}
       '''

    def getSyncCookie(self):
        return {'JSESSIONID': 'JSESSIONID=6538C6EA7DB3ECFC1D3EE8BAF6301896; domain=dynamic.12306.cn; path=/otsweb',
                'BIGipServerotsweb': 'BIGipServerotsweb=2111045898.62495.0000; domain=dynamic.12306.cn; path=/'}

'''
if __name__ == '__main__':
    import sys

    app = QtGui.QApplication(sys.argv)
    c = Ctr()
    b = SimpleBrowsers(c)
    b.bookTicket(c.ticketData, c.condition)
    b.show()
    app.exec_()'''


# -*- coding: UTF-8 -*-

import tensorflow as tf
import numpy as np
from numpy.linalg import cholesky
import matplotlib.pyplot as plt

def get_data(mean, variance, number):
    '''
    生成二维正态分布的采样
    :param mean: 均值
    :param variance: 协方差矩阵
    :param number: 采样数
    :return: 采样点的集合
    '''
    R = cholesky(variance)
    s = np.dot(np.random.randn(number, 2), R) + mean
    return s

def show_data(s1, s2):
    plt.subplot(111)
    # 注意绘制的是散点图，而不是直方图
    plt.plot(s1[:, 0], s1[:, 1], '+', color='blue')
    plt.plot(s2[:, 0], s2[:, 1], 'o', color='purple')
    plt.show()

def show_data_with_line(data, flag, w, b):
    '''

    :param data: 二元组的集合，表示坐标
    :param flag: 与data对应的标签
    :param w: 表示一条直线
    :param b: 表示一条直线
    :return: no return
    '''
    plt.subplot(111)

    i = 0
    for item in data:
        if(flag[i][0] == 1.):
            plt.plot(item[0], item[1], '+', color='blue')
        else:
            plt.plot(item[0], item[1], '*', color='green')
        i += 1

    x = np.linspace(0, 10, 1000)
    y = (-w[0] * x - b)/w[1]
    plt.plot(x, y, color='red')
    plt.show()

def shuffle_data(arr1, arr2):
    '''
    将arr1与arr2合并，并且打乱顺序
    :param arr1:
    :param arr2:
    :return:
    '''
    tt = np.concatenate((arr1, arr2))
    np.random.shuffle(tt)
    return tt

def save_array_to_file(arr1, arr2):
    '''
    将arr1和arr2保存为文件
    :param arr1:
    :param arr2:
    :return:
    '''
    np.save('./test_x.npy', arr1)
    np.save('./test_y.npy', arr2)

def generate_data(number_train, number_validation, number_test):
    '''
    生成实验用的数据
    :param number_train:
    :param number_validation:
    :param number_test:
    :return:
    '''
    d1 = get_data([[6, 3]], [[1, 0], [0, 1]], number_train + number_validation + number_test)
    d2 = get_data([[3, 6]], [[1, 0], [0, 1]], number_train + number_validation + number_test)
    # 两个tmp数组包含标签信息
    tmp1 = np.zeros(shape=(number_train, 3))
    tmp2 = np.zeros(shape=(number_train, 3))

    i = 0
    for item in d1[0:number_train]:
        tmp1[i] = np.concatenate((item, [1]))
        i += 1
    i = 0
    for item in d2[0:number_train]:
        tmp2[i] = np.concatenate((item, [0]))
        i += 1

    tmp = shuffle_data(tmp1, tmp2)
    # 训练数据
    data_for_train = tmp[:, 0:2]
    # 训练数据对应的标签
    flag_for_train = tmp[:, 2:3]

    validation_data_A = d1[number_train: number_train + number_validation]
    test_data_A = d1[number_train + number_validation: ]

    validation_data_B = d2[number_train: number_train + number_validation]
    test_data_B = d2[number_train + number_validation:]

    # 验证用的数据
    data_for_val = np.concatenate((validation_data_A, validation_data_B))
    # 验证用的数据对应的标签
    flag_for_val = []
    for i in range(number_validation):
        flag_for_val.append([1.])
    for i in range(number_validation):
        flag_for_val.append([0.])
    flag_for_train = np.array(flag_for_train)

    # 测试用的数据
    data_for_test = np.concatenate((test_data_A, test_data_B))
    # 测试用的数据对应的标签
    flag_for_test = []
    for i in range(number_test):
        flag_for_test.append([1.])
    for i in range(number_test):
        flag_for_test.append([0.])
    flag_for_train = np.array(flag_for_train)

    save_array_to_file(data_for_test, flag_for_test)

    return data_for_train, flag_for_train, data_for_val, flag_for_val, data_for_test, flag_for_test

def model(X):

    with tf.variable_scope("model", reuse=tf.AUTO_REUSE):
        w = tf.get_variable("w", initializer=tf.random_normal(shape=[2, 1]))
        b = tf.get_variable("b", initializer=tf.random_normal(shape=[1]))

    m = tf.matmul(X, w) + b
    rt = tf.nn.sigmoid(m, name='predict')
    print(rt)
    return rt

def train_model(X):
    m = model(X)
    return m

def test_model(X):
    m = model(X)
    return m

def train_graph(X, Y):
    y = train_model(X)
    loss = tf.reduce_mean(- Y * tf.log(y) - (1 - Y) * tf.log(1 - y))
    # loss = tf.nn.softmax_cross_entropy_with_logits(labels=Y, logits=y)
    # loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=Y, logits=y)
    optimizer = tf.train.GradientDescentOptimizer(0.01)
    train = optimizer.minimize(loss)
    return train

def test_graph(X, Y):
    y = test_model(X)
    loss = tf.reduce_mean(- Y * tf.log(y) - (1 - Y) * tf.log(1 - y))
    # loss = tf.nn.softmax_cross_entropy_with_logits(labels=Y, logits=y)
    # loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=Y, logits=y)
    return y, loss

def work():

    data_for_train, flag_for_train, data_for_val, flag_for_val, data_for_test, flag_for_test = generate_data(100, 30, 30)

    X_train = tf.placeholder(tf.float32, shape=[None, 2], name='x_train')
    Y_train = tf.placeholder(tf.float32, shape=[None, 1], name='y_train')
    train = train_graph(X_train, Y_train)

    X_val = tf.placeholder(tf.float32, shape=[None, 2], name='x_val')
    Y_val = tf.placeholder(tf.float32, shape=[None, 1], name='y_val')
    pred, validation = test_graph(X_val, Y_val)

    saver = tf.train.Saver()

    with tf.Session() as sess:

        sess.run(tf.global_variables_initializer())

        for epoch in range(500):
            # batch_size=20, 每次取20个数据点
            s = (epoch*20) % 200
            e = ((epoch + 1) * 20) % 200
            if e == 0:
                e = 200
            sess.run(train, feed_dict={X_train: data_for_train[s: e],
                                       Y_train: flag_for_train[s: e]})
            if epoch % 100 == 99:
                cur_loss, cur_pred = sess.run([validation, pred], feed_dict={X_val: data_for_val,
                                                             Y_val: flag_for_val})

                i = 0
                error = 0
                for item in flag_for_val:
                    if item - cur_pred[i] > 0.5 or cur_pred[i] - item > 0.5:
                        error += 1
                    i += 1
                acc = (len(flag_for_val) - error) * 1.0 / len(flag_for_val)

                print("epoch{%d}, loss: %s, acc: %s" % (epoch+1, cur_loss, acc))

                saver.save(sess, save_path='/home/gxd/PycharmProjects/DeepLearning/save/my_model',
                           global_step=epoch+1)

        # 显示测试集与训练出的分类器
        with tf.variable_scope("model", reuse=True):
            ww = tf.get_variable('w')
            bb = tf.get_variable('b')
            [www, bbb] = sess.run([ww, bb])
            print(www, bbb)
            show_data_with_line(data_for_val, flag_for_val, www, bbb)

if __name__ == "__main__":
    work()


import math
from time import time as _time
from typing import Generator, Tuple

_YEAR = 33177600
_MOON = 2764800
_DAY = 86400
_HOUR = 3600
_MINUTE = 60
_SECOND = 1
_EORZEA_MINUTE = 60
_EORZEA_BELL = 60
_EORZEA_SUN = 24
_EORZEA_MOON = 32
_EORZEA_YEAR = 12
_EORZEA_TIME_CONST = 3600.0 / 175.0
_MILLISECOND_EORZEA_PER_MINUTE = (2 + 11/12) * 1000

class EorzeaTime:
    """
    EorzeaTime(moon, sun, hour, minute)
    """

    __slots__ = '_moon', '_sun', '_hour', '_minute', '_phase', '_guardian'

    def __new__(cls, moon: int, sun: int, hour: int, minute: int):
        instance = super().__new__(cls)
        instance._hour, instance._minute = _check_time_field(hour, minute)
        moon, instance._sun = _check_date_field(moon, sun)
        instance._moon = _calculate_moon(moon)
        instance._phase = _check_phase_field(_calculate_phase(sun))
        instance._guardian = _the_twelve(moon)
        return instance

    @property
    def moon(self):
        return self._moon

    @property
    def sun(self):
        return self._sun

    @property
    def hour(self):
        return self._hour

    @property
    def minute(self):
        return self._minute

    @property
    def phase(self):
        return self._phase

    @property
    def guardian(self):
        return self._guardian

    @classmethod
    def now(cls):
        """
        get current EorzeaTime

        Returns
        -------
        [EorzeaTime object]
            current EorzeaTime
        """

        t = _time()
        return cls._fromtimestamp(t)

    @classmethod
    def _fromtimestamp(cls, t: float):
        et = t * _EORZEA_TIME_CONST
        moon = math.ceil(et / _MOON % _EORZEA_YEAR)
        sun = math.ceil(et / _DAY % _EORZEA_MOON)
        hh = int(et / _HOUR % _EORZEA_SUN)
        mm = int(et / _MINUTE % _EORZEA_BELL)
        return cls(moon, sun, hh, mm)

    @classmethod
    def weather_period(cls, step=5) -> Generator[float, int, None]:
        """
        generate weather period

        Parameters
        ----------
        step : int, optional
            quantity of period you want, by default 5

        Returns
        -------
        Generator[float, int, None]
            a generator of weather period
        """

        period = _weather_period_generator(cls._weather_period_start(), step)
        return period

    @classmethod
    def _weather_period_start(cls) -> float:
        t = _time()
        et = t * _EORZEA_TIME_CONST
        lt = int(et / 28800) * 28800 / _EORZEA_TIME_CONST
        return lt

    def _cls_to_str(self) -> str:
        return "{}({}, {}, {:02d}, {:02d}, {:.2f}, {})".format(
            self.__class__.__qualname__,
            self.moon, self.sun,
            self.hour, self.minute, self.phase,
            self.guardian)

    def __repr__(self):
        return self._cls_to_str()

    def __str__(self):
        return self._cls_to_str()

def check_int(func):
    def wrap(*values):
        for value in values:
            if not isinstance(value, int):
                raise TypeError("integer argument required")
        return func(*values)
    return wrap

def _weather_period_generator(min, step: int) -> Generator[float, int, None]:
    if not isinstance(step, int):
        raise TypeError("integer argument required")
    n, i = 0, min
    while n < step:
        yield i
        i += 1400
        n += 1

def _the_twelve(moon: int) -> str:
    the_twelve = [
        "Halone",
        "Menphina",
        "Thaliak",
        "Nymeia",
        "Llymlaen",
        "Oschon",
        "Byregot",
        "Rhalgr",
        "Azeyma",
        "Nald'thal",
        "Nophica",
        "Althyk"
    ]
    return the_twelve[moon - 1]

def _calculate_moon(moon: int) -> str:
    th = ["First", "Second", "Third", "Fourth", "Fifth", "Sixth"]
    M_th = th[math.ceil(moon / 2) - 1]
    M_type = _astral_or_embral(moon)
    return "{} {} Moon".format(M_th, M_type)

def _astral_or_embral(moon: int) -> str:
    return "Astral" if moon % 2 else "Embral"

def _calculate_phase(sun: int) -> int:
    if sun <= 20:
        return 1 - int(abs(20 - sun) / 4) / 4
    if sun > 20:
        return int(abs(36 - sun) / 4) / 4

def _calculate_pphase(sun: int) -> int:
    if sun <= 20:
        return round((sun - 1) / 19, 2)
    if sun >= 20:
        return round(1 - (sun - 20) / 13, 2)

@check_int
def _check_time_field(hour: int, minute: int) -> Tuple[int, int]:
    if not 0 <= hour <= 23:
        raise ValueError('hour must be in 0..23', hour)
    if not 0 <= minute <= 59:
        raise ValueError('minute must be in 0..59', minute)
    return hour, minute

@check_int
def _check_date_field(moon: int, sun: int) -> Tuple[int, int]:
    if not 1 <= moon <= 12:
        raise ValueError('moon must be in 1..12', moon)
    if not 1 <= sun <= 32:
        raise ValueError('sun must be in 1..32', sun)
    return moon, sun

def _check_phase_field(phase: float) -> float:
    if not 0 <= phase <= 1:
        raise ValueError('phase must be in 0..1', phase)
    return phase

if __name__ == "__main__":
    while True:
        t = str(EorzeaTime.now())
        print("\r"+t, end="")


# -*- coding: utf-8 -*-
"""
:mod:`re.Filing`
~~~~~~~~~~~~~~~~~~~
Edgar(tm) Renderer was created by staff of the U.S. Securities and Exchange Commission.
Data and content created by government employees within the scope of their employment
are not subject to domestic copyright protection. 17 U.S.C. 105.
"""

from gettext import gettext as _
from collections import defaultdict
import os, re, math, datetime, dateutil.relativedelta, lxml
import arelle.ModelValue, arelle.XbrlConst
import Cube, Embedding, Report, PresentationGroup, Summary, ErrorMgr, Utils, Xlout

def mainFun(controller, modelXbrl, outputFolderName):
    filing = Filing(controller, modelXbrl, outputFolderName)
    filing.populateAndLinkClasses()

    sortedCubeList = sorted(filing.cubeDict.values(), key=lambda cube : cube.definitionText)

    for cube in sortedCubeList:
        filing.mainFunDriverBeforeFlowThroughSuppression(cube)

    if not filing.hasEmbeddings:
        filing.filterOutColumnsWhereAllElementsAreInOtherReports(sortedCubeList)
        #filing.handleFlowThroughSuppression(sortedCubeList)

    # WARNING -- right now, the only place that excludes from numbering is called before flow through, so we can do here
    # might need to do later in the future though.
    # i think flow through can also exclude from numbering?  so do this after flow through for now.
    # number the cubes
    nextFile = controller.nextFile
    for cube in sortedCubeList:
        if not cube.excludeFromNumbering and len(cube.factMemberships) > 0:
            # even though there is embedding, and cubes might have more than one embedding and thus more than one report,
            # we still keep the fileNumber attribute on the cube and not the report, because if there are multiple embeddings
            # they all print in one file.
            cube.fileNumber = nextFile
            nextFile += 1
            filing.addToLog(_('File R{!s} is {} {}')
                              .format(cube.fileNumber, cube.linkroleUri, cube.definitionText)
                              , messageCode='info')

    xlWriter = None
    if filing.hasEmbeddings:
        # sort is a stable sort, so since this is already sorted by definition text, it will now be sorted first by
        # whether it is embedded or not and then by definition text.  Cubes that are embedded will be at the beginning
        # so that their reports will be there when it's time to embed them into cells of reports that are not embedded.
        sortedCubeList = sorted(sortedCubeList, key=lambda cube : cube.isEmbedded, reverse = True)
        if controller.excelXslt:
            filing.addToLog(_("Excel XSLT is not applied to instance {} having embedded commands."
                              ).format(filing.fileNameBase),messageCode='info')
    elif controller.excelXslt:
        xlWriter = controller.xlWriter
        if not xlWriter:
            controller.xlWriter = xlWriter = Xlout.XlWriter(controller, outputFolderName)

    #import win32process
    #print('memory '  + str(int(win32process.GetProcessMemoryInfo(win32process.GetCurrentProcess())['WorkingSetSize'] / (1024*1024))))

    # hang onto a list of skipped facts before cube properties all get dereferenced.
    skippedFactsList = []
    for cube in sortedCubeList:
        skippedFactsList += list(cube.skippedFactMembershipSet)

    # handle the steps after flow through and then emit all of the XML and write the files
    filing.addToLog(_('Generating rendered reports in {}').format(outputFolderName), messageCode='info')
    for cube in sortedCubeList:
        if cube.noFactsOrAllFactsSuppressed:
            for embedding in cube.embeddingList:
                Utils.releaseUnneededMemory(embedding)
        else:
            for embedding in cube.embeddingList:
                if embedding.isEmbeddingOrReportBroken:
                    Utils.releaseUnneededMemory(embedding)
                else:
                    filing.mainFunDriverAfterFlowThroughSuppressionAndReleaseMemory(cube, embedding, xlWriter)

    controller.nextFile = filing.nextFile()
    filing.unusedFactSet = set(modelXbrl.facts) - filing.usedOrBrokenFactSet

    for args in skippedFactsList:
        (fact,role,cube,ignore,shortName) = args
        if fact in filing.unusedFactSet:
            controller.logWarn(cube.strExplainSkippedFact(fact,role,shortName))

    if len(filing.unusedFactSet) > 0:
        filing.handleUncategorizedCube(xlWriter)
        controller.nextUncategorizedFile -= 1

    controller.instanceSummaryList += [Summary.InstanceSummary(filing, modelXbrl)]
    return True

class Filing(object):
    def __init__(self, controller, modelXbrl, outputFolderName):
        self.modelXbrl = modelXbrl

        self.cubeDict = {}
        self.axisDict = {}
        self.memberDict = {}
        self.elementDict = {}
        self.factToQlabelDict = {}
        self.symbolLookupDict = {}
        self.presentationUnitToConceptDict = {}

        self.usedOrBrokenFactSet = set()
        self.unusedFactSet = set()

        self.isRR = False
        self.hasEmbeddings = False
        self.isInvestTaxonomyInDTS = False
        for namespace in self.modelXbrl.namespaceDocs:
            if re.compile('http://xbrl.(sec.gov|us)/rr/20.*').match(namespace):
                self.isRR = True
            elif re.compile('http://xbrl.sec.gov/invest/*').match(namespace):
                self.isInvestTaxonomyInDTS = True
        # These namespaces contain elements treated specially in some layouts.
        self.stmNamespace = next((n for n in self.modelXbrl.namespaceDocs.keys() if re.search('/us-gaap/20',n) is not None),None)
        self.deiNamespace = next((n for n in self.modelXbrl.namespaceDocs.keys() if re.search('/dei/20',n) is not None), None)
        self.builtinEquityColAxes = [('dei',self.deiNamespace,'LegalEntityAxis'),
                                     ('us-gaap',self.stmNamespace,'StatementEquityComponentsAxis'),
                                     ('us-gaap',self.stmNamespace,'PartnerCapitalComponentsAxis'),
                                     ('us-gaap',self.stmNamespace,'StatementClassOfStockAxis')]
        self.builtinEquityRowAxes = [('us-gaap',self.stmNamespace,'CreationDateAxis'),
                                     ('us-gaap',self.stmNamespace,'StatementScenarioAxis'),
                                     ('us-gaap',self.stmNamespace,'AdjustmentsForNewAccountingPronouncementsAxis'),
                                     ('us-gaap',self.stmNamespace,'AdjustmentsForChangeInAccountingPrincipleAxis'),
                                     ('us-gaap',self.stmNamespace,'ErrorCorrectionsAndPriorPeriodAdjustmentsRestatementByRestatementPeriodAndAmountAxis')                                     ]
        self.builtinAxisOrders = [(arelle.ModelValue.QName('us-gaap',self.stmNamespace,'StatementScenarioAxis'),
                                   ['ScenarioPreviouslyReportedMember',
                                    'RestatementAdjustmentMember',
                                    'ChangeInAccountingPrincipleMember'],
                                   ['ScenarioUnspecifiedDomain'])]
        self.builtinLineItems = [arelle.ModelValue.QName('us-gaap',self.stmNamespace,'StatementLineItems')]
        self.segmentHeadingStopList = [arelle.ModelValue.QName(x,y,z) for x,y,z in self.builtinEquityRowAxes]
        # TODO: change flags like isRR, isInvest to contain the actual namespace or None.
        self.factToEmbeddingDict = {}
        self.factFootnoteDict = defaultdict(list)
        self.startEndContextDict = {}

        self.numReports = 0
        self.numBarcharts = 0

        self.controller = controller
        self.reportXmlFormat = 'xml' in controller.reportFormat.casefold()
        self.reportHtmlFormat = 'html' in controller.reportFormat.casefold()
        self.fileNamePrefix = 'R'
        self.fileNameBase = os.path.normpath(os.path.join(os.path.dirname(controller.webCache.normalizeUrl(modelXbrl.fileSource.url)) ,outputFolderName))
        if not os.path.exists(self.fileNameBase):  # This is usually the Reports subfolder.
            os.mkdir(self.fileNameBase)

        if controller.reportXslt:
            self.transform = lxml.etree.XSLT(lxml.etree.parse(controller.reportXslt))
        if controller.summaryXslt:
            self.summary_transform = lxml.etree.XSLT(lxml.etree.parse(controller.summaryXslt))
        self.reportSummaryList = []

        self.rowSeparatorStr = ' | '
        self.titleSeparatorStr = ' - '
        self.verboseHeadingsForDebugging = False
        self.ignoredPreferredLabels = [] # locations where the preferred label role was incompatible with the concept type.
        self.entrypoint = modelXbrl.modelDocument.basename

    def __str__(self):
        return "[Filing "+str(self.entrypoint)+"]"

    def populateAndLinkClasses(self, uncategorizedCube = None):
        duplicateFacts = set()

        if uncategorizedCube is not None:
            for fact in self.unusedFactSet:
                # we know these facts aren't broken, because broken facts weren't added to self.unusedFactSet.
                try:
                    element = self.elementDict[fact.qname]
                except KeyError:
                    element = Element(fact.concept)
                    self.elementDict[fact.qname] = element
                    element.linkCube(uncategorizedCube)
            uncategorizedCube.presentationGroup = PresentationGroup.PresentationGroup(self, uncategorizedCube)
            facts = self.unusedFactSet

        else:
            # build cubes
            for linkroleUri in self.modelXbrl.relationshipSet(arelle.XbrlConst.parentChild).linkRoleUris:
                cube = Cube.Cube(self, linkroleUri)
                self.cubeDict[linkroleUri] = cube
                cube.presentationGroup = PresentationGroup.PresentationGroup(self, cube)

            # initialize elements
            for qname, factSet in self.modelXbrl.factsByQname.items():

                # we are looking to see if we have "duplicate" facts.  a duplicate fact is one with the same qname, context and unit
                # as another fact.  in this case, we keep the first fact with an 'en-US' language, or if there is none, keep the first fact.
                # the others need to be proactively added to the set of unused facts.
                if len(factSet) > 1:
                    sortedFactList = sorted(factSet, key = lambda thing : (thing.contextID, thing.unitID, thing.sourceline))
                    while len(sortedFactList) > 0:

                        firstFact = sortedFactList.pop(0)
                        if firstFact.xmlLang == 'en-US':
                            firstEnUsLangFact = firstFact
                        else:
                            firstEnUsLangFact = None

                        discardedLineNumberList = []
                        counter = 0

                        # finds facts with same qname, context and unit as firstFact
                        while (len(sortedFactList) > 0 and
                               sortedFactList[0].qname == firstFact.qname and
                               sortedFactList[0].context == firstFact.context and
                               sortedFactList[0].unitID == firstFact.unitID):
                            counter += 1
                            fact = sortedFactList.pop(0)
                            if firstEnUsLangFact is None and fact.xmlLang == 'en-US':
                                firstEnUsLangFact = fact # keeping this fact
                            else:
                                duplicateFacts.add(fact) # not keeping this fact
                                discardedLineNumberList += [str(fact.sourceline)] # these are added in sorted order by sourceline

                        if counter > 0:
                            if firstEnUsLangFact is None:
                                lineNumOfFactWeAreKeeping = firstFact.sourceline # there is no en-US fact, keep the first fact we found
                            else:
                                lineNumOfFactWeAreKeeping = firstEnUsLangFact.sourceline # we did find an en-US fact, keep it.
                                if firstFact != firstEnUsLangFact:
                                    duplicateFacts.add(firstFact)
                                    discardedLineNumberList = [str(firstFact.sourceline)] + discardedLineNumberList # prepend to maintain sorted order

                            # start it off because we can assume that these facts have a qname and a context
                            # note that even if we are keeping firstEnUsLangFact, we are only printing qname, contextID and unitID,
                            # which are the same for both firstFact and firstEnUsLangFact.
                            qnameContextIDUnitStr = 'qname {!s}, context {}'.format(firstFact.qname, firstFact.contextID)
                            if firstFact.unit is not None:
                                qnameContextIDUnitStr += ', unit ' + firstFact.unitID
                            message = ErrorMgr.getError('DUPLICATE_FACT_SUPPRESSION').format(qnameContextIDUnitStr, lineNumOfFactWeAreKeeping, ', '.join(discardedLineNumberList))
                            self.addToLog(message, messageCode='warn')

                for fact in factSet: # we only want one thing, but we don't want to pop from the set so we "loop" and then break right away

                    elementBroken = False

                    if fact.concept is None:
                        conceptErrStr = ErrorMgr.getError('FACT_DECLARATION_BROKEN').format(qname)
                        self.addToLog(conceptErrStr, messageCode='warn')
                        elementBroken = True

                    elif fact.concept.type is None:
                        typeErrStr = ErrorMgr.getError('TYPE_DECLARATION_BROKEN').format(qname)
                        self.addToLog(typeErrStr, messageCode='warn')
                        elementBroken = True

                    if fact.context is None or elementBroken: # we will print the error if firstContext is broken later
                        continue # see if there are other facts for this concept with good contexts before we break from the loop, we still might make the Element yet

                    self.elementDict[qname] = Element(fact.concept)
                    break # we don't need to look at more facts from the fact set, we're just trying to make elements.

            # build presentation groups
            for concept in self.modelXbrl.qnameConcepts.values():
                for relationship in self.modelXbrl.relationshipSet(arelle.XbrlConst.parentChild).toModelObject(concept):
                    cube = self.cubeDict[relationship.linkrole]
                    cube.presentationGroup.traverseToRootOrRoots(concept, None, None, None, set())
                    try:
                        element = self.elementDict[concept.qname] # retrieve active Element
                        element.linkCube(cube) # link element to this cube.
                    except KeyError:
                        pass

            # footnotes
            for relationship in self.modelXbrl.relationshipSet('XBRL-footnotes').modelRelationships:
                # relationship.fromModelObject is a fact
                # relationship.toModelObject is a resource
                # make sure Element is active and that no Error is caught by relationshipErrorThrower()
                # if relationship.fromModelObject.qname in self.elementDict and not self.relationshipErrorThrower(relationship, 'Footnote'):
                #if relationship.fromModelObject.qname in self.elementDict:
                self.factFootnoteDict[relationship.fromModelObject].append((relationship.toModelObject, relationship.toModelObject.viewText()))

            facts = self.modelXbrl.facts

        for fact in facts:
            if fact.isTuple:
                tupleErrStr = ErrorMgr.getError('UNSUPPORTED_TUPLE_FOUND').format(fact.qname)
                self.addToLog(tupleErrStr, messageCode='warn')
                self.usedOrBrokenFactSet.add(fact) #now bad fact won't come back to bite us when processing isUncategorizedFacts
                continue

            if fact.context is None:
                contextErrStr1 = ErrorMgr.getError('CONTEXT_BROKEN').format(fact.qname, fact.value)
                self.addToLog(contextErrStr1, messageCode='warn')
                self.usedOrBrokenFactSet.add(fact) #now bad fact won't come back to bite us when processing isUncategorizedFacts
                continue

            if fact.context.scenario is not None:
                scenarioErrStr = ErrorMgr.getError('IMPROPER_CONTEXT_FOUND').format(fact.context.id)
                self.addToLog(scenarioErrStr, messageCode='warn')
            try:
                element = self.elementDict[fact.qname]
            except KeyError:
                self.usedOrBrokenFactSet.add(fact) #now bad fact won't come back to bite us when processing isUncategorizedFacts
                continue # fact was rejected in first loop of this function because of problem with the Element

            # this is after we check for the bad stuff so that we make sure not to put those into usedOrBrokenFactSet
            # so that they don't break when processing isUncategorizedFacts
            if fact in duplicateFacts:
                continue

            # first see if fact's value is an embedded command, then check if it's a qlabel fact.
            if not fact.isNumeric:
                if fact.concept.isTextBlock:
                    isEmbeddedCommand = self.checkForEmbeddedCommandAndProcessIt(fact)
                else:
                    isEmbeddedCommand = False

                if not isEmbeddedCommand and re.compile('[a-zA-Z-]+:[a-zA-Z]+').match(fact.value):
                    try:
                        prefix, ignore, localName = fact.value.partition(':')
                        namespaceURI = self.modelXbrl.prefixedNamespaces[prefix]
                        qname = arelle.ModelValue.QName(prefix, namespaceURI, localName)
                        if qname in self.modelXbrl.qnameConcepts:
                            self.factToQlabelDict[fact] = qname
                    except KeyError:
                        pass

            axisMemberLookupDict = {}

            # add period and unit to axisMemberLookupDict
            startEndContext = None
            con = fact.context
            if fact.context is not None:
                if con.instantDatetime is not None: # is an instant
                    startEndTuple = (None, con.instantDatetime)
                else: # is a startEndContext
                    startEndTuple = (con.startDatetime, con.endDatetime)
                try:
                    startEndContext = self.startEndContextDict[startEndTuple]
                except KeyError:
                    startEndContext = StartEndContext(con, startEndTuple)
                    self.startEndContextDict[startEndTuple] = startEndContext
                axisMemberLookupDict['period'] = startEndContext

            if fact.unit is not None:
                axisMemberLookupDict['unit'] = fact.unit.id

            # add each axis to axisMemberLookupDict
            for arelleDimension in fact.context.qnameDims.values():
                dimensionConcept = arelleDimension.dimension
                memberConcept = arelleDimension.member
                if dimensionConcept is None:
                    errStr1 = ErrorMgr.getError('XBRL_DIMENSIONS_INVALID_AXIS_BROKEN').format(fact.context.id, fact.qname)
                    self.addToLog(errStr1, messageCode='warn')

                elif memberConcept is None:
                    errStr2 = ErrorMgr.getError('XBRL_DIMENSIONS_INVALID_AXIS_MEMBER_BROKEN').format(dimensionConcept.qname, fact.qname, fact.context.id)
                    self.addToLog(errStr2, messageCode='warn')

                else:
                    try:
                        axis = self.axisDict[dimensionConcept.qname]
                    except KeyError:
                        axis = Axis(dimensionConcept)
                        for relationship in self.modelXbrl.relationshipSet(arelle.XbrlConst.dimensionDefault).fromModelObject(dimensionConcept):
                            axis.defaultArelleConcept = relationship.toModelObject
                            break
                        self.axisDict[dimensionConcept.qname] = axis
                    if arelleDimension.isExplicit: # if true, Member exists, else None. there's also isTyped, for typed dims.
                        try:
                            member = self.memberDict[memberConcept.qname]
                        except KeyError:
                            member = Member(memberConcept)
                            self.memberDict[memberConcept.qname] = member
                        member.linkAxis(axis)
                        axis.linkMember(member)
                    axisMemberLookupDict[axis.arelleConcept.qname] = member.arelleConcept.qname

                    # while we're at it, do some other stuff
                    for cube in element.inCubes.values():
                        cube.hasAxes[axis.arelleConcept.qname] = axis
                        cube.hasMembers[member.arelleConcept.qname] = member
                        axis.linkCube(cube)

            for cube in element.inCubes.values():
                # the None in the tuple is only to handle periodStartLabels and periodEndLabels later on
                cube.factMemberships += [(fact, axisMemberLookupDict, None)]
                cube.hasElements.add(fact.concept)
                if fact.unit is not None:
                    cube.unitAxis[fact.unit.id] = fact.unit
                if startEndContext is not None:
                    cube.timeAxis.add(startEndContext)

    def checkForEmbeddedCommandAndProcessIt(self, fact):
        # partition('~') on a string breaks up a string into a tuple with before the first ~, the ~, and then after the ~.
        ignore, tilde, rightOfTilde = fact.value.partition('~')
        if tilde == '':
            return False
        leftOfTilde, tilde, ignore = rightOfTilde.partition('~')
        if tilde == '' or leftOfTilde == '':
            return False
        commandText = leftOfTilde

        # we take out the URI first, because it might have double quotes in it and we want to cleanse the rest of the
        # command of double quotes since the separator command wraps the separator character in double quotes.
        commandTextList = commandText.split(maxsplit=1) # this is a list of length 1 or 2.
        linkroleUri = commandTextList.pop(0) # now commandTextList is a list of length 0 or 1
        try:
            cube = self.cubeDict[linkroleUri]
        except KeyError:
            return False # not a valid linkroleUri
        if len(commandTextList) > 0:
            commandTextList = commandTextList[0].replace('"', ' ')
            commandTextList = commandTextList.split()

        outputList = []
        tokenCounter = 1
        while len(commandTextList) > 0:
            listToAddToOutput = []

            token0 = commandTextList.pop(0)
            tokenCounter += 1
            token0Lower = token0.casefold()
            if token0Lower in {'row', 'column'}:
                listToAddToOutput += [token0Lower]
            else:
                errorStr = Utils.printErrorStringToDisambiguateEmbeddedOrNot(fact)
                message = ErrorMgr.getError('EMBEDDED_COMMAND_TOKEN_NOT_ROW_OR_COLUMN_ERROR').format(token0, tokenCounter, errorStr)
                self.addToLog(message, messageCode='error')
                return False

            token1 = commandTextList.pop(0)
            tokenCounter += 1
            token1Lower = token1.casefold()
            if token1Lower in {'period', 'unit', 'primary'}:
                listToAddToOutput += [token1Lower]
            elif '_' in token1:
                listToAddToOutput += [arelle.ModelValue.qname(fact, token1.replace('_',':',1))] # only replace first _, because qnames can have _

            # separator is not supported, we just pop it off and ignore it
            elif token1Lower == 'separator':
                if commandTextList.pop(0).casefold() == 'segment':
                    commandTextList.pop(0)
                    tokenCounter += 1
                tokenCounter += 1
                errorStr = Utils.printErrorStringToDisambiguateEmbeddedOrNot(fact)
                message = ErrorMgr.getError('EMBEDDED_COMMAND_SEPARATOR_USED_WARNING').format(token1, tokenCounter, errorStr)
                self.addToLog(message, messageCode='warn')
                continue

            else:
                errorStr = Utils.printErrorStringToDisambiguateEmbeddedOrNot(fact)
                message = ErrorMgr.getError('EMBEDDED_COMMAND_INVALID_FIRST_TOKEN_ERROR').format(token1, tokenCounter, errorStr)
                self.addToLog(message, messageCode='error')
                return False

            token2 = commandTextList.pop(0)
            tokenCounter += 1
            token2Lower = token2.casefold()
            if token2Lower in {'compact', 'nodisplay'}:
                listToAddToOutput += [token2Lower]
            elif token2Lower == 'grouped':
                listToAddToOutput += ['compact']
                errorStr = Utils.printErrorStringToDisambiguateEmbeddedOrNot(fact)
                message = ErrorMgr.getError('EMBEDDED_COMMAND_GROUPED_USED_WARNING').format(token2, tokenCounter, errorStr)
                self.addToLog(message, messageCode='warn')
            elif token2Lower == 'unitcell':
                listToAddToOutput += ['compact']
                errorStr = Utils.printErrorStringToDisambiguateEmbeddedOrNot(fact)
                message = ErrorMgr.getError('EMBEDDED_COMMAND_UNITCELL_USED_WARNING').format(token2, tokenCounter, errorStr)
                self.addToLog(message, messageCode='warn')
            else:
                errorStr = Utils.printErrorStringToDisambiguateEmbeddedOrNot(fact)
                message = ErrorMgr.getError('EMBEDDED_COMMAND_INVALID_SECOND_TOKEN_ERROR').format(token2, tokenCounter, errorStr)
                self.addToLog(message, messageCode='error')
                return False

            # there could be multiple members, so grab them all here
            tempList = []
            while len(commandTextList) > 0 and commandTextList[0].casefold() not in {'row', 'column'}:
                tempList += [commandTextList.pop(0)]

            for tokenMember in tempList:
                tokenCounter += 1
                if '_' in tokenMember:
                    listToAddToOutput += [arelle.ModelValue.qname(fact, tokenMember.replace('_',':',1))]
                elif tokenMember == '*' and len(tempList) == 1:
                    listToAddToOutput += [tokenMember]
                else:
                    errorStr = Utils.printErrorStringToDisambiguateEmbeddedOrNot(fact)
                    message = ErrorMgr.getError('EMBEDDED_COMMAND_INVALID_MEMBER_NAME_ERROR').format(tokenMember, tokenCounter, errorStr)
                    self.addToLog(message, messageCode='error')
                    return False

            outputList += [listToAddToOutput]

        cube.isEmbedded = True
        self.hasEmbeddings = True

        embedding = Embedding.Embedding(self, cube, outputList, factThatContainsEmbeddedCommand = fact)
        cube.embeddingList += [embedding]
        self.factToEmbeddingDict[fact] = embedding
        return True

    def handleUncategorizedCube(self, xlWriter):
        # get a fresh start, kill all the old data structures that get built in populateAndLinkClasses()
        for element in self.elementDict.values():
            element.__dict__.clear()
            del element
        self.elementDict = {}

        for cube in self.cubeDict.values():
            cube.__dict__.clear()
            del cube
        self.cubeDict = {}

        for member in self.memberDict.values():
            member.__dict__.clear()
            del member
        self.memberDict = {}

        for axis in self.axisDict.values():
            axis.__dict__.clear()
            del axis
        self.axisDict = {}

        for startEndContext in self.startEndContextDict.values():
            startEndContext.__dict__.clear()
            del startEndContext
        self.startEndContextDict = {}

        uncategorizedCube = Cube.Cube(self, 'http://xbrl.sec.gov/role/uncategorizedFacts')
        uncategorizedCube.fileNumber = self.controller.nextUncategorizedFile
        uncategorizedCube.shortName = uncategorizedCube.definitionText = 'Uncategorized Items - '+self.entrypoint
        uncategorizedCube.isElements = True

        # now run populateAndLinkClasses() again and let it re-populate and re-link everything from scratch but let it do so
        # only with filing.unusedFactSet as it's fact set and with only the uncategorizedCube, and no other cubes.
        self.cubeDict[uncategorizedCube.linkroleUri] = uncategorizedCube
        self.populateAndLinkClasses(uncategorizedCube = uncategorizedCube)

        self.mainFunDriverBeforeFlowThroughSuppression(uncategorizedCube)

        embedding = uncategorizedCube.embeddingList[0]

        self.mainFunDriverAfterFlowThroughSuppressionAndReleaseMemory(uncategorizedCube, embedding, xlWriter)

    def mainFunDriverBeforeFlowThroughSuppression(self, cube):
        if cube.isUncategorizedFacts:
            cube.presentationGroup.generateUncategorizedFactsPresentationGroup()
        else:
            if len(cube.hasElements) == 0 or len(cube.factMemberships) == 0:
                cube.noFactsOrAllFactsSuppressed = True
                return
            cube.presentationGroup.startPreorderTraversal()
            if cube.noFactsOrAllFactsSuppressed:
                return
            cube.checkForTransposedUnlabeledAndElements()
            if len(cube.periodStartEndLabelDict) > 0:
                cube.handlePeriodStartEndLabel() # needs preferred labels from the presentationGroup
            cube.DetermineAxesWhereDefaultFilteredOut()

        cube.populateUnitPseudoaxis()
        cube.populatePeriodPseudoaxis()

        if self.controller.debugMode:
            cube.printCube()

        if len(cube.embeddingList) == 0: # add the non-embedded cube to the end, the rest will plug into this one.
            cube.embeddingList = [Embedding.Embedding(self, cube, [])]

        for embedding in cube.embeddingList:
            embedding.generateStandardEmbeddedCommandsFromPresentationGroup()
            if cube.isTransposed:
                embedding.handleTransposedByModifyingCommandText()
            embedding.buildAndProcessCommands()
            if embedding.isEmbeddingOrReportBroken:
                continue

            embedding.processOrFilterFacts()
            if embedding.isEmbeddingOrReportBroken:
                continue

            embedding.possiblyReorderUnitsAfterTheFactAccordingToPresentationGroup()

            if self.controller.debugMode:
                embedding.printEmbedding()

            report = embedding.report = Report.Report(self, cube, embedding)
            report.generateRowsOrCols('col', sorted(embedding.factAxisMemberGroupList, key=lambda thing: thing.axisMemberPositionTupleColList))

            # this is because if the {Elements} view is used, then you might have lots of facts right next to each other with the same qname
            # this is fine, but each time you render, they might appear in a different order.  so this will sort the facts by value
            # so that each run the same facts don't appear in different orders.
            if cube.isElements:
                sortedFAMGL = sorted(embedding.factAxisMemberGroupList, key=lambda thing: (thing.axisMemberPositionTupleRowList, thing.fact.value))
            else:
                sortedFAMGL = sorted(embedding.factAxisMemberGroupList, key=lambda thing: thing.axisMemberPositionTupleRowList)
            report.generateRowsOrCols('row', sortedFAMGL)

            if not cube.isElements:
                report.promoteAxes()

                if report.isUnitsAxisOnRowsOrColumns == 'row':
                    report.mergeRowsOrColsIfUnitsCompatible('row', report.rowList)
                elif report.isUnitsAxisOnRowsOrColumns == 'col':
                    report.mergeRowsOrColsIfUnitsCompatible('col', report.colList)
                if report.isPeriodAxisOnRowsOrColumns == 'row':
                    report.mergeRowsOrColsInstantsIntoDurationsIfUnitsCompatible('row', report.rowList)
                elif report.isPeriodAxisOnRowsOrColumns == 'col':
                    report.mergeRowsOrColsInstantsIntoDurationsIfUnitsCompatible('col', report.colList)
                report.hideRedundantColumns()

    def mainFunDriverAfterFlowThroughSuppressionAndReleaseMemory(self, cube, embedding, xlWriter):
        report = embedding.report

        report.scaleUnitGlobally()

        if      (len(self.factFootnoteDict) > 0 and
                 not {factAxisMemberGroup.fact for factAxisMemberGroup in embedding.factAxisMemberGroupList}.isdisjoint(self.factFootnoteDict)):
            report.handleFootnotes()
            report.setAndMergeFootnoteRowsAndColumns('row', report.rowList)
            report.setAndMergeFootnoteRowsAndColumns('col', report.colList)

        report.removeVerticalInteriorSymbols()

        #if len(embedding.groupedAxisQnameSet) > 0:
        #    report.handleGrouped()

        if cube.isElements or not cube.isEmbedded:
            if      (report.isPeriodAxisOnRowsOrColumns == 'col' or
                     {command.pseudoAxis for command in embedding.rowCommands}.isdisjoint(self.segmentHeadingStopList)):
                report.makeSegmentTitleRows()

            # if report.rowPrimaryPosition != -1, then primary elements aren't on the rows, so no abstracts to add
            if report.rowPrimaryPosition != -1 and not cube.isUnlabeled:
                report.addAbstracts()

        if report.isPeriodAxisOnRowsOrColumns=='row':
            report.HideAdjacentInstantRows()
        elif cube.isStatementOfCashFlows:
            self.RemoveStuntedCashFlowColumns(report,cube)

        report.generateRowAndOrColHeadings()

        report.emitRFile()

        if xlWriter:
            # we pass the cube's shortname since it doesn't have units and stuff tacked onto the end.
            xlWriter.createWorkSheet(cube.fileNumber, cube.shortName)
            xlWriter.buildWorkSheet(report)

        if not cube.isEmbedded:
            reportSummary = ReportSummary()
            report.createReportSummary(reportSummary)
            report.writeHtmlAndOrXmlFiles(reportSummary)
            self.reportSummaryList += [reportSummary]

            # if the cube is embedded, we need to hold on to it, since we might not have embedded it yet.
            # but once we embed it, we garbage collect it in the cell function insertEmbeddingOrBarChartEmbedding()
            Utils.releaseUnneededMemory(embedding)

    def RemoveStuntedCashFlowColumns(self,report,cube):
        visibleColumns = [col for col in report.colList if not col.isHidden]
        didWeHideAnyCols = False
        if len(visibleColumns)>0:
            remainingVisibleColumns = visibleColumns.copy()
            maxMonths = max(col.startEndContext.numMonths for col in visibleColumns)
            minFacts = min(len(col.factList) for col in visibleColumns if col.startEndContext.numMonths==maxMonths)
            minToKeep = math.floor(.25*minFacts)
            for col in visibleColumns:
                if (col.startEndContext.numMonths < maxMonths
                    and len(col.factList) < minToKeep):
                    self.controller.logInfo(("Columns in cash flow ''{}'' have maximum duration {} months and at least {} values. "+
                                  "Shorter duration columns must have at least one fourth ({}) as many values. "+
                                  "Column '{}' is shorter ({} months) and has only {} values, so it is being removed.")
                                  .format(cube.shortName,maxMonths,minFacts,minToKeep,col.startEndContext,col.startEndContext.numMonths,len(col.factList))
                                  )
                    col.isHidden = True
                    didWeHideAnyCols = True
                    remainingVisibleColumns.remove(col)
                    for fact in col.factList:
                        appearsInOtherColumn = False
                        for otherCol in remainingVisibleColumns:
                            if fact in otherCol.factList:
                                appearsInOtherColumn = True
                                break
                        if not appearsInOtherColumn:
                            cube.factMemberships = [fm for fm in cube.factMemberships if not fact is fm[0]] # local removal
                            # Go look whether the fact is now completely uncategorized
                            if fact in self.usedOrBrokenFactSet:
                                element = self.elementDict[fact.elementQname]
                                appearsInOtherCube = False
                                for c in element.inCubes.values():
                                    if hasattr(c,'factMemberships'): # Some Cube objects seem uninitialized, not sure why.
                                        for fm in c.factMemberships:
                                            if fm[0] is fact: # Assumes that factMemberships is an accurate list of facts presented.
                                                appearsInOtherCube = True
                                                break # Only need to find one other place the fact appears
                                        if appearsInOtherCube is True:
                                            break
                                if appearsInOtherCube is False:
                                    # This was the only place the fact was presented, and now it's hidden.
                                    self.usedOrBrokenFactSet.remove(fact)

        if didWeHideAnyCols:
            Utils.hideEmptyRows(report.rowList)

    # this is only for financial reports, so we know that there are no embedded commands.  this means that for each cube,
    # there is only one embedding object, but we have to get that this way: cube.embeddingList[0].  also, always, in general
    # each embedding only ever has one and only one report.  so for financial reports, report is cube.embeddingList[0].report.
    def filterOutColumnsWhereAllElementsAreInOtherReports(self, sortedCubeList):
        # we separate nonStatementElementsAndElementMemberPairs from all of the cubes because we can do this just once and not run through them each time
        statementCubesList = []
        nonStatementElementsAndElementMemberPairs = set()
        for cube in sortedCubeList:
            if not cube.noFactsOrAllFactsSuppressed and len(cube.embeddingList) == 1 and not cube.embeddingList[0].isEmbeddingOrReportBroken:
                if cube.cubeType == 'statement' and not cube.isStatementOfEquity:
                    statementCubesList += [cube]
                else:
                    nonStatementElementsAndElementMemberPairs.update(cube.embeddingList[0].hasElementsAndElementMemberPairs)

        for i, cube in enumerate(statementCubesList):
            # initialize to all the non statement elements and non statement element member pairs
            elementQnamesInOtherReportsAndElementQnameMemberPairs = nonStatementElementsAndElementMemberPairs.copy()
            # now add other statements
            for j, otherStatement in enumerate(statementCubesList):
                if i != j:
                    elementQnamesInOtherReportsAndElementQnameMemberPairs.update(otherStatement.embeddingList[0].hasElementsAndElementMemberPairs)
            # now elementQnamesInOtherReportsAndElementQnameMemberPairs has all the qnames used in every other report except for the statement we're using

            report = cube.embeddingList[0].report # we know there's no embeddings, so the report is on the first and only embedding
            columnsToKill = []
            nonHiddenColCount = 0

            elementQnamesThatWillBeKeptProvidingThatWeHideTheseCols = set() # if we kill a few cols, we will want to update embedding.hasElements
            for col in report.colList:

                if not col.isHidden:
                    nonHiddenColCount += 1
                    setOfElementQnamesInCol = {fact.qname for fact in col.factList}
                    setOfElementQnamesAndQnameMemberPairsForCol = setOfElementQnamesInCol.union(col.elementQnameMemberForColHidingSet)
                    # the operator <= means subset, not a proper subset.  so if all the facts in the column are elsewhere, then hide column.
                    if setOfElementQnamesAndQnameMemberPairsForCol <= elementQnamesInOtherReportsAndElementQnameMemberPairs:
                        columnsToKill += [col]
                    else:
                        elementQnamesThatWillBeKeptProvidingThatWeHideTheseCols.update(setOfElementQnamesInCol)

            if 0 < len(columnsToKill) < nonHiddenColCount:
                cube.embeddingList[0].hasElements = elementQnamesThatWillBeKeptProvidingThatWeHideTheseCols # update hasElements, might have less now
                for col in columnsToKill:
                    col.isHidden = True
                    self.addToLog('{} column {!s} was just killed'.format(cube.linkroleUri, col.index))

                Utils.hideEmptyRows(report.rowList)

    def addToLog(self,message,messageCode='debug',messageArgs=(),file='Filing.py'):
        self.controller.addToLog(message, messageCode=messageCode, messageArgs=messageArgs, file=file)

    def nextFile(self):
        if len(self.reportSummaryList)==0: return 1
        return self.reportSummaryList[-1].fileNumber + 1

class ReportSummary(object):
    def __init__(self):
        self.order = None
        self.isDefault = None
        self.hasEmbeddedReports = None
        self.longName = None
        self.shortName = None
        self.role = None
        self.logList = None
        self.xmlFileName = None
        self.htmlFileName = None
        self.isUncategorized = False

class StartEndContext(object):
    def __init__(self, context, startEndTuple):
        # be really careful when using this context.  many contexts from the instance can share this object,
        # probably not the right one.
        self.context = context
        self.startTime = startEndTuple[0]
        self.endTime = startEndTuple[1]
        self.label = (self.endTime - datetime.timedelta(days=1)).strftime('%b. %d, %Y')

        if self.startTime == None:
            self.periodTypeStr = 'instant'
            self.startTimePretty = None
            self.numMonths = 0
        else:
            self.periodTypeStr = 'duration'
            self.startTimePretty = (self.startTime).strftime('%Y-%m-%dT%H:%M:%S')
            self.numMonths = self.startEndContextInMonths()
        self.endTimePretty = (self.endTime - datetime.timedelta(days=1)).strftime('%Y-%m-%dT%H:%M:%S')

    def startEndContextInMonths(self):
        modifiedEndTime = self.endTime + datetime.timedelta(days=15) # we add to it because it rounds down
        delta = dateutil.relativedelta.relativedelta(modifiedEndTime, self.startTime)
        return delta.years * 12 + delta.months

    def startOrInstantTime(self):
        if self.startTime is None:
            return self.endTime
        return self.startTime

    def __str__(self):
        if self.periodTypeStr=='instant':
            return "[{}]".format(self.endTimePretty[:10])
        else:
            return "[{} {}m {}]".format(self.startTimePretty[:10],self.numMonths,self.endTimePretty[:10])

class Axis(object):
    def __init__(self, arelleConcept):
        self.inCubes = {}
        self.hasMembers = {}
        self.arelleConcept = arelleConcept
        self.defaultArelleConcept = None
    def linkCube(self, cubeObj):
        self.inCubes[cubeObj.linkroleUri] = cubeObj
    def linkMember(self, memObj):
        self.hasMembers[memObj.arelleConcept.qname] = memObj
    def __repr__(self):
        return "axis(arelleConcept={}, default={})".format(self.arelleConcept, self.defaultArelleConcept)

class Member(object):
    def __init__(self, arelleConcept):
        self.hasMembers = {}
        self.arelleConcept = arelleConcept
        self.axis = None
        self.parent = None
    def linkAxis(self, axisObj):
        self.axis = axisObj
    def linkParent(self, parentObj):
        self.parent = parentObj

class Element(object):
    def __init__(self, arelleConcept):
        self.inCubes = {}
        self.arelleConcept = arelleConcept
    def linkCube(self, cube):
        self.inCubes[cube.linkroleUri] = cube

#===============================================================================
#     # to do, take out linkrole uri's and prints.  only for testing.
#     def handleFlowThroughSuppression(self, sortedCubeList):
#         reportList = []
#         for cube in sortedCubeList:
#             if not cube.noFactsOrAllFactsSuppressed and not cube.isElements:
#                 for embedding in cube.embeddingList:
#                     if not embedding.isEmbeddingOrReportBroken:
#                         report = embedding.report
#                         reportList += [(report, cube.linkroleUri)]
#
#                         # this is just for speed.  better to convert lists to sets only once, rather than each time.
#                         for row in report.rowList:
#                             row.factSet = set(row.factList)
#                         for col in report.colList:
#                             col.factSet = set(col.factList)
#
#         reversedReportList = reportList.copy()
#         reversedReportList.reverse()
#         for reportToTest, reportToTestLinkroleUri in reportList:
#             if not reportToTest.cube.isStatementOfEquity:
#                 self.testEachRowAndCol(reversedReportList, reportToTest, reportToTestLinkroleUri, reportToTest.rowList, 'row')
#             self.testEachRowAndCol(reversedReportList, reportToTest, reportToTestLinkroleUri, reportToTest.colList, 'column')
#
#             if len(reportToTest.factSetForFlowThroughSuppression) == 0:
#                 message = ErrorMgr.getError('FLOWTHROUGH_SUPPRESSION').format(reportToTest.cube.linkroleUri)
#                 self.addToLog(message, messageCode='Info')
#                 reportToTest.embedding.isEmbeddingOrReportBroken = True
#
#
#     def testEachRowAndCol(self, reversedReportList, reportToTest, reportToTestLinkroleUri, rowOrColList, rowOrColStr):
#         for rowOrColToTest in rowOrColList:
#             if not rowOrColToTest.isHidden:
#                 for reportToTestAgainst, reportToTestAgainstLinkroleUri in reversedReportList:
#
#                     # < tests proper subset
#                     # here we are checking a row or col against an entire report.  If the row or col is not in the entire report, stop
#                     if (reportToTest != reportToTestAgainst and
#                         rowOrColToTest.factSet < reportToTestAgainst.factSetForFlowThroughSuppression):
#
#                         # since rowOrColToTest is contained in the report, we check to see if it is a proper subset of
#                         # any of the rows or cols of the report.  it might not be.  if it is, we stop.
#                         returnValueRows = self.findContainingRowOrCol(rowOrColToTest, reportToTest, reportToTestLinkroleUri, rowOrColStr,
#                                                                       reportToTestAgainstLinkroleUri, 'row', reportToTestAgainst.rowList)
#                         if returnValueRows:
#                             break
#
#                         returnValueCols = self.findContainingRowOrCol(rowOrColToTest, reportToTest, reportToTestLinkroleUri, rowOrColStr,
#                                                                       reportToTestAgainstLinkroleUri, 'col', reportToTestAgainst.colList)
#                         if returnValueCols:
#                             break
#
#
#     def findContainingRowOrCol(self, rowOrColToTest, reportToTest, reportToTestLinkroleUri, rowOrColToTestStr,
#                                reportToTestAgainstLinkroleUri, rowOrColToTestAgainstStr, reportToTestAgainstRowOrColList):
#         for rowOrColToTestAgainst in reportToTestAgainstRowOrColList:
#             if not rowOrColToTestAgainst.isHidden and rowOrColToTest.factSet < rowOrColToTestAgainst.factSet:
#
#                 self.addToLog('{} {} {}'.format(rowOrColToTestStr, rowOrColToTest.index, reportToTestLinkroleUri))
#                 for fact in sorted(rowOrColToTest.factList,        key=lambda fact : fact.value):
#                     if fact.value:
#                         self.addToLog(fact.value)
#
#                 self.addToLog('{} {} {}'.format(rowOrColToTestAgainstStr, rowOrColToTestAgainst.index, reportToTestAgainstLinkroleUri))
#                 for fact in sorted(rowOrColToTestAgainst.factList, key=lambda fact : fact.value):
#                     if fact.value:
#                         self.addToLog(fact.value)
#
#                 self.addToLog('\n\n')
#
#                 rowOrColToTest.isHidden = True
#                 reportToTest.factSetForFlowThroughSuppression -= rowOrColToTest.factSet
#                 reportToTest.logList += ['Process Flow-Through: Removing ' + rowOrColToTestStr + ' ' + ' | '.join(rowOrColToTest.headingList)]
#                 return True
#===============================================================================


#!/usr/bin/python

import time
import requests
import subprocess
import json
import re
import sys
import socket
from utils import *
import jinja2
import threading
import random

SERVER_URL = 'http://atxmon.asterix.cz:5000/save'
TESTS_FN = 'tests.conf'
SEND_INTERVAL = 20
TESTS_MAX = 10  # TODO: rename to THREADS_MAX?

def load_tests(fn):
	ret = []

	with open(fn, 'r') as f:
		t = jinja2.Template(f.read())
		rendered = t.render()
	#endwith

	for line in rendered.splitlines():
		line = line.strip()

		if not line: continue
		if line.startswith('#'): continue

		interval, test, *args = line.split(';')
		if interval.endswith('m'):
			interval = float(interval[:-1]) * 60
		else:
			interval = float(interval)
		#endif

		ret.append((interval, test, args))
	#endfor

	return ret
#enddef

def alive():
	return {'ok': 1}
#enddef

def load():
	ret = {}

	with open('/proc/loadavg', 'r') as f:
		loads = [float(i) for i in f.read().split()[:3]]
		ret['1min'] = loads[0]
		ret['5min'] = loads[1]
		ret['15min'] = loads[2]
	#endwith

	return ret
#enddef

def ping(host, ipv6=False):
	if ipv6:
		cmd = 'ping6 -c 5 -q %s 2>/dev/null' % host
	else:
		cmd = 'ping -c 5 -q %s 2>/dev/null' % host
	#endif

	try:
		out = subprocess.check_output(cmd, shell=True).decode()
	except:
		return {'ok': 0}
	#endtry

	packet_loss = None
	rtt_avg = None
	for line in out.splitlines():
		if 'packet loss' in line:
			# TODO: compile this?
			m = re.match('.+, (\d+)% packet loss,.+', line)
			packet_loss = int(m.groups()[0])
		#endif

		if 'rtt min/avg/max/mdev' in line:
			# TODO: compile this?
			m = re.match('rtt min/avg/max/mdev = ([\d.]+)/([\d.]+)/([\d.]+)/([\d.]+) ms', line)
			_, rtt_avg, _, _ = [float(i) for i in m.groups()]
		#endif
	#endfor

	return {'ok': 1, 'packet_loss': packet_loss, 'rtt_avg': rtt_avg}
#enddef

def ping6(host):
	return ping(host, ipv6=True)
#enddef

def iperf3(host):
	ret = {}
	ok = 1

	for direction in ['up', 'down']:
		if direction == 'down':
			cmd = 'timeout 20 iperf3 -c %s -J -R -P 5' % host
		else:
			cmd = 'timeout 20 iperf3 -c %s -J -P 5' % host
		#endif

		try:
			out = subprocess.check_output(cmd, shell=True).decode()
			res = json.loads(out)
			bits_per_second = res['end']['sum_received']['bits_per_second']
			ret['%s/bits_per_second' % direction] = bits_per_second
		except:
			ok = 0
		#endtry
	#endfor

	ret['ok'] = ok
	return ret
#enddef

def url_contains(url, contains=None):
	r = requests.get(url)

	if contains in r.text:
		ok = 1
	else:
		ok = 0
	#endif

	return {'ok': ok}
#enddef

def send(url, data):
	requests.post(url, data=json.dumps(data))
#enddef

class TestThread(threading.Thread):
	def __init__(self, interval, fn, *args):
		threading.Thread.__init__(self)

		self.interval = interval
		self.fn = fn
		self.args = args
		self.res = None
	#enddef

	def run(self):
		self.res = self.fn(*self.args)
	#enddef
#endclass

TEST_MAP = {
	'alive': alive,
	'load': load,
	'ping': ping,
	'ping6': ping6,
	'url_contains': url_contains,
	'iperf3': iperf3,
}

def main():
	data = []
	last_sent = 0

	src = socket.gethostname()
	tests = load_tests(TESTS_FN)

	threads = {}

	last_run = {}
	for interval, test, args in tests:
		# TODO: cut-n-pasted to below
		test_name = '%s/%s' % (src, test)
		if args:
			test_name = '%s/%s' % (test_name, '/'.join(args))
		#endif

		last_run[test_name] = time.time() - interval * random.random()
	#endfor

	while 1:
		t = time.time()

		for test_name, thr in threads.copy().items():
			if thr.is_alive(): continue

			res = thr.res
			interval = thr.interval

			if res is None: continue

			for res_name, v in res.items():
				k_full = '%s/%s' % (test_name, res_name)
				data.append({'path': k_full, 'value': v, 'time': t, 'interval': interval})
				print('%s=%s' % (k_full, v))
			#endfor

			thr.join()
			del threads[test_name]
		#endfor

		for interval, test, args in tests:
			# TODO: cut-n-pasted from above
			test_name = '%s/%s' % (src, test)
			if args:
				test_name = '%s/%s' % (test_name, '/'.join(args))
			#endif

			if t < last_run[test_name] + interval: continue
			if len(threads) >= TESTS_MAX: break

			print('--> %s (%s/%s)' % (test_name, len(threads) + 1, TESTS_MAX))

			fn = TEST_MAP[test]
			thr = TestThread(interval, fn, *args)
			thr.start()
			threads[test_name] = thr
			last_run[test_name] = t
		#endfor

		if data and t > last_sent + SEND_INTERVAL:
			try:
				send(SERVER_URL, data)
				data = []
				last_sent = t
			except Exception as e:
				print('failed to send data: %s -> %s' % (str(e), len(data)))
			#endtry
		#endif

		time.sleep(1)
	#endwhile
#enddef

if __name__ == '__main__':
	main()
#endif