import time

try:
    import boto3
except BaseException:
    boto3 = None

from .model import parse_verbosity
from .util import retry
from . import logs

class SQSQueue(object):

    def __init__(self, name, verbose=10, receive_timeout=300,
                 retry_time=10, region_name=None):
        assert boto3 is not None
        self._client = boto3.client('sqs', region_name=region_name)

        create_q_response = self._client.create_queue(
            QueueName=name)

        self._queue_url = create_q_response['QueueUrl']
        self.logger = logs.getLogger('SQSQueue')
        if verbose is not None:
            self.logger.setLevel(parse_verbosity(verbose))
        self._name = name
        self.logger.info('Creating SQS queue with name ' + name)
        self.logger.info('Queue url = ' + self._queue_url)

        self._receive_timeout = receive_timeout
        self._retry_time = retry_time

    def get_name(self):
        return self._name

    def clean(self, timeout=0):
        while True:
            msg = self.dequeue(timeout=timeout)
            if not msg:
                break

    def enqueue(self, msg):
        self.logger.debug("Sending message {} to queue with url {} "
                          .format(msg, self._queue_url))
        self._client.send_message(
            QueueUrl=self._queue_url,
            MessageBody=msg)

    def has_next(self):
        raise NotImplementedError(
            'Using has_next with distributed queue ' +
            'such as pubsub will bite you in the ass! ' +
            'Use dequeue with timeout instead')

        no_tries = 3
        for _ in range(no_tries):
            response = self._client.receive_message(
                QueueUrl=self._queue_url)

            if 'Messages' not in response.keys():
                time.sleep(5)
                continue
            else:
                break

        msgs = response.get('Messages', [])

        for m in msgs:
            self.logger.debug('Received message {} '.format(m['MessageId']))
            self.hold(m['ReceiptHandle'], 0)

        return any(msgs)

    def dequeue(self, acknowledge=True, timeout=0):
        wait_step = 1
        for waited in range(0, timeout + wait_step, wait_step):
            response = self._client.receive_message(
                QueueUrl=self._queue_url)
            msgs = response.get('Messages', [])
            if any(msgs):
                break
            elif waited == timeout:
                return None
            else:
                self.logger.info(
                    ('No messages found, sleeping for {} ' +
                     ' (total sleep time {})').format(wait_step, waited))
                time.sleep(wait_step)

        msgs = response['Messages']

        if not any(msgs):
            return None

        retval = msgs[0]

        if acknowledge:
            self.acknowledge(retval['ReceiptHandle'])
            self.logger.debug("Message {} received and acknowledged"
                              .format(retval['MessageId']))

            return retval['Body']
        else:
            self.logger.debug("Message {} received, ack_id {}"
                              .format(retval['MessageId'],
                                      retval['ReceiptHandle']))
            return (retval['Body'], retval['ReceiptHandle'])

    def acknowledge(self, ack_id):
        retry(lambda: self._client.delete_message(
            QueueUrl=self._queue_url,
            ReceiptHandle=ack_id),

            sleep_time=10, logger=self.logger)

    def hold(self, ack_id, minutes):
        self._client.change_message_visibility(
            QueueUrl=self._queue_url,
            ReceiptHandle=ack_id,
            VisibilityTimeout=int(minutes * 60))

    def delete(self):
        self._client.delete_queue(QueueUrl=self._queue_url)


# this will download the Dragon Magazine issues from
# https://annarchive.com/dragon.html
# appears that the site is blocking crawlers / scrapers
# regardless of header info
#
# using wget command to retrieve the files
# will keep this as a reminder that sometimes reinventing the wheel
# is not always best

import requests
from bs4 import BeautifulSoup
import urllib.request

def get_magazine():
    dragon_url = 'https://annarchive.com/dragon.html'

    headers = {}
    headers['User-Agent']= 'Mozilla/5.0 (X11; U; Linux i686) Gecko/20071127 Firefox/2.0.0.11'
    req = urllib.request.Request(dragon_url, headers = headers)
    resp = urllib.request.urlopen(req)
    resp_data = resp.read()

    # get the url
    r = requests.get(dragon_url)

    # create the beautifulsoup object
    soup = BeautifulSoup(r.content, 'lxml')

    # find all links
    links = soup.findAll('a')

    # filter the pdfs
    magazines = [dragon_url + link['href'] for link in links if link['href'].endswith('.pdf')]

    return magazines

def download_magazines(magazines):

    for link in magazines:
        urllib.request.urlretrieve(link, '/home/inflatus/Python/Python/Scraping/')

    print('All Magazines Downloaded')
    return

if __name__ == '__main__':
    magazines = get_magazine()
    download_magazines(magazines)


import torch
from torch import nn
import numpy as np
import time
# import wandb
from data_preprocess import Utils
from sklearn.model_selection import TimeSeriesSplit
from torch.utils.data import DataLoader

class GRU(nn.Module):
    def __init__(self, input_dim, hidden_dim, output_dim):
        super(GRU, self).__init__()
        self.gru = nn.GRU(input_dim, hidden_dim)
        self.linear = nn.Linear(hidden_dim, output_dim)
        self.logistic = nn.Sigmoid()

        self.has_features = False

        self.train_window = 20
        self.test_window = 1

        self.epochs = 1
        self.optimizer = None
        self.loss_function = None
        self.model_name = None
        self.need_train = None
        self.k_fold = 5

        self.device = None

    def forward(self, input_seq):
        gru_out, hidden = self.gru(input_seq)
        out = self.linear(hidden)
        out = self.logistic(out)
        return out

    def set_params(self, has_features, train_window, test_window, epoch, optimizer,
                   loss_function, model_name, k_fold, device):
        # 属性
        # self.need_train = need_train
        self.has_features = has_features
        self.train_window = train_window
        self.test_window = test_window
        self.epochs = epoch
        self.optimizer = optimizer
        self.loss_function = loss_function
        self.model_name = model_name
        self.k_fold = k_fold
        self.device = device

    @staticmethod
    def predict_value_nf(self, model, seq):
        predicted_value = []
        begin_time = time.time()
        value = model(seq)
        end_time = time.time()
        infer_time = end_time - begin_time
        value = value.to(self.device)
        predicted_value.append(value)
        predicted_value = torch.Tensor(predicted_value)
        return predicted_value.view(-1, 1), infer_time

    def train_model_nf(self, model: torch.nn.Module, train_X: torch.Tensor, train_Y: torch.Tensor):
        time_start = time.time()
        model = model.to(self.device)
        loss_function = self.loss_function.to(self.device)
        model.train()
        for i in range(self.epochs):
            sum_loss = 0.0
            for data, label in zip(train_X, train_Y):
                self.optimizer.zero_grad()
                data, label = data.to(self.device), label.to(self.device)
                value = model(data)
                each_loss = loss_function(value, label)
                sum_loss = sum_loss + each_loss.item()
                each_loss.backward()
                # wandb.log({"sum_loss": sum_loss})
                # loss = Variable(loss.data, requires_grad=True)
                # loss.requires_grad_(True)
                self.optimizer.step()

            if i % 10 == 1:
                print(f'epoch: {i:3} loss: {sum_loss:10.8f}')
        print(f'epoch: {i:3} loss: {sum_loss:10.10f}')
        # 模型保存
        time_end = time.time()
        torch.save(model.state_dict(), f'./MODEL/GRU/{self.model_name}.pt')
        print('time cost', time_end - time_start, 's')

    def train_model_with_batches(self, model, dataset, batch_size):
        time_start = time.time()
        model = model.to(self.device)
        loss_function = self.loss_function.to(self.device)
        model.train()

        data_loader = DataLoader(dataset, batch_size=batch_size, shuffle=False)
        for i in range(self.epochs):
            sum_loss = 0.0
            for batch_data, batch_label in data_loader:
                self.optimizer.zero_grad()
                batch_data, batch_label = batch_data.to(self.device), batch_label.to(self.device)
                length = batch_data.shape[0]
                batch_data = batch_data.view(-1, length, 1)  # GRU的输入第二维度是batch_size，注意可能会多个余数
                batch_value = model(batch_data)
                batch_value = batch_value.view(length, 1, 1)
                batch_loss = loss_function(batch_value, batch_label)
                sum_loss += batch_loss.item()
                batch_loss.backward()
                self.optimizer.step()

            if i % 10 == 1:
                print(f'epoch: {i:3} loss: {sum_loss / len(data_loader):10.3f}')
            else:
                print(f'epoch: {i:3} loss: {sum_loss / len(data_loader):10.3f}')

        # 模型保存
        time_end = time.time()
        torch.save(model.state_dict(), f'./MODEL/GRU/{self.model_name}.pt')
        print('time cost: {:.3f} s'.format(time_end - time_start))

    def eval_GRU_nf(self, model: torch.nn.Module, test_set: np.ndarray):
        train_window: int = self.train_window
        time_start: float = time.time()
        model.eval()
        # model = model.cpu()
        pred_value: list = []
        t_sum: float = 0.0
        with torch.no_grad():
            for test_idx in range(len(test_set) - train_window):
                test_seq: torch.FloatTensor = torch.FloatTensor(test_set[test_idx: test_idx + train_window, :]).reshape((-1, 1))
                test_seq = test_seq.to(self.device)
                pred, t = self.predict_value_nf(self, model, test_seq)
                t_sum = t + t_sum
                pred_value.append(pred[-1, :])
        time_end: float = time.time()
        print('average cost time per inference: {:.3f} s'.format(t_sum / len(pred_value)))
        print('time cost: {:.3f} s'.format(time_end - time_start))
        return pred_value

    def eval_GRU(self, model, test_set, features, feature_dim):
        time_start = time.time()
        model.eval()
        # model = model.cpu()
        test_window = self.test_window
        train_window = self.train_window
        pred_value = []
        with torch.no_grad():
            for test_idx in range(len(test_set) - train_window):
                test_seq = torch.FloatTensor(test_set[test_idx: test_idx + train_window, :]).reshape((-1, feature_dim))
                pred = self.predict_value(self, model, test_seq, features)
                pred_value.append(pred[-1, :])
        time_end = time.time()
        print('time cost: {:.3f} s'.format(time_end - time_start))
        return pred_value


import logging
from typing import Any
from typing import Callable

import requests
from requests import Response
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry

# fmt: off
# isort: off
# pylint: disable=unused-import
# Keep these imports so that our monkey patch of mailjet_rest allows
# its code to use `requests.exception` and get `RequestException`
# from this module.
from requests import RequestException
import requests.exceptions as exceptions
# pylint: enable=unused-import
# isort: on
# fmt: on

from pcapi import settings

logger = logging.getLogger(__name__)

REQUEST_TIMEOUT_IN_SECOND = 10

def _wrapper(request_func: Callable, method: str, url: str, log_at_error_level=True, **kwargs: Any) -> Response:
    timeout = kwargs.pop("timeout", REQUEST_TIMEOUT_IN_SECOND)
    try:
        response = request_func(method=method, url=url, timeout=timeout, **kwargs)
    except Exception as exc:
        # If the URL is not controlled by us, there is probably no
        # reason to send the error to Sentry. Logging an INFO message
        # is enough if we ever need to debug.
        if log_at_error_level:
            logger_method = logger.exception
        else:
            logger_method = logger.info
        logger_method("Call to external service failed with %s", exc, extra={"method": method, "url": url})
        raise exc
    else:
        logger.info(
            "External service called",
            extra={
                "url": response.url,
                "statusCode": response.status_code,
                "duration": response.elapsed.total_seconds(),
            },
        )

    return response

def get(url: str, **kwargs: Any) -> Response:
    with Session() as session:
        return session.request(method="GET", url=url, **kwargs)

def post(url: str, **kwargs: Any) -> Response:
    with Session() as session:
        return session.request(method="POST", url=url, **kwargs)

def put(url: str, **kwargs: Any) -> Response:
    with Session() as session:
        return session.request(method="PUT", url=url, **kwargs)

def delete(url: str, **kwargs: Any) -> Response:
    with Session() as session:
        return session.request(method="DELETE", url=url, **kwargs)

class _SessionMixin:
    def request(self, method: str, url: str, *args: Any, **kwargs: Any) -> Response:
        return _wrapper(super().request, method, url, *args, **kwargs)

class Session(_SessionMixin, requests.Session):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        # Only sets a retry strategy for safe verbs
        safe_retry_strategy = Retry(total=3, allowed_methods=["HEAD", "GET", "OPTIONS"])
        unsafe_retry_strategy = Retry(total=3)
        safe_adapter = HTTPAdapter(max_retries=safe_retry_strategy)
        unsafe_adapter = HTTPAdapter(max_retries=unsafe_retry_strategy)
        self.mount("https://www.demarches-simplifiees.fr", safe_adapter)
        if settings.JOUVE_API_DOMAIN:
            self.mount(settings.JOUVE_API_DOMAIN, safe_adapter)
        self.mount("https://api.mailjet.com", unsafe_adapter)


# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import errno
import logging
from compare_locales import mozpath
from .project import ProjectConfig
from .matcher import expand
import toml

class ConfigNotFound(EnvironmentError):
    def __init__(self, path):
        super().__init__(
            errno.ENOENT,
            'Configuration file not found',
            path)

class ParseContext:
    def __init__(self, path, env, ignore_missing_includes):
        self.path = path
        self.env = env
        self.ignore_missing_includes = ignore_missing_includes
        self.data = None
        self.pc = ProjectConfig(path)

class TOMLParser:
    def parse(self, path, env=None, ignore_missing_includes=False):
        ctx = self.context(
            path, env=env, ignore_missing_includes=ignore_missing_includes
        )
        self.load(ctx)
        self.processBasePath(ctx)
        self.processEnv(ctx)
        self.processPaths(ctx)
        self.processFilters(ctx)
        self.processIncludes(ctx)
        self.processExcludes(ctx)
        self.processLocales(ctx)
        return self.asConfig(ctx)

    def context(self, path, env=None, ignore_missing_includes=False):
        return ParseContext(
            path,
            env if env is not None else {},
            ignore_missing_includes,
        )

    def load(self, ctx):
        try:
            with open(ctx.path, 'rt') as fin:
                ctx.data = toml.load(fin)
        except (toml.TomlDecodeError, OSError):
            raise ConfigNotFound(ctx.path)

    def processBasePath(self, ctx):
        assert ctx.data is not None
        ctx.pc.set_root(ctx.data.get('basepath', '.'))

    def processEnv(self, ctx):
        assert ctx.data is not None
        ctx.pc.add_environment(**ctx.data.get('env', {}))
        # add parser environment, possibly overwriting file variables
        ctx.pc.add_environment(**ctx.env)

    def processLocales(self, ctx):
        assert ctx.data is not None
        if 'locales' in ctx.data:
            ctx.pc.set_locales(ctx.data['locales'])

    def processPaths(self, ctx):
        assert ctx.data is not None
        for data in ctx.data.get('paths', []):
            paths = {
                "l10n": data['l10n']
            }
            if 'locales' in data:
                paths['locales'] = data['locales']
            if 'reference' in data:
                paths['reference'] = data['reference']
            if 'test' in data:
                paths['test'] = data['test']
            ctx.pc.add_paths(paths)

    def processFilters(self, ctx):
        assert ctx.data is not None
        for data in ctx.data.get('filters', []):
            paths = data['path']
            if isinstance(paths, str):
                paths = [paths]
            rule = {
                "path": paths,
                "action": data['action']
            }
            if 'key' in data:
                rule['key'] = data['key']
            ctx.pc.add_rules(rule)

    def processIncludes(self, ctx):
        for child in self._processChild(ctx, 'includes'):
            ctx.pc.add_child(child)

    def processExcludes(self, ctx):
        for child in self._processChild(ctx, 'excludes'):
            ctx.pc.exclude(child)

    def _processChild(self, ctx, field):
        assert ctx.data is not None
        if field not in ctx.data:
            return
        for child_config in ctx.data[field]:
            # resolve child_config['path'] against our root and env
            p = mozpath.normpath(
                expand(
                    ctx.pc.root,
                    child_config['path'],
                    ctx.pc.environ
                )
            )
            try:
                child = self.parse(
                    p, env=ctx.env,
                    ignore_missing_includes=ctx.ignore_missing_includes
                )
            except ConfigNotFound as e:
                if not ctx.ignore_missing_includes:
                    raise
                (logging
                    .getLogger('compare-locales.io')
                    .error('%s: %s', e.strerror, e.filename))
                continue
            yield child

    def asConfig(self, ctx):
        return ctx.pc


from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import Select
import time
import re

ontologyElement = {}

def runQuery(browser):
#    runOk = browser.find_element(By.ID, "yui-gen5-button")
    runOk = browser.find_element(By.ID, "yui-gen9-button")
    runOk.click()

def pickQueryType(browser, queryTypeList):
    runQuery = browser.find_element(By.ID, "runBox")
    runQuery.click()

    if len(queryTypeList) > 0:
        for queryType in queryTypeList:
            print (queryType)
            currentDivElement = browser.find_element(By.ID, queryType)
            print (currentDivElement.get_attribute('value'))
            currentElement = currentDivElement.find_element(By.TAG_NAME, 'input')
            print (currentElement.get_attribute('value'))
            currentElement.click()

# Open each of the Ontology tree nodes until you get to the desired element
def traverseOntology(browser, nodeStrings):
    global currentElement
    nodeCnt = 0
    for currentNode in nodeStrings:
        print ('Current Node: ' , currentNode)
        #Find all the visible concepts
        '''
        try:
            element = WebDriverWait(browser, 30).until(
                EC.presence_of_element_located((By.CLASS_NAME, "sdxDefaultCONCPT")))
                #EC.presence_of_element_located((By.ID, "myDynamicElement")))
        finally:
            print('Still not visible')
        '''

        sdxConcepts = browser.find_elements(By.CLASS_NAME, "sdxDefaultCONCPT")

        #Add them to a dictionary
        print('Add more nodes...');
        for sdxConcept in sdxConcepts:
            print("sdxConcepts: %s" % sdxConcept.text)
            ontologyElement[sdxConcept.text] = sdxConcept
        #Find the concept that corresponds to your 1st click open it
        print('Open...', currentNode)
        currentElement = ontologyElement[currentNode]
        if ( nodeCnt < len(nodeStrings)):
            currentElement.click()
        nodeCnt = nodeCnt + 1
        time.sleep(5) #webdriver wait does not seem to be working
    #return the last element
    return currentElement

# Open each of the Ontology tree nodes until you get to the desired element
def findValueByExclusion(browser):
    global currentElement
    elementCnt = 0
    tagElements = browser.find_elements(By.TAG_NAME, "tr")

    #Add them to a dictionary
    print('Add more nodes...');
    for tagElement in tagElements:
        print("tagElement Text: %s" % tagElement.text)
'''
        ontologyElement[sdxConcept.text] = sdxConcept
        #Find the concept that corresponds to your 1st click open it
        print('Open...', currentNode)
        currentElement = ontologyElement[currentNode]
        if ( nodeCnt < len(nodeStrings)):
            currentElement.click()
        nodeCnt = nodeCnt + 1
        time.sleep(5) #webdriver wait does not seem to be working
    #return the last element
    return currentElement
'''

# Find the first numerical text
def findValueByNumberOnly(browser, tag_name):
    global currentElement
    elementCnt = 0
    tagElements = browser.find_elements(By.TAG_NAME, tag_name)

    #Add them to a dictionary
    print('Add more nodes...');
    for tagElement in tagElements:
        print("tagElement Text: %s" % tagElement.text)
        if tagElement.text.isnumeric() == True:
            return int(tagElement.text)
    return -1

# use regular expression on innerhtml to find what you are looking for
# This is used because the use of ids and tags is pretty inconsistent
# in the interface
def findValuesByRegularExpression(browser, regExpressions):
    #print ('Element: ' , browser.get_attribute('innerHTML'))
    theText = browser.get_attribute('innerHTML')
    for regExpression in regExpressions:
        print ('Reg Ex: ' ,regExpression)
        value = findValueByRegularExpression(theText, regExpression, 1)
        print ('Value: ' , value)

def findValueByRegularExpression(theText, regularExpression, matchGroup):
    #print ('Element: ' , browser.get_attribute('innerHTML'))
    #print ('Reg Ex: ' ,regularExpression)
    match = re.search(regularExpression, theText )
    if match:
        print('Match: ', match.group(matchGroup))
        return match.group(matchGroup)
    return -1 #NO MATCH

'''
    #Add them to a dictionary
    print('Add more nodes...');
    for tagElement in tagElements:
        print("tagElement Text: %s" % tagElement.getElementName())
        match = re.search(regularExpression, tagElement.getText())
        if match:
            print('Match: ', tagElement.text)
    return -1
'''

browser = webdriver.Chrome()
browser.implicitly_wait(10) # seconds
#browser.get('https://i2b2.act.pitt.edu/webclient/')
browser.get('https://dbmi-ncats-test02.dbmi.pitt.edu/webclient/')
print("Done waiting")
browser.find_element_by_id("loginusr").send_keys(Keys.CONTROL, "a")
browser.find_element_by_id("loginusr").send_keys('username')
browser.find_element_by_id("loginpass").send_keys(Keys.CONTROL, "a")
browser.find_element_by_id("loginpass").send_keys('passwd'+ Keys.RETURN)
nodeStrings = ['ACT Demographics','Race','Asian']
time.sleep(5)

#Drag term to panel
currentNode = traverseOntology(browser, nodeStrings)
queryPanel1 = browser.find_element(By.ID, "QPD1")
ActionChains(browser).drag_and_drop(currentNode, queryPanel1).perform()

#infoQueryStatusText
#Number of patients for 'query name'
#patient_count:

# Select Query Topic
#queryTopicSelect = Select(browser.find_element(By.ID, "queryTopicSelect"))
#queryTopicSelect.select_by_visible_text("Test Pitt Query")

#Run query popup
'''value PATIENT_GENDER_COUNT_XML id crcDlgResultOutputPATIENT_GENDER_COUNT_XML
PATIENT_VITALSTATUS_COUNT_XML
PATIENT_RACE_COUNT_XML
PATIENT_AGE_COUNT_XML
'''
queryTypeList = ['crcDlgResultOutputPATIENT_GENDER_COUNT_XML',
                 'crcDlgResultOutputPATIENT_VITALSTATUS_COUNT_XML',
                 'crcDlgResultOutputPATIENT_RACE_COUNT_XML',
                 'crcDlgResultOutputPATIENT_AGE_COUNT_XML']
pickQueryType(browser, queryTypeList)

#Name Query
runQuery(browser)

# try to find the answer
time.sleep(60)
answer = browser.find_element(By.ID, "chart0")
print(answer.find_element_by_tag_name("tr").text)
findValueByExclusion(answer)
answerNumber = findValueByNumberOnly(answer, 'tr')
print('Query Result',answerNumber)
showQueryStatus = browser.find_element(By.ID, "infoQueryStatusText")
#print('showQueryStatus: ', showQueryStatus.get_attribute('innerHTML'))
#endDateMillsecElem = browser.find_element(By.ID, "endDateMillsecElem")
#print('endDateMillsecElem: ', endDateMillsecElem.get_attribute('innerHTML'))

time.sleep(60)
#findValueByRegularExpression(showQueryStatus, 'div', 'Compute Time: (\w*)')
regExprs = ('Compute Time: (\w*)','patient_count: <font color=\"#0000dd\">(\d*)')
findValuesByRegularExpression(showQueryStatus, regExprs)

'''
#drop Asian on 1st query panel
asian = ontologyElement['Asian']
queryPanel1 = browser.find_element(By.ID, "QPD1")
ActionChains(browser).drag_and_drop(asian, queryPanel1).perform()

# Select Query Topic
queryTopicSelect = Select(browser.find_element(By.ID, "queryTopicSelect"))
queryTopicSelect.select_by_visible_text("Test Pitt Query")

#Run query
runQuery = browser.find_element(By.ID, "runBox")
runQuery.click()
#runOk = browser.find_element(By.ID, "yui-gen5-button")
runOk = browser.find_element(By.ID, "yui-gen9-button")
runOk.click()
'''
# Run Query

#    sdxConcept.click()
#demographics = browser.find_element(By.ID, "ONT_TID-1")
#demographics.click()
#race = browser.find_element(By.ID, "ONT_TID-99")
#race.click()
#asian = browser.find_element(By.ID, "ONT_TID-103")

#ActionChains(browser).drag_and_drop(asian, queryPanel1).perform()

#for sdxConcept in sdxConcepts:
#    print("Value is: %s" % sdxConcept.get_attribute("value"))
#    sdxConcept.click()
'''
WebElement element = browser.findElement(By.name("source"));
WebElement target = browser.findElement(By.name("target"));

(new Actions(browser)).dragAndDrop(element, target).perform();
'''

'''
browser.find_element_by_name('loginForm').send_keys('test')

browser.switch_to.frame('_yuiResizeMonitor')
browser.find_element_by_id('loginusr').send_keys('test')

browser.find_element_by_class_name("input")
browser.find_element_by_id("loginusr").send_keys('username')
browser.find_element_by_id ('ID').send_keys('password')
browser.find_element_by_id('submit').click()

elem = browser.findElement(By.id("uname"));
#elem = browser.find_element_by_name('uname')  # Login
elem.send_keys('username' + Keys.RETURN)
elem = browser.find_element_by_name('input#loginpass')  # Login
elem.send_keys('password' + Keys.RETURN)
elem = browser.find_element_by_name('input')  # Login
elem.send_keys(Keys.RETURN)
'''
#browser.quit()
'''
male
250 icd9
e13 icd10
36 icd9p
03 icd10p
1012974 cpt
va med 161 acetam
loinc 2345-7
visit 2 days
inc e13 exc 250
01/01/2014 e08-e13 exclude 250 no date
male all breakdowns
retired 71010
'''

''' Make Methods
 Read in json that describes the test
--select
--breakdown
--set date
--set exclude
--add to more that one panel
--login
'''


import os, sys
import itertools
from Crypto.Util.number import bytes_to_long, long_to_bytes
import puzzle; reload(puzzle); from puzzle import *;
import signature; reload(signature); from signature import *

# Parameters
puz_params = dict(
    NUM_BLOCKS=1024*1024*8*2,
    BLOCK_SIZE=1024/2,
    ITERS=20,
    )

serv_params = dict(
    CHUNK=100
    )

globals().update(puz_params)

# Default low-level file access
def make_file(fname, num_blocks, block_size):
    fd = os.open(fname, os.O_RDONLY | os.O_DIRECT) # Avoids filesystem cache
    #fd = os.open(fname, os.O_RDONLY)
    f = os.fdopen(fd, 'rb')
    def query(idx):
        assert idx < num_blocks
        f.seek(idx*block_size)
        b = 0
        r = ''
        while b < block_size:
            c = min(2048, block_size-b)
            r += f.read(c)
            b += c
        assert len(r) == block_size
        return r
    return query

def make_PoRMiner(puzzle, sigscheme, pkey, skey):
    index_from_state = lambda sol: hash(sol) % puzzle.num_blocks

    def attempt(puz, nonce):
        state = '%s:%s:%s' % (puz, pkey, nonce)
        sigs = []
        for i in xrange(puzzle.iters):
            # Access the file
            idx = index_from_state(state)
            block = puzzle.fquery(idx)
            # Sign the data
            message = '%s:%s:%s' % (puz, nonce, hash(block))
            sig = sigscheme.sign(message, skey)
            sigs.append(sig)
            state = puz + nonce + sig

        final = SHA.SHA1Hash(state).digest()
        return final, sigs

    def mine(puz, tries=None):
        assert type(puz) is str
        if tries is None: print("Mining indefinitely")
        else: print("Mining, %d tries" % tries)
        for i in itertools.count() if tries is None else xrange(tries):
            if (i % 10 == 0): print("Attempt %d..." % i)
            nonce = os.urandom(20)
            final, sigs = attempt(puz, nonce)
            if check_difficulty(difficulty, final):
                print("Solution found!")
                return nonce, sigs
    return mine

# Tests
def make_random_file(fname, num_bytes):
    ctr = 0
    block_size = 1024*1024
    f = open(fname, 'wb')
    print("Writing %d random bytes to %s..." % (num_bytes, fname))
    while ctr < num_bytes:
        b = min(block_size, num_bytes-ctr)
        ctr += b
        f.write(os.urandom(b))
    print("Done")

ssd_fname = "random_8gb.dat"
harddisk_fname = "/media/amiller/asus-backup-7_12/home/blockplayerdata/harddisk_8gb.dat"
fquery = make_file(ssd_fname, NUM_BLOCKS, BLOCK_SIZE)
difficulty = 32.
sigscheme = DegenerateSignatureScheme(1024)
puzzle = PoRPuzzle(fquery, sigscheme, NUM_BLOCKS, BLOCK_SIZE, ITERS, difficulty)
pkey, skey = sigscheme.generate()
miner = make_PoRMiner(puzzle, sigscheme, pkey, skey)

class Streamer(object):
    """
    """
    def __init__(self, callback):
        self.callback = callback

    def query(self, index):
        raise NotImplemented

class SequentialStreamer(Streamer):
    def __init__(self, callback, filepath):
        self.callback = callback
    def query(self, index):
        pass


import os
import string
from inspect import stack
from itertools import combinations

import networkx as nx
from matplotlib import pyplot as plt
from networkx.drawing.nx_pydot import graphviz_layout

LIVE_DRAW_ENABLED = False  # Disable the function draw for live renderings
DRAW_ENABLED = True  # Disable the function draw, but not draw_multi

def config(enable_live_draw=False, enable_draw=True):
    global LIVE_DRAW_ENABLED, DRAW_ENABLED
    LIVE_DRAW_ENABLED = enable_live_draw
    DRAW_ENABLED = enable_draw

class PhyloNet(nx.DiGraph):
    """
    Helper class for a phylogenetic network (of instance NetworkX.DiGraph)
    """

    def __new__(cls, di_graph=nx.DiGraph()):
        """ Create a PhyloNet instance. The first argument can be used to define either a DiGraph or MultiDiGraph. """
        if not isinstance(di_graph, nx.DiGraph):
            print(di_graph, type(di_graph), isinstance(di_graph, nx.DiGraph))
        #     raise TypeError('PhyloNet should be of type DiGraph (or MultiDiGraph).')
        return di_graph.__new__(cls)

    def __init__(self, type1=None):
        super().__init__(type1)
        self._z_edges = None

    def copy(self, as_view=False):
        return super().copy(as_view)

    def get_root(self):
        """
        Get the root of the network
        :return: The unique root
        """
        for n in self.nodes:
            if self.in_degree(n) == 0:
                return n

    def get_ret_num(self):
        """
        Get the number of binary hybrid nodes/reticulations
        :return: The number of reticulations
        """
        return sum(self.in_degree(n) - 1 for n in self.nodes if self.in_degree(n) >= 2)

    def get_level(self):
        """ Get the level of the network, i.e. the maximal number of reticulations per bi-connected component. """
        g = self.to_undirected()
        level = 0
        for cc in nx.biconnected_components(g):
            if len(cc) <= 2:
                continue
            cc_ret_num = len([n for n in cc if self.in_degree(n) >= 2])
            level = max(level, cc_ret_num)
        return level

    def get_leaves(self):
        """
        For the given acyclic digraph, get the leaf nodes
        :return: A sorted list of leaves
        """
        return sorted(n for n in self.nodes() if self.out_degree(n) == 0)

    def get_dummy_leaves(self):
        """
        Get the dummy leaves
        :return: A list of names
        """
        return sorted(n for n in self.nodes() if len(n) >= 2 and n[0:2] == "d_")

    def get_hardwired_cluster(self, source: str, filter_name=False):
        """
        Get the hardwired cluster below the given source node
        :param source: A node
        :return: The hardwired cluster below the given source/node
        """
        if source not in self.nodes:
            return None
        # If the source is a leaf
        if source[0] != '_':
            return [source]
        # assert source[0] == '_':
        # The source is an internal node
        if filter_name:
            return sorted([n for n in nx.descendants(self, source) if n[0] != '_'])
        return sorted([n for n in nx.descendants(self, source) if self.out_degree(n) == 0])

    # NICE TODO: Implement a more efficient method
    def _get_soft_wired_clusters(self, source):
        """
        Get all soft-wired clusters
        :param source: A node
        :return: A set of the soft-wired clusters below the given source/node
        """

        def _select_ret_inputs(ret_ins: dict):
            """
            In the network net, select one incoming edge per reticulation to leave a tree
            :param ret_ins: A dictionary:
                Each (r: i) determines that the i-th (starting from 0) edge is selected for reticulation r.
            :return: The resulting tree
            """
            _tree = self.copy()
            for r, i in ret_ins.items():
                E_in = list(_tree.in_edges(r))
                E_in.__delitem__(i)
                _tree.remove_edges_from(E_in)
            for n in list(_tree.nodes):
                if _tree.out_degree(n) == 0 and n[0] == '_':
                    _tree.remove_node(n)
            # Remove singletons
            _tree.remove_nodes_from(list(nx.isolates(_tree)))
            return _tree

        # Find all reticulations below the source
        rets = [n for n in nx.descendants(self, source) if self.in_degree(n) >= 2]

        # If there are no reticulations below, then return the hardwired cluster
        if len(rets) == 0:
            c = self.get_hardwired_cluster(source)
            if c:
                return [c]
            return set()

        # Now find all soft-wired clusters
        c_strings = set()
        done = False
        ret_inputs = {r: 0 for r in rets}
        while not done:
            # For each reticulation, let 1 incoming edge remain
            tree = _select_ret_inputs(ret_inputs)
            c = tree.get_hardwired_cluster(source, filter_name=True)
            if c:
                c_strings.add(','.join(c))
            # Select the next tree configuration
            for r in rets:
                if ret_inputs[r] + 1 < self.in_degree(r):
                    ret_inputs[r] += 1
                    break
                elif r == rets[-1]:
                    done = True
                else:
                    ret_inputs[r] = 0
        clusters = []
        for c_s in c_strings:
            clusters.append(c_s.split(','))
        return clusters

    def get_st_sets(self, source: str):
        """
        Given that this network is a tree, return all ST-sets from a given source node and below it
        :param source: A source node
        :return: A list of lists (ST-sets)
        """
        # If the source is a leaf
        if source[0] != '_':
            return source
        sts = [self.get_hardwired_cluster(source)]
        childs = list(self.successors(source))
        # For each combination of children, add the hardwired cluster as ST-set
        for k in range(2, len(childs)):
            # Choose k children
            for combi in combinations(childs, k):
                # Get the hardwired cluster for this combination
                cl = []
                for n in combi:
                    cl.extend(self.get_hardwired_cluster(n))  # Singletons will be included
                sts.append(sorted(cl))
        # Recursively search for the ST-sets below the child split nodes
        for n in childs:
            if n[0] == '_':  # If n is a split node
                sts.extend(self.get_st_sets(n))
            else:
                sts.append([n])
        return sts

    def represents_old(self, C: set):
        """
        Check if the given network represents all clusters in C
        :param C: A set of clusters
        :param print_instr: Print instructions T/F
        :return: Does the PN represent the clusters [T/F]?
        """
        # Make a copy of the clusters C and remove each cluster that is represented. Note: This is inefficient!
        _C = C.copy()
        for node in self.nodes:
            if self.out_degree(node) == 0:
                continue
            clusters = self._get_soft_wired_clusters(node)
            for c in clusters:
                if c in _C:
                    _C.remove(c)

        return len(_C) == 0

    def represents(self, cs, x_added, lcas, print_instr=False):
        """
        Does this network represent the cluster defined in ClusterSet cs? T/F
        :param cs: A ClusterSet
        :param x_added:
        :param print_instr: Print instructions T/F
        :return: T/F
        """
        return self.represents_old(cs.C)

    def represents_cl(self, cluster, lcas):
        """ Check if the given cluster is represented """
        cl = list(cluster)
        if len(cl) == 1:
            return True
        c_s = ','.join(cluster)
        if c_s not in lcas.keys():
            # Check if the cluster is in the network
            print(f'ERROR: I do not see cluster {c_s}. Is it there?')
            self.draw(True, '?', all_labels=True)
        for ca in lcas[c_s]:  # For each common ancestor
            # Get the hardwired cluster
            hw_cl = self.get_hardwired_cluster(ca)
            if hw_cl == cluster:
                return True
        # Create a copy of the network & remove the leaves that are not in the cluster
        net = self.copy()
        # Remove each (parent) node that is not part of the cluster
        X_not = set(self.get_leaves()).difference(cluster)
        X_not_p = {list(net.predecessors(x))[0] for x in X_not if net.in_degree(x) == 1}
        net.remove_nodes_from(X_not)
        net.clean()
        while X_not_p:
            X_not = X_not_p
            X_not_1p = [x for x in X_not if
                        self.in_degree(x) == 1 and self.out_degree(x) <= 2]  # All x in X_not having 1 parent
            X_not_p = [list(self.predecessors(x))[0] for x in X_not_1p]
            net.remove_nodes_from(X_not)
            net.clean()
        for ca in lcas[c_s]:  # For each common ancestor
            # Get the hardwired cluster
            hw_cl = net.get_hardwired_cluster(ca)
            if not hw_cl:
                continue
            if hw_cl == cl:
                return True

        return False

    def get_lcas_2(self, n1: str, n2: str):
        """ Get the LCA of the 2 given nodes """
        if n1[0] != '_':
            # n1 is a leaf
            n1 = self.predecessors(n1).__next__()
        if n2[0] != '_':
            # n2 is a leaf
            n2 = self.predecessors(n2).__next__()
        if n2 in nx.ancestors(self, n1):
            lcas = set()
            lcas.add(n2)
            return lcas
        if n1 in nx.ancestors(self, n2):
            lcas = set()
            lcas.add(n1)
            return lcas
        # n1 is not an ancestor of n2, nor vice versa
        P1 = [n1]
        P2 = [n2]
        lcas = set()
        net = self.copy()
        while True:
            P1_new = set()
            for n in P1:
                if n not in lcas:
                    P1_new.update(net.predecessors(n))
            if not P1_new:
                break
            P1 = P1_new
            for p in P1:
                # If node 2 is reachable from p, then p is an LCA
                if n2 in nx.descendants(net, p):
                    lcas.add(p)
                    net.remove_node(p)
            P2_new = set()
            for n in P2:
                if n not in lcas:
                    P2_new.update(net.predecessors(n))
            if not P2_new:
                break
            P2 = P2_new
            for p in P2:
                # If node 1 is reachable from p, then p is an LCA
                if n1 in nx.descendants(net, p):
                    lcas.add(p)
                    net.remove_node(p)
        if lcas:
            return lcas
        self.draw(True, f'LCA({n1}, {n2}) = ?', all_labels=True)

    def _subdivide_edge(self, e):
        """
        Insert node v into edge e in the given network
        :param e: An edge
        :return: The added node (name)
        """
        # Let v be the new node
        v = self.new_node_name()
        # Remove the old edge
        self.remove_edge(e[0], e[1])
        # Add the new edges
        self.add_edge(e[0], v)
        self.add_edge(v, e[1])
        return v

    def new_node_name(self) -> str:
        """
        Get a name for a new (internal) node (splitting or reticulation) to insert in the given network
        :return: A new node name
        """
        node_nrs = [int(n[1:]) for n in self.nodes if n[1:].isnumeric() and n[0] == '_']
        if not node_nrs:
            return '_1'
        return '_' + str(max(node_nrs) + 1)

    def add_new_root(self):
        """
        Add a new root as parent of the current root and return the labels of the new and old root, respectively.

        :return: The labels of the new and old root, respectively
        """
        root = self.get_root()
        label = self.new_node_name()
        self.add_edge(label, root)
        return label, root

    def add_ret_below_2(self, parents: list, x: str):
        """
        Add a reticulation below two of the given parent nodes
        If no parents are given, then insert a reticulation instead of two dummy leaves, and a dummy leaf below it
        :return: A list of networks having a new reticulation
        """
        # If dummy leaves are given, then search their parents
        if parents[0][0:2] == 'd_':
            # Search for the dummy leaves & their parents
            p = {}
            for dum in parents:
                p[dum] = list(self.predecessors(dum))[0]
            parents = p
        else:
            parents = dict.fromkeys(parents, None)
        parent_combis = combinations(parents, 2)
        nets = []
        # t becomes the new reticulation/hybrid node
        t = self.new_node_name()
        for ret_parents in parent_combis:
            net = self.copy()
            # If dummy leaves are selected, then hang the reticulation below its parent & remove the dummy leaf
            p = parents[ret_parents[0]]
            if p:
                # Preserve one dummy root. Subdivide its incoming edge and make it a reticulation.
                net.remove_node(ret_parents[1])
                # If there is another dummy leaf in the network, then add a dummy leaf below the new reticulation
                if len(parents) >= 3:
                    v = net._subdivide_edge((p, ret_parents[0]))
                    net.add_edge(parents[ret_parents[1]], v)
                else:
                    # #parents = 2 => Add leaf x below reticulation t
                    net.remove_node(ret_parents[0])
                    net.add_edge(p, t)
                    net.add_edge(parents[ret_parents[1]], t)
                    net.add_edge(t, x)
            else:
                # Add the edges going to t
                net.add_edges_from((v, t) for v in ret_parents)
                # Add dummy leaves
                parents_new = list(parents)
                parents_new.remove(ret_parents[0])
                parents_new.remove(ret_parents[1])
                parents_new.append(t)
                net.add_edges_from((d, f'd_{i}') for i, d in enumerate(parents_new))
            nets.append(net)
        return nets

    def add_leaf_below_ret(self, x: str, edges, cs, stay_binary: bool = False, print_info=False):
        """
        Add a leaf below a reticulation below the given edges as follows.
        Create a reticulation t, a leaf l labeled x and an edge from t to l. Then, for each edge e_i, i = 1,2,... in the
        list, insert a node v_i into e_i and add an edge from v_i to t
        :param x: A leaf
        :param cs: A set of clusters (ClusterSet)
        :param edges: A list of edges
        :param stay_binary: Let the returned network(s) be binary
        :param print_info:
        :return: The new PN
        """
        parent_nodes = []
        E_done = []
        for e in edges:
            # Note: it can happen that an edge occurs multiple times in E
            if e in E_done:
                parent_nodes.append(parent_nodes[E_done.index(e)])
                continue
            # If e represents a node, i.e. e = (u, u)
            if e[0] == e[1]:
                parent_nodes.append(e[0])
            else:
                parent_nodes.append(self._subdivide_edge(e))
            E_done.append(e)
        # If the new leaf (or ST-set) is to be hung below exactly 1 edge (or node)
        if len(edges) == 1:
            # Hang the leaf below this edge (or node)
            self.add_edge(parent_nodes[0], x)
        else:
            # If leaf x should be hung below 3 edges or more and if the network should stay binary, then add 1 ret. now
            if len(parent_nodes) > 2 and stay_binary:
                # Print.var(parent_nodes, print_info)
                # Choose two nodes to create a reticulation below, and a dummy leaf below it
                return self.add_ret_below_2(parent_nodes, x)

            # t becomes the new reticulation/hybrid node
            t = self.new_node_name()
            # Add the edges going to t
            self.add_edges_from((v, t) for v in parent_nodes)
            # Add edge (t, x)
            self.add_edge(t, x)

        return [self]

    def get_e_out(self, cs_new, cs_in, x: str, lcas):
        """
        Get the lowest edge below which a new reticulation should be hung, such that all clusters below the edge will be
        represented in the newly formed network
        :param cs_new: The new ClusterSet
        :param cs_in: The minimal clusters containing the ST-set
        :param x: A leaf
        :return:
        """
        # If cs_in is empty, then return None
        if not cs_in:
            raise RuntimeError('cs_in should not be empty')

        # If the cluster X\{x} is in C, then E_out = the root edge
        Xm = list(cs_new.X)
        Xm.remove(x)
        if Xm in cs_new.C:
            r = self.get_root()
            return cs_new.X, list(self.out_edges(r))[0]

        # Check if the cluster X_in\st exists as (not necessarily strict) subset of any cluster in C.
        # Make a list (set) of all such edges.
        c_out = set(cs_in.get_used_taxa())
        c_out.remove(x)
        E_out = set()
        cl_out = set()
        # for c in cs_new.maximal().C:
        for c in cs_new.C:
            # If cluster c contains c_out and not leaf x
            if c_out.issubset(c) and x not in c:
                cl_out.update(c)

                # Cluster c in C contains c_out. Hence, the ST-set should be hung below an edge that is above cluster c.
                c_s = ','.join(c)
                if c_s in lcas.keys():
                    lca = lcas[c_s]
                    e = list(self.in_edges(lca))[0]
                else:
                    e = self.get_cluster_edge(c)
                if not e:
                    # Cluster c is not represented
                    print(f'ERROR: Cluster {c} is not represented')
                    return None, None
                E_out.add(e)

        # Check the length of E_out and return if possible
        _l = len(E_out)
        if _l == 1:
            return cl_out, list(E_out)[0]
        if _l == 0:
            return None, None

        # Find the highest E
        cl_nodes = {v for u, v in E_out}
        lca = self.get_comm_anc(cl_nodes)
        # if len(lcas) > 2:
        #     Print.var(cl_nodes)
        #     Print.var(lcas)
        #     self.draw(True, f'LCA({cl_nodes}) ?= {lcas}', all_labels=True)
        # lca = lcas[0]
        e_in = self.in_edges(lca)
        if len(e_in) == 0:
            # This node has no incoming edge. It must be the root
            e_out = self.out_edges(lca)
            return cl_out, list(e_out)[0]
        return cl_out, list(e_in)[0]

    def get_cluster_edge(self, c: list):
        """
        Find the lowest edge that represents the cluster
        :param c: A cluster
        :return: An edge
        """
        # Get the cluster leaves
        if len(c) == 1:
            comm_anc = c[0]
        else:
            # Quickly check if the root edge is present
            r = self.get_root()
            assert r not in c

            # Get the (lowest) common ancestor and then its incoming edge
            comm_anc = self.get_comm_anc(c)
            if not comm_anc:
                # This cluster is not represented
                return None
        e_in = self.in_edges(comm_anc)
        if len(e_in) == 0:
            # This node has no incoming edge. It must be the root
            return None, comm_anc
        return list(e_in)[0]

    def get_comm_anc(self, nodes):
        """
        Get the common ancestor of the given nodes
        :param nodes: A list of nodes
        :return: A node
        """

        assert nodes
        nodes = list(nodes)
        # Print.var(nodes, True)
        # If all leaves in the network are given, then LCA = the child of the root
        if nodes == self.get_leaves():
            r = self.get_root()
            return list(self.out_edges(r))[0]
        net = self.copy()
        # If the given nodes are leaves
        if net.out_degree(nodes[0]) == 0:
            # Remove each (parent) node that is not part of the cluster
            X_not = set(self.get_leaves()).difference(nodes)
            X_not_p = {list(net.predecessors(x))[0] for x in X_not}
            while X_not_p:
                X_not = X_not_p
                X_not_1p = [x for x in X_not if
                            self.in_degree(x) == 1 and self.out_degree(x) <= 2]  # All x in X_not having 1 parent
                X_not_p = [list(self.predecessors(x))[0] for x in X_not_1p]
                net.remove_nodes_from(X_not)
                net.clean()
            # net.draw(True, 'OK?', all_labels=True)
        comm_anc = None
        while comm_anc is None:  # TODO: will this always work well?
            # Search the common ancestor for each pair of nodes
            new_comm_ancs = set()
            for combi in combinations(nodes, 2):
                lcas = net.get_lcas_2(combi[0], combi[1])
                if not lcas:
                    print(f'ERROR: LCA({combi[0]}, {combi[1]}) = ?')
                    self.draw(True, f'LCA({combi[0]}, {combi[1]}) = ?', all_labels=True)
                # lcas = sorted(lcas)
                new_comm_ancs.update(lcas)
            if not new_comm_ancs:
                # Check if for some combination, a wrong CA is taken
                print(comm_ancs)
                self.draw(True, f'Check CA({nodes})', all_labels=True)
            comm_ancs = new_comm_ancs
            if len(comm_ancs) == 0:
                return None
            if len(comm_ancs) == 1:
                comm_anc = comm_ancs.pop()
            else:
                # If they are not all the same, then search for the common ancestor of these nodes!
                nodes = comm_ancs
        return comm_anc

    def _get_ret_ret_in_edges(self, node, incl_intermediate=True):
        """
        Suppose reticulation [node] has a reticulation v as parent, then return the union of the non-ret. parents
        of both [node] and v.
        """
        in_edges = []
        rets = [node]
        for r in rets:
            for e in self.in_edges(r):
                if self.in_degree(e[0]) > 1:
                    rets.append(e[0])
                    if incl_intermediate:
                        in_edges.append(e)
                else:
                    in_edges.append(e)
        return in_edges

    def get_hang_edges(self, cs_new, x: str, lcas, filter_edges: bool = True, guess_edges: bool = True,
                       stay_bin=False, full_cx=False, print_instr=False):
        """
        Get a list of lists of edges per cluster in Cin, to hang the ST-set below
        :param cs_new: A set of clusters (of type ClusterSet)
        :param x: A leaf to add
        :param lcas: The LCAs per cluster
        :param filter_edges: Filter the edges T/F
        :param guess_edges:
        :param stay_bin: Stay binary T/F
        :param full_cx: Use Cx as containing all clusters containing x
        :param print_instr: T/F
        :return: A list of edges
        """
        # Init
        leaves = self.get_leaves()
        if len(cs_new.X) != len(leaves) + 1:
            self.draw(True, 'Check #leaves', all_labels=True)
            return {}
        # st_str = BitString.from_bool(x in st for x in X)
        cl_edges = {}
        cl_parents = {}
        meta_candidates = {}  # Candidates for being a meta-node to be split
        cs_in = cs_new.get_cs_in(x, full_cx)

        # Find the edge e_out
        if full_cx:
            _cs_in = cs_in.minimal()
            cl_out, e_out = self.get_e_out(cs_new, _cs_in, x, lcas)
        else:
            cl_out, e_out = self.get_e_out(cs_new, cs_in, x, lcas)
        if e_out:
            E_cl = set()
            # E_cl.add(e_out)
            # Add the edges above e_out
            E_cl.update(self.out_edges(e_out[0]))
            c_s_out = ','.join(cl_out)  # Cluster as string
            cl_edges[c_s_out] = E_cl
            P = set()
            P.add(e_out[0])
            cl_parents[c_s_out] = P

        # For each cluster c in cs_in, get the lowest edge in the network that represents it.
        # Then, find all edges E_cl below which x may be hung.
        self.update_z_edges()
        for c in cs_in.C:
            if len(c) == len(cs_new.X):
                continue
            # cl = cl \ {x}
            c.remove(x)
            if filter_edges:
                E_cl, meta_candidate = self._get_cluster_ext_edges(c, cs_new, lcas, full_cx)
                # assert len(set(E_cl)) == len(E_cl)  # There should be no duplicate edges
                if not E_cl:  # If this cluster c is not represented, then return
                    return {}
            else:
                E_cl = list(self.edges)
                meta_candidate = None

            c_s = ','.join(c)  # Cluster as string
            cl_edges[c_s] = E_cl
            cl_parents[c_s] = set(e[0] for e in E_cl)
            if meta_candidate:
                meta_candidates[c_s] = meta_candidate
        # Do not use only minimal clusters in Cx, but all clusters containing x
        if full_cx:
            # Filter the edge sets: only preserve the minimal edge sets
            cl_edges_new = {}
            # Print.var(cl_edges, True)
            for c_s1, E1 in cl_edges.items():
                E1 = set(E1)
                c1 = set(c_s1.split(','))
                issupset = False
                for c_s2, E2 in reversed(cl_edges.items()):
                    if c_s2 == c_s1:
                        break
                    if E1.issuperset(E2):
                        issupset = True
                        break
                if 2 <= len(c1) < len(cs_new.X) - 1:
                    for _x in c1:
                        E1.update(self.in_edges(_x))
                if not issupset:
                    cl_edges_new[c_s1] = E1
            cl_edges = cl_edges_new
            # Update cl_parents
            for c_s in reversed(list(cl_parents.keys())):
                if c_s not in cl_edges.keys():
                    del cl_parents[c_s]
        if not guess_edges:
            return cl_edges

        for c_s1, e in meta_candidates.items():
            p = e[0]
            # Is this parent p a parent of an edge in another hang-edge group?
            # has_overlap = False
            for c_s2, P in cl_parents.items():
                if p in P:
                    E = list(filter(lambda _e: _e[0] == p, cl_edges[c_s2]))
                    assert E
                    e_div = E[0]
                    if e_div == e:
                        continue
                    # If v in e = (u, v) is not a reticulation, then continue
                    if self.in_degree(e_div[1]) == 1:
                        continue
                    # Regard p as meta node & split it
                    n_new = self._subdivide_edge(e_div)
                    # In order to minimize the number of reticulations (according to our Greedy approach, we should
                    # subdivide edge e_div and hang cluster (c_s1) below the new node n_new
                    self.remove_edge(e[0], e[1])
                    self.add_edge(n_new, e[1])
                    # Set the edges E_cl below which the new leaf should be hung
                    E_cl = set()
                    E_cl.add((e[0], n_new))
                    E_cl.add((n_new, e[1]))
                    E_cl.add((n_new, e_div[1]))
                    cl_edges[c_s1] = E_cl

                    # Add 0-edges
                    ret = e_div[1]
                    while ret:
                        succ = list(self.successors(ret))
                        if len(succ) == 0:
                            self.draw(True, f'Check ret {ret}', all_labels=True)
                            raise RuntimeError(f'Reticulation {ret} does not have any childs.')
                        desc = succ[0]
                        # If its child is a reticulation too, then add the 0-edge between the two rets.
                        if self.in_degree(desc) >= 2:
                            cl_edges[c_s1].add((ret, desc))
                            ret = desc
                        else:
                            ret = None

                    # Clean cl_edges
                    for _c_s in cl_edges.keys():
                        if e_div in cl_edges[_c_s]:
                            cl_edges[_c_s].remove(e_div)
                        if e in cl_edges[_c_s]:
                            cl_edges[_c_s].remove(e)

                    # For cluster 2, change edge e_div to (n_new, e_div[1])
                    if e_div not in cl_edges[c_s2]:
                        # In this case, there are multiple possibilities to split the metanode!
                        #  Which one to choose?
                        continue
                    cl_edges[c_s2].add((n_new, e_div[1]))
                    break
        return cl_edges

    def update_z_edges(self):
        """
        Get the zero-edges
        :return:
        """
        self._z_edges = set()
        for n in self.nodes:
            if self.in_degree(n) >= 2:  # If n is a ret.
                for e in self.in_edges(n):
                    self._z_edges.add(e)
                    p = e[0]
                    # If p has only child reticulations
                    onlyRets = True
                    for c in self.successors(p):
                        if self.in_degree(c) == 1:
                            onlyRets = False
                            break
                    if onlyRets:
                        self._z_edges.add(list(self.in_edges(p))[0])

    def get_z_edges_below(self, node: str):
        z_edges = set()
        for e in self.out_edges(node):
            if e in self._z_edges:
                z_edges.add(e)
                z_edges.update(self.get_z_edges_below(e[1]))
        return z_edges

    def _get_cluster_ext_edges(self, c: list, cs_new, lcas, full_cx=False):
        """
        Get the edges below which another cluster cl2 can be hung, such that cluster (cl1 union cl2) will be represented
        :param c: The cluster to search the 'extension edges' for
        :param cs_new: The set of clusters to be represented after hanging cluster cl back
        :return: A list of 'extension edges'
        """
        # Get the cluster edge
        if len(c) == 1:
            lca = c[0]
        else:
            c_str = ','.join(c)
            lca = lcas[c_str]
        e = list(self.in_edges(lca))[0]
        if not e:
            # Cluster c is not represented
            return [], None
        E_cl = []

        # If edge e is the root edge, then return
        if len(c) >= 2:
            # Add 0-edges connected to v
            E_cl.extend(self.get_z_edges_below(e[1]))
        # Add 0-edges connecting the subtree below the LCA with a reticulation
        for x in c:
            p = list(self.predecessors(x))[0]  # Parent of leaf x
            rest_only_z_outs = True
            for _e in self.out_edges(p):
                if _e[1] != x and _e not in self._z_edges:
                    rest_only_z_outs = False
                    break
            if rest_only_z_outs:
                for _e in self.out_edges(p):
                    if _e in self._z_edges:
                        if _e not in E_cl:
                            E_cl.append(_e)
                        for _ez in self.get_z_edges_below(_e[1]):
                            if _ez not in E_cl:
                                E_cl.append(_ez)
            else:
                E_cl.append((p, x))

        if self.out_degree(e[0]) == 1:  # If u has out-degree 1, then (u is a ret, so) add edge e
            E_cl.append(e)
            # Add all in-edges of the reticulation (and of its parent reticulations, etc.)
            E_cl.extend(self._get_ret_ret_in_edges(e[0]))
            return list(reversed(E_cl)), None
        # u of e = (u, v) is a split node.
        # If the other children of 'u' are only reticulations, then add (w, u)
        neighbors = list(self.neighbors(e[0]))

        # Consider the other children of u. Are they all reticulations?
        neighbors.remove(e[1])
        only_rets = True
        has_ret = False
        for n in neighbors:
            if self.in_degree(n) == 1:  # If node n is a splitting
                only_rets = False
                if has_ret:
                    break
            else:  # If node n is a reticulation
                has_ret = True
                if not only_rets:
                    break
        if only_rets and self.in_degree(e[0]) == 1:
            # # Add (w, u)
            E_cl.extend(self.in_edges(e[0]))
            # Add (u, u); let the new node be hung below node u!
            E_cl.append((e[0], e[0]))
            # The other children of node e[0] are all reticulations
            return E_cl, None
        # u is a splitting and there is a splitting among the neighbours of v
        if len(c) == 1:
            # If this cluster cl is a single leaf, then surely, x should be hung below this edge,
            # unless there is a ret. below e[0]
            if e not in E_cl:
                E_cl.append(e)
            if has_ret:
                # Node u has a child ret.
                return E_cl, e  # Node u of edge e=(u,v) is a meta-node candidate
            return E_cl, None
        # Otherwise, add e
        assert e[1][0] == '_', self.draw(True, f'Edge {e}')
        if e not in E_cl:
            if c in cs_new.C or full_cx:
                E_cl.append(e)
        if self.in_degree(e[1]) == 1 and not E_cl:
            E_cl.append((e[1], e[1]))
        return E_cl, None

    def clean(self, rem_dummy_root=False):
        """
        Clean the network from redundant edges, etc.
        :param rem_dummy_root: Remove the dummy root T/F
        :return: The cleaned PN
        """
        # If multiple edges go from v1 to v2, then remove redundant edges among them
        E = []
        for i, e in reversed(list(enumerate(self.edges()))):
            if not e in E:
                E.append(e)

        # If a node has both in- and out-degree 1, then remove the node and connect its ancestor with its descendant
        for node in list(self.nodes):
            if self.in_degree(node) == 1 and self.out_degree(node) == 1:
                e1 = list(self.in_edges(node))[0]
                e2 = list(self.out_edges(node))[0]
                self.remove_node(e1[1])
                self.add_edge(e1[0], e2[1])

        if rem_dummy_root:
            self.remove_dummy_root()

    def remove_dummy_root(self):
        r = self.get_root()
        if self.out_degree(r) == 1:
            self.remove_node(r)

    def merge(self, other: "PhyloNet"):
        X_other = other.get_leaves()
        # Search the node in this network to be replaced by the other network
        for n in self.nodes:
            if self.out_degree(n) <= 1:  # Only consider splittings
                continue
            # Node n is an internal node
            c = self.get_hardwired_cluster(n)
            if X_other == c:
                # Remove node n and its descendants
                nodes2rem = [n]
                nodes2rem.extend(nx.descendants(self, n))
                preds = list(self.predecessors(n))
                if len(preds) == 0:  # If n does not have a parent, then return the other tree!
                    self.remove_nodes_from(list(self.nodes))
                    self.add_edges_from(other.edges)
                    return
                p_n = preds[0]  # The parent of n
                # For each node containing 3 or more child leaves in the other network, replace it by the
                # pendant subtree in this network
                self.remove_nodes_from(nodes2rem)
                r_other = other.get_root()
                # This is the i-th merged subnetwork
                subnet_prefixes = {n[0:3] for n in self.nodes if n[0:2] == '__'}
                i = len(subnet_prefixes)
                # Add the subnetwork below the parent of node n
                prefix = f"__{string.ascii_uppercase[i]}"
                # Relabel the internal nodes to prevent duplicate labels
                mapping = {}
                for n in other.nodes:
                    if n[0] == '_':  # If node n is an internal node
                        mapping[n] = prefix + n
                other = nx.relabel_nodes(other, mapping)
                self.add_edges_from(other.edges)
                self.add_edge(p_n, prefix + r_other)
                return
            if set(X_other).issubset(c):
                # Remove the children of node n that are not reticulations
                child_leaves = sorted(_n for _n in self.successors(n) if self.in_degree(_n) == 1 and _n[0] != '_')
                if not child_leaves:
                    continue
                if set(X_other).issubset(child_leaves):
                    # Merge the networks
                    self.remove_nodes_from(X_other)
                    r_other = other.get_root()
                    # This is the i-th merged subnetwork
                    subnet_prefixes = {n[0:3] for n in self.nodes if n[0:2] == '__'}
                    i = len(subnet_prefixes)
                    # Add the subnetwork below the parent of node n
                    prefix = f"__{string.ascii_uppercase[i]}"
                    # Relabel the internal nodes to prevent duplicate labels
                    mapping = {}
                    for _n in other.nodes:
                        if _n[0] == '_':  # If node n is an internal node
                            mapping[_n] = prefix + _n
                    other = nx.relabel_nodes(other, mapping)
                    self.add_edges_from(other.edges)
                    self.add_edge(n, prefix + r_other)
                    return
        print('ERROR: How should I merge these networks?')
        # self.draw(True, 'Final Net', all_labels=True)
        # other.draw(True, 'Merge It', all_labels=True)

    def _get_newick(self, _root=None, _hybrids=None):
        """
        Convert the phylogenetic network to a string in (e-)Newick format.

        :param _root: A root
        :param _hybrids: A list of hybrid nodes
        :return: A Newick-formatted string
        """
        if _hybrids is None:
            _hybrids = {}
        if _root is None:
            _root = self.get_root()
        sub_str = []
        for child in self[_root]:
            # Is this a hybrid node?
            if self.in_degree(child) > 1:
                if child in _hybrids.keys():
                    hybrid_label = _hybrids[child]
                    sub_str.append(hybrid_label)
                else:
                    hybrid_label = '#H' + str(len(_hybrids) + 1)
                    _hybrids[child] = hybrid_label
                    sub_str.append(self._get_newick(_root=child, _hybrids=_hybrids) + hybrid_label)
            elif len(self[child]) > 0:
                sub_str.append(self._get_newick(_root=child, _hybrids=_hybrids))
            else:
                sub_str.append(child)
        return "(" + ','.join(sub_str) + ")"

    def save_net(self, filename):
        """
        Save the given network in the Newick (or e-Newick) format.

        :param filename: A file name
        """
        nwk = self._get_newick()
        # Create the path if it does not exist
        path = os.path.dirname(filename)
        if len(path) and not os.path.exists(path):
            os.makedirs(path)

        with open(filename, 'w') as f:
            f.write(nwk + ';')
        print(f'SUCCESS: The network is saved as {filename}.')

    def draw(self, hold=False, title=None, axes=None, filename=None, all_labels=False, labels=None, edge_color=None,
             node_color=None):
        """
        Draw the given NetworkX digraph
        :param hold: Show the figure until the user closes it
        :param title: A title
        :param axes: Figure axes
        :param filename: A filename to store the figure
        :param edge_color:
        :param labels:
        :param all_labels:
        :param node_color:
        """
        if not LIVE_DRAW_ENABLED and not hold:
            print('WARNING: Drawing live is disabled.')
            return
        if not DRAW_ENABLED and stack()[1].function != 'draw_multi':  # TODO: remove stack
            print('WARNING: Drawing is disabled.')
            return
        if axes is None:
            plt.clf()  # HERE the program halts often :( WHY?
            plt.cla()

        def _decrease_leaf_labels():
            ys = set(y for (x, y) in pos.values())
            y_shift = (max(ys) - min(ys)) / (2 * len(ys))
            return {_n: (x, y - y_shift * (self.out_degree(_n) == 0)) for _n, (x, y) in pos.items()}

        # If self is a digraph, then nicely color its nodes
        if isinstance(self, nx.DiGraph):
            # Color the nodes: distinguish leaf nodes, hybrid nodes/reticulations and splits
            color_leaf = 'green'
            color_ret = 'red'
            color_splt = 'black' if not all_labels else 'lightGrey'
            color_lookup = {}
            node_size = []
            for node in self.nodes():
                if ',' in node:
                    raise RuntimeError(f'A node may not include a comma (I see node {node})')
                if self.out_degree(node) == 0:  # If this is a leaf
                    color_lookup[node] = color_leaf
                    node_size.append(100)
                elif self.in_degree(node) >= 2:  # If this is a hybrid node
                    color_lookup[node] = color_ret
                    node_size.append(60)
                else:
                    color_lookup[node] = color_splt
                    node_size.append(60)
                if node_color and node in node_color.keys():
                    color_lookup[node] = node_color[node]

            # Create a nice network layout
            pos = graphviz_layout(self, prog="dot")
            # Draw the network
            nx.draw_networkx(self, pos, True, with_labels=False, node_color=list(color_lookup.values()),
                             node_size=node_size, ax=axes, edge_color=edge_color)
            pos_nodes = _decrease_leaf_labels()
            # Only show the labels of the taxa/leaves
            if labels is None:
                labels = {}
                for n in self.nodes:
                    # If n is a leaf, then show the name
                    if self.out_degree(n) == 0 or all_labels:
                        labels[n] = n
            else:
                # Filter the labels for the leaves in this network
                nodes = self.get_leaves()
                labels = {n: l for n, l in labels.items() if n in nodes}
            nx.draw_networkx_labels(self, pos=pos_nodes, labels=labels, ax=axes)
        else:
            if labels is not None:
                # Filter the labels for the leaves in this network
                nodes = list(self.nodes)
                labels = {n: l for n, l in labels.items() if n in nodes}
            # Create a nice network layout
            nx.draw_circular(self, with_labels=True, edge_color=edge_color, node_color='lightGrey', ax=axes,
                             labels=labels)
            # If there are edge labels, show them
            e_labels = nx.get_edge_attributes(self, 'weight')
            if len(e_labels) > 0:
                nx.draw_networkx_edge_labels(self, nx.circular_layout(self), e_labels, ax=axes)

        # Set the title
        if title is not None:
            plt.gcf().suptitle(title)
        plt.subplots_adjust(left=0, bottom=0, right=1)
        if filename is not None:
            plt.pause(1E-9)
            plt.savefig(f'output/{filename}', bbox_inches='tight')
        if axes is None:
            if hold:
                plt.show()
            else:
                plt.pause(1E-9)


# -*- coding:utf-8 -*-

import os
import re
import time
import logging
import pdfkit
import requests
from bs4 import BeautifulSoup
from PyPDF2 import PdfFileMerger
import os.path
import shutil
from datetime import date

html_template = """
<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
</head>
<body>
{content}
</body>
</html>

"""

def get_url_list():
    response = requests.get("http://vitalik.ca//")
    soup = BeautifulSoup(response.content, "html.parser")
    urls = []
    for v in soup.find_all(class_="post-link"):
        url = "http://vitalik.ca/" + v.get('href')
        urls.append(url)
    return urls

def parse_url_to_html(url, name):
    try:
        response = requests.get(url)
        soup = BeautifulSoup(response.content, 'html.parser')

        inner = soup.find_all(class_="markdown-body")
        if inner.__len__() == 0:
            return False

        body = inner[0]

        body.find("div", {"id": "color-mode-switch"}).decompose()

        for img in body.find_all("img"):
            img_style = img.get("style")
            style = "display: block; margin-left: auto; margin-right: auto;"
            if img_style is not None:
                style += img_style
            img['style'] = style

        html = str(body)

        pattern = "(<img .*?src=\")(.*?)(\")"
        def func(m):
            return m.group(0).replace("../../../..", "https://vitalik.ca")
        html = re.compile(pattern).sub(func, html)

        html = html_template.format(content=html)
        with open(name, 'wb') as f:
            f.write(html.encode())
        return True

    except Exception as e:
        logging.error("parse error ", exc_info=True)

def save_pdf(htmls, file_name):
    options = {
        'page-size': 'A4',
        'margin-top': '0.75in',
        'margin-right': '0.75in',
        'margin-bottom': '0.75in',
        'margin-left': '0.75in',
        'encoding': "UTF-8",
        'custom-header': [
            ('Accept-Encoding', 'gzip')
        ],
        'cookie': [
            ('cookie-name1', 'cookie-value1'),
            ('cookie-name2', 'cookie-value2'),
        ],
        'minimum-font-size': 14,
    }
    try:
        pdfkit.from_file(htmls, file_name, options=options)
    except Exception as e:
        logging.error("covert error ", exc_info=True)

def main():
    shutil.rmtree("html") if os.path.exists("html") else None
    os.mkdir("html") if not os.path.exists("html") else None
    shutil.rmtree("pdf") if os.path.exists("pdf") else None
    os.mkdir("pdf") if not os.path.exists("pdf") else None

    start = time.time()
    urls = get_url_list()
    wrongUrl = []
    for index, url in enumerate(urls):
        name = str(index) + ".html"
        name = os.path.join("html", name)
        res = parse_url_to_html(url, name)
        if res is False:
            wrongUrl.append(str(index))
        print(u"download " + str(index) + ' html')

    htmls = []
    pdfs = []
    print("wrongUrl", wrongUrl)
    for i in range(0, urls.__len__()):
        if str(i) in wrongUrl:
            continue
        htmlName = str(i) + ".html"
        htmlName = os.path.join("html", htmlName)
        name = str(i) + '.pdf'
        name = os.path.join("pdf", name)
        htmls.append(htmlName)
        pdfs.append(name)

        save_pdf(htmlName, name)
        print(u"covert " + str(i) + ' html')

    merger = PdfFileMerger()
    for pdf in pdfs:
        merger.append(open(pdf, 'rb'))
        print(u"merge " + str(i) + ' pdf')

    todays_date = date.today()
    #name = "vitalik-blog-" + str(todays_date) + ".pdf"
    name = "vitalik-blog.pdf"
    output = open(name, "wb")
    merger.write(output)

    print(u"success")

    # for html in htmls:
    #     os.remove(html)
    #     print(u"delete " + html)

    # for pdf in pdfs:
    #     os.remove(pdf)
    #     print(u"delete " + pdf)

    total_time = time.time() - start
    print(u"total spend：%f second" % total_time)

if __name__ == '__main__':
    main()


"""Evaluate the one-scale performance on MSCOCO dataset"""
import os
import argparse
import logging
import cv2
import matplotlib.pyplot as plt
import time
import warnings
import datetime
import numpy as np
import json
import torch
import torchvision
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
import config
import data
import transforms
import models
import logs
from visualization import show
import decoder
from utils.util import AverageMeter

try:
    from apex.parallel import DistributedDataParallel as DDP
    from apex.fp16_utils import *
    from apex import amp
    import apex.optimizers as apex_optim
    from apex.multi_tensor_apply import multi_tensor_applier
except ImportError:
    raise ImportError(
        "Please install apex from https://www.github.com/nvidia/apex to run this example.")

# os.environ['CUDA_VISIBLE_DEVICES'] = "1"  # choose the available GPUs

LOG = logging.getLogger(__name__)

ANNOTATIONS_TRAIN = 'data/link2COCO2017/annotations/person_keypoints_train2017.json'
ANNOTATIONS_VAL = 'data/link2COCO2017/annotations/person_keypoints_val2017.json'
IMAGE_DIR_TRAIN = 'data/link2COCO2017/train2017'
IMAGE_DIR_VAL = 'data/link2COCO2017/val2017'

ANNOTATIONS_TESTDEV = 'data/link2COCO2017/annotations_trainval_info/image_info_test-dev2017.json'
ANNOTATIONS_TEST = 'data/link2COCO2017/annotations_trainval_info/image_info_test2017.json'
IMAGE_DIR_TEST = 'data/link2COCO2017/test2017/'

def evaluate_cli():
    parser = argparse.ArgumentParser(
        # __doc__: current module's annotation (or module.a_function's annotation)
        description=__doc__,
        # --help text with default values, e.g., gaussian threshold (default: 0.1)
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    logs.cli(parser)
    models.net_cli(parser)
    decoder.decoder_cli(parser)

    parser.add_argument('--dump-name',  # TODO: edit this name each evaluation
                        default='hourglass104_focal_l2_instance_l1_sqrt_epoch_77__distmax40_640_input_1scale_flip_hmpoff_gamma2_thre004',
                        type=str, help='detection file name')

    parser.add_argument('--dataset', choices=('val', 'test', 'test-dev'), default='val',
                        help='dataset to evaluate')
    parser.add_argument('--batch-size', default=8, type=int,
                        help='batch size')
    parser.add_argument('--long-edge', default=640, type=int,
                        help='long edge of input images')
    parser.add_argument('--fixed-height', action='store_true', default=False,
                        help='resize input images to the fixed height of long_edge')
    parser.add_argument('--flip-test', action='store_true', default=False,
                        help='flip augmentation during testing')
    parser.add_argument('--cat-flip-offset', action='store_true', default=False,
                        help='offset flip merge of increasing to 4D vector space')
    parser.add_argument('--loader-workers', default=8, type=int,
                        help='number of workers for data loading')
    parser.add_argument('--all-images', default=False, action='store_true',
                        help='run over all images irrespective of catIds')

    parser.add_argument('--resume', '-r', action='store_true', default=False,
                        help='resume from checkpoint')
    parser.add_argument('--checkpoint-path', '-p',
                        default='link2checkpoints_storage',
                        help='folder path checkpoint storage of the whole pose estimation model')
    parser.add_argument('--show-detected-poses', action='store_true', default=False,
                        help='show the final results')

    group = parser.add_argument_group('apex configuration')
    group.add_argument("--local_rank", default=0, type=int)
    # full precision O0  # mixture precision O1 # half precision O2
    group.add_argument('--opt-level', type=str, default='O2')
    group.add_argument('--keep-batchnorm-fp32', type=str, default=None)
    group.add_argument('--loss-scale', type=str, default=None)  # '1.0'
    # todo: channel last may lead to 22% speed up however not obvious here
    #  channels Last memory format is implemented for 4D NCWH Tensors only.
    group.add_argument('--channels-last', default=False, action='store_true',
                       help='channel last may lead to 22% speed up')
    group.add_argument('--print-freq', '-f', default=10, type=int, metavar='N',
                       help='print frequency (default: 10)')

    # args = parser.parse_args(
    #     '--checkpoint-whole link2checkpoints_storage/PoseNet_18_epoch.pth --resume --no-pretrain'.split())
    args = parser.parse_args()

    if args.dataset == 'val':
        args.image_dir = IMAGE_DIR_VAL
        args.annotation_file = ANNOTATIONS_VAL
    elif args.dataset == 'test':
        args.image_dir = IMAGE_DIR_TEST
        args.annotation_file = ANNOTATIONS_TEST
    elif args.dataset == 'test-dev':
        args.image_dir = IMAGE_DIR_TEST
        args.annotation_file = ANNOTATIONS_TESTDEV
    else:
        raise Exception

    if args.dataset in ('test', 'test-dev') and not args.all_images:
        print('force to use --all-images for this dataset because catIds are unknown')
        args.all_images = True

    return args

def run_images():
    result_keypoints = []
    result_image_ids = []

    print(f"\nopt_level = {args.opt_level}")
    print(f"keep_batchnorm_fp32 = {args.keep_batchnorm_fp32}")
    print(f"CUDNN VERSION: {torch.backends.cudnn.version()}\n")

    torch.backends.cudnn.benchmark = True
    # print(vars(args))
    args.pin_memory = False
    if torch.cuda.is_available():
        args.pin_memory = True

    args.world_size = 1

    if args.fixed_height:
        try:
            assert args.batch_size == 1
        except AssertionError:
            warnings.warn(f'forcibly fix the height of the input rgb_img to '
                          f'{args.long_edge: d} and set batch_size=1"')
            args.batch_size = 1

        preprocess_transformations = [
            transforms.NormalizeAnnotations(),
            # transforms.RescaleRelative(scale_factor=1),  # original input
            transforms.RescaleHighAbsolute(args.long_edge),
            transforms.RightDownPad(args.max_stride),
        ]
    else:
        LOG.info('you resize the longer edge of the input rgb_img to %d ', args.long_edge)
        preprocess_transformations = [
            transforms.NormalizeAnnotations(),
            transforms.RescaleLongAbsolute(args.long_edge),
            transforms.CenterPad(args.long_edge),
        ]

    preprocess_transformations += [
        transforms.ImageTransform(torchvision.transforms.ToTensor()),
        transforms.ImageTransform(
            torchvision.transforms.Normalize(mean=config.data_mean,
                                             std=config.data_std)),
    ]
    preprocess = transforms.Compose(preprocess_transformations)

    dataset = data.CocoKeypoints(
        args.image_dir,
        annFile=args.annotation_file,
        preprocess=preprocess,
        all_persons=True,  # we do not know if people are exitst in each rgb_img of test-dev
        all_images=args.all_images,
    )
    data_loader = torch.utils.data.DataLoader(
        dataset, batch_size=args.batch_size,
        pin_memory=args.pin_memory,
        num_workers=args.loader_workers,
        collate_fn=data.collate_images_anns_meta)

    model, _ = models.model_factory(args)

    model.cuda()  # .to(memory_format=torch.channels_last)

    if args.resume:
        model, _, start_epoch, best_loss, _ = models.networks.load_model(
            model, args.checkpoint_whole, optimizer=None, resume_optimizer=False,
            drop_layers=False, load_amp=False)

    processor = decoder.decoder_factory(args)

    # Initialize Amp.  Amp accepts either values or strings for the optional override arguments,
    # for convenient interpretation with argparse.
    # make processor as Module (rewrite forward method) and wrap it 测试后发现没有明显加速效果
    [model] = amp.initialize([model],  # , processor
                             opt_level=args.opt_level,
                             keep_batchnorm_fp32=args.keep_batchnorm_fp32,
                             loss_scale=args.loss_scale)

    model.eval()

    batch_time = AverageMeter()
    end = time.time()  # Notice: CenterNet 和 pifpaf 应该没有包含数据加载的时间，所以确认一下这个计时，考虑换个位置
    for batch_idx, (images, annos, metas) in enumerate(data_loader):

        images = images.cuda(non_blocking=True)  # .to(memory_format=memory_format)

        if args.flip_test:
            images = torch.cat((images, torch.flip(images, [-1])))  # images tensor shape = (N, 3, H, W)

        with torch.no_grad():
            outputs = model(images)  # outputs of multiple headnets

        # post-processing for generating individual poses
        batch_poses = processor.generate_poses(
            outputs,
            flip_test=args.flip_test,
            cat_flip_offs=args.cat_flip_offset
        )

        # #########################################################################
        # ############# inverse the keypoint into the original rgb_img space ############
        # #########################################################################
        for index, (image_poses, image_meta) in enumerate(zip(batch_poses, metas)):
            subset = preprocess.annotations_inverse(image_poses, image_meta)
            batch_poses[index] = subset  # numpy array [M * [ 17 * [x, y, v, s, limb_score, ind]]]

            image_id = image_meta['image_id']

            # #########################################################################
            # ############# collect the detected person poses and rgb_img ids ###########
            # #########################################################################
            result_image_ids.append(image_id)
            # last dim of subset: [x, y, v, s, limb_score, ind]
            subset[:, :, :2] = np.around(subset[:, :, :2], 2)
            for i, person in enumerate(subset.astype(float)):
                keypoints_list = []
                v = []
                for xyv in person[:, :3]:
                    v.append(xyv[2])
                    keypoints_list += [
                        xyv[0], xyv[1], 1 if xyv[0] > 0 or xyv[1] > 0 else 0
                    ]

                result_keypoints.append({
                    'image_id': image_id,
                    'category_id': 1,  # person category
                    'keypoints': keypoints_list,
                    'score': sum(v) / len(v),
                    #  the criterion of person pose score
                    #  we actually did at decoder.group.GreedyGroup._delete_sort
                })

            # force at least one annotation per rgb_img following PIFPAF
            # but make no difference to the result
            if not len(subset):
                result_keypoints.append({
                    'image_id': image_id,
                    'category_id': 1,
                    'keypoints': np.zeros((17 * 3,)).tolist(),
                    'score': 0.01,
                })

        if args.show_detected_poses:
            image_poses = batch_poses[0]

            image_path = metas[0]['image_path']
            orig_img = cv2.imread(image_path)
            rgb_img = cv2.cvtColor(orig_img, cv2.COLOR_BGR2RGB)
            skeleton = processor.skeleton
            keypoint_painter = show.KeypointPainter(
                show_box=False,
                # color_connections=True, linewidth=5,
            )
            with show.image_canvas(rgb_img,
                                   # output_path + '.keypoints.png',
                                   show=True,
                                   # fig_width=args.figure_width,
                                   # dpi_factor=args.dpi_factor
                                   ) as ax:
                keypoint_painter.keypoints(ax, image_poses[:, :, :3], skeleton=skeleton)

        if batch_idx % args.print_freq == 0:
            # to_python_float incurs a host<->device sync
            torch.cuda.synchronize()
            batch_time.update((time.time() - end) / args.print_freq)
            end = time.time()

            print('==================> Epoch: [{0}][{1}/{2}]\t'
                  'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                  'Speed {3:.3f} ({4:.3f}) <================ \t'.format(
                0, batch_idx, len(data_loader),
                args.world_size * args.batch_size / batch_time.val,
                args.world_size * args.batch_size / batch_time.avg,
                batch_time=batch_time))

    return result_keypoints, result_image_ids

def validation(annFile, dump_name, dataset):
    prefix = 'person_keypoints'

    dataDir = 'data/link2COCO2017'

    print(annFile)
    cocoGt = COCO(annFile)

    resFile = '%s/results/%s_%s_%s_results.json'
    resFile = resFile % (dataDir, prefix, dataset, dump_name)
    print('======================>')
    print('The path of detected keypoint file is: ', resFile)
    os.makedirs(os.path.dirname(resFile), exist_ok=True)

    results_keypoints, validation_ids = run_images()

    json.dump(results_keypoints, open(resFile, 'w'))

    # ####################  COCO Evaluation ################
    cocoDt = cocoGt.loadRes(resFile)
    cocoEval = COCOeval(cocoGt, cocoDt, iouType='keypoints')
    cocoEval.params.imgIds = validation_ids  # only part of the person images are evaluated
    cocoEval.evaluate()
    cocoEval.accumulate()
    cocoEval.summarize()
    return cocoEval

if __name__ == '__main__':
    log_level = logging.INFO  # logging.DEBUG
    # set RootLogger
    logging.basicConfig(
        level=log_level,
    )

    global best_loss, args
    best_loss = float('inf')
    args = evaluate_cli()

    print(args.dump_name)   # todo: 添加 NN inference 和 decoding time记录
    eval_result_original = validation(args.annotation_file,
                                      dump_name=args.dump_name,
                                      dataset=args.dataset)
    print('\nEvaluation finished!')


"""Add table for comments

Revision ID: 0366ba6575ca
Revises: 1093835a1051
Create Date: 2020-08-14 00:46:54.161120

"""
from alembic import op  # noqa: I001
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = "0366ba6575ca"
down_revision = "1093835a1051"
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table(
        "comments",
        sa.Column("id", sa.Integer(), nullable=False),
        sa.Column("type", sa.String(length=80), nullable=True),
        sa.Column("content", sa.Text(), nullable=True),
        sa.Column("date", sa.DateTime(), nullable=True),
        sa.Column("author_id", sa.Integer(), nullable=True),
        sa.Column("challenge_id", sa.Integer(), nullable=True),
        sa.Column("user_id", sa.Integer(), nullable=True),
        sa.Column("team_id", sa.Integer(), nullable=True),
        sa.Column("page_id", sa.Integer(), nullable=True),
        sa.ForeignKeyConstraint(["author_id"], ["users.id"], ondelete="CASCADE"),
        sa.ForeignKeyConstraint(
            ["challenge_id"], ["challenges.id"], ondelete="CASCADE"
        ),
        sa.ForeignKeyConstraint(["page_id"], ["pages.id"], ondelete="CASCADE"),
        sa.ForeignKeyConstraint(["team_id"], ["teams.id"], ondelete="CASCADE"),
        sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
        sa.PrimaryKeyConstraint("id"),
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table("comments")
    # ### end Alembic commands ###


# Imports
import os
import hikari
import aiohttp
import dotenv
import lightbulb

# Loading the .env file
dotenv.load_dotenv()

# Functions
from utils.functions import array_to_string, codestring

# Initialize the bot
bot = lightbulb.BotApp(os.environ["TOKEN"],
prefix=",",
help_slash_command=True,
default_enabled_guilds=int(os.environ["GUILD_ID"]), # You can do default_enabled_guild=(guild_id1, guild_id2, ...)
case_insensitive_prefix_commands=True,
intents=hikari.Intents.ALL,
help_class=None
)

@bot.listen()
async def on_starting(event: hikari.StartingEvent) -> None:
    bot.d.aio_session = aiohttp.ClientSession()

@bot.listen()
async def on_stopping(event: hikari.StoppingEvent) -> None:
    await bot.d.aio_session.close()

# General Error Handler
@bot.listen(lightbulb.CommandErrorEvent)
async def on_error(event: lightbulb.CommandErrorEvent) -> None:
    if isinstance(event.exception, lightbulb.CommandInvocationError):
        await event.context.respond(f"Something went wrong during invocation of command `{event.context.command.name}`.")
        raise event.exception

    # Unwrap the exception to get the original cause
    exception = event.exception.__cause__ or event.exception

    if isinstance(exception, lightbulb.NotOwner):
        await event.context.respond("Command restricted to bot owner only.")
    elif isinstance(exception, lightbulb.CommandIsOnCooldown):
        await event.context.respond(f"This command is on cooldown. Retry in `{exception.retry_after:.2f}` seconds.")
    elif isinstance(exception, lightbulb.NotEnoughArguments):
        args_missing = []
        for option in exception.missing_options:
            args_missing.append(
                f'{codestring(f"{option.name}: {option.arg_type}")}')

        await event.context.respond(f'Command invocation is missing one or more required arguments.\n- Missing arguments: {array_to_string(array=args_missing, iterable=", ")}.')
    elif isinstance(exception, lightbulb.CommandNotFound):
        return
    else:
        raise exception

# Command sample that returns the bot ping in ms.
@bot.command
@lightbulb.command("ping", description="The bot's ping")
@lightbulb.implements(lightbulb.PrefixCommand)
async def ping(ctx: lightbulb.Context) -> None:
    await ctx.respond(f"Pong! Latency: {bot.heartbeat_latency*1000:.2f}ms")

# Load the extension from the extensions folder
bot.load_extensions_from("./extensions/", must_exist=True)

# Linux thing
if __name__ == "__main__":
    if os.name != "nt":
        import uvloop

        uvloop.install()

# Running the bot
bot.run(
    activity=hikari.Activity(
        type=hikari.ActivityType.WATCHING,
        name=f'Monke swim for 10 hours :)'
    ),
    status=hikari.Status.DO_NOT_DISTURB
)