import pytest
import sys
import json
import mock

from types import GeneratorType

from requests.exceptions import HTTPError, ConnectionError
from pypuppetdb.errors import EmptyResponseError
from requests import Response
from werkzeug.exceptions import NotFound, InternalServerError

from puppetboard import utils
from puppetboard import app

from bs4 import BeautifulSoup
import logging

def test_json_format():
    demo = [{'foo': 'bar'}, {'bar': 'foo'}]
    sample = json.dumps(demo, indent=2, separators=(',', ': '))

    assert sample == utils.jsonprint(demo), "Json formatting has changed"

def test_format_val_str():
    x = "some string"
    assert x == utils.formatvalue(x), "Should return same value"

def test_format_val_array():
    x = ['a', 'b', 'c']
    assert "a, b, c" == utils.formatvalue(x)

def test_format_val_dict_one_layer():
    x = {'a': 'b'}
    assert "a => b,<br/>" == utils.formatvalue(x)

def test_format_val_tuple():
    x = ('a', 'b')
    assert str(x) == utils.formatvalue(x)

def test_get():
    x = "hello world"

    def test_get_or_abort():
        return x

    assert x == utils.get_or_abort(test_get_or_abort)

def test_pretty_print():
    test_data = [{'hello': 'world'}]

    html = utils.prettyprint(test_data)
    soup = BeautifulSoup(html, 'html.parser')

    assert soup.th.text == 'hello'

@pytest.fixture
def mock_log(mocker):
    return mocker.patch('logging.log')

@pytest.fixture
def mock_info_log(mocker):
    logger = logging.getLogger('puppetboard.utils')
    return mocker.patch.object(logger, 'info')

@pytest.fixture
def mock_err_log(mocker):
    logger = logging.getLogger('puppetboard.utils')
    return mocker.patch.object(logger, 'error')

def test_http_error(mock_log):
    err = "NotFound"

    def raise_http_error():
        x = Response()
        x.status_code = 404
        x.reason = err
        raise HTTPError(err, response=x)

    with pytest.raises(NotFound):
        utils.get_or_abort(raise_http_error)
        mock_log.error.assert_called_once_with(err)

def test_http_connection_error(mock_log):
    err = "ConnectionError"

    def connection_error():
        x = Response()
        x.status_code = 500
        x.reason = err
        raise ConnectionError(err, response=x)

    with pytest.raises(InternalServerError):
        utils.get_or_abort(connection_error)
        mock_log.error.assert_called_with(err)

def test_db_version_good(mocker, mock_info_log):
    mocker.patch.object(app.puppetdb, 'current_version', return_value='4.2.0')
    err = 'PuppetDB Version %d.%d.%d' % (4, 2, 0)
    result = utils.get_db_version(app.puppetdb)
    mock_info_log.assert_called_with(err)
    assert (4, 0, 0) < result
    assert (4, 2, 0) == result
    assert (3, 2, 0) < result
    assert (4, 3, 0) > result
    assert (5, 0, 0) > result
    assert (4, 2, 1) > result

def test_db_invalid_version(mocker, mock_err_log):
    mocker.patch.object(app.puppetdb, 'current_version', return_value='4')
    err = u"Unable to determine version from string: '%s'" % (4)
    result = utils.get_db_version(app.puppetdb)
    mock_err_log.assert_called_with(err)
    assert (4, 0, 0) < result
    assert (4, 2, 0) == result

def test_db_http_error(mocker, mock_err_log):
    err = "NotFound"

    def raise_http_error():
        x = Response()
        x.status_code = 404
        x.reason = err
        raise HTTPError(err, response=x)

    mocker.patch.object(app.puppetdb, 'current_version',
                        side_effect=raise_http_error)
    result = utils.get_db_version(app.puppetdb)
    mock_err_log.assert_called_with(err)
    assert result == ()

def test_db_connection_error(mocker, mock_err_log):
    err = "ConnectionError"

    def connection_error():
        x = Response()
        x.status_code = 500
        x.reason = err
        raise ConnectionError(err, response=x)

    mocker.patch.object(app.puppetdb, 'current_version',
                        side_effect=connection_error)
    result = utils.get_db_version(app.puppetdb)
    mock_err_log.assert_called_with(err)
    assert result == ()

def test_db_empty_response(mocker, mock_err_log):
    err = "Empty Response"

    def connection_error():
        raise EmptyResponseError(err)

    mocker.patch.object(app.puppetdb, 'current_version',
                        side_effect=connection_error)
    result = utils.get_db_version(app.puppetdb)
    mock_err_log.assert_called_with(err)
    assert result == ()

def test_iter():
    test_list = (0, 1, 2, 3)

    def my_generator():
        for i in test_list:
            yield i

    gen = utils.yield_or_stop(my_generator())
    assert isinstance(gen, GeneratorType)

    i = 0
    for val in gen:
        assert i == val
        i = i + 1

def test_stop_empty():
    def my_generator():
        yield 1
        raise EmptyResponseError
        yield 2

    gen = utils.yield_or_stop(my_generator())
    for val in gen:
        assert 1 == val

def test_stop_conn_error():
    def my_generator():
        yield 1
        raise ConnectionError
        yield 2
    gen = utils.yield_or_stop(my_generator())
    for val in gen:
        assert 1 == val

def test_stop_http_error():
    def my_generator():
        yield 1
        raise HTTPError
        yield 2
    gen = utils.yield_or_stop(my_generator())
    for val in gen:
        assert 1 == val


__author__ = 'jstorm'

import io

from transmedia.util import PixelArray
from transmedia.io import (write_data_to_file, write_pixels_to_png)

def test_write_png_produces_file_of_expected_size():
    expected_size = 77  # Precomputed size of resulting PNG
    white = [255, 255, 255]
    black = [0, 0, 0]
    pixel_rows = [black + white + black,
                  white + black + white,
                  black + white + black]

    pixel_array = PixelArray(pixel_rows)
    file = io.BytesIO()
    write_pixels_to_png(pixel_array, file)

    assert len(file.getvalue()) == expected_size

def test_write_data_produces_expected_output():
    data = bytearray.fromhex('ffffe3990012')
    file = io.BytesIO()
    write_data_to_file(data, file)

    assert file.getvalue() == data


from typing import List

class Solution:

    def dfs(self, board: List[List[str]], i: int, j: int, k: int, word: str) -> bool:
        row = len(board)
        col = len(board[0])

        if i < 0 or i >= row or j < 0 or j >= col or board[i][j] != word[k]:
            return False

        if k == len(word) - 1:
            return True

        tmp = board[i][j]
        board[i][j] = '*'

        sub_res = self.dfs(board, i + 1, j, k + 1, word) or self.dfs(board, i - 1, j, k + 1, word) \
                  or self.dfs(board, i, j + 1, k + 1, word) or self.dfs(board, i, j - 1, k + 1, word)

        board[i][j] = tmp

        return sub_res

    def exist(self, board: List[List[str]], word: str) -> bool:
        for i in range(len(board)):
            for j in range(len(board[0])):
                if self.dfs(board, i, j, 0, word):
                    return True
        return False

剑指offer系列/剑指 Offer 12. 矩阵中的路径/solution.py


import sys
import os
import re
import gm2fr.src.io as io
from gm2fr.src.Results import Results
import argparse
import numpy as np

# ==================================================================================================

def merge_results(results, output_dir = None, output_name = None, indices = None, sort_column = "index", transforms = False):

  # Look for numerical indices in each folder name, e.g. "Calo12" -> 12. Default to -1 if none found.
  if indices is None:
    indices = [
      (index if index is not None else -1)
      for index in io.find_indices([os.path.basename(os.path.dirname(result)) for result in results])
    ]

  if output_name is None:
    output_name = "merged_results"

  merged_results = Results()
  merged_transforms = {}

  unique_indices = indices if len(np.unique(indices)) == len(indices) else np.arange(len(indices))

  # Loop through the folders, load each results file, and merge them into a cumulative array.
  for i, result in enumerate(results):

    if os.path.isfile(result):
      merged_results.append(Results.load(result))

      if transforms:
        parent_dir = os.path.dirname(result)
        transform_file = f"{parent_dir}/transform.npz"
        if os.path.isfile(transform_file):
          transform_data = np.load(transform_file, allow_pickle = True)
          for element in ("edges", "heights", "cov"):
            merged_transforms[f"{unique_indices[i]}/{element}"] = transform_data[f"transform_x/{element}"]

    else:
      del indices[i]

  merged_results.table["index"] = indices

  # Put the new 'index' column first, and sort the rows by the values in the specified column.
  column_order = ["index"] + [col for col in merged_results.table.columns if col != "index"]
  merged_results.table = merged_results.table[column_order].sort_values(by = sort_column)

  # Save (if output specified) and return merged Results object.
  if output_dir is not None:
    merged_results.save(output_dir, output_name)
    if transforms:
      np.savez(f"{output_dir}/{output_name.replace('_results', '') + '_transforms'}", **merged_transforms)

  if not transforms:
    return merged_results
  else:
    return merged_results, merged_transforms

# ==================================================================================================

if __name__ == "__main__":

  parser = argparse.ArgumentParser()
  parser.add_argument("--output", "-o", required = True)
  parser.add_argument("--name", "-n", default = "merged_results")
  parser.add_argument("--results", "-r", nargs = "+")
  parser.add_argument("--indices", "-i", default = None, choices = [None, "parent"])
  parser.add_argument("--sort", "-s", default = "index")
  parser.add_argument("--transforms", "-t", action = "store_true")
  args = parser.parse_args()

  if args.indices == "parent":
    indices = [os.path.basename(os.path.dirname(result)) for result in args.results]
  else:
    indices = args.indices

  merge_results(args.results, args.output, args.name, indices, args.sort, args.transforms)


from typing import Tuple

import numpy as np
from matplotlib.axes import Axes
from matplotlib.backend_bases import MouseEvent, MouseButton

from ...analysis import DataPacket
from ...support import log

class VicarEvent:
    line_has_data: bool = False
    fit_x_start: int = -1
    fit_x_end: int = -1
    fit_degree: int = 2
    dpkt: DataPacket

    def __init__(self, data: np.ndarray, data_axis: Axes, line_axis: Axes, area: Tuple[int, int]):
        self.data = data
        self.area = area

        from ...support import wrap_axes

        self.data_axis = wrap_axes(data_axis)
        self.line_axis = wrap_axes(line_axis)

        self.cid = data_axis.figure.canvas.mpl_connect('button_press_event', self)
        self.dpkt = DataPacket(data)

        self.err = None
        self.rect = None

    def __call__(self, event: MouseEvent):
        if event.canvas.cursor().shape() != 0:
            return
        if event.inaxes == self.line_axis and self.line_has_data:
            if self.err:
                self.err.remove()
                self.err = None
            if event.button == MouseButton.LEFT and self.fit_x_start == -1 and self.fit_x_end == -1:
                self.fit_x_start = event.xdata
                self.line_axis.set_left(self.line_axis.get_first_left())
                self.line_axis.append_left("set end ")
            elif event.button == MouseButton.LEFT and self.fit_x_end == -1:
                self.fit_x_end = event.xdata
                if self.fit_x_start == self.fit_x_end:
                    return
                elif self.fit_x_end < self.fit_x_start:
                    temp = self.fit_x_end
                    self.fit_x_end = self.fit_x_start
                    self.fit_x_start = temp

                self.line_axis.set_left(self.line_axis.get_first_left())

                d = self.dpkt.fit(
                    self.fit_x_start,
                    self.fit_x_end,
                    in_kwargs={'color': 'black', 'linewidth': 3},
                    out_kwargs={'color': 'gray', 'linewidth': 3}
                )

                self.line_axis.append_left(
                    d['FIT']['title']
                    + fr' mse: {d["FIT"]["mse"]}'
                    + '\n'
                    + d['BG']['title'] + fr' mse: {d["BG"]["mse"]}'
                )

                self.line_axis.add_line(d['FIT']['line'])
                self.line_axis.add_line(d['BG']['line'])
                self.err = self.line_axis.scatter(
                    d['BG']['out'][0],
                    d['BG']['out'][1],
                    c='white',
                    s=4,
                    marker='.'
                )

                self.line_axis.refresh()
            else:
                if event.button == MouseButton.LEFT:
                    self.fit_x_start = event.xdata
                    self.line_axis.set_left(self.line_axis.get_first_left())
                    self.line_axis.append_left("set end ")
                else:
                    self.fit_x_start = -1
                    self.line_axis.set_left(self.line_axis.get_first_left())
                    self.line_axis.append_left("set start ")
                self.fit_x_end = -1
                self.line_axis.clear_lines()
            self.line_axis.figure.canvas.draw()
        elif event.inaxes == self.data_axis:
            self.clear_line_soft()
            if self.rect:
                self.rect.remove()
                self.rect = None
            if event.button in {MouseButton.LEFT, MouseButton.RIGHT}:
                width = self.area[0]
                window = self.area[1]
                row = int(event.ydata)
                col = int(event.xdata)
                log.debug(f"Click detected at {row},{col}")
                self.dpkt.configure(width, window, 2)
                vertical = event.button == MouseButton.RIGHT
                if self.line_axis.axes_modifier is not None:
                    self.line_axis.axes_modifier(self.line_axis, vertical=vertical)
                r = self.dpkt.select(
                    col,
                    row,
                    vertical=vertical,
                    lw=1,
                    fill=False,
                )
                if event.button == MouseButton.LEFT:
                    r.set_color('b')
                    self.line_axis.set_left(f"HORIZONTAL slice with HEIGHT: {row - width} <= y <= {row + width}")
                    self.dpkt.scatter(self.line_axis, s=8, c='b')
                else:
                    r.set_color('r')
                    self.line_axis.set_left(f"VERTICAL slice with WIDTH: {col - width} <= x <= {col + width}")
                    self.dpkt.scatter(self.line_axis, s=8, c='r')
                self.rect = self.data_axis.add_patch(r)
                self.line_has_data = True
            else:
                self.line_has_data = False
            self.line_axis.refresh()

    def clear_line_soft(self):
        self.line_axis.clear()
        self.line_axis.secondary_yaxis(location="right")

    def clear_line(self):
        self.clear_line_soft()
        self.line_has_data = False
        self.fit_x_start = -1
        self.fit_x_end = -1
        self.rect = None
        self.err = None

    def detach(self):
        try:
            self.data_axis.figure.canvas.mpl_disconnect(self.cid)
        except Exception as e:
            log.exception("Exception in detach", exc_info=e)


#!/usr/bin/env python3
import collections
import os
import sys

import numpy as np
import tensorflow as tf
import tensorflow_addons as tfa

from mapping import Mapping
import mtool_evaluate
from sentence import Sentence

NODE_NO, NODE_PARENT, NODE_CHILD = range(3)

def get_layers(s):
    # Shortest paths from start, compute anchors
    paths = [0] * s.n_tokens() + [None] * s.n_nodes()
    anchors = [{i} if i < s.n_tokens() else set() for i in range(s.n_tokens() + s.n_nodes())]
    queue = collections.deque(range(s.n_tokens()))

    while queue:
        n = queue.popleft()
        for e in s.parents[n]:
            p = s.factors[s.EDGE_PARENTS][e]
            anchors[p] |= anchors[n]
            if not paths[p]:
                paths[p] = paths[n] + 1
                queue.append(p)

    # Compute a sequence of created nodes
    layers = []

    nodes = set(range(s.n_tokens()))
    while len(nodes) < s.n_nodes() + s.n_tokens():
        new_nodes, new_layer = set(), []
        for n in nodes:
            best, best_e = None, None
            for e in s.parents[n]:
                p = s.factors[s.EDGE_PARENTS][e]
                if p not in nodes and p not in new_nodes:
                    if not best or len(anchors[p]) < len(anchors[best]) or sorted(anchors[p]) < sorted(anchors[best]):
                        best, best_e = p, e
            for e in s.children[n]:
                p = s.factors[s.EDGE_CHILDREN][e]
                if paths[p]: continue
                if p not in nodes and p not in new_nodes:
                    if not best or len(anchors[p]) < len(anchors[best]) or sorted(anchors[p]) < sorted(anchors[best]):
                        best, best_e = p, e
            if best:
                new_nodes.add(best)
                new_layer.append((n, best, best_e))

        if not new_nodes: break
        nodes |= new_nodes
        layers.append(new_layer)

    if len(nodes) != s.n_nodes() + s.n_tokens():
        raise ValueError("Cannot get layers for sentence {}".format(s.id))

    return layers

class Network:
    def __init__(self, train, args):
        # Create required layers
        self.layers = tf.keras.Model()

        # Encoder
        self.layers.encoder_we = tf.keras.layers.Embedding(len(train.mappings[Sentence.TOKENS].strings), args.we_dim, mask_zero=True)
        self.layers.encoder_cle = tf.keras.layers.Embedding(len(train.mappings[Sentence.TOKENS].alphabet), args.cle_dim, mask_zero=True)
        self.layers.encoder_cle_rnn = tf.keras.layers.Bidirectional(tf.keras.layers.GRU(args.cle_dim), merge_mode="concat")
        self.layers.encoder_token_values = [
            tf.keras.layers.Embedding(len(train.mappings[Sentence.TOKEN_VALUES][prop].strings), 128, mask_zero=True)
            for prop in train.token_properties
        ]
        self.layers.encoder_rnns = [
            tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(args.encoder_dim, return_sequences=True), merge_mode="sum")
            for _ in range(args.encoder_rnn_layers)
        ]
        self.layers.encoder_node_values = [
            tf.keras.layers.Embedding(len(train.mappings[Sentence.NODE_VALUES][prop].strings), args.encoder_dim)
            for prop in train.node_properties
        ]
        self.edge_values = [len(train.mappings[Sentence.EDGE_VALUES][prop].strings) for prop in train.edge_properties]
        self.layers.encoder_edge_values = [
            tf.keras.layers.Embedding(2 * edge_values, args.encoder_dim)
            for edge_values in self.edge_values
        ]
        self.layers.encoder_token_transfer = [
            tf.keras.layers.Dense(args.encoder_dim, activation=tf.nn.tanh)
            for _ in range(args.decoder_iterations)]

        # Decoder
        self.layers.decoder_node_operation = [
            tf.keras.layers.Dense(3, activation=tf.nn.softmax) for _ in range(args.decoder_iterations)]
        self.layers.decoder_node_values = [
            [tf.keras.layers.Dense(len(train.mappings[Sentence.NODE_VALUES][prop].strings), activation=tf.nn.softmax) for prop in train.node_properties]
            for _ in range(args.decoder_iterations)]
#         self.layers.decoder_edge_values = [
#             [tf.keras.layers.Dense(len(train.mappings[Sentence.EDGE_VALUES][prop].strings), activation=tf.nn.softmax) for prop in train.edge_properties]
#             for _ in range(args.decoder_iterations)]
        self.layers.decoder_edge_parents = [
            tf.keras.layers.Dense(args.edge_dim)
            for _ in range(args.decoder_iterations)]
        self.layers.decoder_edge_children = [
            tf.keras.layers.Dense(args.edge_dim)
            for _ in range(args.decoder_iterations)]
        self.layers.decoder_edge_arc = [
            tf.keras.layers.Dense(2, activation=tf.nn.softmax)
            for _ in range(args.decoder_iterations)]
        if args.highway:
            self.layers.decoder_edge_highway = [
                tf.keras.layers.Dense(args.edge_dim, activation=tf.nn.tanh)
                for _ in range(args.decoder_iterations)]
        self.layers.decoder_deprel_parents = [
            tf.keras.layers.Dense(args.deprel_dim)
            for _ in range(args.decoder_iterations)]
        self.layers.decoder_deprel_children = [
            tf.keras.layers.Dense(args.deprel_dim)
            for _ in range(args.decoder_iterations)]
        self.layers.decoder_deprel_values = [
            [tf.keras.layers.Dense(len(train.mappings[Sentence.EDGE_VALUES][prop].strings), activation=tf.nn.softmax) for prop in train.edge_properties]
            for _ in range(args.decoder_iterations)]
        if args.highway:
            self.layers.decoder_deprel_highway = [
                tf.keras.layers.Dense(args.edge_dim, activation=tf.nn.tanh)
                for _ in range(args.decoder_iterations)]
        self.layers.decoder_tops = tf.keras.layers.Dense(2, activation=tf.nn.softmax)

        # Generic helpers
        self.layers.dropout = tf.keras.layers.Dropout(args.dropout)
        self.layers.concat = tf.keras.layers.Concatenate()
        self.layers.sum = tf.keras.layers.Add()
        self.word_dropout = args.word_dropout
        self.optimizer_step_sentences = len(train.sentence_lens) / ((len(train.sentence_lens) - 1) // (args.batch_size * args.batch_aggregation) + 1)

        if args.predict: return

        # Create optimizer, losses, summary writer and accuracies
        self.optimizer = tfa.optimizers.LazyAdam(beta_2=args.beta_2)
        self.loss_sce = tf.keras.losses.SparseCategoricalCrossentropy()
        self.writer = tf.summary.create_file_writer(args.logdir, flush_millis=10*1000)
        self.accuracies = dict([
            ("node/ops", tf.keras.metrics.SparseCategoricalAccuracy())] + [
            ("node/" + prop, tf.keras.metrics.SparseCategoricalAccuracy()) for prop in train.node_properties] + [
            ("edge/arc", tf.keras.metrics.SparseCategoricalAccuracy())] + [
            ("edge/" + prop, tf.keras.metrics.SparseCategoricalAccuracy()) for prop in train.edge_properties] + [
            ("edge/tops", tf.keras.metrics.SparseCategoricalAccuracy())])

    def load_weights(self, path):
        self.layers.load_weights(path)

    def save_weights(self, path):
        self.layers.save_weights(path)

    def _encoder(self, token_ids, token_charseqs, token_charseq_ids, token_values, token_additionals, training):
        if training and self.word_dropout:
            word_dropout_mask = tf.cast(tf.math.greater_equal(tf.random.uniform(token_ids.shape), self.word_dropout), tf.int32)
            unk_mask = tf.cast(tf.math.not_equal(token_ids, 0), tf.int32) * Mapping.UNK
            token_ids = word_dropout_mask * token_ids + (1 - word_dropout_mask) * unk_mask
        inputs = [self.layers.encoder_we(token_ids)]

        characters_embedded = self.layers.encoder_cle(token_charseqs)
        characters_embedded = self.layers.dropout(characters_embedded, training=training)
        charseqs = self.layers.encoder_cle_rnn(characters_embedded)
        charseqs = tf.gather(charseqs, token_charseq_ids)
        inputs.append(charseqs)

        for i, embedding in enumerate(self.layers.encoder_token_values):
            inputs.append(embedding(token_values[:, :, i]))

        inputs.extend(token_additionals)

        hidden = self.layers.concat(inputs)
        hidden = self.layers.dropout(hidden, training=training)
        for i, rnn in enumerate(self.layers.encoder_rnns):
            rnn_output = rnn(hidden)
            rnn_output = self.layers.dropout(rnn_output, training=training)
            if i == 0:
                hidden = rnn_output
            else:
                hidden = self.layers.sum([hidden, rnn_output])

        delattr(hidden, "_keras_mask")
        return hidden

    class State:
        def __init__(self, network, sentence, tokens, training):
            self.network = network
            self.sentence = sentence
            self.tokens = tokens
            self.training = training

            self.nodes = tokens
            self.n_nodes = sentence.n_tokens()
            self.nodes_token = list(range(sentence.n_tokens()))

            self.n_nodes_cached = self.n_nodes
            self.node_mapping = {i: i for i in range(self.n_nodes)}

        def add_node(self, values, predecessor, mapping=None):
            self.sentence.add_node(values)
            self.nodes_token.append(self.nodes_token[predecessor])
            if mapping is not None:
                self.node_mapping[mapping] = self.n_nodes
            self.n_nodes += 1

        def add_edge(self, parent, child, values):
            if values[0] == Mapping.ANCHOR and child >= self.sentence.n_tokens(): return
            if values[0] == Mapping.ANCHOR and parent < self.sentence.n_tokens(): return
            self.sentence.add_edge(parent, child, values)

        @staticmethod
        def recompute_nodes(nodes, states, iteration):
            nodes_token, nodes_values = [], []
            start = 0
            for state in states:
                nodes_token.extend([start + t for t in state.nodes_token[state.n_nodes_cached:]])
                nodes_values.extend(state.sentence.factors[Sentence.NODE_VALUES][state.n_nodes_cached - state.sentence.n_tokens():])
                start += state.n_nodes_cached

            new_nodes = states[0].network.layers.encoder_token_transfer[iteration](tf.gather(nodes, np.array(nodes_token, np.int32)))
            for i, encoder in enumerate(states[0].network.layers.encoder_node_values):
                new_nodes += encoder(np.array([node_values[i] for node_values in nodes_values], np.int32))

            new_nodes = states[0].network.layers.dropout(new_nodes, training=states[0].training)

            start = 0
            for state in states:
                num_nodes = state.n_nodes - state.n_nodes_cached
                if num_nodes:
                    state.nodes = tf.concat([state.nodes, new_nodes[start:start + num_nodes]], axis=0)
                start += num_nodes
                state.n_nodes_cached = state.n_nodes

        @staticmethod
        def recompute_edges(ori_nodes, states, iteration):
            node_ids, edge_values = [], []
            start = 0
            for i, state in enumerate(states):
                for n in range(ori_nodes[i], state.n_nodes):
                    for e in state.sentence.children[n]:
                        node_ids.append(start + n - ori_nodes[i])
                        edge_values.append(state.sentence.factors[Sentence.EDGE_VALUES][e])
                    for e in state.sentence.parents[n]:
                        node_ids.append(start + n - ori_nodes[i])
                        edge_values.append([a + b for a, b in zip(
                            state.network.edge_values, state.sentence.factors[Sentence.EDGE_VALUES][e])])
                start += state.n_nodes - ori_nodes[i]

            edge_embeddings = 0
            for i, encoder in enumerate(states[0].network.layers.encoder_edge_values):
                edge_embeddings += encoder(np.array([edge_values[i] for edge_values in edge_values], np.int32))

            node_values = tf.math.segment_mean(edge_embeddings, node_ids)
            max_node_id = max(node_ids)
            if max_node_id + 1 < start:
                node_values = tf.pad(node_values, [[0, start - max_node_id - 1], [0, 0]])

            start = 0
            for i, state in enumerate(states):
                if state.n_nodes == ori_nodes[i]: continue
                state.nodes += tf.pad(node_values[start:start + state.n_nodes - ori_nodes[i]],
                                      [[ori_nodes[i], 0], [0, 0]])
                start += state.n_nodes - ori_nodes[i]

    def _summary_step(self):
        return int(int(self.optimizer.iterations) * self.optimizer_step_sentences)

    def train_epoch(self, train, learning_rate, args):
        self.optimizer.learning_rate = learning_rate

        skipped = 0
        num_gradients = 0
        while not train.epoch_finished():
            batch = train.next_batch(args.batch_size)
            sentences = batch["sentences"]

            layers = []
            for sentence in sentences:
                layers.append(get_layers(sentence))

            for accuracy in self.accuracies.values():
                accuracy.reset_states()

            with tf.GradientTape() as tape:
                encoded_tokens = self._encoder(
                    token_ids=batch["batch_factors"][Sentence.TOKENS],
                    token_charseqs=batch["charseqs"],
                    token_charseq_ids=batch["charseq_ids"],
                    token_values=batch["batch_factors"][Sentence.TOKEN_VALUES],
                    token_additionals=[batch[x] for x in ["bert", "fasttext", "word2vec"] if x in batch],
                    training=True)

                states = []
                for i, sentence in enumerate(sentences):
                    states.append(self.State(self, sentence.empty_copy(), encoded_tokens[i][:sentence.n_tokens()], True))

                loss = 0
                for iteration in range(args.decoder_iterations):
                    # 1) New nodes
                    ori_nodes = [s.n_nodes for s in states]
                    sum_nodes = sum(ori_nodes)
                    nodes = tf.concat([s.nodes for s in states], axis=0)

                    target_ops = np.zeros([sum_nodes], np.int32)
                    target_node_values = np.zeros([sum_nodes, len(train.node_properties)], np.int32)
                    target_node_values_mask = np.zeros([sum_nodes, 1], np.int32)
#                     target_edge_values = np.zeros([sum_nodes, len(train.edge_properties)], np.int32)
#                     target_edge_values_mask = np.zeros([sum_nodes, 1], np.int32)

                    start = 0
                    for i in range(len(sentences)):
                        if iteration < len(layers[i]):
                            state, layer, sentence = states[i], layers[i][iteration], sentences[i]
                            for n, target, edge in layer:
                                n = state.node_mapping[n]
                                target_ops[start + n] = NODE_PARENT if sentence.factors[Sentence.EDGE_PARENTS][edge] == target else NODE_CHILD
                                target_node_values[start + n] = sentence.factors[Sentence.NODE_VALUES][target - sentence.n_tokens()]
                                target_node_values_mask[start + n] = 1
#                                 target_edge_values[start + n] = sentence.factors[Sentence.EDGE_VALUES][edge]
#                                 target_edge_values_mask[start + n] = 1 if n >= state.sentence.n_tokens() else 0
                                state.add_node(target_node_values[start + n], n, target)
                        start += states[i].n_nodes_cached
                    self.State.recompute_nodes(nodes, states, iteration)

                    predictions = self.layers.decoder_node_operation[iteration](nodes)
                    loss += self.loss_sce(target_ops, predictions)
                    self.accuracies["node/ops"](target_ops, predictions)
                    for i, prop in enumerate(train.node_properties):
                        predictions = self.layers.decoder_node_values[iteration][i](nodes)
                        loss += self.loss_sce(target_node_values[:, i], predictions, target_node_values_mask)
                        self.accuracies["node/" + prop](target_node_values[:, i], predictions, target_node_values_mask)
#                     for i, prop in enumerate(train.edge_properties):
#                         predictions = self.layers.decoder_edge_values[iteration][i](nodes)
#                         loss += self.loss_sce(target_edge_values[:, i], predictions, target_edge_values_mask)
#                         self.accuracies["edge/" + prop](target_edge_values[:, i], predictions, target_edge_values_mask)

                    # 2) Edges
                    sum_nodes = sum(s.n_nodes for s in states)
                    nodes = tf.concat([s.nodes for s in states], axis=0)
                    target_indices_a, target_indices_b = [], []
                    target_a_parent, target_a_child = [], []
                    target_deprel_parents, target_deprel_children = [], []
                    target_deprel_values = []

                    start = 0
                    for i in range(len(sentences)):
                        state, sentence = states[i], sentences[i]
                        offset = len(target_indices_a)
                        for j in range(ori_nodes[i], state.n_nodes):
                            target_indices_a.append(start + np.repeat(np.int32(j), state.n_nodes))
                            target_indices_b.append(start + np.arange(0, state.n_nodes, dtype=np.int32))
                            target_a_parent.append(np.zeros(state.n_nodes, np.float32))
                            target_a_child.append(np.zeros(state.n_nodes, np.float32))
                        for n_ori, n in state.node_mapping.items():
                            if n >= ori_nodes[i]:
                                for e in sentence.parents[n_ori]:
                                    if sentence.factors[Sentence.EDGE_VALUES][e][0] == Mapping.ROOT: continue
                                    p = sentence.factors[sentence.EDGE_PARENTS][e]
                                    if p in state.node_mapping:
                                        p = state.node_mapping[p]
                                        if not args.no_anchors or not sentence.factors[Sentence.EDGE_VALUES][e][0] == Mapping.ANCHOR:
                                            target_a_child[offset + n - ori_nodes[i]][p] = 1
                                        target_deprel_parents.append(start + p)
                                        target_deprel_children.append(start + n)
                                        target_deprel_values.append(sentence.factors[Sentence.EDGE_VALUES][e])
                                        state.add_edge(p, n, target_deprel_values[-1])
                                for e in sentence.children[n_ori]:
                                    if sentence.factors[Sentence.EDGE_VALUES][e][0] == Mapping.ROOT: continue
                                    c = sentence.factors[sentence.EDGE_CHILDREN][e]
                                    if c in state.node_mapping:
                                        c = state.node_mapping[c]
                                        if not args.no_anchors or not sentence.factors[Sentence.EDGE_VALUES][e][0] == Mapping.ANCHOR:
                                            target_a_parent[offset + n-ori_nodes[i]][c] = 1
                                        target_deprel_parents.append(start + n)
                                        target_deprel_children.append(start + c)
                                        target_deprel_values.append(sentence.factors[Sentence.EDGE_VALUES][e])
                                        state.add_edge(n, c, target_deprel_values[-1])
                        start += state.n_nodes

                    if not target_indices_a or not target_deprel_parents:
                        continue
                    target_indices_a = np.concatenate(target_indices_a, axis=0)
                    target_indices_b = np.concatenate(target_indices_b, axis=0)
                    target_a_parent = np.concatenate(target_a_parent, axis=0)
                    target_a_child = np.concatenate(target_a_child, axis=0)
                    target_deprel_parents = np.array(target_deprel_parents, np.int32)
                    target_deprel_children = np.array(target_deprel_children, np.int32)
                    target_deprel_values = np.array(target_deprel_values, np.int32)

                    # 2.1) Compute arcs
                    edge_parents = self.layers.decoder_edge_parents[iteration](nodes)
                    edge_children = self.layers.decoder_edge_children[iteration](nodes)

                    a_parent = tf.nn.tanh(self.layers.sum([tf.gather(edge_parents, target_indices_a),
                                                           tf.gather(edge_children, target_indices_b)]))
                    if args.highway:
                        a_parent += self.layers.decoder_edge_highway[iteration](a_parent)
                    a_parent = self.layers.decoder_edge_arc[iteration](a_parent)
                    loss += self.loss_sce(target_a_parent, a_parent)
                    self.accuracies["edge/arc"](target_a_parent, a_parent)

                    a_child = tf.nn.tanh(self.layers.sum([tf.gather(edge_parents, target_indices_b),
                                                          tf.gather(edge_children, target_indices_a)]))
                    if args.highway:
                        a_child += self.layers.decoder_edge_highway[iteration](a_child)
                    a_child = self.layers.decoder_edge_arc[iteration](a_child)
                    loss += self.loss_sce(target_a_child, a_child)
                    self.accuracies["edge/arc"](target_a_child, a_child)

                    # 2.2) Compute deprels
                    deprel_parents = self.layers.decoder_deprel_parents[iteration](nodes)
                    deprel_children = self.layers.decoder_deprel_children[iteration](nodes)
                    deprel_weights = tf.nn.tanh(self.layers.sum(
                        [tf.gather(deprel_parents, target_deprel_parents),
                         tf.gather(deprel_children, target_deprel_children)]))
                    if args.highway:
                        deprel_weights += self.layers.decoder_deprel_highway[iteration](deprel_weights)
                    for i, prop in enumerate(train.edge_properties):
                        predictions = self.layers.decoder_deprel_values[iteration][i](deprel_weights)
                        loss += self.loss_sce(target_deprel_values[:, i], predictions)
                        self.accuracies["edge/" + prop](target_deprel_values[:, i], predictions)

                    self.State.recompute_edges(ori_nodes, states, iteration)

                # Tops
                sum_nodes = sum(s.n_nodes for s in states)
                nodes = tf.concat([s.nodes for s in states], axis=0)
                target_tops = np.zeros([sum_nodes], np.int32)
                start = 0
                for i, sentence in enumerate(sentences):
                    for e in sentence.children[0]:
                        if sentence.factors[Sentence.EDGE_VALUES][e][0] == Mapping.ROOT:
                            c = sentence.factors[Sentence.EDGE_CHILDREN][e]
                            if c in states[i].node_mapping:
                                target_tops[start + states[i].node_mapping[c]] = 1
                    start += states[i].n_nodes
                predictions = self.layers.decoder_tops(nodes)
                loss += self.loss_sce(target_tops, predictions)
                self.accuracies["edge/tops"](target_tops, predictions)

            tg = tape.gradient(loss, self.layers.trainable_variables)
            tg_none = [variable.name for g, variable in zip(tg, self.layers.trainable_variables) if g is None]
            if tg_none:
                print("Skipping a batch with None gradient for variables {}".format(tg_none), file=sys.stderr, flush=True)
                continue

            if num_gradients == 0:
                gradients = [g.numpy() if not isinstance(g, tf.IndexedSlices) else [(g.values.numpy(), g.indices.numpy())] for g in tg]
            else:
                for g, ng in zip(gradients, tg):
                    if isinstance(g, list):
                        g.append((ng.values.numpy(), ng.indices.numpy()))
                    else:
                        g += ng.numpy()
            num_gradients += 1
            if num_gradients == args.batch_aggregation or len(train._permutation) == 0:
                gradients = [tf.IndexedSlices(*map(np.concatenate, zip(*g))) if isinstance(g, list) else g for g in gradients]
                self.optimizer.apply_gradients(zip(gradients, self.layers.trainable_variables))
                num_gradients = 0
                if int(self.optimizer.iterations) % 100 == 0:
                    tf.summary.experimental.set_step(self._summary_step())
                    with self.writer.as_default():
                        for name, accuracy in self.accuracies.items():
                            tf.summary.scalar("train/" + name, accuracy.result())

        tf.summary.experimental.set_step(self._summary_step())
        with self.writer.as_default():
            tf.summary.scalar("train/skipped", skipped)

    def predict(self, data, args):
        output = []

        while not data.epoch_finished():
            batch = data.next_batch(args.batch_size)
            sentences = batch["sentences"]

            encoded_tokens = self._encoder(
                token_ids=batch["batch_factors"][Sentence.TOKENS],
                token_charseqs=batch["charseqs"],
                token_charseq_ids=batch["charseq_ids"],
                token_values=batch["batch_factors"][Sentence.TOKEN_VALUES],
                token_additionals=[batch[x] for x in ["bert", "fasttext", "word2vec"] if x in batch],
                training=False)

            states = []
            for i, sentence in enumerate(sentences):
                states.append(self.State(self, sentence.empty_copy(), encoded_tokens[i][:sentence.n_tokens()], False))

            for iteration in range(args.decoder_iterations):
                # 1) New nodes
                ori_nodes = [s.n_nodes for s in states]
                sum_nodes = sum(ori_nodes)
                nodes = tf.concat([s.nodes for s in states], axis=0)

                prediction_ops = tf.argmax(self.layers.decoder_node_operation[iteration](nodes), axis=1).numpy()
                prediction_node_values = np.stack(
                    [tf.argmax(decoder(nodes), axis=1).numpy() for decoder in self.layers.decoder_node_values[iteration]], axis=1)
#                 prediction_edge_values = np.stack(
#                     [tf.argmax(decoder(nodes), axis=1).numpy() for decoder in self.layers.decoder_edge_values[iteration]], axis=1)
                start = 0
                for i, state in enumerate(states):
                    for n in range(1, ori_nodes[i]):
                        if prediction_ops[start + n] == NODE_NO: continue
                        state.add_node(prediction_node_values[start + n], n)
#                         edge_values = prediction_edge_values[start + n] if n >= state.sentence.n_tokens() else [Mapping.ANCHOR] + [Mapping.NONE] * (len(data.edge_properties) - 1)
#                         if prediction_ops[start + n] == NODE_PARENT:
#                             state.add_edge(state.n_nodes - 1, n, edge_values)
#                         else:
#                             state.add_edge(n, state.n_nodes - 1, edge_values)
                    start += ori_nodes[i]
                self.State.recompute_nodes(nodes, states, iteration)

                # 2) Edges
                sum_nodes = sum(s.n_nodes for s in states)
                nodes = tf.concat([s.nodes for s in states], axis=0)
                indices_a, indices_b = [], []
                indices_parents, indices_children = [], []

                start = 0
                for i, state in enumerate(states):
                    for j in range(ori_nodes[i], state.n_nodes):
                        indices_a.append(start + np.repeat(np.int32(j), state.n_nodes))
                        indices_b.append(start + np.arange(0, state.n_nodes, dtype=np.int32))
                    start += state.n_nodes
                if not indices_a: continue

                indices_a = np.concatenate(indices_a, axis=0)
                indices_b = np.concatenate(indices_b, axis=0)

                # 2.1) Compute arcs
                edge_parents = self.layers.decoder_edge_parents[iteration](nodes)
                edge_children = self.layers.decoder_edge_children[iteration](nodes)

                a_parent = tf.nn.tanh(self.layers.sum([tf.gather(edge_parents, indices_a),
                                                       tf.gather(edge_children, indices_b)]))
                if args.highway:
                    a_parent += self.layers.decoder_edge_highway[iteration](a_parent)
                a_parent = tf.argmax(self.layers.decoder_edge_arc[iteration](a_parent), axis=1).numpy()
                a_child = tf.nn.tanh(self.layers.sum([tf.gather(edge_parents, indices_b),
                                                      tf.gather(edge_children, indices_a)]))
                if args.highway:
                    a_child += self.layers.decoder_edge_highway[iteration](a_child)
                a_child = tf.argmax(self.layers.decoder_edge_arc[iteration](a_child), axis=1).numpy()

                start, offset = 0, 0
                for i, state in enumerate(states):
                    node = 0
                    for n in range(1, ori_nodes[i]):
                        if prediction_ops[start + n] == NODE_NO: continue
                        if prediction_ops[start + n] == NODE_PARENT:
                            a_parent[offset + node * state.n_nodes + n] = 1
                        else:
                            a_child[offset + node * state.n_nodes + n] = 1
                        node += 1
                    start += ori_nodes[i]
                    offset += (state.n_nodes - ori_nodes[i]) * state.n_nodes
                for i in np.where(a_parent > 0)[0]:
                    indices_parents.append(indices_a[i])
                    indices_children.append(indices_b[i])
                for i in np.where(a_child > 0)[0]:
                    indices_parents.append(indices_b[i])
                    indices_children.append(indices_a[i])
                indices_parents = np.array(indices_parents, np.int32)
                indices_children = np.array(indices_children, np.int32)

                # 2.2) Compute deprels
                deprel_parents = self.layers.decoder_deprel_parents[iteration](nodes)
                deprel_children = self.layers.decoder_deprel_children[iteration](nodes)
                deprel_weights = tf.nn.tanh(self.layers.sum([tf.gather(deprel_parents, indices_parents),
                                                             tf.gather(deprel_children, indices_children)]))
                if args.highway:
                    deprel_weights += self.layers.decoder_deprel_highway[iteration](deprel_weights)
                edge_predictions = []
                for i, prop in enumerate(data.edge_properties):
                    edge_predictions.append(tf.argmax(
                        self.layers.decoder_deprel_values[iteration][i](deprel_weights), axis=1).numpy())
                edge_predictions = np.stack(edge_predictions, axis=1)

                offset = 0
                for _ in range(2):
                    start = 0
                    for state in states:
                        while offset < len(indices_parents) and indices_parents[offset] >= start and indices_parents[offset] < start + state.n_nodes:
                            state.add_edge(
                                indices_parents[offset] - start,
                                indices_children[offset] - start,
                                edge_predictions[offset])
                            offset += 1
                        start += state.n_nodes
                assert offset == len(indices_parents)

                self.State.recompute_edges(ori_nodes, states, iteration)

            # Tops
            sum_nodes = sum(s.n_nodes for s in states)
            nodes = tf.concat([s.nodes for s in states], axis=0)
            predictions = tf.argmax(self.layers.decoder_tops(nodes), axis=1)
            start = 0
            for i, state in enumerate(states):
                for n in range(1, state.n_nodes):
                    if predictions[start + n]:
                        state.add_edge(0, n, [Mapping.ROOT] + [Mapping.NONE] * (len(data.edge_properties) - 1))
                start += state.n_nodes

            for state in states:
                output.append(state.sentence.write(data.mappings))
                output.append("\n")

        return "".join(output)

    @staticmethod
    def predict_ensemble(networks, data, args):
        output = []

        while not data.epoch_finished():
            batch = data.next_batch(args.batch_size)
            sentences = batch["sentences"]

            for network in networks:
                encoded_tokens = network._encoder(
                    token_ids=batch["batch_factors"][Sentence.TOKENS],
                    token_charseqs=batch["charseqs"],
                    token_charseq_ids=batch["charseq_ids"],
                    token_values=batch["batch_factors"][Sentence.TOKEN_VALUES],
                    token_additionals=[batch[x] for x in ["bert", "fasttext", "word2vec"] if x in batch],
                    training=False)

                network.states = []
                for i, sentence in enumerate(sentences):
                    network.states.append(network.State(network, sentence.empty_copy(), encoded_tokens[i][:sentence.n_tokens()], False))

            for iteration in range(args.decoder_iterations):
                # 1) New nodes
                ori_nodes = [s.n_nodes for s in networks[0].states]
                sum_nodes = sum(ori_nodes)
                for network in networks:
                    network.nodes = tf.concat([s.nodes for s in network.states], axis=0)

                prediction_ops = np.argmax(
                    np.mean([network.layers.decoder_node_operation[iteration](network.nodes).numpy() for network in networks], axis=0),
                    axis=1)
                prediction_node_values = np.stack([
                    np.argmax(
                        np.mean([network.layers.decoder_node_values[iteration][i](network.nodes).numpy() for network in networks], axis=0),
                        axis=1)
                    for i in range(len(networks[0].layers.decoder_node_values[iteration]))
                ], axis=1)

                for network in networks:
                    start = 0
                    for i, state in enumerate(network.states):
                        for n in range(1, ori_nodes[i]):
                            if prediction_ops[start + n] == NODE_NO: continue
                            state.add_node(prediction_node_values[start + n], n)
                        start += ori_nodes[i]
                    network.State.recompute_nodes(network.nodes, network.states, iteration)

                # 2) Edges
                sum_nodes = sum(s.n_nodes for s in networks[0].states)
                for network in networks:
                    network.nodes = tf.concat([s.nodes for s in network.states], axis=0)
                indices_a, indices_b = [], []
                indices_parents, indices_children = [], []

                start = 0
                for i, state in enumerate(networks[0].states):
                    for j in range(ori_nodes[i], state.n_nodes):
                        indices_a.append(start + np.repeat(np.int32(j), state.n_nodes))
                        indices_b.append(start + np.arange(0, state.n_nodes, dtype=np.int32))
                    start += state.n_nodes
                if not indices_a: continue

                indices_a = np.concatenate(indices_a, axis=0)
                indices_b = np.concatenate(indices_b, axis=0)

                # 2.1) Compute arcs
                a_parent, a_child = [], []
                for network in networks:
                    edge_parents = network.layers.decoder_edge_parents[iteration](network.nodes)
                    edge_children = network.layers.decoder_edge_children[iteration](network.nodes)

                    a_parent.append(tf.nn.tanh(network.layers.sum([tf.gather(edge_parents, indices_a),
                                                                   tf.gather(edge_children, indices_b)])))
                    if args.highway:
                        a_parent[-1] += network.layers.decoder_edge_highway[iteration](a_parent[-1])
                    a_parent[-1] = network.layers.decoder_edge_arc[iteration](a_parent[-1]).numpy()

                    a_child.append(tf.nn.tanh(network.layers.sum([tf.gather(edge_parents, indices_b),
                                                                   tf.gather(edge_children, indices_a)])))
                    if args.highway:
                        a_child[-1] += network.layers.decoder_edge_highway[iteration](a_child[-1])
                    a_child[-1] = network.layers.decoder_edge_arc[iteration](a_child[-1]).numpy()
                a_parent = np.argmax(np.mean(a_parent, axis=0), axis=1)
                a_child = np.argmax(np.mean(a_child, axis=0), axis=1)

                start, offset = 0, 0
                for i, state in enumerate(networks[0].states):
                    node = 0
                    for n in range(1, ori_nodes[i]):
                        if prediction_ops[start + n] == NODE_NO: continue
                        if prediction_ops[start + n] == NODE_PARENT:
                            a_parent[offset + node * state.n_nodes + n] = 1
                        else:
                            a_child[offset + node * state.n_nodes + n] = 1
                        node += 1
                    start += ori_nodes[i]
                    offset += (state.n_nodes - ori_nodes[i]) * state.n_nodes
                for i in np.where(a_parent > 0)[0]:
                    indices_parents.append(indices_a[i])
                    indices_children.append(indices_b[i])
                for i in np.where(a_child > 0)[0]:
                    indices_parents.append(indices_b[i])
                    indices_children.append(indices_a[i])
                indices_parents = np.array(indices_parents, np.int32)
                indices_children = np.array(indices_children, np.int32)

                # 2.2) Compute deprels
                edge_predictions = []
                for i, prop in enumerate(data.edge_properties):
                    predictions = []
                    for network in networks:
                        deprel_parents = network.layers.decoder_deprel_parents[iteration](network.nodes)
                        deprel_children = network.layers.decoder_deprel_children[iteration](network.nodes)
                        deprel_weights = tf.nn.tanh(network.layers.sum([tf.gather(deprel_parents, indices_parents),
                                                                        tf.gather(deprel_children, indices_children)]))
                        if args.highway:
                            deprel_weights += network.layers.decoder_deprel_highway[iteration](deprel_weights)
                        predictions.append(network.layers.decoder_deprel_values[iteration][i](deprel_weights).numpy())
                    edge_predictions.append(np.argmax(np.mean(predictions, axis=0), axis=1))
                edge_predictions = np.stack(edge_predictions, axis=1)

                for network in networks:
                    offset = 0
                    for _ in range(2):
                        start = 0
                        for state in network.states:
                            while offset < len(indices_parents) and indices_parents[offset] >= start and indices_parents[offset] < start + state.n_nodes:
                                state.add_edge(
                                    indices_parents[offset] - start,
                                    indices_children[offset] - start,
                                    edge_predictions[offset])
                                offset += 1
                            start += state.n_nodes
                    assert offset == len(indices_parents)

                    network.State.recompute_edges(ori_nodes, network.states, iteration)

            # Tops
            sum_nodes = sum(s.n_nodes for s in networks[0].states)
            for network in networks:
                network.nodes = tf.concat([s.nodes for s in network.states], axis=0)
            predictions = np.argmax(
                np.mean([network.layers.decoder_tops(network.nodes).numpy() for network in networks], axis=0),
                axis=1)
            for network in networks:
                start = 0
                for i, state in enumerate(network.states):
                    for n in range(1, state.n_nodes):
                        if predictions[start + n]:
                            state.add_edge(0, n, [Mapping.ROOT] + [Mapping.NONE] * (len(data.edge_properties) - 1))
                    start += state.n_nodes

            for state in networks[0].states:
                output.append(state.sentence.write(data.mappings))
                output.append("\n")

        return "".join(output)

    def evaluate(self, dataset, data, gold_path, args):
        step = self._summary_step()

        predicted_path = os.path.join(args.logdireval, "{}_{:07d}.mrp".format(dataset, step))
        assert not os.path.exists(predicted_path)

        with open(predicted_path, "w", encoding="utf-8") as predicted_file:
            predicted = self.predict(data, args)
            print(predicted, end="", file=predicted_file)

        if dataset in ["dev", "bdev", "lpss"]:
            mtool_evaluate.mrp_eval(
                gold_path, predicted_path, "{}_eval".format(dataset), rrhc=4, mces=100000, parallelize=8, logdir=args.logdireval, logstep=step, distributed=True)


#!/usr/bin/env python

from setuptools import setup, find_packages

name = 'd2d'

version = '0.1.0'

setup(
    name=name,
    version=version,
    author='inytar',
    packages=find_packages(),
    namespace_packages=name.split('.')[:-1],
    include_package_data=True,
    zip_safe=False,
    platforms='any',
    install_requires=[
        'morepath',
    ],
    entry_points=dict(
        console_scripts=[
            'run-app = d2d.__main__:run',
        ],
    ),
    classifiers=[
        'Intended Audience :: Developers',
        'Environment :: Web Environment',
        'Topic :: Internet :: WWW/HTTP :: WSGI',
        'Programming Language :: Python :: 3.5',
    ]
)


import subprocess
import sys
import time
if sys.platform == "darwin":
    from AppKit import NSWorkspace
    from Quartz import (
        CGWindowListCopyWindowInfo,
        kCGWindowListOptionOnScreenOnly,
        kCGWindowListOptionIncludingWindow,
        kCGNullWindowID
    )
import numpy as np
import cv2
import pyautogui

class BookwormEnv():
    def __init__(self):
        self.osType = sys.platform
        self.windowNumber = None

    def _launchProgram(self):
        if self.osType == "darwin":
            retcode = subprocess.call(['open', '-a', '/Applications/Bookworm.app'])
            print(retcode)
        if retcode != 0:
            raise Exception("Could not find Bookworm!")

    def _getWindow(self):
        return CGWindowListCopyWindowInfo(
            kCGWindowListOptionIncludingWindow,
            self.windowNumber
        )[0]

    def getCoordinates(self):
        coord = self._getWindow()['kCGWindowBounds']
        return (
            coord["X"],
            coord["Y"],
            coord["X"] + coord["Width"],
            coord["Y"] + coord["Height"]
        )

    def _findWindow(self, pid):
        if self.windowNumber != None:
            return
        print('Finding window number...')
        window = None
        while True:
            options = kCGWindowListOptionOnScreenOnly
            windowList = CGWindowListCopyWindowInfo(options, kCGNullWindowID)
            pidList = [app['kCGWindowOwnerPID'] for app in windowList]
            if pid in pidList:
                window = windowList[pidList.index(pid)]
                break
        self.windowNumber = window['kCGWindowNumber']

    def _getPid(self):
        active_app_name = None
        app_pid = None
        print('Getting pid...')
        # Wait until application is frontmost
        while active_app_name != "Bookworm":
            apps = NSWorkspace.sharedWorkspace().runningApplications()
            for app in apps:
                active_app_name = app.localizedName()
                app_pid = app.processIdentifier()
        return app_pid

    def _translateCoordinates(self, x, y):
        coords = self.getCoordinates()
        return (coords[0] + x, coords[1] + y)

    def moveMouse(self, x, y):
        coords = self._translateCoordinates(x, y)
        pyautogui.moveTo(coords[0], coords[1])

    def getWindowShot(self, region=None):
        if region == None:
            region = self.getCoordinates()
        coord = (region[:])
        print(coord)
        img = pyautogui.screenshot(region=region)
        img = np.array(img)
        return cv2.cvtColor(img, cv2.COLOR_RGBA2BGR)

    def getProgram(self):
        if sys.platform == "darwin":
            if self.windowNumber != None:
                return
            self._launchProgram()
            pid = self._getPid()
            self._findWindow(pid)
            geometry = self.getCoordinates()
            print(geometry)

    def _checkContinueBar(self):
        x, y = self._translateCoordinates(357, 171)
        img = self.getWindowShot(region=(x, y, 110, 22))
        filtered = cv2.inRange(img, np.array([0, 200, 200]), np.array([0, 255, 255]))
        _, counts = np.unique(filtered, return_counts=True)
        if len(counts) == 2:
            self.moveMouse(507, 337)
            pyautogui.click()

    def selectClassicMode(self):
        if self.windowNumber == None:
            self.handleLoadingScreen()
        self.moveMouse(504, 118)
        pyautogui.click()
        self._checkContinueBar()

    def handleLoadingScreen(self):
        self.getProgram()
        while True:
            img = self.getWindowShot()
            print(img.shape)
            filtered = cv2.inRange(img, np.array([0, 240, 240]), np.array([0, 255, 255]))
            _, counts = np.unique(filtered, return_counts=True)
            if len(counts) == 2 and counts[1] >= 947:
                self.moveMouse(335, 480)
                pyautogui.click()
                break


#encoding: utf-8
#!/usr/bin/env python3.5
#import smbus
from fnmatch import fnmatch
from operations.mqtt import MosquittoServer, Mosquitto
import config #configurações dos controladores rgb
import linecache
import time
import threading

import signal
import serial
import serial.tools.list_ports
global serial_data
import json
from omxplayer.player import OMXPlayer
from instance.config import Config
#from operations.telegram import Bot
from pathlib import Path
from time import sleep
import logging
import subprocess
import datetime
import os
import filecmp
import sys
import termios, tty
TOPICO = "capela/crato"
from check_internet import have_internet
volume = 1
root = os.path.dirname(os.path.abspath(__file__))
pattern = "*.py"
inicio_chuva_petalas = 218
acende_scenario = 365
inicio_pulsa_cruz = 340
fim_pulsa_cruz = 350
inicio_sequencia = inicio_chuva_petalas - 5
inicio_fumaca = inicio_chuva_petalas - 5 - 5
for path, subdirs, files in os.walk(root):
	for name in files:
		if fnmatch(name, pattern):
			if not ".debris" in path:
				try:
					#os.system("rm " + os.path.join(path, name+"_")) #remover todos
					if not(filecmp.cmp (os.path.join(path, name) ,os.path.join(path, "."+name+"_"))):
						os.system("sudo cp " + os.path.join(path, name)+" "+os.path.join(path, "." + name + "_"))

				except Exception as e:
					os.system("sudo cp " + os.path.join(path, name)+" "+os.path.join(path, "." + name + "_"))
					print("Novo arquivo criado: ",os.path.join(path, name))

def printException():
	exc_type, exc_obj, tb = sys.exc_info()
	f = tb.tb_frame
	lineno = tb.tb_lineno
	filename = f.f_code.co_filename
	linecache.checkcache(filename)
	line = linecache.getline(filename, lineno, f.f_globals)
	print ('EXCEPTION IN ({}, LINE {} "{}"): {}'.format(filename, lineno, line.strip(), exc_obj))

def buttonPressed(status):
	'''ATRIBUI A CERIMONIA AO BOTAO PRESSIONADO NO PAINEL'''
	global CONFIGURACAO
	global videoDuration

	if status == 1:
		CONFIGURACAO = config.CERIMONIA_1
		try:
			player.load(config.video_1,  pause=False)
			time.sleep(1)
			player.pause()
			player.set_aspect_mode('stretch')

			#player.set_video_pos(200, 0, 680, 480)
			print(player.video_pos())
		except Exception as ex:
			print(ex)

	if status == 2:
		CONFIGURACAO = config.CERIMONIA_2
		print(CONFIGURACAO)
		try:
			player.load(config.video_2,  pause=False)
			time.sleep(1)
			player.pause()
			player.set_aspect_mode('stretch')
		except Exception as ex:
			print(ex)

	if status == 3:
		CONFIGURACAO = config.CERIMONIA_3
		print(CONFIGURACAO)
		try:
			player.load(config.video_3,  pause=False)
			time.sleep(1)
			player.pause()
			player.set_aspect_mode('stretch')
		except Exception as ex:
			print(ex)

	if status == 4:
		CONFIGURACAO = config.CERIMONIA_4
		print(CONFIGURACAO)
		try:
			player.load(config.video_4,  pause=False)
			time.sleep(1)
			#player.pause()
			player.set_aspect_mode('stretch')
		except Exception as ex:
			print(ex)

	if status == 5:
		CONFIGURACAO = config.CERIMONIA_5
		print(CONFIGURACAO)
		try:
			player.load(config.video_5,  pause=False)
			time.sleep(1)
			player.pause()
			player.set_aspect_mode('stretch')
		except Exception as ex:
			print(ex)

	videoDuration = player.duration()
	#~ while player.playback_status() != "Paused":
		#~ player.pause()
		#~ time.sleep(.01)
	time.sleep(.1)
	#~ tempo = time.time()
	player.set_volume(volume)
	print("Volume " + str(player.volume()))
	print("O vídeo %d tem %.2f segundos de duração "%(status, player.duration()))

def process_serial(dados):
	if dados != "":
		global CONFIGURACAO
		print("Dados serial " +str(dados))
		global tempo
		global intervalo
		global lastIntervalo

		if dados == 'GUARDAMESA':
			print("player position", player.position())
			pass

		if dados == 'ABRINDO A PORTA':
			# ~ print("Player position abre", player.position())
			pass

		if dados == 'FECHA PORTA':
			'''ACENDE A CAPELA APÓS O TEMPO config.ACENDE_SCENARIO'''
			print("Player position inicio fechamento", player.position())
			pass

		if dados == 'PORTA FECHADA':
			'''ACENDE A CAPELA APÓS O TEMPO config.ACENDE_SCENARIO'''
			print("Player position final fechamento", player.position())
			pass

		if dados == 'TIME':
			print(tempo - time.time())
			tempo = time.time()
			pass
		if dados == 'APAGA_CABINE':
			#~ J2=Job('change', "SANCA_CABINE", ' -C gradual 90 "(0,0,0)"')
			#~ J2.start()

			pass
		if dados != 'ONLINE':
			pass

		if dados == 'button_counter = 2':
			serial_mega.write(b'GPIO,42,1\n')

		if dados == 'ESTEIRA DESLIGADA':
			serial_mega.write(b'LED_ESTEIRA,0\n')

		if dados == 'CERIMONIA_COMPLETA':
			# ~ try:
				# ~ telegram_bot.envia_telegram_single("334240998", "CERIMONIA INICIADA")
			# ~ except:
				# ~ print("ERRO TELEGRAM")
			global iniciada
			iniciada = True
			passedScenario1 = False
			passedScenario2 = False
			serial_mega.write(b'COMPLETA\n')
			serial_mega.write(b'<0,0,0,0,0,8,0,0,0,0>\n')
			time.sleep(.1)

			cmd = {"display_off":"display_off"}
			cmd = json.dumps(cmd)
			mqttLocal.mqttc.publish("capela/ihm", cmd, qos=2)
			serial_mega.write(b'B_C\n')
			# ~ for a in range(22):
				# ~ print("aguardando", 22 - a, "segundos", end = "\r")
				# ~ time.sleep(1)

			#time.sleep(25) #delay da cortina

			#
			player.set_position(1.)

			# ~ player.set_position(200)
			# ~ player.seek(280)
			player.play()
			#player.load(source, pause=False)

class Liga_manual (threading.Thread):
##  '''VERIFICAR SE A THREAD RODA SEM O PCF8574 INSTALADO'''
	def __init__(self):
		threading.Thread.__init__(self)
		# The shutdown_flag is a threading.Event object that
		# indicates whether the thread should be terminated.
		self.shutdown_flag = threading.Event()

	def run(self):
		serial_mega.write(b'DIMER,1,100\n')
		time.sleep(2)
		serial_mega.write(b'DIMER,2,100\n')
		time.sleep(2)
		serial_mega.write(b'DIMER,3,100\n')
		time.sleep(2)
		serial_mega.write(b'DIMER,4,100\n')
		time.sleep(2)
		serial_mega.write(b'DIMER,5,100\n')
		time.sleep(2)
		serial_mega.write(b'DIMER,6,100\n')
		time.sleep(2)
		serial_mega.write(b'DIMER,8,100\n')
		time.sleep(2)
		serial_mega.write(b'DIMER,9,100\n')
		time.sleep(2)
		serial_mega.write(b'DIMER,10,100\n')
		time.sleep(2)
		serial_mega.write(b'GPIO,42,0\n')
		time.sleep(10)
		serial_mega.write(b'T_DIM,50\n')

class readSerial(threading.Thread):
##  '''VERIFICAR SE A THREAD RODA SEM O PCF8574 INSTALADO'''
	def __init__(self):
		threading.Thread.__init__(self)
		# The shutdown_flag is a threading.Event object that
		# indicates whether the thread should be terminated.
		self.shutdown_flag = threading.Event()

	def run(self):
			#liga_todos()
			while not self.shutdown_flag.is_set():
				try:
					global serial_mega
					if serial_mega.isOpen():
						reading = serial_mega.readline().decode("utf-8").rstrip()
						process_serial(reading)

				except:
					pass

class ServiceExit(Exception):
	"""
	Custom exception which is used to trigger the clean exit
	of all running threads and the main program.
	"""
	pass

def service_shutdown(signum, frame):
	print('Caught signal %d' % signum)
	raise ServiceExit

def main():
	global iniciada
	global videoDuration
	global intervalo
	global lastIntervalo
	global CONFIGURACAO
	global passedScenario1
	global passedScenario2
	# Register the signal handlers
	signal.signal(signal.SIGTERM, service_shutdown)
	signal.signal(signal.SIGINT, service_shutdown)

	print('Starting main program')

	# Start the job threads
	buttonPressed(1)
	try:

		pass
		time_video = player.position()
		ler_serial = readSerial()
		ler_serial.start()

		looping = time.time()
		lastLooping = time.time()
		time_cortinas = time.time()

		while True:
			if mqttLocal.desligar == True:
				time.sleep(2)
				os.system("sudo poweroff")

			if player.playback_status() == "Playing" and looping < time.time():
				time_video = player.position()
				looping = time.time() + .5

				if time_video >= inicio_fumaca and time_video <= inicio_fumaca + 1:
					serial_mega.write(b'fum,12000\n')

				if time_video >= inicio_chuva_petalas and time_video <= inicio_chuva_petalas +1:
					print("chuva_petalas")
					serial_mega.write(b'CHUVAPETALAS,128\n')
					serial_mega.write(b'LED_ESTEIRA,255\n')
					#serial_mega.write(b'<0,0,0,0,0,100,0,0,0,100>\n')

				if time_video >= inicio_sequencia and time_video <= inicio_sequencia + 1:
					print(" sequencia")
					serial_mega.write(b'sequencia\n')
					time.sleep(1)
					serial_mega.write(b'LED_CABINE,255,255,0\n')

				if time_video >= inicio_pulsa_cruz and time_video <= inicio_pulsa_cruz + 1:
					print("pulsa cruz")
					serial_mega.write(b'EN_RGB_GERAL,1\n')# habilita o loop pulsante vermelho
					time.sleep(1)
					serial_mega.write(b'[100,0,0,100,0,0,5,0,0,5,0,0]\n')

				if time_video >= fim_pulsa_cruz and time_video <= fim_pulsa_cruz + 1:
					print(" desabilita pulsa cruz")
					serial_mega.write(b'EN_RGB_GERAL,0\n')
					time.sleep(1)
					serial_mega.write(b'(0,0,0,0,0,0,0,0,0)\n')

				if time_video >= acende_scenario and time_video <= acende_scenario + 1:
					print(" fim de cerimonia")
					serial_mega.write(b'S_C\n')
					serial_mega.write(b'T_DIM,100\n')
					# ~ serial_mega.write(b'<100,100,100,100,100,100,100,100,100,100>\n')#luz no fim da cerimonia
					time.sleep(1)
					serial_mega.write(b'(255,255,255,0,0,0,255,255,255)\n')
					iluminacao_manual = Liga_manual()
					iluminacao_manual.start()

				print(time_video, end = '\r')

			if player.can_control() and time_video > videoDuration - 2:
				while player.playback_status() != "Paused":
					print("pausando video")
					player.pause()

				#videoDuration = player.duration()
				time.sleep(.5)

				iniciada = False

	except ServiceExit:
		pass
		# Terminate the running threads.
		# Set the shutdown flag on each thread to trigger a clean shutdown of each thread.
##        j1.shutdown_flag.set()

		# Wait for the threads to close...
##        j1.join()

	print('Exiting main program')

try:
	os.system("sudo killall omxplayer.bin")
except:
	pass

logging.basicConfig(level=logging.INFO)

source = "/usr/share/APP/videos/video_1.mp4"
try:
	player_log = logging.getLogger("Player 1")
	player = OMXPlayer(source, args=[  '-o', 'local', '--no-osd','-b'], dbus_name='org.mpris.MediaPlayer2.omxplayer1')
	player.set_aspect_mode('stretch')
except:# NameError:# dbus.exceptions.DBusException:
	print("Reiniciando App")
	os.execv(sys.executable, ['python3'] + sys.argv)
else:
	print("Exception")

#~ player.set_aspect_mode('fill')

#time.sleep(20)
while player.playback_status() != "Paused":
	player.pause()
tempo = time.time()
print("video pausado")
player.set_volume(volume)
#player.mute()
print("Player mute")
print("Volume " + str(player.volume()))
print("O vídeo tem %.2f segundos de duração "%player.duration())

serial_mega = None
serial_nano = None

def config_serial():
	global serial_mega
	global serial_nano
	print("function")
	lista_porta=[]
	portas_abertas=[]
	ports = serial.tools.list_ports.comports()

	for port, desc, hwid in sorted(ports):
	        if "USB" in port:
	            lista_porta.append(port)

	for portas in lista_porta:
		'''
		Abre todas as portas
		'''
		try:
			portas_abertas.append(serial.Serial(portas, 9600, timeout=1))
		except:
			print(portas, "Acesso negado")

	try:

		while serial_mega == None or serial_nano == None:
			for channel in portas_abertas:
				if channel != serial_mega and channel != serial_nano:
					channel.write(b'MODEL\n')
					time.sleep(.1)
					string = channel.readline().decode("utf-8", "ignore")
					if serial_nano == None:
						if "NANO" in string:
							#print("Porta NANO", lista_porta[portas_abertas.index(channel)])
							serial_nano = channel
					if serial_mega == None:
						if "MEGA" in string:
							#print("Porta MEGA", lista_porta[portas_abertas.index(channel)])
							serial_mega = channel
		print("Porta serial MEGA = {}\nPorta serial NANO = {}".format(lista_porta[portas_abertas.index(serial_mega)], lista_porta[portas_abertas.index(serial_nano)]))
		mqttServer.serial_nano = serial_nano
		mqttServer.serial_mega = serial_mega
		mqttLocal.serial_nano = serial_nano
		mqttLocal.serial_mega = serial_mega
	except Exception as e:
		print(e)
		print("Porta Serial não aberta")

thread = threading.Thread(target=config_serial)
thread.start()
mqttLocal = Mosquitto(TOPICO)
#mqttLocal.player = player
mqttServer = MosquittoServer(TOPICO)
#mqttServer.player = player

if __name__ == '__main__':
	main()

# ~ Traceback (most recent call last):
  # ~ File "video_popen_4_comando_rasp.py", line 430, in <module>
    # ~ main()
  # ~ File "video_popen_4_comando_rasp.py", line 353, in main
    # ~ if player.can_control() and time_video > videoDuration - 2:
  # ~ File "</usr/local/lib/python3.7/dist-packages/decorator.py:decorator-gen-22>", line 2, in can_control
# ~ Dados serial B_C
  # ~ File "/usr/local/lib/python3.7/dist-packages/omxplayer/player.py", line 48, in wrapped
    # ~ return fn(self, *args, **kwargs)
  # ~ File "</usr/local/lib/python3.7/dist-packages/decorator.py:decorator-gen-21>", line 2, in can_control
  # ~ File "/usr/local/lib/python3.7/dist-packages/omxplayer/player.py", line 85, in wrapped
    # ~ return from_dbus_type(fn(self, *args, **kwargs))
  # ~ File "/usr/local/lib/python3.7/dist-packages/omxplayer/player.py", line 342, in can_control
    # ~ return self._player_interface_property('CanControl')
  # ~ File "/usr/local/lib/python3.7/dist-packages/omxplayer/player.py", line 829, in _player_interface_property
    # ~ return self._interface_property(self._player_interface.dbus_interface, prop, val)
  # ~ File "/usr/local/lib/python3.7/dist-packages/omxplayer/player.py", line 823, in _interface_property
    # ~ return self._properties_interface.Get(interface, prop)
  # ~ File "/usr/local/lib/python3.7/dist-packages/dbus/proxies.py", line 147, in __call__
    # ~ **keywords)
  # ~ File "/usr/local/lib/python3.7/dist-packages/dbus/connection.py", line 653, in call_blocking
    # ~ message, timeout)
# ~ dbus.exceptions.DBusException: org.freedesktop.DBus.Error.NoReply: Message recipient disconnected from message bus without replying
# ~ INFO:omxplayer.player:OMXPlayer process is dead, all DBus calls from here will fail

#Player position inicio fechamento 291.791414
#Player position final fechamento 332.850824