<commit_before>package discovery

import (
	"context"
	"fmt"
	"time"

	"github.com/golang/protobuf/jsonpb"
	"github.com/improbable-eng/kedge/pkg/k8s"
	"github.com/improbable-eng/kedge/pkg/sharedflags"
	pb_config "github.com/improbable-eng/kedge/protogen/kedge/config"
	"github.com/jpillora/backoff"
	"github.com/mwitkow/go-flagz/protobuf"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const (
	selectorKeySuffix           = "kedge-exposed"
	hostMatcherAnnotationSuffix = "host-matcher"
)

var (
	// TODO(bplotka): Consider moving to regex with .namespace .name variables.
	flagExternalDomainSuffix = sharedflags.Set.String("discovery_external_domain_suffix", "", "Required suffix "+
		"that will be added to service name to constructs external domain for director route")
	flagAnnotationLabelPrefix = sharedflags.Set.String("discovery_label_annotation_prefix", "kedge.com/",
		"Expected annotation/label prefix for all kedge annotations and kedge-exposed label")
	streamRetryBackoff = &backoff.Backoff{
		Min:    50 * time.Millisecond,
		Jitter: true,
		Factor: 2,
		Max:    2 * time.Second,
	}
)

// RoutingDiscovery allows to get fresh director and backendpool configuration filled with autogenerated routings based on service annotations.
// See more details about it in server/README.md#Running_with_Dynaming_Routing_Discovery
type RoutingDiscovery struct {
	logger                logrus.FieldLogger
	serviceClient         serviceClient
	baseBackendpool       *pb_config.BackendPoolConfig
	baseDirector          *pb_config.DirectorConfig
	labelSelectorKey      string
	externalDomainSuffix  string
	labelAnnotationPrefix string
}

// NewFromFlags creates new RoutingDiscovery flow flags.
func NewFromFlags(logger logrus.FieldLogger, baseDirector *pb_config.DirectorConfig, baseBackendpool *pb_config.BackendPoolConfig) (*RoutingDiscovery, error) {
	if *flagExternalDomainSuffix == "" {
		return nil, errors.Errorf("required flag 'discovery_external_domain_suffix' is not specified.")
	}

	apiClient, err := k8s.NewFromFlags()
	if err != nil {
		return nil, err
	}
	return NewWithClient(logger, baseDirector, baseBackendpool, &client{k8sClient: apiClient}), nil
}

// NewWithClient returns a new Kubernetes RoutingDiscovery using given k8s.APIClient configured to be used against kube-apiserver.
func NewWithClient(logger logrus.FieldLogger, baseDirector *pb_config.DirectorConfig, baseBackendpool *pb_config.BackendPoolConfig, serviceClient serviceClient) *RoutingDiscovery {
	return &RoutingDiscovery{
		logger:                logger,
		baseBackendpool:       baseBackendpool,
		baseDirector:          baseDirector,
		serviceClient:         serviceClient,
		labelSelectorKey:      fmt.Sprintf("%s%s", *flagAnnotationLabelPrefix, selectorKeySuffix),
		externalDomainSuffix:  *flagExternalDomainSuffix,
		labelAnnotationPrefix: *flagAnnotationLabelPrefix,
	}
}

// DiscoverOnce returns director & backendpool configs filled with mix of persistent routes & backends given in base configs and dynamically discovered ones.
func (d *RoutingDiscovery) DiscoverOnce(ctx context.Context, timeToWait time.Duration) (*pb_config.DirectorConfig, *pb_config.BackendPoolConfig, error) {
	ctx, cancel := context.WithTimeout(ctx, timeToWait) // Let's give 4 seconds to gather all changes.
	defer cancel()

	watchResultCh := make(chan watchResult)
	defer close(watchResultCh)

	err := startWatchingServicesChanges(ctx, d.labelSelectorKey, d.serviceClient, watchResultCh)
	if err != nil {
		return nil, nil, errors.Wrapf(err, "Failed to start watching services by %s selector stream", d.labelSelectorKey)
	}

	updater := newUpdater(
		d.baseDirector,
		d.baseBackendpool,
		d.externalDomainSuffix,
		d.labelAnnotationPrefix,
	)

	var resultDirectorConfig *pb_config.DirectorConfig
	var resultBackendPool *pb_config.BackendPoolConfig
	for {
		var event event
		select {
		case <-ctx.Done():
			if resultBackendPool == nil {
				resultBackendPool = d.baseBackendpool
			}
			if resultDirectorConfig == nil {
				resultDirectorConfig = d.baseDirector
			}
			// Time is up, let's return what we have so far.
			return resultDirectorConfig, resultBackendPool, nil
		case r := <-watchResultCh:
			if r.err != nil {
				return nil, nil, errors.Wrap(r.err, "error on reading event stream")
			}
			event = *r.ep
		}

		resultDirectorConfig, resultBackendPool, err = updater.onEvent(event)
		if err != nil {
			return nil, nil, errors.Wrapf(err, "error on updating routing on event %v", event)
		}
	}
}

// DiscoverAndSetFlags constantly watches service list endpoint for changes and sets director and backendpool
// flags to change routing configuration. Having it set by flagz is giving us possibility to see current values in
// debug/flagz page and perform proper apply in different place (where we are parsing flags).
func (d *RoutingDiscovery) DiscoverAndSetFlags(
	ctx context.Context,
	directorFlagz *protoflagz.DynProto3Value,
	backendpoolFlagz *protoflagz.DynProto3Value,
) error {
	watchResultCh := make(chan watchResult)
	defer close(watchResultCh)

	for ctx.Err() == nil {
		streamCtx, streamCancel := context.WithCancel(ctx)
		// All errors from watching service or watchResultCh are irrecoverable.
		err := startWatchingServicesChanges(streamCtx, d.labelSelectorKey, d.serviceClient, watchResultCh)
		if err != nil {
			streamCancel()
			d.logger.WithError(err).Errorf("discovery: Failed to start watching services by %s selector stream", d.labelSelectorKey)

			time.Sleep(streamRetryBackoff.Duration())
			continue
		}
		streamRetryBackoff.Reset()

		updater := newUpdater(
			d.baseDirector,
			d.baseBackendpool,
			d.externalDomainSuffix,
			d.labelAnnotationPrefix,
		)

		aggregateDelay := 300 * time.Millisecond
		maxAggrUpdates := 50
		for ctx.Err() == nil {
			// Watch loop.

			director, backendPool, err := d.nextWithAggregate(ctx, updater, watchResultCh, aggregateDelay, maxAggrUpdates)
			if err != nil {
				d.logger.WithError(err).Error("discovery: Error on watching event stream. Retrying stream... ")
				break
			}

			d.logger.Infof("Setting director and backendpool configs with %d (httpScheme) and %d (grpcScheme) backends.",
				len(backendPool.Http.Backends), len(backendPool.Grpc.Backends))

			// Only first update should aggregate that much, to not erase and create backend unnecessarily on watch endpoint EOF (which is every 15 minutes).
			aggregateDelay = 100 * time.Millisecond
			maxAggrUpdates = 10

			err = d.setFlags(director, directorFlagz, backendPool, backendpoolFlagz)
			if err != nil {
				streamCancel()
				// This is critical error, since retry will not help. Not much we can do, abort dynamic discovery.
				return errors.Wrap(err, "discovery: Critical error on updating flags from director and backendpool configs. Discover will not work!")
			}
		}
		streamCancel()
	}

	return nil
}

// nextWithAggregate is waiting for next watcher update. It does not immediately return valid full director and
// backendPool configs that includes this single change. Instead it aggregates all changes that are gathered within next "aggregateDelay"
// after last update to aggregate these.
// Parameter called "maxAggrUpdates" is to have safeguard in case of constant changes. In theory time waiting for an aggregated
// changes in our configs is maxAggrUpdates * aggregateDelay if we have constant changes (unlikely).
// NOTE that error from next() method are irrecoverable.
func (d *RoutingDiscovery) nextWithAggregate(ctx context.Context, updater *updater, watchResultCh <-chan watchResult, aggregateDelay time.Duration, maxAggrUpdates int) (*pb_config.DirectorConfig, *pb_config.BackendPoolConfig, error) {
	var firstUpdateAt time.Time

	var director *pb_config.DirectorConfig
	var backendPool *pb_config.BackendPoolConfig
	for {
		var event event
		select {
		case <-ctx.Done():
			return nil, nil, ctx.Err()
		case r := <-watchResultCh:
			if r.err != nil {
				return nil, nil, r.err
			}

			if firstUpdateAt.IsZero() {
				firstUpdateAt = time.Now()
			}
			event = *r.ep
		case <-time.After(aggregateDelay):
			if firstUpdateAt.IsZero() {
				// Still waiting for any update.
				continue
			}
			return director, backendPool, nil
		}

		// Safe guard in case of constant changes.
		maxAggrUpdates--
		if maxAggrUpdates <= 0 {
			return director, backendPool, nil
		}

		var err error
		director, backendPool, err = updater.onEvent(event)
		if err != nil {
			// There is possibility we missed event, so retry stream.
			return nil, nil, errors.Wrapf(err, "internal error on updating routing on event %v.", event)
		}
	}
}

func (d *RoutingDiscovery) setFlags(
	directorConfig *pb_config.DirectorConfig,
	directorFlagz *protoflagz.DynProto3Value,
	backenpoolConfig *pb_config.BackendPoolConfig,
	backendpoolFlagz *protoflagz.DynProto3Value,
) error {
	director, err := (&jsonpb.Marshaler{}).MarshalToString(directorConfig)
	if err != nil {
		return errors.Wrap(err, "failed to marshal directorConfig")
	}
	err = directorFlagz.Set(string(director))
	if err != nil {
		return errors.Wrap(err, "failed to set directorConfig into protoflagz flag")
	}

	backendpool, err := (&jsonpb.Marshaler{}).MarshalToString(backenpoolConfig)
	if err != nil {
		return errors.Wrap(err, "failed to marshal backenpoolConfig")
	}
	err = backendpoolFlagz.Set(backendpool)
	if err != nil {
		return errors.Wrap(err, "failed to set backenpoolConfig into protoflagz flag")
	}
	return nil
}
<commit_msg>Change error log in retryable loop to warn log.<commit_after>package discovery

import (
	"context"
	"fmt"
	"time"

	"github.com/golang/protobuf/jsonpb"
	"github.com/improbable-eng/kedge/pkg/k8s"
	"github.com/improbable-eng/kedge/pkg/sharedflags"
	pb_config "github.com/improbable-eng/kedge/protogen/kedge/config"
	"github.com/jpillora/backoff"
	"github.com/mwitkow/go-flagz/protobuf"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const (
	selectorKeySuffix           = "kedge-exposed"
	hostMatcherAnnotationSuffix = "host-matcher"
)

var (
	// TODO(bplotka): Consider moving to regex with .namespace .name variables.
	flagExternalDomainSuffix = sharedflags.Set.String("discovery_external_domain_suffix", "", "Required suffix "+
		"that will be added to service name to constructs external domain for director route")
	flagAnnotationLabelPrefix = sharedflags.Set.String("discovery_label_annotation_prefix", "kedge.com/",
		"Expected annotation/label prefix for all kedge annotations and kedge-exposed label")
	streamRetryBackoff = &backoff.Backoff{
		Min:    50 * time.Millisecond,
		Jitter: true,
		Factor: 2,
		Max:    2 * time.Second,
	}
)

// RoutingDiscovery allows to get fresh director and backendpool configuration filled with autogenerated routings based on service annotations.
// See more details about it in server/README.md#Running_with_Dynaming_Routing_Discovery
type RoutingDiscovery struct {
	logger                logrus.FieldLogger
	serviceClient         serviceClient
	baseBackendpool       *pb_config.BackendPoolConfig
	baseDirector          *pb_config.DirectorConfig
	labelSelectorKey      string
	externalDomainSuffix  string
	labelAnnotationPrefix string
}

// NewFromFlags creates new RoutingDiscovery flow flags.
func NewFromFlags(logger logrus.FieldLogger, baseDirector *pb_config.DirectorConfig, baseBackendpool *pb_config.BackendPoolConfig) (*RoutingDiscovery, error) {
	if *flagExternalDomainSuffix == "" {
		return nil, errors.Errorf("required flag 'discovery_external_domain_suffix' is not specified.")
	}

	apiClient, err := k8s.NewFromFlags()
	if err != nil {
		return nil, err
	}
	return NewWithClient(logger, baseDirector, baseBackendpool, &client{k8sClient: apiClient}), nil
}

// NewWithClient returns a new Kubernetes RoutingDiscovery using given k8s.APIClient configured to be used against kube-apiserver.
func NewWithClient(logger logrus.FieldLogger, baseDirector *pb_config.DirectorConfig, baseBackendpool *pb_config.BackendPoolConfig, serviceClient serviceClient) *RoutingDiscovery {
	return &RoutingDiscovery{
		logger:                logger,
		baseBackendpool:       baseBackendpool,
		baseDirector:          baseDirector,
		serviceClient:         serviceClient,
		labelSelectorKey:      fmt.Sprintf("%s%s", *flagAnnotationLabelPrefix, selectorKeySuffix),
		externalDomainSuffix:  *flagExternalDomainSuffix,
		labelAnnotationPrefix: *flagAnnotationLabelPrefix,
	}
}

// DiscoverOnce returns director & backendpool configs filled with mix of persistent routes & backends given in base configs and dynamically discovered ones.
func (d *RoutingDiscovery) DiscoverOnce(ctx context.Context, timeToWait time.Duration) (*pb_config.DirectorConfig, *pb_config.BackendPoolConfig, error) {
	ctx, cancel := context.WithTimeout(ctx, timeToWait) // Let's give 4 seconds to gather all changes.
	defer cancel()

	watchResultCh := make(chan watchResult)
	defer close(watchResultCh)

	err := startWatchingServicesChanges(ctx, d.labelSelectorKey, d.serviceClient, watchResultCh)
	if err != nil {
		return nil, nil, errors.Wrapf(err, "Failed to start watching services by %s selector stream", d.labelSelectorKey)
	}

	updater := newUpdater(
		d.baseDirector,
		d.baseBackendpool,
		d.externalDomainSuffix,
		d.labelAnnotationPrefix,
	)

	var resultDirectorConfig *pb_config.DirectorConfig
	var resultBackendPool *pb_config.BackendPoolConfig
	for {
		var event event
		select {
		case <-ctx.Done():
			if resultBackendPool == nil {
				resultBackendPool = d.baseBackendpool
			}
			if resultDirectorConfig == nil {
				resultDirectorConfig = d.baseDirector
			}
			// Time is up, let's return what we have so far.
			return resultDirectorConfig, resultBackendPool, nil
		case r := <-watchResultCh:
			if r.err != nil {
				return nil, nil, errors.Wrap(r.err, "error on reading event stream")
			}
			event = *r.ep
		}

		resultDirectorConfig, resultBackendPool, err = updater.onEvent(event)
		if err != nil {
			return nil, nil, errors.Wrapf(err, "error on updating routing on event %v", event)
		}
	}
}

// DiscoverAndSetFlags constantly watches service list endpoint for changes and sets director and backendpool
// flags to change routing configuration. Having it set by flagz is giving us possibility to see current values in
// debug/flagz page and perform proper apply in different place (where we are parsing flags).
func (d *RoutingDiscovery) DiscoverAndSetFlags(
	ctx context.Context,
	directorFlagz *protoflagz.DynProto3Value,
	backendpoolFlagz *protoflagz.DynProto3Value,
) error {
	watchResultCh := make(chan watchResult)
	defer close(watchResultCh)

	for ctx.Err() == nil {
		streamCtx, streamCancel := context.WithCancel(ctx)
		// All errors from watching service or watchResultCh are irrecoverable.
		err := startWatchingServicesChanges(streamCtx, d.labelSelectorKey, d.serviceClient, watchResultCh)
		if err != nil {
			streamCancel()
			d.logger.WithError(err).Warnf("discovery: Failed to start watching services by %s selector stream", d.labelSelectorKey)

			time.Sleep(streamRetryBackoff.Duration())
			continue
		}
		streamRetryBackoff.Reset()

		updater := newUpdater(
			d.baseDirector,
			d.baseBackendpool,
			d.externalDomainSuffix,
			d.labelAnnotationPrefix,
		)

		aggregateDelay := 300 * time.Millisecond
		maxAggrUpdates := 50
		for ctx.Err() == nil {
			// Watch loop.

			director, backendPool, err := d.nextWithAggregate(ctx, updater, watchResultCh, aggregateDelay, maxAggrUpdates)
			if err != nil {
				d.logger.WithError(err).Error("discovery: Error on watching event stream. Retrying stream... ")
				break
			}

			d.logger.Infof("Setting director and backendpool configs with %d (httpScheme) and %d (grpcScheme) backends.",
				len(backendPool.Http.Backends), len(backendPool.Grpc.Backends))

			// Only first update should aggregate that much, to not erase and create backend unnecessarily on watch endpoint EOF (which is every 15 minutes).
			aggregateDelay = 100 * time.Millisecond
			maxAggrUpdates = 10

			err = d.setFlags(director, directorFlagz, backendPool, backendpoolFlagz)
			if err != nil {
				streamCancel()
				// This is critical error, since retry will not help. Not much we can do, abort dynamic discovery.
				return errors.Wrap(err, "discovery: Critical error on updating flags from director and backendpool configs. Discover will not work!")
			}
		}
		streamCancel()
	}

	return nil
}

// nextWithAggregate is waiting for next watcher update. It does not immediately return valid full director and
// backendPool configs that includes this single change. Instead it aggregates all changes that are gathered within next "aggregateDelay"
// after last update to aggregate these.
// Parameter called "maxAggrUpdates" is to have safeguard in case of constant changes. In theory time waiting for an aggregated
// changes in our configs is maxAggrUpdates * aggregateDelay if we have constant changes (unlikely).
// NOTE that error from next() method are irrecoverable.
func (d *RoutingDiscovery) nextWithAggregate(ctx context.Context, updater *updater, watchResultCh <-chan watchResult, aggregateDelay time.Duration, maxAggrUpdates int) (*pb_config.DirectorConfig, *pb_config.BackendPoolConfig, error) {
	var firstUpdateAt time.Time

	var director *pb_config.DirectorConfig
	var backendPool *pb_config.BackendPoolConfig
	for {
		var event event
		select {
		case <-ctx.Done():
			return nil, nil, ctx.Err()
		case r := <-watchResultCh:
			if r.err != nil {
				return nil, nil, r.err
			}

			if firstUpdateAt.IsZero() {
				firstUpdateAt = time.Now()
			}
			event = *r.ep
		case <-time.After(aggregateDelay):
			if firstUpdateAt.IsZero() {
				// Still waiting for any update.
				continue
			}
			return director, backendPool, nil
		}

		// Safe guard in case of constant changes.
		maxAggrUpdates--
		if maxAggrUpdates <= 0 {
			return director, backendPool, nil
		}

		var err error
		director, backendPool, err = updater.onEvent(event)
		if err != nil {
			// There is possibility we missed event, so retry stream.
			return nil, nil, errors.Wrapf(err, "internal error on updating routing on event %v.", event)
		}
	}
}

func (d *RoutingDiscovery) setFlags(
	directorConfig *pb_config.DirectorConfig,
	directorFlagz *protoflagz.DynProto3Value,
	backenpoolConfig *pb_config.BackendPoolConfig,
	backendpoolFlagz *protoflagz.DynProto3Value,
) error {
	director, err := (&jsonpb.Marshaler{}).MarshalToString(directorConfig)
	if err != nil {
		return errors.Wrap(err, "failed to marshal directorConfig")
	}
	err = directorFlagz.Set(string(director))
	if err != nil {
		return errors.Wrap(err, "failed to set directorConfig into protoflagz flag")
	}

	backendpool, err := (&jsonpb.Marshaler{}).MarshalToString(backenpoolConfig)
	if err != nil {
		return errors.Wrap(err, "failed to marshal backenpoolConfig")
	}
	err = backendpoolFlagz.Set(backendpool)
	if err != nil {
		return errors.Wrap(err, "failed to set backenpoolConfig into protoflagz flag")
	}
	return nil
}
<|endoftext|>


<commit_before>// +build linux

/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kvm

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"strings"
	"text/template"

	"github.com/docker/machine/libmachine/log"
	libvirt "github.com/libvirt/libvirt-go"
	"github.com/pkg/errors"
)

// Replace with hardcoded range with CIDR
// https://play.golang.org/p/m8TNTtygK0
const networkTmpl = `
<network>
  <name>{{.PrivateNetwork}}</name>
  <dns enable='no'/>
  <ip address='192.168.39.1' netmask='255.255.255.0'>
    <dhcp>
      <range start='192.168.39.2' end='192.168.39.254'/>
    </dhcp>
  </ip>
</network>
`

// setupNetwork ensures that the network with `name` is started (active)
// and has the autostart feature set.
func setupNetwork(conn *libvirt.Connect, name string) error {
	n, err := conn.LookupNetworkByName(name)
	if err != nil {
		return errors.Wrapf(err, "checking network %s", name)
	}

	// always ensure autostart is set on the network
	autostart, err := n.GetAutostart()
	if err != nil {
		return errors.Wrapf(err, "checking network %s autostart", name)
	}
	if !autostart {
		if err := n.SetAutostart(true); err != nil {
			return errors.Wrapf(err, "setting autostart for network %s", name)
		}
	}

	// always ensure the network is started (active)
	active, err := n.IsActive()
	if err != nil {
		return errors.Wrapf(err, "checking network status for %s", name)
	}
	if !active {
		if err := n.Create(); err != nil {
			return errors.Wrapf(err, "starting network %s", name)
		}
	}

	return nil
}

// ensureNetwork is called on start of the VM
func (d *Driver) ensureNetwork() error {
	conn, err := getConnection()
	if err != nil {
		return errors.Wrap(err, "getting libvirt connection")
	}
	defer conn.Close()

	// network: default

	// Start the default network
	// It is assumed that the libvirt/kvm installation has already created this network
	log.Infof("Ensuring network %s is active", defaultNetworkName)
	if err := setupNetwork(conn, defaultNetworkName); err != nil {
		return err
	}

	// network: private

	// Start the private network
	log.Infof("Ensuring network %s is active", d.PrivateNetwork)
	if err := setupNetwork(conn, d.PrivateNetwork); err != nil {
		return err
	}

	return nil
}

// createNetwork is called during creation of the VM only (and not on start)
func (d *Driver) createNetwork() error {
	conn, err := getConnection()
	if err != nil {
		return errors.Wrap(err, "getting libvirt connection")
	}
	defer conn.Close()

	// network: default
	// It is assumed that the libvirt/kvm installation has already created this network
	if _, err := conn.LookupNetworkByName(d.Network); err != nil {
		return errors.Wrapf(err, "network %s doesn't exist", d.Network)
	}

	// network: private
	// Only create the private network if it does not already exist
	if _, err := conn.LookupNetworkByName(d.PrivateNetwork); err != nil {
		// create the XML for the private network from our networkTmpl
		tmpl := template.Must(template.New("network").Parse(networkTmpl))
		var networkXML bytes.Buffer
		if err := tmpl.Execute(&networkXML, d); err != nil {
			return errors.Wrap(err, "executing network template")
		}

		// define the network using our template
		network, err := conn.NetworkDefineXML(networkXML.String())
		if err != nil {
			return errors.Wrapf(err, "defining network from xml: %s", networkXML.String())
		}

		// and finally create it
		if err := network.Create(); err != nil {
			return errors.Wrapf(err, "creating network %s", d.PrivateNetwork)
		}
	}

	return nil
}

func (d *Driver) deleteNetwork() error {
	type source struct {
		//XMLName xml.Name `xml:"source"`
		Network string `xml:"network,attr"`
	}
	type iface struct {
		//XMLName xml.Name `xml:"interface"`
		Source source `xml:"source"`
	}
	type result struct {
		//XMLName xml.Name `xml:"domain"`
		Name       string  `xml:"name"`
		Interfaces []iface `xml:"devices>interface"`
	}

	conn, err := getConnection()
	if err != nil {
		return errors.Wrap(err, "getting libvirt connection")
	}
	defer conn.Close()

	// network: default
	// It is assumed that the OS manages this network

	// network: private
	log.Debugf("Checking if network %s exists...", d.PrivateNetwork)
	network, err := conn.LookupNetworkByName(d.PrivateNetwork)
	if err != nil {
		// TODO: decide if we really wanna throw an error?
		return errors.Wrap(err, "network %s does not exist")
	}
	log.Debugf("Network %s exists", d.PrivateNetwork)

	// iterate over every (also turned off) domains, and check if it
	// is using the private network. Do *not* delete the network if
	// that is the case
	log.Debug("Trying to list all domains...")
	doms, err := conn.ListAllDomains(0)
	if err != nil {
		return errors.Wrap(err, "list all domains")
	}
	log.Debugf("Listed all domains: total of %d domains", len(doms))

	// fail if there are 0 domains
	if len(doms) == 0 {
		log.Warn("list of domains is 0 length")
	}

	for _, dom := range doms {
		// get the name of the domain we iterate over
		log.Debug("Trying to get name of domain...")
		name, err := dom.GetName()
		if err != nil {
			return errors.Wrap(err, "failed to get name of a domain")
		}
		log.Debugf("Got domain name: %s", name)

		// skip the domain if it is our own machine
		if name == d.MachineName {
			log.Debug("Skipping domain as it is us...")
			continue
		}

		// unfortunately, there is no better way to retrieve a list of all defined interfaces
		// in domains than getting it from the defined XML of all domains
		// NOTE: conn.ListAllInterfaces does not help in this case
		log.Debugf("Getting XML for domain %s...", name)
		xmlString, err := dom.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
		if err != nil {
			return errors.Wrapf(err, "failed to get XML of domain '%s'", name)
		}
		log.Debugf("Got XML for domain %s", name)

		v := result{}
		err = xml.Unmarshal([]byte(xmlString), &v)
		if err != nil {
			return errors.Wrapf(err, "failed to unmarshal XML of domain '%s", name)
		}
		log.Debugf("Unmarshaled XML for domain %s: %#v", name, v)

		// iterate over the found interfaces
		for _, i := range v.Interfaces {
			if i.Source.Network == d.PrivateNetwork {
				log.Debugf("domain %s DOES use network %s, aborting...", name, d.PrivateNetwork)
				return fmt.Errorf("network still in use at least by domain '%s',", name)
			}
			log.Debugf("domain %s does not use network %s", name, d.PrivateNetwork)
		}
	}

	// when we reach this point, it means it is safe to delete the network
	log.Debugf("Trying to destroy network %s...", d.PrivateNetwork)
	err = network.Destroy()
	if err != nil {
		return errors.Wrap(err, "network destroy")
	}
	log.Debugf("Trying to undefine network %s...", d.PrivateNetwork)
	err = network.Undefine()
	if err != nil {
		return errors.Wrap(err, "network undefine")
	}

	return nil
}

func (d *Driver) lookupIP() (string, error) {
	conn, err := getConnection()
	if err != nil {
		return "", errors.Wrap(err, "getting connection and domain")
	}

	defer conn.Close()

	libVersion, err := conn.GetLibVersion()
	if err != nil {
		return "", errors.Wrap(err, "getting libversion")
	}

	// Earlier versions of libvirt use a lease file instead of a status file
	if libVersion < 1002006 {
		return d.lookupIPFromLeasesFile()
	}

	// TODO: for everything > 1002006, there is direct support in the libvirt-go for handling this
	return d.lookupIPFromStatusFile(conn)
}

func (d *Driver) lookupIPFromStatusFile(conn *libvirt.Connect) (string, error) {
	network, err := conn.LookupNetworkByName(d.PrivateNetwork)
	if err != nil {
		return "", errors.Wrap(err, "looking up network by name")
	}

	bridge, err := network.GetBridgeName()
	if err != nil {
		log.Warnf("Failed to get network bridge: %v", err)
		return "", err
	}
	statusFile := fmt.Sprintf("/var/lib/libvirt/dnsmasq/%s.status", bridge)
	statuses, err := ioutil.ReadFile(statusFile)
	if err != nil {
		return "", errors.Wrap(err, "reading status file")
	}
	type StatusEntry struct {
		IPAddress  string `json:"ip-address"`
		MacAddress string `json:"mac-address"`
	}

	var statusEntries []StatusEntry

	// If the status file is empty, parsing will fail, ignore this error.
	_ = json.Unmarshal(statuses, &statusEntries)

	ipAddress := ""
	for _, status := range statusEntries {
		if status.MacAddress == d.PrivateMAC {
			ipAddress = status.IPAddress
		}
	}

	return ipAddress, nil
}

func (d *Driver) lookupIPFromLeasesFile() (string, error) {
	leasesFile := fmt.Sprintf("/var/lib/libvirt/dnsmasq/%s.leases", d.PrivateNetwork)
	leases, err := ioutil.ReadFile(leasesFile)
	if err != nil {
		return "", errors.Wrap(err, "reading leases file")
	}
	ipAddress := ""
	for _, lease := range strings.Split(string(leases), "\n") {
		if len(lease) == 0 {
			continue
		}
		// format for lease entry
		// ExpiryTime MAC IP Hostname ExtendedMAC
		entry := strings.Split(lease, " ")
		if len(entry) != 5 {
			return "", fmt.Errorf("malformed leases entry: %s", entry)
		}
		if entry[1] == d.PrivateMAC {
			ipAddress = entry[2]
		}
	}
	return ipAddress, nil
}
<commit_msg>Fix kvm ensureNetwork<commit_after>// +build linux

/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kvm

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"strings"
	"text/template"

	"github.com/docker/machine/libmachine/log"
	libvirt "github.com/libvirt/libvirt-go"
	"github.com/pkg/errors"
)

// Replace with hardcoded range with CIDR
// https://play.golang.org/p/m8TNTtygK0
const networkTmpl = `
<network>
  <name>{{.PrivateNetwork}}</name>
  <dns enable='no'/>
  <ip address='192.168.39.1' netmask='255.255.255.0'>
    <dhcp>
      <range start='192.168.39.2' end='192.168.39.254'/>
    </dhcp>
  </ip>
</network>
`

// setupNetwork ensures that the network with `name` is started (active)
// and has the autostart feature set.
func setupNetwork(conn *libvirt.Connect, name string) error {
	n, err := conn.LookupNetworkByName(name)
	if err != nil {
		return errors.Wrapf(err, "checking network %s", name)
	}

	// always ensure autostart is set on the network
	autostart, err := n.GetAutostart()
	if err != nil {
		return errors.Wrapf(err, "checking network %s autostart", name)
	}
	if !autostart {
		if err := n.SetAutostart(true); err != nil {
			return errors.Wrapf(err, "setting autostart for network %s", name)
		}
	}

	// always ensure the network is started (active)
	active, err := n.IsActive()
	if err != nil {
		return errors.Wrapf(err, "checking network status for %s", name)
	}
	if !active {
		if err := n.Create(); err != nil {
			return errors.Wrapf(err, "starting network %s", name)
		}
	}

	return nil
}

// ensureNetwork is called on start of the VM
func (d *Driver) ensureNetwork() error {
	conn, err := getConnection()
	if err != nil {
		return errors.Wrap(err, "getting libvirt connection")
	}
	defer conn.Close()

	// network: default

	// Start the default network
	// It is assumed that the libvirt/kvm installation has already created this network
	log.Infof("Ensuring network %s is active", d.Network)
	if err := setupNetwork(conn, d.Network); err != nil {
		return err
	}

	// network: private

	// Start the private network
	log.Infof("Ensuring network %s is active", d.PrivateNetwork)
	if err := setupNetwork(conn, d.PrivateNetwork); err != nil {
		return err
	}

	return nil
}

// createNetwork is called during creation of the VM only (and not on start)
func (d *Driver) createNetwork() error {
	conn, err := getConnection()
	if err != nil {
		return errors.Wrap(err, "getting libvirt connection")
	}
	defer conn.Close()

	// network: default
	// It is assumed that the libvirt/kvm installation has already created this network
	if _, err := conn.LookupNetworkByName(d.Network); err != nil {
		return errors.Wrapf(err, "network %s doesn't exist", d.Network)
	}

	// network: private
	// Only create the private network if it does not already exist
	if _, err := conn.LookupNetworkByName(d.PrivateNetwork); err != nil {
		// create the XML for the private network from our networkTmpl
		tmpl := template.Must(template.New("network").Parse(networkTmpl))
		var networkXML bytes.Buffer
		if err := tmpl.Execute(&networkXML, d); err != nil {
			return errors.Wrap(err, "executing network template")
		}

		// define the network using our template
		network, err := conn.NetworkDefineXML(networkXML.String())
		if err != nil {
			return errors.Wrapf(err, "defining network from xml: %s", networkXML.String())
		}

		// and finally create it
		if err := network.Create(); err != nil {
			return errors.Wrapf(err, "creating network %s", d.PrivateNetwork)
		}
	}

	return nil
}

func (d *Driver) deleteNetwork() error {
	type source struct {
		//XMLName xml.Name `xml:"source"`
		Network string `xml:"network,attr"`
	}
	type iface struct {
		//XMLName xml.Name `xml:"interface"`
		Source source `xml:"source"`
	}
	type result struct {
		//XMLName xml.Name `xml:"domain"`
		Name       string  `xml:"name"`
		Interfaces []iface `xml:"devices>interface"`
	}

	conn, err := getConnection()
	if err != nil {
		return errors.Wrap(err, "getting libvirt connection")
	}
	defer conn.Close()

	// network: default
	// It is assumed that the OS manages this network

	// network: private
	log.Debugf("Checking if network %s exists...", d.PrivateNetwork)
	network, err := conn.LookupNetworkByName(d.PrivateNetwork)
	if err != nil {
		// TODO: decide if we really wanna throw an error?
		return errors.Wrap(err, "network %s does not exist")
	}
	log.Debugf("Network %s exists", d.PrivateNetwork)

	// iterate over every (also turned off) domains, and check if it
	// is using the private network. Do *not* delete the network if
	// that is the case
	log.Debug("Trying to list all domains...")
	doms, err := conn.ListAllDomains(0)
	if err != nil {
		return errors.Wrap(err, "list all domains")
	}
	log.Debugf("Listed all domains: total of %d domains", len(doms))

	// fail if there are 0 domains
	if len(doms) == 0 {
		log.Warn("list of domains is 0 length")
	}

	for _, dom := range doms {
		// get the name of the domain we iterate over
		log.Debug("Trying to get name of domain...")
		name, err := dom.GetName()
		if err != nil {
			return errors.Wrap(err, "failed to get name of a domain")
		}
		log.Debugf("Got domain name: %s", name)

		// skip the domain if it is our own machine
		if name == d.MachineName {
			log.Debug("Skipping domain as it is us...")
			continue
		}

		// unfortunately, there is no better way to retrieve a list of all defined interfaces
		// in domains than getting it from the defined XML of all domains
		// NOTE: conn.ListAllInterfaces does not help in this case
		log.Debugf("Getting XML for domain %s...", name)
		xmlString, err := dom.GetXMLDesc(libvirt.DOMAIN_XML_INACTIVE)
		if err != nil {
			return errors.Wrapf(err, "failed to get XML of domain '%s'", name)
		}
		log.Debugf("Got XML for domain %s", name)

		v := result{}
		err = xml.Unmarshal([]byte(xmlString), &v)
		if err != nil {
			return errors.Wrapf(err, "failed to unmarshal XML of domain '%s", name)
		}
		log.Debugf("Unmarshaled XML for domain %s: %#v", name, v)

		// iterate over the found interfaces
		for _, i := range v.Interfaces {
			if i.Source.Network == d.PrivateNetwork {
				log.Debugf("domain %s DOES use network %s, aborting...", name, d.PrivateNetwork)
				return fmt.Errorf("network still in use at least by domain '%s',", name)
			}
			log.Debugf("domain %s does not use network %s", name, d.PrivateNetwork)
		}
	}

	// when we reach this point, it means it is safe to delete the network
	log.Debugf("Trying to destroy network %s...", d.PrivateNetwork)
	err = network.Destroy()
	if err != nil {
		return errors.Wrap(err, "network destroy")
	}
	log.Debugf("Trying to undefine network %s...", d.PrivateNetwork)
	err = network.Undefine()
	if err != nil {
		return errors.Wrap(err, "network undefine")
	}

	return nil
}

func (d *Driver) lookupIP() (string, error) {
	conn, err := getConnection()
	if err != nil {
		return "", errors.Wrap(err, "getting connection and domain")
	}

	defer conn.Close()

	libVersion, err := conn.GetLibVersion()
	if err != nil {
		return "", errors.Wrap(err, "getting libversion")
	}

	// Earlier versions of libvirt use a lease file instead of a status file
	if libVersion < 1002006 {
		return d.lookupIPFromLeasesFile()
	}

	// TODO: for everything > 1002006, there is direct support in the libvirt-go for handling this
	return d.lookupIPFromStatusFile(conn)
}

func (d *Driver) lookupIPFromStatusFile(conn *libvirt.Connect) (string, error) {
	network, err := conn.LookupNetworkByName(d.PrivateNetwork)
	if err != nil {
		return "", errors.Wrap(err, "looking up network by name")
	}

	bridge, err := network.GetBridgeName()
	if err != nil {
		log.Warnf("Failed to get network bridge: %v", err)
		return "", err
	}
	statusFile := fmt.Sprintf("/var/lib/libvirt/dnsmasq/%s.status", bridge)
	statuses, err := ioutil.ReadFile(statusFile)
	if err != nil {
		return "", errors.Wrap(err, "reading status file")
	}
	type StatusEntry struct {
		IPAddress  string `json:"ip-address"`
		MacAddress string `json:"mac-address"`
	}

	var statusEntries []StatusEntry

	// If the status file is empty, parsing will fail, ignore this error.
	_ = json.Unmarshal(statuses, &statusEntries)

	ipAddress := ""
	for _, status := range statusEntries {
		if status.MacAddress == d.PrivateMAC {
			ipAddress = status.IPAddress
		}
	}

	return ipAddress, nil
}

func (d *Driver) lookupIPFromLeasesFile() (string, error) {
	leasesFile := fmt.Sprintf("/var/lib/libvirt/dnsmasq/%s.leases", d.PrivateNetwork)
	leases, err := ioutil.ReadFile(leasesFile)
	if err != nil {
		return "", errors.Wrap(err, "reading leases file")
	}
	ipAddress := ""
	for _, lease := range strings.Split(string(leases), "\n") {
		if len(lease) == 0 {
			continue
		}
		// format for lease entry
		// ExpiryTime MAC IP Hostname ExtendedMAC
		entry := strings.Split(lease, " ")
		if len(entry) != 5 {
			return "", fmt.Errorf("malformed leases entry: %s", entry)
		}
		if entry[1] == d.PrivateMAC {
			ipAddress = entry[2]
		}
	}
	return ipAddress, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package flexvolume

const (
	VolIdKey  = "volumeID"
	FsTypeKey = "fsType"

	HyperFlexvolumeDataFile = "hyper-flexvolume.json"

	// Cinder flexvolume
	CinderConfigKey  = "cinderConfig"
	CinderConfigFile = "/etc/kubernetes/cinder.conf"

	// GCE PD flexvolume
	ZoneKey      = "zone"
	ProjectKey   = "project"
	DivcePathKey = "devicePath"

	// Build-in fsType key of flexvolume
	SystemFsTypeKey = "kubernetes.io/fsType"
)

// CinderVolumeOptsData is the struct of json file
type CinderVolumeOptsData struct {
	AccessMode   string   `json:"access_mode"`
	AuthUserName string   `json:"auth_username"`
	AuthEnabled  bool     `json:"auth_enabled"`
	ClusterName  string   `json:"cluster_name"`
	Encrypted    bool     `json:"encrypted"`
	FsType       string   `json:"fsType"`
	Hosts        []string `json:"hosts"`
	Keyring      string   `json:"keyring"`
	Name         string   `json:"name"`
	Ports        []string `json:"ports"`
	SecretUUID   string   `json:"secret_uuid"`
	SecretType   string   `json:"secret_type"`
	VolumeID     string   `json:"volumeID"`
	VolumeType   string   `json:"volume_type"`
}

// GCEPDOptsData is the struct of json file
type GCEPDOptsData struct {
	DevicePath   string `json:"devicePath"`
	SystemFsType string `json:"kubernetes.io/fsType"`
}
<commit_msg>flexvolume: make volume opts data match its usage<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package flexvolume

const (
	VolIdKey  = "volumeID"
	FsTypeKey = "fsType"

	HyperFlexvolumeDataFile = "hyper-flexvolume.json"

	// Cinder flexvolume
	CinderConfigKey  = "cinderConfig"
	CinderConfigFile = "/etc/kubernetes/cinder.conf"

	// GCE PD flexvolume
	ZoneKey      = "zone"
	ProjectKey   = "project"
	DivcePathKey = "devicePath"

	// Build-in fsType key of flexvolume
	SystemFsTypeKey = "kubernetes.io/fsType"
)

// CinderVolumeOptsData is the struct of json file
type CinderVolumeOptsData struct {
	// Needed to reconstruct new cinder clients
	ConfigKey string `json:"cinderConfig"`
	VolumeID  string `json:"volumeID"`

	// rbd volume details
	VolumeType string   `json:"volume_type"`
	Name       string   `json:"name"`
	FsType     string   `json:"fsType"`
	Hosts      []string `json:"hosts"`
	Ports      []string `json:"ports"`
}

// GCEPDOptsData is the struct of json file
type GCEPDOptsData struct {
	// Needed for unmount
	VolumeID string `json:"volumeID"`
	Zone     string `json:"zone"`
	Project  string `json:"project"`

	// gce pd volume details
	DevicePath   string `json:"devicePath"`
	SystemFsType string `json:"kubernetes.io/fsType"`
}
<|endoftext|>


<commit_before>package outputs

import "log"

type Output interface {
	Init(map[string]interface{}, *log.Logger) error
	Write([]byte)
	Close() error
}
type Initializer func() Output

var Outputs = map[string]Initializer{}

func Register(name string, initFn Initializer) {
	Outputs[name] = initFn
}
<commit_msg>add Metrics() to output interface{}<commit_after>package outputs

import (
	"log"

	"github.com/prometheus/client_golang/prometheus"
)

type Output interface {
	Init(map[string]interface{}, *log.Logger) error
	Write([]byte)
	Close() error
	Metrics() []prometheus.Collector
}
type Initializer func() Output

var Outputs = map[string]Initializer{}

func Register(name string, initFn Initializer) {
	Outputs[name] = initFn
}
<|endoftext|>


<commit_before>package treerack

import (
	"os"
	"testing"
	"time"
)

func TestBoot(t *testing.T) {
	b, err := initBoot(bootSyntaxDefs)
	if err != nil {
		t.Error(err)
		return
	}

	f, err := os.Open("mml.parser")
	if err != nil {
		t.Error(err)
		return
	}

	defer f.Close()

	start := time.Now()
	_, err = b.Parse(f)
	t.Log("duration:", time.Now().Sub(start))

	if err != ErrNotImplemented {
		t.Error(err)
	}

	// if err != nil {
	// 	t.Error(err)
	// 	return
	// }

	// s0 := NewSyntax()
	// if err := define(s0, n0); err != nil {
	// 	t.Error(err)
	// 	return
	// }

	// _, err = f.Seek(0, 0)
	// if err != nil {
	// 	t.Error(err)
	// 	return
	// }

	// err = s0.Init()
	// if err != nil {
	// 	t.Error(err)
	// 	return
	// }

	// n1, err := s0.Parse(f)
	// if err != nil {
	// 	t.Error(err)
	// 	return
	// }

	// checkNode(t, n1, n0)
	// if t.Failed() {
	// 	return
	// }

	// s1 := NewSyntax()
	// if err := define(s1, n1); err != nil {
	// 	t.Error(err)
	// 	return
	// }

	// _, err = f.Seek(0, 0)
	// if err != nil {
	// 	t.Error(err)
	// 	return
	// }

	// n2, err := s1.Parse(f)
	// if err != nil {
	// 	t.Error(err)
	// 	return
	// }

	// checkNode(t, n2, n1)
}
<commit_msg>measure multiple parse average<commit_after>package treerack

import (
	"os"
	"testing"
	"time"
)

func TestBoot(t *testing.T) {
	b, err := initBoot(bootSyntaxDefs)
	if err != nil {
		t.Error(err)
		return
	}

	f, err := os.Open("mml.parser")
	if err != nil {
		t.Error(err)
		return
	}

	defer f.Close()

	var d time.Duration
	const n = 120
	for i := 0; i < n; i++ {
		if _, err := f.Seek(0, 0); err != nil {
			t.Error(err)
			return
		}

		start := time.Now()
		_, err = b.Parse(f)
		d += time.Now().Sub(start)

		if err != ErrNotImplemented {
			t.Error(err)
			return
		}
	}

	t.Log("duration:", d/n)

	// if err != nil {
	// 	t.Error(err)
	// 	return
	// }

	// s0 := NewSyntax()
	// if err := define(s0, n0); err != nil {
	// 	t.Error(err)
	// 	return
	// }

	// _, err = f.Seek(0, 0)
	// if err != nil {
	// 	t.Error(err)
	// 	return
	// }

	// err = s0.Init()
	// if err != nil {
	// 	t.Error(err)
	// 	return
	// }

	// n1, err := s0.Parse(f)
	// if err != nil {
	// 	t.Error(err)
	// 	return
	// }

	// checkNode(t, n1, n0)
	// if t.Failed() {
	// 	return
	// }

	// s1 := NewSyntax()
	// if err := define(s1, n1); err != nil {
	// 	t.Error(err)
	// 	return
	// }

	// _, err = f.Seek(0, 0)
	// if err != nil {
	// 	t.Error(err)
	// 	return
	// }

	// n2, err := s1.Parse(f)
	// if err != nil {
	// 	t.Error(err)
	// 	return
	// }

	// checkNode(t, n2, n1)
}
<|endoftext|>


<commit_before>package bootstrap

import (
	"encoding/json"
	"fmt"
	"log"
	"sync"
	"time"

	"github.com/tinzenite/channel"
	"github.com/tinzenite/shared"
)

/*
Success is the callback that will be called once the bootstrap is complete.
*/
type Success func()

/*
Bootstrap is a temporary peer object that allows to bootstrap into an existing
Tinzenite network. NOTE: bootstrapping is only capable for now to trusted peers.
*/
type Bootstrap struct {
	path       string           // root path
	cInterface *chaninterface   // internal hidden struct for channel callbacks
	channel    *channel.Channel // tox communication channel
	peer       *shared.Peer     // self peer
	bootstrap  map[string]bool  // stores address of peers we need to bootstrap
	onDone     Success          // callback for when done
	wg         sync.WaitGroup   // stuff for background thread
	stop       chan bool        // stuff for background thread
}

/*
Start begins a bootstrap process to the given address.
*/
func (b *Bootstrap) Start(address string) error {
	// send own peer
	msg, err := json.Marshal(b.peer)
	if err != nil {
		return err
	}
	// send request
	return b.channel.RequestConnection(address, string(msg))
}

/*
Address returns the full address of this peer.
*/
func (b *Bootstrap) Address() (string, error) {
	return b.channel.ConnectionAddress()
}

/*
Store writes a bootstrapped .TINZENITEDIR to disk. Call this if you want
persistant bootstrapping (and why wouldn't you?).
*/
func (b *Bootstrap) Store() error {
	err := shared.MakeDotTinzenite(b.path)
	if err != nil {
		return err
	}
	// write self peer
	err = b.peer.StoreTo(b.path + "/" + shared.STOREPEERDIR)
	if err != nil {
		return err
	}
	// store local peer info with toxdata
	toxData, err := b.channel.ToxData()
	if err != nil {
		return err
	}
	toxPeerDump := &shared.ToxPeerDump{
		SelfPeer: b.peer,
		ToxData:  toxData}
	err = toxPeerDump.StoreTo(b.path + "/" + shared.STORETOXDUMPDIR)
	if err != nil {
		return err
	}
	return nil
}

/*
PrintStatus returns a formatted string of the peer status.
*/
func (b *Bootstrap) PrintStatus() string {
	var out string
	out += "Online:\n"
	addresses, err := b.channel.FriendAddresses()
	if err != nil {
		out += "channel.FriendAddresses failed!"
	} else {
		var count int
		for _, address := range addresses {
			online, err := b.channel.IsAddressOnline(address)
			var insert string
			if err != nil {
				insert = "ERROR"
			} else {
				insert = fmt.Sprintf("%v", online)
			}
			out += address[:16] + " :: " + insert + "\n"
			count++
		}
		out += "Total friends: " + fmt.Sprintf("%d", count)
	}
	return out
}

/*
IsTrusted can be used to read whether this bootstrap is creating an encrypted or
a trusted peer.
*/
func (b *Bootstrap) IsTrusted() bool {
	return b.peer.Trusted
}

/*
Close cleanly closes everything underlying.
*/
func (b *Bootstrap) Close() {
	// send stop signal
	b.stop <- true
	// wait for it to close
	b.wg.Wait()
	// finally close channel
	b.channel.Close()
}

/*
Run is the background thread that keeps checking if it can bootstrap.
*/
func (b *Bootstrap) run() {
	defer func() { log.Println("Bootstrap:", "Background process stopped.") }()
	online := false
	var interval time.Duration
	for {
		// this ensures 2 different tick spans depending on whether someone is online or not
		if online {
			interval = tickSpanOnline
		} else {
			interval = tickSpanNone
		}
		select {
		case <-b.stop:
			b.wg.Done()
			return
		case <-time.Tick(interval):
			online = false
			addresses, err := b.channel.OnlineAddresses()
			if err != nil {
				log.Println("Check:", err)
				break
			}
			if len(addresses) == 0 {
				log.Println("None available yet.")
				break
			}
			if len(addresses) > 1 {
				// since we'll always only try connecting to one, warn specifically!
				log.Println("WARNING: Multiple online! Will try connecting to ", addresses[0][:16], " only.")
			}
			online = true
			// yo, we want to bootstrap!
			rm := shared.CreateRequestMessage(shared.OtModel, shared.IDMODEL)
			b.channel.Send(addresses[0], rm.JSON())
		} // select
	} // for
}
<commit_msg>small stuff<commit_after>package bootstrap

import (
	"encoding/json"
	"fmt"
	"log"
	"sync"
	"time"

	"github.com/tinzenite/channel"
	"github.com/tinzenite/shared"
)

/*
Success is the callback that will be called once the bootstrap is complete.
*/
type Success func()

/*
Bootstrap is a temporary peer object that allows to bootstrap into an existing
Tinzenite network. NOTE: bootstrapping is only capable for now to trusted peers.
*/
type Bootstrap struct {
	path       string           // root path
	cInterface *chaninterface   // internal hidden struct for channel callbacks
	channel    *channel.Channel // tox communication channel
	peer       *shared.Peer     // self peer
	bootstrap  map[string]bool  // stores address of peers we need to bootstrap
	onDone     Success          // callback for when done
	wg         sync.WaitGroup   // stuff for background thread
	stop       chan bool        // stuff for background thread
}

/*
Start begins a bootstrap process to the given address.
*/
func (b *Bootstrap) Start(address string) error {
	// send own peer
	msg, err := json.Marshal(b.peer)
	if err != nil {
		return err
	}
	// send request
	return b.channel.RequestConnection(address, string(msg))
}

/*
Address returns the full address of this peer.
*/
func (b *Bootstrap) Address() (string, error) {
	return b.channel.ConnectionAddress()
}

/*
Store writes a bootstrapped .TINZENITEDIR to disk. Call this if you want
persistant bootstrapping (and why wouldn't you?).
*/
func (b *Bootstrap) Store() error {
	err := shared.MakeDotTinzenite(b.path)
	if err != nil {
		return err
	}
	// write self peer
	err = b.peer.StoreTo(b.path + "/" + shared.STOREPEERDIR)
	if err != nil {
		return err
	}
	// store local peer info with toxdata
	toxData, err := b.channel.ToxData()
	if err != nil {
		return err
	}
	toxPeerDump := &shared.ToxPeerDump{
		SelfPeer: b.peer,
		ToxData:  toxData}
	err = toxPeerDump.StoreTo(b.path + "/" + shared.STORETOXDUMPDIR)
	if err != nil {
		return err
	}
	return nil
}

/*
PrintStatus returns a formatted string of the peer status.
*/
func (b *Bootstrap) PrintStatus() string {
	var out string
	out += "Online:\n"
	addresses, err := b.channel.FriendAddresses()
	if err != nil {
		out += "channel.FriendAddresses failed!"
	} else {
		var count int
		for _, address := range addresses {
			online, err := b.channel.IsAddressOnline(address)
			var insert string
			if err != nil {
				insert = "ERROR"
			} else {
				insert = fmt.Sprintf("%v", online)
			}
			out += address[:16] + " :: " + insert + "\n"
			count++
		}
		out += "Total friends: " + fmt.Sprintf("%d", count)
	}
	return out
}

/*
IsTrusted can be used to read whether this bootstrap is creating an encrypted or
a trusted peer.
*/
func (b *Bootstrap) IsTrusted() bool {
	return b.peer.Trusted
}

/*
Close cleanly closes everything underlying.
*/
func (b *Bootstrap) Close() {
	// send stop signal
	b.stop <- true
	// wait for it to close
	b.wg.Wait()
	// finally close channel
	b.channel.Close()
}

/*
Run is the background thread that keeps checking if it can bootstrap.
*/
func (b *Bootstrap) run() {
	defer func() { log.Println("Bootstrap:", "Background process stopped.") }()
	online := false
	var interval time.Duration
	for {
		// this ensures 2 different tick spans depending on whether someone is online or not
		if online {
			interval = tickSpanOnline
		} else {
			interval = tickSpanNone
		}
		select {
		case <-b.stop:
			b.wg.Done()
			return
		case <-time.Tick(interval):
			online = false
			addresses, err := b.channel.OnlineAddresses()
			if err != nil {
				log.Println("Check:", err)
				break
			}
			if len(addresses) == 0 {
				log.Println("None available yet.")
				break
			}
			if len(addresses) > 1 {
				// since we'll always only try connecting to one, warn specifically!
				log.Println("WARNING: Multiple online! Will try connecting to ", addresses[0][:8], " only.")
			}
			online = true
			// TODO for encrypted: do nothing except prepare structure?
			// yo, we want to bootstrap!
			rm := shared.CreateRequestMessage(shared.OtModel, shared.IDMODEL)
			b.channel.Send(addresses[0], rm.JSON())
		} // select
	} // for
}
<|endoftext|>


<commit_before>package model

type Expression interface {
	Interprete(map[string]Expression) bool
	//Interprete(Map<String,Expression> variables) bool
}

type VariableExpression struct {
	Value int
}

func Variable(value int) *VariableExpression {
	return &VariableExpression{Value: value}
}

func (s *VariableExpression) Interprete(variables map[string]Expression) int {
	return s.Value
}

func (s *VariableExpression) Interprete(variables map[string]Expression) bool {
	return true
}

type EqualsExpression struct {
	leftOperand  Expression
	rightOperand Expression
}

func Equals(leftOperand Expression, rightOperand Expression) *EqualsExpression {
	return &EqualsExpression{leftOperand: leftOperand, rightOperand: rightOperand}
}

func (s *EqualsExpression) Interprete(variables map[string]Expression) bool {
	return s.leftOperand.Interprete(variables) == s.rightOperand.Interprete(variables)
}

type OrExpression struct {
	leftOperand  Expression
	rightOperand Expression
}

func Or(leftOperand Expression, rightOperand Expression) *OrExpression {
	return &OrExpression{leftOperand: leftOperand, rightOperand: rightOperand}
}

func (s *OrExpression) Interprete(variables map[string]Expression) bool {
	return s.leftOperand.Interprete(variables) || s.rightOperand.Interprete(variables)
}

type AndExpression struct {
	leftOperand  Expression
	rightOperand Expression
}

func And(leftOperand Expression, rightOperand Expression) *AndExpression {
	return &AndExpression{leftOperand: leftOperand, rightOperand: rightOperand}
}

func (s *AndExpression) Interprete(variables map[string]Expression) bool {
	return s.leftOperand.Interprete(variables) && s.rightOperand.Interprete(variables)
}
<commit_msg>Autoformat test<commit_after>package model

/* we deviate from the Constructor convention (New...) by intention, to enable concise
   expressions like
	a := And(Equals(Variable(7), Variable(6)), Equals(Variable(5), Variable(5)))
*/
type Expression interface {

	    Interprete(map[string]Expression) bool
	//Interprete(Map<String,Expression> variables) bool
}

type VariableExpression struct {
	Value int
}

func Variable(value int) *VariableExpression {
	return &VariableExpression{Value: value}
}

func (s *VariableExpression) Interprete(variables map[string]Expression) int {
	return s.Value
}

func (s *VariableExpression) Interprete(variables map[string]Expression) bool {
	return true
}

type EqualsExpression struct {
	leftOperand  Expression
	rightOperand Expression
}

func Equals(leftOperand Expression, rightOperand Expression) *EqualsExpression {
	return &EqualsExpression{leftOperand: leftOperand, rightOperand: rightOperand}
}

func (s *EqualsExpression) Interprete(variables map[string]Expression) bool {
	return s.leftOperand.Interprete(variables) == s.rightOperand.Interprete(variables)
}

type OrExpression struct {
	leftOperand  Expression
	rightOperand Expression
}

func Or(leftOperand Expression, rightOperand Expression) *OrExpression {
	return &OrExpression{leftOperand: leftOperand, rightOperand: rightOperand}
}

func (s *OrExpression) Interprete(variables map[string]Expression) bool {
	return s.leftOperand.Interprete(variables) || s.rightOperand.Interprete(variables)
}

type AndExpression struct {
	leftOperand  Expression
	rightOperand Expression
}

func And(leftOperand Expression, rightOperand Expression) *AndExpression {
	return &AndExpression{leftOperand: leftOperand, rightOperand: rightOperand}
}

func (s *AndExpression) Interprete(variables map[string]Expression) bool {
	return s.leftOperand.Interprete(variables) && s.rightOperand.Interprete(variables)
}
<|endoftext|>


<commit_before>package models

import (
	"database/sql"
	"errors"

	"github.com/pborman/uuid"
)

/*Preference entity data*/
type Preference struct {
	ID     uuid.UUID       `json:"id"`
	UserID uuid.UUID       `json:"userId" binding:"required"`
	Email  PreferenceState `json:"email" binding:"required"`
}

/*NewPreference contstructs and returns a new preference entity with it's id*/
func NewPreference(userID uuid.UUID) *Preference {
	return &Preference{
		ID:     uuid.NewUUID(),
		UserID: userID,
		Email:  SUBSCRIBED,
	}
}

/*PreferencesFromSQL returns a preference splice from sql rows*/
func PreferencesFromSQL(rows *sql.Rows) ([]*Preference, error) {
	preferences := make([]*Preference, 0)

	for rows.Next() {
		p := &Preference{}
		var email string
		rows.Scan(&p.ID, &p.UserID, &email)

		var ok bool
		p.Email, ok = toPreferenceState(email)
		if !ok {
			return nil, errors.New("invalid email preference")
		}
		preferences = append(preferences, p)
	}

	return preferences, nil
}

func toPreferenceState(s string) (PreferenceState, bool) {
	switch s {
	case SUBSCRIBED:
		return SUBSCRIBED, true
	case UNSUBSCRIBED:
		return UNSUBSCRIBED, true
	case MINIMAL:
		return MINIMAL, true
	default:
		return "", false
	}
}

/*PreferenceState wraps valid preference state strings*/
type PreferenceState string

/*valid preference states*/
const (
	SUBSCRIBED   = "SUBSCRIBED"
	UNSUBSCRIBED = "UNSUBSCRIBED"
	MINIMAL      = "MINIMAL"
)
<commit_msg>Set EMAIL to not required<commit_after>package models

import (
	"database/sql"
	"errors"

	"github.com/pborman/uuid"
)

/*Preference entity data*/
type Preference struct {
	ID     uuid.UUID       `json:"id"`
	UserID uuid.UUID       `json:"userId" binding:"required"`
	Email  PreferenceState `json:"email"`
}

/*NewPreference contstructs and returns a new preference entity with it's id*/
func NewPreference(userID uuid.UUID) *Preference {
	return &Preference{
		ID:     uuid.NewUUID(),
		UserID: userID,
		Email:  SUBSCRIBED,
	}
}

/*PreferencesFromSQL returns a preference splice from sql rows*/
func PreferencesFromSQL(rows *sql.Rows) ([]*Preference, error) {
	preferences := make([]*Preference, 0)

	for rows.Next() {
		p := &Preference{}
		var email string
		rows.Scan(&p.ID, &p.UserID, &email)

		var ok bool
		p.Email, ok = toPreferenceState(email)
		if !ok {
			return nil, errors.New("invalid email preference")
		}
		preferences = append(preferences, p)
	}

	return preferences, nil
}

func toPreferenceState(s string) (PreferenceState, bool) {
	switch s {
	case SUBSCRIBED:
		return SUBSCRIBED, true
	case UNSUBSCRIBED:
		return UNSUBSCRIBED, true
	case MINIMAL:
		return MINIMAL, true
	default:
		return "", false
	}
}

/*PreferenceState wraps valid preference state strings*/
type PreferenceState string

/*valid preference states*/
const (
	SUBSCRIBED   = "SUBSCRIBED"
	UNSUBSCRIBED = "UNSUBSCRIBED"
	MINIMAL      = "MINIMAL"
)
<|endoftext|>


<commit_before>// Package bq includes all code related to BigQuery.
//
// NB: NOTES ON MEMORY USE AND HTTP SIZE
// The bigquery library uses JSON encoding of data, which appears to be the only
// option at this time.  Furthermore, it uses intermediate data representations,
// eventually creating a map[string]Value (unless you pass that in to begin with).
// In general, when we start pumping large volumes of data, both the map and the
// JSON will cause some memory pressure, and likely pretty severe limits on the size
// of the insert we can send, likely on the order of a couple MB of actual row footprint
// in BQ.
//
// Passing in slice of structs makes memory pressure a bit worse, but probably isn't
// worth worrying about.

package bq

import (
	"log"
	"os"
	"sync"
	"time"

	"cloud.google.com/go/bigquery"
	"golang.org/x/net/context"

	"github.com/m-lab/etl/etl"
	"github.com/m-lab/etl/metrics"
)

// TODO(dev) Use a more thoughtful setting for buffer size.
// TODO(P3) Include the project name in the parameters.
// For now, 10K per row times 100 results is 1MB, which is an order of
// magnitude below our 10MB max, so 100 might not be such a bad
// default.
func NewInserter(dataset string, dt etl.DataType, partition time.Time) (etl.Inserter, error) {
	suffix := ""
	table := etl.DataTypeToTable[dt]
	if time.Since(partition) < 30*24*time.Hour {
		// If within past 30 days, we can stream directly to partition.
		suffix = "$" + partition.Format("20060102")
	} else {
		// Otherwise, we use a templated table, and must merge it later.
		suffix = "_" + partition.Format("20060102")
	}

	return NewBQInserter(
		etl.InserterParams{Dataset: dataset, Table: table, Suffix: suffix,
			Timeout: 15 * time.Minute, BufferSize: 10}, nil)
}

// TODO - improve the naming between here and NewInserter.
// Pass in nil uploader for normal use, custom uploader for custom behavior
func NewBQInserter(params etl.InserterParams, uploader etl.Uploader) (etl.Inserter, error) {
	if uploader == nil {
		client := MustGetClient(params.Timeout)
		table := params.Table
		if params.Suffix[0] == '$' {
			// Suffix starting with $ is just a partition spec.
			table += params.Suffix
		}
		u := client.Dataset(params.Dataset).Table(table).Uploader()
		if params.Suffix[0] == '_' {
			// Suffix starting with _ is a template suffix.
			u.TableTemplateSuffix = params.Suffix
		}
		// This avoids problems when a single row of the insert has invalid
		// data.  We then have to carefully parse the returned error object.
		u.SkipInvalidRows = true
		uploader = u
	}
	in := BQInserter{params: params, uploader: uploader, timeout: params.Timeout}
	in.rows = make([]interface{}, 0, in.params.BufferSize)
	return &in, nil
}

//===============================================================================
var (
	clientOnce sync.Once // This avoids a race on setting bqClient.
	bqClient   *bigquery.Client
)

// Returns the Singleton bigquery client for this process.
// TODO - is there any advantage to using more than one client?
func MustGetClient(timeout time.Duration) *bigquery.Client {
	// We do this here, instead of in init(), because we only want to do it
	// when we actually want to access the bigquery backend.
	clientOnce.Do(func() {
		ctx, _ := context.WithTimeout(context.Background(), timeout)
		project, ok := os.LookupEnv("BIGQUERY_PROJECT")
		if !ok {
			project = os.Getenv("GCLOUD_PROJECT")
		}
		log.Printf("Using project: %s\n", project)
		// Heavyweight!
		var err error
		bqClient, err = bigquery.NewClient(ctx, project)
		if err != nil {
			panic(err.Error())
		}
	})
	return bqClient
}

//===============================================================================

// Generic implementation of bq.ValueSaver, based on map.  This avoids extra
// conversion steps in the bigquery library (except for the JSON conversion).
// IMPLEMENTS: bigquery.ValueSaver
type MapSaver struct {
	Values map[string]bigquery.Value
}

func (s *MapSaver) Save() (row map[string]bigquery.Value, insertID string, err error) {
	return s.Values, "", nil
}

//----------------------------------------------------------------------------

type BQInserter struct {
	etl.Inserter
	params   etl.InserterParams
	uploader etl.Uploader // May be a BQ Uploader, or a test Uploader
	timeout  time.Duration
	rows     []interface{}
	inserted int // Number of rows successfully inserted.
	badRows  int // Number of row failures, including rows in full failures.
	failures int // Number of complete insert failures.
}

// Caller should check error, and take appropriate action before calling again.
func (in *BQInserter) InsertRow(data interface{}) error {
	return in.InsertRows([]interface{}{data})
}

// Caller should check error, and take appropriate action before calling again.
// TODO - should this return a specific error to indicate that a flush is needed
// instead of flushing internally?  The "handle errors in the middle" would
// be easier, though other complications would ensue.
func (in *BQInserter) InsertRows(data []interface{}) error {
	metrics.WorkerState.WithLabelValues("insert").Inc()
	defer metrics.WorkerState.WithLabelValues("insert").Dec()

	for len(data)+len(in.rows) >= in.params.BufferSize {
		// space >= len(data)
		space := cap(in.rows) - len(in.rows)
		var add []interface{}
		add, data = data[:space], data[space:] // does this break?
		in.rows = append(in.rows, add...)
		err := in.Flush()
		if err != nil {
			// TODO - handle errors in middle better?
			return err
		}
	}
	in.rows = append(in.rows, data...)
	return nil
}

func (in *BQInserter) HandleInsertErrors(err error) error {
	switch typedErr := err.(type) {
	case bigquery.PutMultiError:
		if len(typedErr) == len(in.rows) {
			log.Printf("%v\n", err)
			metrics.BackendFailureCount.WithLabelValues(
				in.TableBase(), "failed insert").Inc()
			in.failures += 1
		}
		// If ALL rows failed, and number of rows is large, just report single failure.
		if len(typedErr) > 10 && len(typedErr) == len(in.rows) {
			log.Printf("Insert error: %v\n", err)
			metrics.ErrorCount.WithLabelValues(
				in.TableBase(), "unknown", "insert row error").
				Add(float64(len(typedErr)))
		} else {
			// Handle each error individually.
			// TODO Should we try to handle large numbers of row errors?
			for _, rowError := range typedErr {
				// These are rowInsertionErrors
				log.Printf("%v\n", rowError)
				// rowError.Errors is a MultiError
				for _, oneErr := range rowError.Errors {
					log.Printf("Insert error: %v\n", oneErr)
					metrics.ErrorCount.WithLabelValues(
						in.TableBase(), "unknown", "insert row error").Inc()
				}
			}
		}
		in.inserted += len(in.rows) - len(typedErr)
		in.badRows += len(typedErr)
		err = nil
	default:
		log.Printf("Unhandled insert error %v\n", typedErr)
		metrics.BackendFailureCount.WithLabelValues(
			in.TableBase(), "failed insert").Inc()
		metrics.ErrorCount.WithLabelValues(
			in.TableBase(), "unknown", "UNHANDLED insert error").Inc()
		// TODO - Conservative, but possibly not correct.
		// This at least preserves the count invariance.
		in.badRows += len(in.rows)
		err = nil
	}
	// Allocate new slice of rows.  Any failed rows are lost.
	in.rows = make([]interface{}, 0, in.params.BufferSize)
	return err
}

// TODO(dev) Should have a recovery mechanism for failed inserts.
func (in *BQInserter) Flush() error {
	metrics.WorkerState.WithLabelValues("flush").Inc()
	defer metrics.WorkerState.WithLabelValues("flush").Dec()

	if len(in.rows) == 0 {
		return nil
	}

	// This is heavyweight, and may run forever without a context deadline.
	ctx, _ := context.WithTimeout(context.Background(), in.timeout)
	err := in.uploader.Put(ctx, in.rows)
	if err == nil {
		in.inserted += len(in.rows)
		in.rows = make([]interface{}, 0, in.params.BufferSize)
	} else {
		// This adjusts the inserted count, failure count, and updates in.rows.
		err = in.HandleInsertErrors(err)
	}
	return err
}

func (in *BQInserter) FullTableName() string {
	return in.TableBase() + in.TableSuffix()
}
func (in *BQInserter) TableBase() string {
	return in.params.Table
}

// The $ or _ suffix.
func (in *BQInserter) TableSuffix() string {
	return in.params.Suffix
}
func (in *BQInserter) Dataset() string {
	return in.params.Dataset
}
func (in *BQInserter) RowsInBuffer() int {
	return len(in.rows)
}
func (in *BQInserter) Accepted() int {
	return in.inserted + in.badRows + len(in.rows)
}
func (in *BQInserter) Committed() int {
	return in.inserted
}
func (in *BQInserter) Failed() int {
	return in.badRows
}

//----------------------------------------------------------------------------

type NullInserter struct {
	etl.Inserter
}

func (in *NullInserter) InsertRow(data interface{}) error {
	return nil
}
func (in *NullInserter) InsertRows(data []interface{}) error {
	return nil
}
func (in *NullInserter) Flush() error {
	return nil
}
func (in *NullInserter) FullTableName() string {
	return "base_suffix"
}
func (in *NullInserter) TableBase() string {
	return "base"
}
func (in *NullInserter) TableSuffix() string {
	return "_suffix"
}
func (in *NullInserter) Dataset() string {
	return ""
}
func (in *NullInserter) RowsInBuffer() int {
	return 0
}
func (in *NullInserter) Count() int {
	return 0
}

//----------------------------------------------------------------------------

// Inserter wrapper that handles flush metrics.
// TODO - add prometheus counters for attempts, number of rows.
type DurationWrapper struct {
	etl.Inserter
}

func (dw DurationWrapper) Flush() error {
	t := time.Now()
	status := "succeed"
	err := dw.Inserter.Flush()
	if err != nil {
		status = "fail"
	}
	metrics.InsertionHistogram.WithLabelValues(
		dw.TableBase(), status).Observe(time.Since(t).Seconds())
	return err
}
<commit_msg>add space<commit_after>// Package bq includes all code related to BigQuery.
//
// NB: NOTES ON MEMORY USE AND HTTP SIZE
// The bigquery library uses JSON encoding of data, which appears to be the only
// option at this time.  Furthermore, it uses intermediate data representations,
// eventually creating a map[string]Value (unless you pass that in to begin with).
// In general, when we start pumping large volumes of data, both the map and the
// JSON will cause some memory pressure, and likely pretty severe limits on the size
// of the insert we can send, likely on the order of a couple MB of actual row footprint
// in BQ.
//
// Passing in slice of structs makes memory pressure a bit worse, but probably isn't
// worth worrying about.

package bq

import (
	"log"
	"os"
	"sync"
	"time"

	"cloud.google.com/go/bigquery"
	"golang.org/x/net/context"

	"github.com/m-lab/etl/etl"
	"github.com/m-lab/etl/metrics"
)

// TODO(dev) Use a more thoughtful setting for buffer size.
// TODO(P3) Include the project name in the parameters.
// For now, 10K per row times 100 results is 1MB, which is an order of
// magnitude below our 10MB max, so 100 might not be such a bad
// default.
func NewInserter(dataset string, dt etl.DataType, partition time.Time) (etl.Inserter, error) {
	suffix := ""
	table := etl.DataTypeToTable[dt]
	if time.Since(partition) < 30*24*time.Hour {
		// If within past 30 days, we can stream directly to partition.
		suffix = "$" + partition.Format("20060102")
	} else {
		// Otherwise, we use a templated table, and must merge it later.
		suffix = "_" + partition.Format("20060102")
	}

	return NewBQInserter(
		etl.InserterParams{Dataset: dataset, Table: table, Suffix: suffix,
			Timeout: 15 * time.Minute, BufferSize: 10}, nil)
}

// TODO - improve the naming between here and NewInserter.
// Pass in nil uploader for normal use, custom uploader for custom behavior
func NewBQInserter(params etl.InserterParams, uploader etl.Uploader) (etl.Inserter, error) {
	if uploader == nil {
		client := MustGetClient(params.Timeout)
		table := params.Table
		if params.Suffix[0] == '$' {
			// Suffix starting with $ is just a partition spec.
			table += params.Suffix
		}
		u := client.Dataset(params.Dataset).Table(table).Uploader()
		if params.Suffix[0] == '_' {
			// Suffix starting with _ is a template suffix.
			u.TableTemplateSuffix = params.Suffix
		}
		// This avoids problems when a single row of the insert has invalid
		// data.  We then have to carefully parse the returned error object.
		u.SkipInvalidRows = true
		uploader = u
	}
	in := BQInserter{params: params, uploader: uploader, timeout: params.Timeout}
	in.rows = make([]interface{}, 0, in.params.BufferSize)
	return &in, nil
}

//===============================================================================
var (
	clientOnce sync.Once // This avoids a race on setting bqClient.
	bqClient   *bigquery.Client
)

// Returns the Singleton bigquery client for this process.
// TODO - is there any advantage to using more than one client?
func MustGetClient(timeout time.Duration) *bigquery.Client {
	// We do this here, instead of in init(), because we only want to do it
	// when we actually want to access the bigquery backend.
	clientOnce.Do(func() {
		ctx, _ := context.WithTimeout(context.Background(), timeout)
		project, ok := os.LookupEnv("BIGQUERY_PROJECT")
		if !ok {
			project = os.Getenv("GCLOUD_PROJECT")
		}

		log.Printf("Using project: %s\n", project)
		// Heavyweight!
		var err error
		bqClient, err = bigquery.NewClient(ctx, project)
		if err != nil {
			panic(err.Error())
		}
	})
	return bqClient
}

//===============================================================================

// Generic implementation of bq.ValueSaver, based on map.  This avoids extra
// conversion steps in the bigquery library (except for the JSON conversion).
// IMPLEMENTS: bigquery.ValueSaver
type MapSaver struct {
	Values map[string]bigquery.Value
}

func (s *MapSaver) Save() (row map[string]bigquery.Value, insertID string, err error) {
	return s.Values, "", nil
}

//----------------------------------------------------------------------------

type BQInserter struct {
	etl.Inserter
	params   etl.InserterParams
	uploader etl.Uploader // May be a BQ Uploader, or a test Uploader
	timeout  time.Duration
	rows     []interface{}
	inserted int // Number of rows successfully inserted.
	badRows  int // Number of row failures, including rows in full failures.
	failures int // Number of complete insert failures.
}

// Caller should check error, and take appropriate action before calling again.
func (in *BQInserter) InsertRow(data interface{}) error {
	return in.InsertRows([]interface{}{data})
}

// Caller should check error, and take appropriate action before calling again.
// TODO - should this return a specific error to indicate that a flush is needed
// instead of flushing internally?  The "handle errors in the middle" would
// be easier, though other complications would ensue.
func (in *BQInserter) InsertRows(data []interface{}) error {
	metrics.WorkerState.WithLabelValues("insert").Inc()
	defer metrics.WorkerState.WithLabelValues("insert").Dec()

	for len(data)+len(in.rows) >= in.params.BufferSize {
		// space >= len(data)
		space := cap(in.rows) - len(in.rows)
		var add []interface{}
		add, data = data[:space], data[space:] // does this break?
		in.rows = append(in.rows, add...)
		err := in.Flush()
		if err != nil {
			// TODO - handle errors in middle better?
			return err
		}
	}
	in.rows = append(in.rows, data...)
	return nil
}

func (in *BQInserter) HandleInsertErrors(err error) error {
	switch typedErr := err.(type) {
	case bigquery.PutMultiError:
		if len(typedErr) == len(in.rows) {
			log.Printf("%v\n", err)
			metrics.BackendFailureCount.WithLabelValues(
				in.TableBase(), "failed insert").Inc()
			in.failures += 1
		}
		// If ALL rows failed, and number of rows is large, just report single failure.
		if len(typedErr) > 10 && len(typedErr) == len(in.rows) {
			log.Printf("Insert error: %v\n", err)
			metrics.ErrorCount.WithLabelValues(
				in.TableBase(), "unknown", "insert row error").
				Add(float64(len(typedErr)))
		} else {
			// Handle each error individually.
			// TODO Should we try to handle large numbers of row errors?
			for _, rowError := range typedErr {
				// These are rowInsertionErrors
				log.Printf("%v\n", rowError)
				// rowError.Errors is a MultiError
				for _, oneErr := range rowError.Errors {
					log.Printf("Insert error: %v\n", oneErr)
					metrics.ErrorCount.WithLabelValues(
						in.TableBase(), "unknown", "insert row error").Inc()
				}
			}
		}
		in.inserted += len(in.rows) - len(typedErr)
		in.badRows += len(typedErr)
		err = nil
	default:
		log.Printf("Unhandled insert error %v\n", typedErr)
		metrics.BackendFailureCount.WithLabelValues(
			in.TableBase(), "failed insert").Inc()
		metrics.ErrorCount.WithLabelValues(
			in.TableBase(), "unknown", "UNHANDLED insert error").Inc()
		// TODO - Conservative, but possibly not correct.
		// This at least preserves the count invariance.
		in.badRows += len(in.rows)
		err = nil
	}
	// Allocate new slice of rows.  Any failed rows are lost.
	in.rows = make([]interface{}, 0, in.params.BufferSize)
	return err
}

// TODO(dev) Should have a recovery mechanism for failed inserts.
func (in *BQInserter) Flush() error {
	metrics.WorkerState.WithLabelValues("flush").Inc()
	defer metrics.WorkerState.WithLabelValues("flush").Dec()

	if len(in.rows) == 0 {
		return nil
	}

	// This is heavyweight, and may run forever without a context deadline.
	ctx, _ := context.WithTimeout(context.Background(), in.timeout)
	err := in.uploader.Put(ctx, in.rows)
	if err == nil {
		in.inserted += len(in.rows)
		in.rows = make([]interface{}, 0, in.params.BufferSize)
	} else {
		// This adjusts the inserted count, failure count, and updates in.rows.
		err = in.HandleInsertErrors(err)
	}
	return err
}

func (in *BQInserter) FullTableName() string {
	return in.TableBase() + in.TableSuffix()
}
func (in *BQInserter) TableBase() string {
	return in.params.Table
}

// The $ or _ suffix.
func (in *BQInserter) TableSuffix() string {
	return in.params.Suffix
}
func (in *BQInserter) Dataset() string {
	return in.params.Dataset
}
func (in *BQInserter) RowsInBuffer() int {
	return len(in.rows)
}
func (in *BQInserter) Accepted() int {
	return in.inserted + in.badRows + len(in.rows)
}
func (in *BQInserter) Committed() int {
	return in.inserted
}
func (in *BQInserter) Failed() int {
	return in.badRows
}

//----------------------------------------------------------------------------

type NullInserter struct {
	etl.Inserter
}

func (in *NullInserter) InsertRow(data interface{}) error {
	return nil
}
func (in *NullInserter) InsertRows(data []interface{}) error {
	return nil
}
func (in *NullInserter) Flush() error {
	return nil
}
func (in *NullInserter) FullTableName() string {
	return "base_suffix"
}
func (in *NullInserter) TableBase() string {
	return "base"
}
func (in *NullInserter) TableSuffix() string {
	return "_suffix"
}
func (in *NullInserter) Dataset() string {
	return ""
}
func (in *NullInserter) RowsInBuffer() int {
	return 0
}
func (in *NullInserter) Count() int {
	return 0
}

//----------------------------------------------------------------------------

// Inserter wrapper that handles flush metrics.
// TODO - add prometheus counters for attempts, number of rows.
type DurationWrapper struct {
	etl.Inserter
}

func (dw DurationWrapper) Flush() error {
	t := time.Now()
	status := "succeed"
	err := dw.Inserter.Flush()
	if err != nil {
		status = "fail"
	}
	metrics.InsertionHistogram.WithLabelValues(
		dw.TableBase(), status).Observe(time.Since(t).Seconds())
	return err
}
<|endoftext|>


<commit_before>package kumoru

import (
	"bytes"
	"encoding/json"
	"net/http"
	"strings"
)

// NewRequest sets the appropriate header and appropriate request content
func (k *Client) NewRequest() (*http.Request, error) {
	switch k.Method {
	case POST, PUT, PATCH:
		if k.TargetType == "json" {

			var contentJSON []byte

			if len(k.Data) != 0 {
				contentJSON, _ = json.Marshal(k.Data)
			}

			contentReader := bytes.NewReader(contentJSON)
			req, err := http.NewRequest(k.Method, k.URL, contentReader)
			req.Header.Set("Content-Type", "application/json")
			return req, err
		} else if k.TargetType == "form" {
			req, err := http.NewRequest(k.Method, k.URL, bytes.NewBufferString(k.RawString))
			req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
			return req, err
		} else if k.TargetType == "text" {
			req, err := http.NewRequest(k.Method, k.URL, strings.NewReader(k.RawString))
			req.Header.Set("Content-Type", "text/plain")
			return req, err
		} else if k.TargetType == "xml" {
			req, err := http.NewRequest(k.Method, k.URL, strings.NewReader(k.RawString))
			req.Header.Set("Content-Type", "application/xml")
			return req, err
		}
	case GET, HEAD, DELETE:
		return http.NewRequest(k.Method, k.URL, nil)
	}

	return nil, nil
}
<commit_msg>TargetType JSON no longer Marshals a struct<commit_after>package kumoru

import (
	"bytes"
	"net/http"
	"strings"
)

// NewRequest sets the appropriate header and appropriate request content
func (k *Client) NewRequest() (*http.Request, error) {
	switch k.Method {
	case POST, PUT, PATCH:
		if k.TargetType == "json" {
			req, err := http.NewRequest(k.Method, k.URL, strings.NewReader(k.RawString))
			req.Header.Set("Content-Type", "application/json")
			return req, err
		} else if k.TargetType == "form" {
			req, err := http.NewRequest(k.Method, k.URL, bytes.NewBufferString(k.RawString))
			req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
			return req, err
		} else if k.TargetType == "text" {
			req, err := http.NewRequest(k.Method, k.URL, strings.NewReader(k.RawString))
			req.Header.Set("Content-Type", "text/plain")
			return req, err
		} else if k.TargetType == "xml" {
			req, err := http.NewRequest(k.Method, k.URL, strings.NewReader(k.RawString))
			req.Header.Set("Content-Type", "application/xml")
			return req, err
		}
	case GET, HEAD, DELETE:
		return http.NewRequest(k.Method, k.URL, nil)
	}

	return nil, nil
}
<|endoftext|>


<commit_before>// Package memberlist implements the raftor.ClusterChangeNotifier interface along with the memberlist.EventDelegate interface. The Notifier interface provides for a simple solution allowing the two packages to communicate membership changes.
package memberlist

import (
	"github.com/blacklabeldata/raftor"
	"github.com/blacklabeldata/raftor/pkg/murmur"
	"github.com/hashicorp/memberlist"
)

// BlockingNotifier blocks until each event has been consumed. It also implements the memberlist.EventDelegate interface which is used to process cluster membership events.
func BlockingNotifier() Notifier {
	return &notifier{make(chan raftor.ClusterChangeEvent), false}
}

// BufferedNotifier will buffer events up to the size given. After the buffered channel is full it will block until an event has been consumed.
func BufferedNotifier(size int) Notifier {
	return &notifier{make(chan raftor.ClusterChangeEvent, size), false}
}

// A Notifier subscribes to memberlist notifications and emits them as raftor.ClusterChangeEvents.
type Notifier interface {
	raftor.ClusterChangeNotifier
	memberlist.EventDelegate
}

// notifier notifies the receiver of the cluster change.
type notifier struct {
	channel chan raftor.ClusterChangeEvent
	closed  bool
}

// NotifyChange sends raftor.ClusterChangeEvents over the given channel when a node joins, leaves or is updated in the cluster.
func (n *notifier) NotifyChange() <-chan raftor.ClusterChangeEvent {
	return n.channel
}

// NotifyJoin is invoked when a node is detected to have joined.
// The Node argument must not be modified.
func (n *notifier) NotifyJoin(other *memberlist.Node) {
	n.send(raftor.ClusterChangeEvent{
		Type:   raftor.AddMember,
		Member: raftor.NewMember(n.hash(other), other.Meta),
	})
	return
}

// NotifyLeave is invoked when a node is detected to have left.
// The Node argument must not be modified.
func (n *notifier) NotifyLeave(other *memberlist.Node) {
	n.send(raftor.ClusterChangeEvent{
		Type:   raftor.RemoveMember,
		Member: raftor.NewMember(n.hash(other), other.Meta),
	})
	return
}

// NotifyUpdate is invoked when a node is detected to have
// updated, usually involving the meta data. The Node argument
// must not be modified.
func (n *notifier) NotifyUpdate(other *memberlist.Node) {
	n.send(raftor.ClusterChangeEvent{
		Type:   raftor.UpdateMember,
		Member: raftor.NewMember(n.hash(other), other.Meta),
	})
	return
}

// Stop closes the notifier channel
func (n *notifier) Stop() {
	n.closed = true
	close(n.channel)
}

// send sends an event over the channel
func (n *notifier) send(evt raftor.ClusterChangeEvent) {
	if !n.closed {
		n.channel <- evt
	}
}

// hash performs a Murmur3 hash on the memberlist.Node
func (n *notifier) hash(other *memberlist.Node) uint64 {
	return uint64(murmur.Murmur3(other.Meta, murmur.M3Seed))
}
<commit_msg>Revert "Add Stop method to ClusterChangeNotifier"<commit_after>package memberlist

import "github.com/blacklabeldata/raftor"
import "github.com/blacklabeldata/raftor/pkg/murmur"

// ClusterChangeNotifier notifies the receiver of the cluster change.
type notifier struct {
	channel <-chan raftor.ClusterChangeEvent
}

// NotifyChange sends raftor.ClusterChangeEvents over the given channel when a node joins, leaves or is updated in the cluster.
func (n *notifier) NotifyChange() <-chan raftor.ClusterChangeEvent {
	return n.channel
}

// NotifyJoin is invoked when a node is detected to have joined.
// The Node argument must not be modified.
func (n *notifier) NotifyJoin(other *memberlist.Node) {
	n.channel <- raftor.ClusterChangeEvent{
		Type:   raftor.AddMember,
		Member: raftor.NewMember(n.hash(other), other.Meta),
	}
	return
}

// NotifyLeave is invoked when a node is detected to have left.
// The Node argument must not be modified.
func (n *notifier) NotifyLeave(other *memberlist.Node) {
	n.channel <- raftor.ClusterChangeEvent{
		Type:   raftor.RemoveMember,
		Member: raftor.NewMember(n.hash(other), other.Meta),
	}
	return
}

// NotifyUpdate is invoked when a node is detected to have
// updated, usually involving the meta data. The Node argument
// must not be modified.
func (n *notifier) NotifyUpdate(other *memberlist.Node) {
	n.channel <- raftor.ClusterChangeEvent{
		Type:   raftor.UpdateMember,
		Member: raftor.NewMember(n.hash(other), other.Meta),
	}
	return
}

// hash performs a Murmur3 hash on the memberlist.Node
func (n *notifier) hash(other *memberlist.Node) uint64 {
	return uint64(murmur.Murmur3(other.Meta, murmur.M3Seed))
}
<|endoftext|>


<commit_before>package crawler

import (
	"bytes"
	"errors"
	"unicode"
	"unicode/utf8"

	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
	"golang.org/x/text/unicode/rangetable"
)

var (
	// ErrMinificationTextUnexpectedNodeType - found unexpected node type
	ErrMinificationTextUnexpectedNodeType = errors.New("minification_text.minificationText.parseNode: unexpected node type")
	// ErrMinificationTextUnexpectedTag - found unexpected tag
	ErrMinificationTextUnexpectedTag = errors.New("minification_text.minificationText.parseElements: unexpected tag")
)

var notSeparatorRT = rangetable.New(
	'&', '-', '@', '_', '+', '\'',
	'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
	'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k',
	'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
	'w', 'x', 'y', 'z',
	'а', 'б', 'в', 'г', 'д', 'е', 'ё', 'ж', 'з', 'и', 'й',
	'к', 'л', 'м', 'н', 'о', 'п', 'р', 'с', 'т', 'у', 'ф',
	'х', 'ц', 'ч', 'ш', 'щ', 'ъ', 'ы', 'ь', 'э', 'ю', 'я')

type minificationText struct {
}

func (m *minificationText) processText(in string) string {
	var buffer bytes.Buffer
	var rRaw, r rune
	var size int
	prevIsSeparator := false
	prevRune := ' '
	isFirst := true
	for len(in) > 0 {
		rRaw, size = utf8.DecodeRuneInString(in)
		r = unicode.ToLower(rRaw)

		if unicode.Is(notSeparatorRT, r) {
			if prevIsSeparator {
				prevIsSeparator = false
				if !isFirst {
					_ = buffer.WriteByte(' ')
				}
			}
			_, _ = buffer.WriteRune(r)
			isFirst = false
		} else {
			if !prevIsSeparator {
				prevIsSeparator = true
				// digits
				if '0' <= prevRune && prevRune <= '9' && (r == '.' || r == ',' || r == ':') {
					rRaw, _ = utf8.DecodeRuneInString(in[size:])
					if '0' <= rRaw && rRaw <= '9' {
						_, _ = buffer.WriteRune(r)
						prevIsSeparator = false
					}
				}
			} else {
				prevIsSeparator = true
			}
		}

		prevRune = r
		in = in[size:]
	}

	return buffer.String()
}

func (m *minificationText) parseText(node *html.Node) (*html.Node, error) {
	next := node.NextSibling
	text := m.processText(node.Data)
	if len(text) != 0 {
		node.Data = text
	} else {
		node.Parent.RemoveChild(node)
	}
	return next, nil
}

func (m *minificationText) parseChildren(node *html.Node) (*html.Node, error) {
	var err error
	for it := node.FirstChild; it != nil; {
		it, err = m.parseNode(it)
		if err != nil {
			return node.NextSibling, err
		}
	}

	return node.NextSibling, nil
}

func (m *minificationText) parseElements(node *html.Node) (*html.Node, error) {
	switch node.DataAtom {
	case atom.Head, atom.Html:
		return m.parseChildren(node)
	case atom.Body, atom.Div:
		next, err := m.parseChildren(node)
		if err == nil {
			child := node.FirstChild
			if child == nil {
				node.Parent.RemoveChild(node)
			} else if child == node.LastChild && child.DataAtom == atom.Div {
				for it := child.FirstChild; it != nil; it = it.NextSibling {
					it.Parent = node
				}
				node.FirstChild = child.FirstChild
				node.LastChild = child.LastChild
				child.FirstChild = nil
				child.LastChild = nil
				child.Parent = nil
			}
		}
		return next, err
	default:
		return nil, ErrMinificationTextUnexpectedTag
	}
}

func (m *minificationText) parseNode(node *html.Node) (*html.Node, error) {
	switch node.Type {
	case html.DocumentNode: // +children -attr (first node)
		return m.parseChildren(node)
	case html.ElementNode: // +children +attr
		return m.parseElements(node)
	case html.TextNode: // -children -attr
		return m.parseText(node)
	default: // ErrorNode, CommentNode, DoctypeNode
		return nil, ErrMinificationTextUnexpectedNodeType
	}
}

// RunMinification2 - start minification node
func RunMinification2(node *html.Node) error {
	m := minificationText{}
	_, err := m.parseNode(node)
	return err
}
<commit_msg>раскрытие и удаление тегов в минификации текста<commit_after>package crawler

import (
	"bytes"
	"errors"
	"unicode"
	"unicode/utf8"

	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
	"golang.org/x/text/unicode/rangetable"
)

var (
	// ErrMinificationTextUnexpectedNodeType - found unexpected node type
	ErrMinificationTextUnexpectedNodeType = errors.New("minification_text.minificationText.parseNode: unexpected node type")
	// ErrMinificationTextUnexpectedTag - found unexpected tag
	ErrMinificationTextUnexpectedTag = errors.New("minification_text.minificationText.parseElements: unexpected tag")
)

var notSeparatorRT = rangetable.New(
	'&', '-', '@', '_', '+', '\'',
	'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
	'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k',
	'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
	'w', 'x', 'y', 'z',
	'а', 'б', 'в', 'г', 'д', 'е', 'ё', 'ж', 'з', 'и', 'й',
	'к', 'л', 'м', 'н', 'о', 'п', 'р', 'с', 'т', 'у', 'ф',
	'х', 'ц', 'ч', 'ш', 'щ', 'ъ', 'ы', 'ь', 'э', 'ю', 'я')

type minificationText struct {
}

func (m *minificationText) processText(in string) string {
	var buffer bytes.Buffer
	var rRaw, r rune
	var size int
	prevIsSeparator := false
	prevRune := ' '
	isFirst := true
	for len(in) > 0 {
		rRaw, size = utf8.DecodeRuneInString(in)
		r = unicode.ToLower(rRaw)

		if unicode.Is(notSeparatorRT, r) {
			if prevIsSeparator {
				prevIsSeparator = false
				if !isFirst {
					_ = buffer.WriteByte(' ')
				}
			}
			_, _ = buffer.WriteRune(r)
			isFirst = false
		} else {
			if !prevIsSeparator {
				prevIsSeparator = true
				// digits
				if '0' <= prevRune && prevRune <= '9' && (r == '.' || r == ',' || r == ':') {
					rRaw, _ = utf8.DecodeRuneInString(in[size:])
					if '0' <= rRaw && rRaw <= '9' {
						_, _ = buffer.WriteRune(r)
						prevIsSeparator = false
					}
				}
			} else {
				prevIsSeparator = true
			}
		}

		prevRune = r
		in = in[size:]
	}

	return buffer.String()
}

func (m *minificationText) openTag(node *html.Node) {
	parent := node.Parent
	for it := node.FirstChild; it != nil; it = it.NextSibling {
		it.Parent = parent
	}
	parent.FirstChild = node.FirstChild
	parent.LastChild = node.LastChild
	node.FirstChild = nil
	node.LastChild = nil
	node.Parent = nil
}

func (m *minificationText) removeTag(node *html.Node) *html.Node {
	prev, next, parent := node.PrevSibling, node.NextSibling, node.Parent
	prevText := prev != nil && prev.Type == html.TextNode
	nextText := next != nil && next.Type == html.TextNode
	result := next

	if prevText && nextText {
		text := m.processText(next.Data)
		if len(text) != 0 {
			prev.Data = prev.Data + " " + text
		}
		result = next.NextSibling
		parent.RemoveChild(next)
	}
	parent.RemoveChild(node)

	return result
}

func (m *minificationText) parseText(node *html.Node) (*html.Node, error) {
	next := node.NextSibling
	text := m.processText(node.Data)
	if len(text) != 0 {
		node.Data = text
	} else {
		node.Parent.RemoveChild(node)
	}
	return next, nil
}

func (m *minificationText) parseChildren(node *html.Node) (*html.Node, error) {
	var err error
	for it := node.FirstChild; it != nil; {
		it, err = m.parseNode(it)
		if err != nil {
			return node.NextSibling, err
		}
	}

	return node.NextSibling, nil
}

func (m *minificationText) parseElements(node *html.Node) (*html.Node, error) {
	switch node.DataAtom {
	case atom.Head, atom.Html:
		return m.parseChildren(node)
	case atom.Body, atom.Div:
		next, err := m.parseChildren(node)
		if err == nil {
			child := node.FirstChild
			if child == nil {
				next = m.removeTag(node)
			} else if child == node.LastChild && child.DataAtom == atom.Div {
				m.openTag(child)
			}
		}
		return next, err
	default:
		return nil, ErrMinificationTextUnexpectedTag
	}
}

func (m *minificationText) parseNode(node *html.Node) (*html.Node, error) {
	switch node.Type {
	case html.DocumentNode: // +children -attr (first node)
		return m.parseChildren(node)
	case html.ElementNode: // +children +attr
		return m.parseElements(node)
	case html.TextNode: // -children -attr
		return m.parseText(node)
	default: // ErrorNode, CommentNode, DoctypeNode
		return nil, ErrMinificationTextUnexpectedNodeType
	}
}

// RunMinification2 - start minification node
func RunMinification2(node *html.Node) error {
	m := minificationText{}
	_, err := m.parseNode(node)
	return err
}
<|endoftext|>