<commit_before>// Copyright 2017 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !nobcache

package collector

import (
	"fmt"

	// https://godoc.org/github.com/prometheus/client_golang/prometheus
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/procfs/bcache"
)

func init() {
	registerCollector("bcache", defaultEnabled, NewBcacheCollector)
}

// A bcacheCollector is a Collector which gathers metrics from Linux bcache.
type bcacheCollector struct {
	fs bcache.FS
}

// NewBcacheCollector returns a newly allocated bcacheCollector.
// It exposes a number of Linux bcache statistics.
func NewBcacheCollector() (Collector, error) {
	fs, err := bcache.NewFS(*sysPath)
	if err != nil {
		return nil, fmt.Errorf("failed to open sysfs: %v", err)
	}

	return &bcacheCollector{
		fs: fs,
	}, nil
}

// Update reads and exposes bcache stats.
// It implements the Collector interface.
func (c *bcacheCollector) Update(ch chan<- prometheus.Metric) error {
	stats, err := c.fs.Stats()
	if err != nil {
		return fmt.Errorf("failed to retrieve bcache stats: %v", err)
	}

	for _, s := range stats {
		c.updateBcacheStats(ch, s)
	}
	return nil
}

type bcacheMetric struct {
	name            string
	desc            string
	value           float64
	metricType      prometheus.ValueType
	extraLabel      []string
	extraLabelValue string
}

func bcachePeriodStatsToMetric(ps *bcache.PeriodStats, labelValue string) []bcacheMetric {
	label := []string{"backing_device"}

	metrics := []bcacheMetric{
		{
			name:            "bypassed_bytes_total",
			desc:            "Amount of IO (both reads and writes) that has bypassed the cache.",
			value:           float64(ps.Bypassed),
			metricType:      prometheus.CounterValue,
			extraLabel:      label,
			extraLabelValue: labelValue,
		},
		{
			name:            "cache_hits_total",
			desc:            "Hits counted per individual IO as bcache sees them.",
			value:           float64(ps.CacheHits),
			metricType:      prometheus.CounterValue,
			extraLabel:      label,
			extraLabelValue: labelValue,
		},
		{
			name:            "cache_misses_total",
			desc:            "Misses counted per individual IO as bcache sees them.",
			value:           float64(ps.CacheMisses),
			metricType:      prometheus.CounterValue,
			extraLabel:      label,
			extraLabelValue: labelValue,
		},
		{
			name:            "cache_bypass_hits_total",
			desc:            "Hits for IO intended to skip the cache.",
			value:           float64(ps.CacheBypassHits),
			metricType:      prometheus.CounterValue,
			extraLabel:      label,
			extraLabelValue: labelValue,
		},
		{
			name:            "cache_bypass_misses_total",
			desc:            "Misses for IO intended to skip the cache.",
			value:           float64(ps.CacheBypassMisses),
			metricType:      prometheus.CounterValue,
			extraLabel:      label,
			extraLabelValue: labelValue,
		},
		{
			name:            "cache_miss_collisions_total",
			desc:            "Instances where data insertion from cache miss raced with write (data already present).",
			value:           float64(ps.CacheMissCollisions),
			metricType:      prometheus.CounterValue,
			extraLabel:      label,
			extraLabelValue: labelValue,
		},
		{
			name:            "cache_readaheads_total",
			desc:            "Count of times readahead occurred.",
			value:           float64(ps.CacheReadaheads),
			metricType:      prometheus.CounterValue,
			extraLabel:      label,
			extraLabelValue: labelValue,
		},
	}
	return metrics
}

// UpdateBcacheStats collects statistics for one bcache ID.
func (c *bcacheCollector) updateBcacheStats(ch chan<- prometheus.Metric, s *bcache.Stats) {

	const (
		subsystem = "bcache"
	)

	var (
		devLabel   = []string{"uuid"}
		allMetrics []bcacheMetric
		metrics    []bcacheMetric
	)

	allMetrics = []bcacheMetric{
		// metrics in /sys/fs/bcache/<uuid>/
		{
			name:       "average_key_size_sectors",
			desc:       "Average data per key in the btree (sectors).",
			value:      float64(s.Bcache.AverageKeySize),
			metricType: prometheus.GaugeValue,
		},
		{
			name:       "btree_cache_size_bytes",
			desc:       "Amount of memory currently used by the btree cache.",
			value:      float64(s.Bcache.BtreeCacheSize),
			metricType: prometheus.GaugeValue,
		},
		{
			name:       "cache_available_percent",
			desc:       "Percentage of cache device without dirty data, usable for writeback (may contain clean cached data).",
			value:      float64(s.Bcache.CacheAvailablePercent),
			metricType: prometheus.GaugeValue,
		},
		{
			name:       "congested",
			desc:       "Congestion.",
			value:      float64(s.Bcache.Congested),
			metricType: prometheus.GaugeValue,
		},
		{
			name:       "root_usage_percent",
			desc:       "Percentage of the root btree node in use (tree depth increases if too high).",
			value:      float64(s.Bcache.RootUsagePercent),
			metricType: prometheus.GaugeValue,
		},
		{
			name:       "tree_depth",
			desc:       "Depth of the btree.",
			value:      float64(s.Bcache.TreeDepth),
			metricType: prometheus.GaugeValue,
		},
		// metrics in /sys/fs/bcache/<uuid>/internal/
		{
			name:       "active_journal_entries",
			desc:       "Number of journal entries that are newer than the index.",
			value:      float64(s.Bcache.Internal.ActiveJournalEntries),
			metricType: prometheus.GaugeValue,
		},
		{
			name:       "btree_nodes",
			desc:       "Total nodes in the btree.",
			value:      float64(s.Bcache.Internal.BtreeNodes),
			metricType: prometheus.GaugeValue,
		},
		{
			name:       "btree_read_average_duration_seconds",
			desc:       "Average btree read duration.",
			value:      float64(s.Bcache.Internal.BtreeReadAverageDurationNanoSeconds) * 1e-9,
			metricType: prometheus.GaugeValue,
		},
		{
			name:       "cache_read_races_total",
			desc:       "Counts instances where while data was being read from the cache, the bucket was reused and invalidated - i.e. where the pointer was stale after the read completed.",
			value:      float64(s.Bcache.Internal.CacheReadRaces),
			metricType: prometheus.CounterValue,
		},
	}

	for _, bdev := range s.Bdevs {
		// metrics in /sys/fs/bcache/<uuid>/<bdev>/
		metrics = []bcacheMetric{
			{
				name:            "dirty_data_bytes",
				desc:            "Amount of dirty data for this backing device in the cache.",
				value:           float64(bdev.DirtyData),
				metricType:      prometheus.GaugeValue,
				extraLabel:      []string{"backing_device"},
				extraLabelValue: bdev.Name,
			},
		}
		allMetrics = append(allMetrics, metrics...)

		// metrics in /sys/fs/bcache/<uuid>/<bdev>/stats_total
		metrics := bcachePeriodStatsToMetric(&bdev.Total, bdev.Name)
		allMetrics = append(allMetrics, metrics...)

	}

	for _, cache := range s.Caches {
		metrics = []bcacheMetric{
			// metrics in /sys/fs/bcache/<uuid>/<cache>/
			{
				name:            "io_errors",
				desc:            "Number of errors that have occurred, decayed by io_error_halflife.",
				value:           float64(cache.IOErrors),
				metricType:      prometheus.GaugeValue,
				extraLabel:      []string{"cache_device"},
				extraLabelValue: cache.Name,
			},
			{
				name:            "metadata_written_bytes_total",
				desc:            "Sum of all non data writes (btree writes and all other metadata).",
				value:           float64(cache.MetadataWritten),
				metricType:      prometheus.CounterValue,
				extraLabel:      []string{"cache_device"},
				extraLabelValue: cache.Name,
			},
			{
				name:            "written_bytes_total",
				desc:            "Sum of all data that has been written to the cache.",
				value:           float64(cache.Written),
				metricType:      prometheus.CounterValue,
				extraLabel:      []string{"cache_device"},
				extraLabelValue: cache.Name,
			},
			// metrics in /sys/fs/bcache/<uuid>/<cache>/priority_stats
			{
				name:            "priority_stats_unused_percent",
				desc:            "The percentage of the cache that doesn't contain any data.",
				value:           float64(cache.Priority.UnusedPercent),
				metricType:      prometheus.GaugeValue,
				extraLabel:      []string{"cache_device"},
				extraLabelValue: cache.Name,
			},
			{
				name:            "priority_stats_metadata_percent",
				desc:            "Bcache's metadata overhead.",
				value:           float64(cache.Priority.MetadataPercent),
				metricType:      prometheus.GaugeValue,
				extraLabel:      []string{"cache_device"},
				extraLabelValue: cache.Name,
			},
		}
		allMetrics = append(allMetrics, metrics...)
	}

	for _, m := range allMetrics {
		labels := append(devLabel, m.extraLabel...)

		desc := prometheus.NewDesc(
			prometheus.BuildFQName(namespace, subsystem, m.name),
			m.desc,
			labels,
			nil,
		)

		labelValues := []string{s.Name}
		if m.extraLabelValue != "" {
			labelValues = append(labelValues, m.extraLabelValue)
		}

		ch <- prometheus.MustNewConstMetric(
			desc,
			m.metricType,
			m.value,
			labelValues...,
		)
	}
}
<commit_msg>collector: remove commented-out import from bcache collector<commit_after>// Copyright 2017 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !nobcache

package collector

import (
	"fmt"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/procfs/bcache"
)

func init() {
	registerCollector("bcache", defaultEnabled, NewBcacheCollector)
}

// A bcacheCollector is a Collector which gathers metrics from Linux bcache.
type bcacheCollector struct {
	fs bcache.FS
}

// NewBcacheCollector returns a newly allocated bcacheCollector.
// It exposes a number of Linux bcache statistics.
func NewBcacheCollector() (Collector, error) {
	fs, err := bcache.NewFS(*sysPath)
	if err != nil {
		return nil, fmt.Errorf("failed to open sysfs: %v", err)
	}

	return &bcacheCollector{
		fs: fs,
	}, nil
}

// Update reads and exposes bcache stats.
// It implements the Collector interface.
func (c *bcacheCollector) Update(ch chan<- prometheus.Metric) error {
	stats, err := c.fs.Stats()
	if err != nil {
		return fmt.Errorf("failed to retrieve bcache stats: %v", err)
	}

	for _, s := range stats {
		c.updateBcacheStats(ch, s)
	}
	return nil
}

type bcacheMetric struct {
	name            string
	desc            string
	value           float64
	metricType      prometheus.ValueType
	extraLabel      []string
	extraLabelValue string
}

func bcachePeriodStatsToMetric(ps *bcache.PeriodStats, labelValue string) []bcacheMetric {
	label := []string{"backing_device"}

	metrics := []bcacheMetric{
		{
			name:            "bypassed_bytes_total",
			desc:            "Amount of IO (both reads and writes) that has bypassed the cache.",
			value:           float64(ps.Bypassed),
			metricType:      prometheus.CounterValue,
			extraLabel:      label,
			extraLabelValue: labelValue,
		},
		{
			name:            "cache_hits_total",
			desc:            "Hits counted per individual IO as bcache sees them.",
			value:           float64(ps.CacheHits),
			metricType:      prometheus.CounterValue,
			extraLabel:      label,
			extraLabelValue: labelValue,
		},
		{
			name:            "cache_misses_total",
			desc:            "Misses counted per individual IO as bcache sees them.",
			value:           float64(ps.CacheMisses),
			metricType:      prometheus.CounterValue,
			extraLabel:      label,
			extraLabelValue: labelValue,
		},
		{
			name:            "cache_bypass_hits_total",
			desc:            "Hits for IO intended to skip the cache.",
			value:           float64(ps.CacheBypassHits),
			metricType:      prometheus.CounterValue,
			extraLabel:      label,
			extraLabelValue: labelValue,
		},
		{
			name:            "cache_bypass_misses_total",
			desc:            "Misses for IO intended to skip the cache.",
			value:           float64(ps.CacheBypassMisses),
			metricType:      prometheus.CounterValue,
			extraLabel:      label,
			extraLabelValue: labelValue,
		},
		{
			name:            "cache_miss_collisions_total",
			desc:            "Instances where data insertion from cache miss raced with write (data already present).",
			value:           float64(ps.CacheMissCollisions),
			metricType:      prometheus.CounterValue,
			extraLabel:      label,
			extraLabelValue: labelValue,
		},
		{
			name:            "cache_readaheads_total",
			desc:            "Count of times readahead occurred.",
			value:           float64(ps.CacheReadaheads),
			metricType:      prometheus.CounterValue,
			extraLabel:      label,
			extraLabelValue: labelValue,
		},
	}
	return metrics
}

// UpdateBcacheStats collects statistics for one bcache ID.
func (c *bcacheCollector) updateBcacheStats(ch chan<- prometheus.Metric, s *bcache.Stats) {

	const (
		subsystem = "bcache"
	)

	var (
		devLabel   = []string{"uuid"}
		allMetrics []bcacheMetric
		metrics    []bcacheMetric
	)

	allMetrics = []bcacheMetric{
		// metrics in /sys/fs/bcache/<uuid>/
		{
			name:       "average_key_size_sectors",
			desc:       "Average data per key in the btree (sectors).",
			value:      float64(s.Bcache.AverageKeySize),
			metricType: prometheus.GaugeValue,
		},
		{
			name:       "btree_cache_size_bytes",
			desc:       "Amount of memory currently used by the btree cache.",
			value:      float64(s.Bcache.BtreeCacheSize),
			metricType: prometheus.GaugeValue,
		},
		{
			name:       "cache_available_percent",
			desc:       "Percentage of cache device without dirty data, usable for writeback (may contain clean cached data).",
			value:      float64(s.Bcache.CacheAvailablePercent),
			metricType: prometheus.GaugeValue,
		},
		{
			name:       "congested",
			desc:       "Congestion.",
			value:      float64(s.Bcache.Congested),
			metricType: prometheus.GaugeValue,
		},
		{
			name:       "root_usage_percent",
			desc:       "Percentage of the root btree node in use (tree depth increases if too high).",
			value:      float64(s.Bcache.RootUsagePercent),
			metricType: prometheus.GaugeValue,
		},
		{
			name:       "tree_depth",
			desc:       "Depth of the btree.",
			value:      float64(s.Bcache.TreeDepth),
			metricType: prometheus.GaugeValue,
		},
		// metrics in /sys/fs/bcache/<uuid>/internal/
		{
			name:       "active_journal_entries",
			desc:       "Number of journal entries that are newer than the index.",
			value:      float64(s.Bcache.Internal.ActiveJournalEntries),
			metricType: prometheus.GaugeValue,
		},
		{
			name:       "btree_nodes",
			desc:       "Total nodes in the btree.",
			value:      float64(s.Bcache.Internal.BtreeNodes),
			metricType: prometheus.GaugeValue,
		},
		{
			name:       "btree_read_average_duration_seconds",
			desc:       "Average btree read duration.",
			value:      float64(s.Bcache.Internal.BtreeReadAverageDurationNanoSeconds) * 1e-9,
			metricType: prometheus.GaugeValue,
		},
		{
			name:       "cache_read_races_total",
			desc:       "Counts instances where while data was being read from the cache, the bucket was reused and invalidated - i.e. where the pointer was stale after the read completed.",
			value:      float64(s.Bcache.Internal.CacheReadRaces),
			metricType: prometheus.CounterValue,
		},
	}

	for _, bdev := range s.Bdevs {
		// metrics in /sys/fs/bcache/<uuid>/<bdev>/
		metrics = []bcacheMetric{
			{
				name:            "dirty_data_bytes",
				desc:            "Amount of dirty data for this backing device in the cache.",
				value:           float64(bdev.DirtyData),
				metricType:      prometheus.GaugeValue,
				extraLabel:      []string{"backing_device"},
				extraLabelValue: bdev.Name,
			},
		}
		allMetrics = append(allMetrics, metrics...)

		// metrics in /sys/fs/bcache/<uuid>/<bdev>/stats_total
		metrics := bcachePeriodStatsToMetric(&bdev.Total, bdev.Name)
		allMetrics = append(allMetrics, metrics...)

	}

	for _, cache := range s.Caches {
		metrics = []bcacheMetric{
			// metrics in /sys/fs/bcache/<uuid>/<cache>/
			{
				name:            "io_errors",
				desc:            "Number of errors that have occurred, decayed by io_error_halflife.",
				value:           float64(cache.IOErrors),
				metricType:      prometheus.GaugeValue,
				extraLabel:      []string{"cache_device"},
				extraLabelValue: cache.Name,
			},
			{
				name:            "metadata_written_bytes_total",
				desc:            "Sum of all non data writes (btree writes and all other metadata).",
				value:           float64(cache.MetadataWritten),
				metricType:      prometheus.CounterValue,
				extraLabel:      []string{"cache_device"},
				extraLabelValue: cache.Name,
			},
			{
				name:            "written_bytes_total",
				desc:            "Sum of all data that has been written to the cache.",
				value:           float64(cache.Written),
				metricType:      prometheus.CounterValue,
				extraLabel:      []string{"cache_device"},
				extraLabelValue: cache.Name,
			},
			// metrics in /sys/fs/bcache/<uuid>/<cache>/priority_stats
			{
				name:            "priority_stats_unused_percent",
				desc:            "The percentage of the cache that doesn't contain any data.",
				value:           float64(cache.Priority.UnusedPercent),
				metricType:      prometheus.GaugeValue,
				extraLabel:      []string{"cache_device"},
				extraLabelValue: cache.Name,
			},
			{
				name:            "priority_stats_metadata_percent",
				desc:            "Bcache's metadata overhead.",
				value:           float64(cache.Priority.MetadataPercent),
				metricType:      prometheus.GaugeValue,
				extraLabel:      []string{"cache_device"},
				extraLabelValue: cache.Name,
			},
		}
		allMetrics = append(allMetrics, metrics...)
	}

	for _, m := range allMetrics {
		labels := append(devLabel, m.extraLabel...)

		desc := prometheus.NewDesc(
			prometheus.BuildFQName(namespace, subsystem, m.name),
			m.desc,
			labels,
			nil,
		)

		labelValues := []string{s.Name}
		if m.extraLabelValue != "" {
			labelValues = append(labelValues, m.extraLabelValue)
		}

		ch <- prometheus.MustNewConstMetric(
			desc,
			m.metricType,
			m.value,
			labelValues...,
		)
	}
}
<|endoftext|>


<commit_before>package collector

import (
	"github.com/golang/glog"
	"github.com/prometheus/client_golang/prometheus"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

var (
	oplogStatusCount = prometheus.NewGauge(prometheus.GaugeOpts{
		Namespace:	Namespace,
		Subsystem:	"replset_oplog",
		Name:		"items_total",
		Help:		"The total number of changes in the oplog",
	})
	oplogStatusHeadTimestamp = prometheus.NewGauge(prometheus.GaugeOpts{
		Namespace:	Namespace,
		Subsystem:	"replset_oplog",
		Name:		"head_timestamp",
		Help:		"The timestamp of the newest change in the oplog",
	})
	oplogStatusTailTimestamp = prometheus.NewGauge(prometheus.GaugeOpts{
		Namespace:	Namespace,
		Subsystem:	"replset_oplog",
		Name:		"tail_timestamp",
		Help:		"The timestamp of the oldest change in the oplog",
	})
	oplogStatusSizeBytes = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace:	Namespace,
		Subsystem:	"replset_oplog",
		Name:		"size_bytes",
		Help:		"Size of oplog in bytes",
	}, []string{"type"})
)

type OplogCollectionStats struct {
	Count		float64	`bson:"count"`
	Size		float64	`bson:"size"`
	StorageSize	float64 `bson:"storageSize"`
}

type OplogStatus struct {
	TailTimestamp	float64
	HeadTimestamp	float64
	CollectionStats	*OplogCollectionStats
}

// there's gotta be a better way to do this, but it works for now :/
func BsonMongoTimestampToUnix(timestamp bson.MongoTimestamp) float64 {
	return float64(timestamp >> 32)
}

func GetOplogTimestamp(session *mgo.Session, returnHead bool) (float64, error) {
	var sortBy string = "$natural"
	if returnHead {
		sortBy = "-$natural"
	}

	var err error
	var tries int    = 0
	var maxTries int = 2
	var result struct { Timestamp bson.MongoTimestamp `bson:"ts"` }
	for tries < maxTries {
		err = session.DB("local").C("oplog.rs").Find(nil).Sort(sortBy).Limit(1).One(&result)
		if err != nil {
			tries += 1
			time.Sleep(500 * time.Millisecond)
		} else {
			return BsonMongoTimestampToUnix(result.Timestamp), err
		}
	}

	return nil, err
}

func GetOplogCollectionStats(session *mgo.Session) (*OplogCollectionStats, error) {
	results := &OplogCollectionStats{}
	err := session.DB("local").Run(bson.M{ "collStats" : "oplog.rs" }, &results)
	return results, err
}

func (status *OplogStatus) Export(ch chan<- prometheus.Metric) {
	oplogStatusSizeBytes.WithLabelValues("current").Set(0)
	oplogStatusSizeBytes.WithLabelValues("storage").Set(0)
	if status.CollectionStats != nil {
		oplogStatusCount.Set(status.CollectionStats.Count)
		oplogStatusSizeBytes.WithLabelValues("current").Set(status.CollectionStats.Size)
		oplogStatusSizeBytes.WithLabelValues("storage").Set(status.CollectionStats.StorageSize)
	}
	if status.HeadTimestamp != nil && status.TailTimestamp != nil {
		oplogStatusHeadTimestamp.Set(status.HeadTimestamp)
		oplogStatusTailTimestamp.Set(status.TailTimestamp)
	}

	oplogStatusCount.Collect(ch)
	oplogStatusHeadTimestamp.Collect(ch)
	oplogStatusTailTimestamp.Collect(ch)
	oplogStatusSizeBytes.Collect(ch)
}

func (status *OplogStatus) Describe(ch chan<- *prometheus.Desc) {
	oplogStatusCount.Describe(ch)
	oplogStatusHeadTimestamp.Describe(ch)
	oplogStatusTailTimestamp.Describe(ch)
	oplogStatusSizeBytes.Describe(ch)
}

func GetOplogStatus(session *mgo.Session) *OplogStatus {
	oplogStatus := &OplogStatus{}
	collectionStats, err := GetOplogCollectionStats(session)
	if err != nil {
		glog.Error("Failed to get local.oplog_rs collection stats.")
		return nil
	}

	tailTimestamp, err := GetOplogTimestamp(session, false)
	headTimestamp, err := GetOplogTimestamp(session, true)
	if err != nil {
		glog.Error("Failed to get oplog head or tail timestamps.")
		return nil
	}

	oplogStatus.CollectionStats = collectionStats
	oplogStatus.TailTimestamp   = tailTimestamp
	oplogStatus.HeadTimestamp   = headTimestamp

	return oplogStatus
}
<commit_msg>missing time import<commit_after>package collector

import (
	"time"
	"github.com/golang/glog"
	"github.com/prometheus/client_golang/prometheus"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

var (
	oplogStatusCount = prometheus.NewGauge(prometheus.GaugeOpts{
		Namespace:	Namespace,
		Subsystem:	"replset_oplog",
		Name:		"items_total",
		Help:		"The total number of changes in the oplog",
	})
	oplogStatusHeadTimestamp = prometheus.NewGauge(prometheus.GaugeOpts{
		Namespace:	Namespace,
		Subsystem:	"replset_oplog",
		Name:		"head_timestamp",
		Help:		"The timestamp of the newest change in the oplog",
	})
	oplogStatusTailTimestamp = prometheus.NewGauge(prometheus.GaugeOpts{
		Namespace:	Namespace,
		Subsystem:	"replset_oplog",
		Name:		"tail_timestamp",
		Help:		"The timestamp of the oldest change in the oplog",
	})
	oplogStatusSizeBytes = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace:	Namespace,
		Subsystem:	"replset_oplog",
		Name:		"size_bytes",
		Help:		"Size of oplog in bytes",
	}, []string{"type"})
)

type OplogCollectionStats struct {
	Count		float64	`bson:"count"`
	Size		float64	`bson:"size"`
	StorageSize	float64 `bson:"storageSize"`
}

type OplogStatus struct {
	TailTimestamp	float64
	HeadTimestamp	float64
	CollectionStats	*OplogCollectionStats
}

// there's gotta be a better way to do this, but it works for now :/
func BsonMongoTimestampToUnix(timestamp bson.MongoTimestamp) float64 {
	return float64(timestamp >> 32)
}

func GetOplogTimestamp(session *mgo.Session, returnHead bool) (float64, error) {
	var sortBy string = "$natural"
	if returnHead {
		sortBy = "-$natural"
	}

	var err error
	var tries int    = 0
	var maxTries int = 2
	var result struct { Timestamp bson.MongoTimestamp `bson:"ts"` }
	for tries < maxTries {
		err = session.DB("local").C("oplog.rs").Find(nil).Sort(sortBy).Limit(1).One(&result)
		if err != nil {
			tries += 1
			time.Sleep(500 * time.Millisecond)
		} else {
			return BsonMongoTimestampToUnix(result.Timestamp), err
		}
	}

	return nil, err
}

func GetOplogCollectionStats(session *mgo.Session) (*OplogCollectionStats, error) {
	results := &OplogCollectionStats{}
	err := session.DB("local").Run(bson.M{ "collStats" : "oplog.rs" }, &results)
	return results, err
}

func (status *OplogStatus) Export(ch chan<- prometheus.Metric) {
	oplogStatusSizeBytes.WithLabelValues("current").Set(0)
	oplogStatusSizeBytes.WithLabelValues("storage").Set(0)
	if status.CollectionStats != nil {
		oplogStatusCount.Set(status.CollectionStats.Count)
		oplogStatusSizeBytes.WithLabelValues("current").Set(status.CollectionStats.Size)
		oplogStatusSizeBytes.WithLabelValues("storage").Set(status.CollectionStats.StorageSize)
	}
	if status.HeadTimestamp != nil && status.TailTimestamp != nil {
		oplogStatusHeadTimestamp.Set(status.HeadTimestamp)
		oplogStatusTailTimestamp.Set(status.TailTimestamp)
	}

	oplogStatusCount.Collect(ch)
	oplogStatusHeadTimestamp.Collect(ch)
	oplogStatusTailTimestamp.Collect(ch)
	oplogStatusSizeBytes.Collect(ch)
}

func (status *OplogStatus) Describe(ch chan<- *prometheus.Desc) {
	oplogStatusCount.Describe(ch)
	oplogStatusHeadTimestamp.Describe(ch)
	oplogStatusTailTimestamp.Describe(ch)
	oplogStatusSizeBytes.Describe(ch)
}

func GetOplogStatus(session *mgo.Session) *OplogStatus {
	oplogStatus := &OplogStatus{}
	collectionStats, err := GetOplogCollectionStats(session)
	if err != nil {
		glog.Error("Failed to get local.oplog_rs collection stats.")
		return nil
	}

	tailTimestamp, err := GetOplogTimestamp(session, false)
	headTimestamp, err := GetOplogTimestamp(session, true)
	if err != nil {
		glog.Error("Failed to get oplog head or tail timestamps.")
		return nil
	}

	oplogStatus.CollectionStats = collectionStats
	oplogStatus.TailTimestamp   = tailTimestamp
	oplogStatus.HeadTimestamp   = headTimestamp

	return oplogStatus
}
<|endoftext|>


<commit_before>package commands

import (
	"os"
	"path/filepath"
	"sync/atomic"

	"github.com/git-lfs/git-lfs/config"
	"github.com/git-lfs/git-lfs/errors"
	"github.com/git-lfs/git-lfs/git"
	"github.com/git-lfs/git-lfs/lfs"
	"github.com/git-lfs/git-lfs/tools"
	"github.com/spf13/cobra"
)

var (
	dedupFlags = struct {
		test bool
	}{}
	dedupStats = &struct {
		totalProcessedCount int64
		totalProcessedSize  int64
	}{}
)

func dedupTestCommand(*cobra.Command, []string) {
	requireInRepo()

	if supported, err := tools.CheckCloneFileSupported(cfg.TempDir()); err != nil || !supported {
		if err == nil {
			err = errors.New("Unknown reason.")
		}
		Exit("This system does not support deduplication. %s", err)
	}

	Print("OK: This platform and repository support file de-duplication.")
}

func dedupCommand(cmd *cobra.Command, args []string) {
	if dedupFlags.test {
		dedupTestCommand(cmd, args)
		return
	}

	requireInRepo()
	if gitDir, err := git.GitDir(); err != nil {
		ExitWithError(err)
	} else if supported, err := tools.CheckCloneFileSupported(gitDir); err != nil || !supported {
		Exit("This system does not support deduplication.")
	}

	if dirty, err := git.IsWorkingCopyDirty(); err != nil {
		ExitWithError(err)
	} else if dirty {
		Exit("Working tree is dirty. Please commit or reset your change.")
	}

	// We assume working tree is clean.
	gitScanner := lfs.NewGitScanner(config.New(), func(p *lfs.WrappedPointer, err error) {
		if err != nil {
			Exit("Could not scan for Git LFS tree: %s", err)
			return
		}

		if success, err := dedup(p); err != nil {
			Error("Skipped: %s (Size: %d)\n          %s", p.Name, p.Size, err)
		} else if !success {
			Error("Skipped: %s (Size: %d)", p.Name, p.Size)
		} else if success {
			Print("Success: %s (Size: %d)", p.Name, p.Size)

			atomic.AddInt64(&dedupStats.totalProcessedCount, 1)
			atomic.AddInt64(&dedupStats.totalProcessedSize, p.Size)
		}
	})
	defer gitScanner.Close()

	if err := gitScanner.ScanTree("HEAD"); err != nil {
		ExitWithError(err)
	}

	Print("\n\nSuccessfully finished.\n"+
		"  De-duplicated  size: %d bytes\n"+
		"                count: %d",
		dedupStats.totalProcessedSize,
		dedupStats.totalProcessedCount)
}

// dedup executes
// Precondition: working tree MUST clean. We can replace working tree files from mediafile safely.
func dedup(p *lfs.WrappedPointer) (success bool, err error) {
	// PRECONDITION, check ofs object exists or skip this file.
	if !cfg.LFSObjectExists(p.Oid, p.Size) { // Not exists,
		// Basically, this is not happens because executing 'git status' in `git.IsWorkingCopyDirty()` recover it.
		return false, errors.New("mediafile is not exist")
	}

	// DO de-dup
	// Gather original state
	originalStat, err := os.Stat(p.Name)
	if err != nil {
		return false, err
	}

	// Do clone
	srcFile := cfg.Filesystem().ObjectPathname(p.Oid)
	dstFile := filepath.Join(cfg.LocalWorkingDir(), p.Name)

	// Clone the file. This overwrites the destination if it exists.
	if ok, err := tools.CloneFileByPath(dstFile, srcFile); err != nil {
		return false, err
	} else if !ok {
		return false, errors.Errorf("unknown clone file error")
	}

	// Recover original state
	if err := os.Chmod(dstFile, originalStat.Mode()); err != nil {
		return false, err
	}

	return true, nil
}

func init() {
	RegisterCommand("dedup", dedupCommand, func(cmd *cobra.Command) {
		cmd.Flags().BoolVarP(&dedupFlags.test, "test", "t", false, "test")
	})
}
<commit_msg>command/command_dedup.go: exit if extentions exist<commit_after>package commands

import (
	"os"
	"path/filepath"
	"sync/atomic"

	"github.com/git-lfs/git-lfs/config"
	"github.com/git-lfs/git-lfs/errors"
	"github.com/git-lfs/git-lfs/git"
	"github.com/git-lfs/git-lfs/lfs"
	"github.com/git-lfs/git-lfs/tools"
	"github.com/spf13/cobra"
)

var (
	dedupFlags = struct {
		test bool
	}{}
	dedupStats = &struct {
		totalProcessedCount int64
		totalProcessedSize  int64
	}{}
)

func dedupTestCommand(*cobra.Command, []string) {
	requireInRepo()

	if supported, err := tools.CheckCloneFileSupported(cfg.TempDir()); err != nil || !supported {
		if err == nil {
			err = errors.New("Unknown reason.")
		}
		Exit("This system does not support deduplication. %s", err)
	}

	if len(cfg.Extensions()) > 0 {
		Exit("This platform supports file de-duplication, however, Git LFS extensions are configured and therefore de-duplication can not be used.")
	}

	Print("OK: This platform and repository support file de-duplication.")
}

func dedupCommand(cmd *cobra.Command, args []string) {
	if dedupFlags.test {
		dedupTestCommand(cmd, args)
		return
	}

	requireInRepo()
	if gitDir, err := git.GitDir(); err != nil {
		ExitWithError(err)
	} else if supported, err := tools.CheckCloneFileSupported(gitDir); err != nil || !supported {
		Exit("This system does not support deduplication.")
	}

	if len(cfg.Extensions()) > 0 {
		Exit("This platform supports file de-duplication, however, Git LFS extensions are configured and therefore de-duplication can not be used.")
	}

	if dirty, err := git.IsWorkingCopyDirty(); err != nil {
		ExitWithError(err)
	} else if dirty {
		Exit("Working tree is dirty. Please commit or reset your change.")
	}

	// We assume working tree is clean.
	gitScanner := lfs.NewGitScanner(config.New(), func(p *lfs.WrappedPointer, err error) {
		if err != nil {
			Exit("Could not scan for Git LFS tree: %s", err)
			return
		}

		if success, err := dedup(p); err != nil {
			Error("Skipped: %s (Size: %d)\n          %s", p.Name, p.Size, err)
		} else if !success {
			Error("Skipped: %s (Size: %d)", p.Name, p.Size)
		} else if success {
			Print("Success: %s (Size: %d)", p.Name, p.Size)

			atomic.AddInt64(&dedupStats.totalProcessedCount, 1)
			atomic.AddInt64(&dedupStats.totalProcessedSize, p.Size)
		}
	})
	defer gitScanner.Close()

	if err := gitScanner.ScanTree("HEAD"); err != nil {
		ExitWithError(err)
	}

	Print("\n\nSuccessfully finished.\n"+
		"  De-duplicated  size: %d bytes\n"+
		"                count: %d",
		dedupStats.totalProcessedSize,
		dedupStats.totalProcessedCount)
}

// dedup executes
// Precondition: working tree MUST clean. We can replace working tree files from mediafile safely.
func dedup(p *lfs.WrappedPointer) (success bool, err error) {
	// PRECONDITION, check ofs object exists or skip this file.
	if !cfg.LFSObjectExists(p.Oid, p.Size) { // Not exists,
		// Basically, this is not happens because executing 'git status' in `git.IsWorkingCopyDirty()` recover it.
		return false, errors.New("mediafile is not exist")
	}

	// DO de-dup
	// Gather original state
	originalStat, err := os.Stat(p.Name)
	if err != nil {
		return false, err
	}

	// Do clone
	srcFile := cfg.Filesystem().ObjectPathname(p.Oid)
	dstFile := filepath.Join(cfg.LocalWorkingDir(), p.Name)

	// Clone the file. This overwrites the destination if it exists.
	if ok, err := tools.CloneFileByPath(dstFile, srcFile); err != nil {
		return false, err
	} else if !ok {
		return false, errors.Errorf("unknown clone file error")
	}

	// Recover original state
	if err := os.Chmod(dstFile, originalStat.Mode()); err != nil {
		return false, err
	}

	return true, nil
}

func init() {
	RegisterCommand("dedup", dedupCommand, func(cmd *cobra.Command) {
		cmd.Flags().BoolVarP(&dedupFlags.test, "test", "t", false, "test")
	})
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"os/exec"
	"github.com/dotcloud/docker/pkg/beam"
	"github.com/dotcloud/docker/pkg/beam/data"
	"github.com/dotcloud/docker/pkg/term"
	"text/template"
	"fmt"
	"sync"
	"os"
	"strings"
	"path"
	"bufio"
	"net"
	"net/url"
)

func CmdLogger(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	if err := os.MkdirAll("logs", 0700); err != nil {
		fmt.Fprintf(stderr, "%v\n", err)
		return
	}
	var tasks sync.WaitGroup
	defer tasks.Wait()
	var n int = 1
	r := beam.NewRouter(out)
	r.NewRoute().HasAttachment().KeyStartsWith("cmd", "log").Handler(func (payload []byte, attachment *os.File) error {
		tasks.Add(1)
		go func(n int) {
			defer tasks.Done()
			defer attachment.Close()
			var streamname string
			if cmd := data.Message(payload).Get("cmd"); len(cmd) == 1 || cmd[1] == "stdout" {
				streamname = "stdout"
			} else {
				streamname = cmd[1]
			}
			if fromcmd := data.Message(payload).Get("fromcmd"); len(fromcmd) != 0 {
				streamname = fmt.Sprintf("%s-%s", strings.Replace(strings.Join(fromcmd, "_"), "/", "_", -1), streamname)
			}
			logfile, err := os.OpenFile(path.Join("logs", fmt.Sprintf("%d-%s", n, streamname)), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0700)
			if err != nil {
				fmt.Fprintf(stderr, "%v\n", err)
				return
			}
			defer logfile.Close()
			io.Copy(logfile, attachment)
			logfile.Sync()
		}(n)
		n++
		return nil
	}).Tee(out)
	if _, err := beam.Copy(r, in); err != nil {
		fmt.Fprintf(stderr, "%v\n", err)
		return
	}
}

func CmdRender(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	if len(args) != 2 {
		fmt.Fprintf(stderr, "Usage: %s FORMAT\n", args[0])
		out.Send(data.Empty().Set("status", "1").Bytes(), nil)
		return
	}
	txt := args[1]
	if !strings.HasSuffix(txt, "\n") {
		txt += "\n"
	}
	t := template.Must(template.New("render").Parse(txt))
	for {
		payload, attachment, err := in.Receive()
		if err != nil {
			return
		}
		msg, err := data.Decode(string(payload))
		if err != nil {
			fmt.Fprintf(stderr, "decode error: %v\n")
		}
		if err := t.Execute(stdout, msg); err != nil {
			fmt.Fprintf(stderr, "rendering error: %v\n", err)
			out.Send(data.Empty().Set("status", "1").Bytes(), nil)
			return
		}
		if err := out.Send(payload, attachment); err != nil {
			return
		}
	}
}

func CmdDevnull(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	for {
		_, attachment, err := in.Receive()
		if err != nil {
			return
		}
		if attachment != nil {
			attachment.Close()
		}
	}
}

func CmdPrompt(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	if len(args) < 2 {
		fmt.Fprintf(stderr, "usage: %s PROMPT...\n", args[0])
		return
	}
	if !term.IsTerminal(0) {
		fmt.Fprintf(stderr, "can't prompt: no tty available...\n")
		return
	}
	fmt.Printf("%s: ", strings.Join(args[1:], " "))
	oldState, _ := term.SaveState(0)
	term.DisableEcho(0, oldState)
	line, _, err := bufio.NewReader(os.Stdin).ReadLine()
	if err != nil {
		fmt.Fprintln(stderr, err.Error())
		return
	}
	val := string(line)
	fmt.Printf("\n")
	term.RestoreTerminal(0, oldState)
	out.Send(data.Empty().Set("fromcmd", args...).Set("value", val).Bytes(), nil)
}

func CmdStdio(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	var tasks sync.WaitGroup
	defer tasks.Wait()

	r := beam.NewRouter(out)
	r.NewRoute().HasAttachment().KeyStartsWith("cmd", "log").Handler(func(payload []byte, attachment *os.File) error {
		tasks.Add(1)
		go func() {
			defer tasks.Done()
			defer attachment.Close()
			io.Copy(os.Stdout, attachment)
			attachment.Close()
		}()
		return nil
	}).Tee(out)

	if _, err := beam.Copy(r, in); err != nil {
		Fatal(err)
		fmt.Fprintf(stderr, "%v\n", err)
		return
	}
}

func CmdEcho(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	fmt.Fprintln(stdout, strings.Join(args[1:], " "))
}

func CmdPass(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	for {
		payload, attachment, err := in.Receive()
		if err != nil {
			return
		}
		if err := out.Send(payload, attachment); err != nil {
			if attachment != nil {
				attachment.Close()
			}
			return
		}
	}
}

func CmdIn(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	os.Chdir(args[1])
	GetHandler("pass")([]string{"pass"}, stdout, stderr, in, out)
}

func CmdExec(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	cmd := exec.Command(args[1], args[2:]...)
	cmd.Stdout = stdout
	cmd.Stderr = stderr
	cmd.Stdin = os.Stdin
	execErr := cmd.Run()
	var status string
	if execErr != nil {
		status = execErr.Error()
	} else {
		status = "ok"
	}
	out.Send(data.Empty().Set("status", status).Set("cmd", args...).Bytes(), nil)
}

func CmdTrace(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	for {
		p, a, err := in.Receive()
		if err != nil {
			return
		}
		var msg string
		if pretty := data.Message(string(p)).Pretty(); pretty != "" {
			msg = pretty
		} else {
			msg = string(p)
		}
		if a != nil {
			msg = fmt.Sprintf("%s [%d]", msg, a.Fd())
		}
		fmt.Printf("===> %s\n", msg)
		out.Send(p, a)
	}
}

func CmdEmit(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	out.Send(data.Parse(args[1:]).Bytes(), nil)
}

func CmdPrint(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	for {
		payload, a, err := in.Receive()
		if err != nil {
			return
		}
		// Skip commands
		if a != nil && data.Message(payload).Get("cmd") == nil {
			dup, err := beam.SendPipe(out, payload)
			if err != nil {
				a.Close()
				return
			}
			io.Copy(io.MultiWriter(os.Stdout, dup), a)
			dup.Close()
		} else {
			if err := out.Send(payload, a); err != nil {
				return
			}
		}
	}
}

func CmdMultiprint(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	var tasks sync.WaitGroup
	for {
		payload, a, err := in.Receive()
		if err != nil {
			return
		}
		if a != nil {
			tasks.Add(1)
			go func(payload []byte, attachment *os.File) {
				defer tasks.Done()
				msg := data.Message(string(payload))
				input := bufio.NewScanner(attachment)
				for input.Scan() {
					fmt.Printf("[%s] %s\n", msg.Pretty(), input.Text())
				}
			}(payload, a)
		}
	}
	tasks.Wait()
}

func CmdListen(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	if len(args) != 2 {
		out.Send(data.Empty().Set("status", "1").Set("message", "wrong number of arguments").Bytes(), nil)
		return
	}
	u, err := url.Parse(args[1])
	if err != nil {
		out.Send(data.Empty().Set("status", "1").Set("message", err.Error()).Bytes(), nil)
		return
	}
	l, err := net.Listen(u.Scheme, u.Host)
	if err != nil {
		out.Send(data.Empty().Set("status", "1").Set("message", err.Error()).Bytes(), nil)
		return
	}
	for {
		conn, err := l.Accept()
		if err != nil {
			out.Send(data.Empty().Set("status", "1").Set("message", err.Error()).Bytes(), nil)
			return
		}
		f, err := connToFile(conn)
		if err != nil {
			conn.Close()
			continue
		}
		out.Send(data.Empty().Set("type", "socket").Set("remoteaddr", conn.RemoteAddr().String()).Bytes(), f)
	}
}

func CmdBeamsend(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	if len(args) < 2 {
		if err := out.Send(data.Empty().Set("status", "1").Set("message", "wrong number of arguments").Bytes(), nil); err != nil {
			Fatal(err)
		}
		return
	}
	var connector func(string) (chan net.Conn, error)
	connector = dialer
	connections, err := connector(args[1])
	if err != nil {
		out.Send(data.Empty().Set("status", "1").Set("message", err.Error()).Bytes(), nil)
		return
	}
	// Copy in to conn
	SendToConn(connections, in)
}

func CmdBeamreceive(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	if len(args) != 2 {
		if err := out.Send(data.Empty().Set("status", "1").Set("message", "wrong number of arguments").Bytes(), nil); err != nil {
			Fatal(err)
		}
		return
	}
	var connector func(string) (chan net.Conn, error)
	connector = listener
	connections, err := connector(args[1])
	if err != nil {
		out.Send(data.Empty().Set("status", "1").Set("message", err.Error()).Bytes(), nil)
		return
	}
	// Copy in to conn
	ReceiveFromConn(connections, out)
}

func CmdConnect(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	if len(args) != 2 {
		out.Send(data.Empty().Set("status", "1").Set("message", "wrong number of arguments").Bytes(), nil)
		return
	}
	u, err := url.Parse(args[1])
	if err != nil {
		out.Send(data.Empty().Set("status", "1").Set("message", err.Error()).Bytes(), nil)
		return
	}
	var tasks sync.WaitGroup
	for {
		_, attachment, err := in.Receive()
		if err != nil {
			break
		}
		if attachment == nil {
			continue
		}
		Logf("connecting to %s/%s\n", u.Scheme, u.Host)
		conn, err := net.Dial(u.Scheme, u.Host)
		if err != nil {
			out.Send(data.Empty().Set("cmd", "msg", "connect error: " + err.Error()).Bytes(), nil)
			return
		}
		out.Send(data.Empty().Set("cmd", "msg", "connection established").Bytes(), nil)
		tasks.Add(1)
		go func(attachment *os.File, conn net.Conn) {
			defer tasks.Done()
			// even when successful, conn.File() returns a duplicate,
			// so we must close the original
			var iotasks sync.WaitGroup
			iotasks.Add(2)
			go func(attachment *os.File, conn net.Conn) {
				defer iotasks.Done()
				io.Copy(attachment, conn)
			}(attachment, conn)
			go func(attachment *os.File, conn net.Conn) {
				defer iotasks.Done()
				io.Copy(conn, attachment)
			}(attachment, conn)
			iotasks.Wait()
			conn.Close()
			attachment.Close()
		}(attachment, conn)
	}
	tasks.Wait()
}

func CmdOpenfile(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	for _, name := range args {
		f, err := os.Open(name)
		if err != nil {
			continue
		}
		if err := out.Send(data.Empty().Set("path", name).Set("type", "file").Bytes(), f); err != nil {
			f.Close()
		}
	}
}
<commit_msg>beam/examples/beamsh: use beam.Router to simplify 'trace'<commit_after>package main

import (
	"io"
	"os/exec"
	"github.com/dotcloud/docker/pkg/beam"
	"github.com/dotcloud/docker/pkg/beam/data"
	"github.com/dotcloud/docker/pkg/term"
	"text/template"
	"fmt"
	"sync"
	"os"
	"strings"
	"path"
	"bufio"
	"net"
	"net/url"
)

func CmdLogger(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	if err := os.MkdirAll("logs", 0700); err != nil {
		fmt.Fprintf(stderr, "%v\n", err)
		return
	}
	var tasks sync.WaitGroup
	defer tasks.Wait()
	var n int = 1
	r := beam.NewRouter(out)
	r.NewRoute().HasAttachment().KeyStartsWith("cmd", "log").Handler(func (payload []byte, attachment *os.File) error {
		tasks.Add(1)
		go func(n int) {
			defer tasks.Done()
			defer attachment.Close()
			var streamname string
			if cmd := data.Message(payload).Get("cmd"); len(cmd) == 1 || cmd[1] == "stdout" {
				streamname = "stdout"
			} else {
				streamname = cmd[1]
			}
			if fromcmd := data.Message(payload).Get("fromcmd"); len(fromcmd) != 0 {
				streamname = fmt.Sprintf("%s-%s", strings.Replace(strings.Join(fromcmd, "_"), "/", "_", -1), streamname)
			}
			logfile, err := os.OpenFile(path.Join("logs", fmt.Sprintf("%d-%s", n, streamname)), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0700)
			if err != nil {
				fmt.Fprintf(stderr, "%v\n", err)
				return
			}
			defer logfile.Close()
			io.Copy(logfile, attachment)
			logfile.Sync()
		}(n)
		n++
		return nil
	}).Tee(out)
	if _, err := beam.Copy(r, in); err != nil {
		fmt.Fprintf(stderr, "%v\n", err)
		return
	}
}

func CmdRender(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	if len(args) != 2 {
		fmt.Fprintf(stderr, "Usage: %s FORMAT\n", args[0])
		out.Send(data.Empty().Set("status", "1").Bytes(), nil)
		return
	}
	txt := args[1]
	if !strings.HasSuffix(txt, "\n") {
		txt += "\n"
	}
	t := template.Must(template.New("render").Parse(txt))
	for {
		payload, attachment, err := in.Receive()
		if err != nil {
			return
		}
		msg, err := data.Decode(string(payload))
		if err != nil {
			fmt.Fprintf(stderr, "decode error: %v\n")
		}
		if err := t.Execute(stdout, msg); err != nil {
			fmt.Fprintf(stderr, "rendering error: %v\n", err)
			out.Send(data.Empty().Set("status", "1").Bytes(), nil)
			return
		}
		if err := out.Send(payload, attachment); err != nil {
			return
		}
	}
}

func CmdDevnull(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	for {
		_, attachment, err := in.Receive()
		if err != nil {
			return
		}
		if attachment != nil {
			attachment.Close()
		}
	}
}

func CmdPrompt(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	if len(args) < 2 {
		fmt.Fprintf(stderr, "usage: %s PROMPT...\n", args[0])
		return
	}
	if !term.IsTerminal(0) {
		fmt.Fprintf(stderr, "can't prompt: no tty available...\n")
		return
	}
	fmt.Printf("%s: ", strings.Join(args[1:], " "))
	oldState, _ := term.SaveState(0)
	term.DisableEcho(0, oldState)
	line, _, err := bufio.NewReader(os.Stdin).ReadLine()
	if err != nil {
		fmt.Fprintln(stderr, err.Error())
		return
	}
	val := string(line)
	fmt.Printf("\n")
	term.RestoreTerminal(0, oldState)
	out.Send(data.Empty().Set("fromcmd", args...).Set("value", val).Bytes(), nil)
}

func CmdStdio(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	var tasks sync.WaitGroup
	defer tasks.Wait()

	r := beam.NewRouter(out)
	r.NewRoute().HasAttachment().KeyStartsWith("cmd", "log").Handler(func(payload []byte, attachment *os.File) error {
		tasks.Add(1)
		go func() {
			defer tasks.Done()
			defer attachment.Close()
			io.Copy(os.Stdout, attachment)
			attachment.Close()
		}()
		return nil
	}).Tee(out)

	if _, err := beam.Copy(r, in); err != nil {
		Fatal(err)
		fmt.Fprintf(stderr, "%v\n", err)
		return
	}
}

func CmdEcho(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	fmt.Fprintln(stdout, strings.Join(args[1:], " "))
}

func CmdPass(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	for {
		payload, attachment, err := in.Receive()
		if err != nil {
			return
		}
		if err := out.Send(payload, attachment); err != nil {
			if attachment != nil {
				attachment.Close()
			}
			return
		}
	}
}

func CmdIn(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	os.Chdir(args[1])
	GetHandler("pass")([]string{"pass"}, stdout, stderr, in, out)
}

func CmdExec(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	cmd := exec.Command(args[1], args[2:]...)
	cmd.Stdout = stdout
	cmd.Stderr = stderr
	cmd.Stdin = os.Stdin
	execErr := cmd.Run()
	var status string
	if execErr != nil {
		status = execErr.Error()
	} else {
		status = "ok"
	}
	out.Send(data.Empty().Set("status", status).Set("cmd", args...).Bytes(), nil)
}

func CmdTrace(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	r := beam.NewRouter(out)
	r.NewRoute().All().Handler(func(payload []byte, attachment *os.File) error {
		fmt.Printf("===> %s\n", beam.MsgDesc(payload, attachment))
		out.Send(payload, attachment)
		return nil
	})
	beam.Copy(r, in)
}

func CmdEmit(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	out.Send(data.Parse(args[1:]).Bytes(), nil)
}

func CmdPrint(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	for {
		payload, a, err := in.Receive()
		if err != nil {
			return
		}
		// Skip commands
		if a != nil && data.Message(payload).Get("cmd") == nil {
			dup, err := beam.SendPipe(out, payload)
			if err != nil {
				a.Close()
				return
			}
			io.Copy(io.MultiWriter(os.Stdout, dup), a)
			dup.Close()
		} else {
			if err := out.Send(payload, a); err != nil {
				return
			}
		}
	}
}

func CmdMultiprint(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	var tasks sync.WaitGroup
	for {
		payload, a, err := in.Receive()
		if err != nil {
			return
		}
		if a != nil {
			tasks.Add(1)
			go func(payload []byte, attachment *os.File) {
				defer tasks.Done()
				msg := data.Message(string(payload))
				input := bufio.NewScanner(attachment)
				for input.Scan() {
					fmt.Printf("[%s] %s\n", msg.Pretty(), input.Text())
				}
			}(payload, a)
		}
	}
	tasks.Wait()
}

func CmdListen(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	if len(args) != 2 {
		out.Send(data.Empty().Set("status", "1").Set("message", "wrong number of arguments").Bytes(), nil)
		return
	}
	u, err := url.Parse(args[1])
	if err != nil {
		out.Send(data.Empty().Set("status", "1").Set("message", err.Error()).Bytes(), nil)
		return
	}
	l, err := net.Listen(u.Scheme, u.Host)
	if err != nil {
		out.Send(data.Empty().Set("status", "1").Set("message", err.Error()).Bytes(), nil)
		return
	}
	for {
		conn, err := l.Accept()
		if err != nil {
			out.Send(data.Empty().Set("status", "1").Set("message", err.Error()).Bytes(), nil)
			return
		}
		f, err := connToFile(conn)
		if err != nil {
			conn.Close()
			continue
		}
		out.Send(data.Empty().Set("type", "socket").Set("remoteaddr", conn.RemoteAddr().String()).Bytes(), f)
	}
}

func CmdBeamsend(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	if len(args) < 2 {
		if err := out.Send(data.Empty().Set("status", "1").Set("message", "wrong number of arguments").Bytes(), nil); err != nil {
			Fatal(err)
		}
		return
	}
	var connector func(string) (chan net.Conn, error)
	connector = dialer
	connections, err := connector(args[1])
	if err != nil {
		out.Send(data.Empty().Set("status", "1").Set("message", err.Error()).Bytes(), nil)
		return
	}
	// Copy in to conn
	SendToConn(connections, in)
}

func CmdBeamreceive(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	if len(args) != 2 {
		if err := out.Send(data.Empty().Set("status", "1").Set("message", "wrong number of arguments").Bytes(), nil); err != nil {
			Fatal(err)
		}
		return
	}
	var connector func(string) (chan net.Conn, error)
	connector = listener
	connections, err := connector(args[1])
	if err != nil {
		out.Send(data.Empty().Set("status", "1").Set("message", err.Error()).Bytes(), nil)
		return
	}
	// Copy in to conn
	ReceiveFromConn(connections, out)
}

func CmdConnect(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	if len(args) != 2 {
		out.Send(data.Empty().Set("status", "1").Set("message", "wrong number of arguments").Bytes(), nil)
		return
	}
	u, err := url.Parse(args[1])
	if err != nil {
		out.Send(data.Empty().Set("status", "1").Set("message", err.Error()).Bytes(), nil)
		return
	}
	var tasks sync.WaitGroup
	for {
		_, attachment, err := in.Receive()
		if err != nil {
			break
		}
		if attachment == nil {
			continue
		}
		Logf("connecting to %s/%s\n", u.Scheme, u.Host)
		conn, err := net.Dial(u.Scheme, u.Host)
		if err != nil {
			out.Send(data.Empty().Set("cmd", "msg", "connect error: " + err.Error()).Bytes(), nil)
			return
		}
		out.Send(data.Empty().Set("cmd", "msg", "connection established").Bytes(), nil)
		tasks.Add(1)
		go func(attachment *os.File, conn net.Conn) {
			defer tasks.Done()
			// even when successful, conn.File() returns a duplicate,
			// so we must close the original
			var iotasks sync.WaitGroup
			iotasks.Add(2)
			go func(attachment *os.File, conn net.Conn) {
				defer iotasks.Done()
				io.Copy(attachment, conn)
			}(attachment, conn)
			go func(attachment *os.File, conn net.Conn) {
				defer iotasks.Done()
				io.Copy(conn, attachment)
			}(attachment, conn)
			iotasks.Wait()
			conn.Close()
			attachment.Close()
		}(attachment, conn)
	}
	tasks.Wait()
}

func CmdOpenfile(args []string, stdout, stderr io.Writer, in beam.Receiver, out beam.Sender) {
	for _, name := range args {
		f, err := os.Open(name)
		if err != nil {
			continue
		}
		if err := out.Send(data.Empty().Set("path", name).Set("type", "file").Bytes(), f); err != nil {
			f.Close()
		}
	}
}
<|endoftext|>


<commit_before>package commands

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/github/git-lfs/lfs"
	"github.com/github/git-lfs/vendor/_nuts/github.com/spf13/cobra"
)

var (
	trackCmd = &cobra.Command{
		Use:   "track",
		Short: "Manipulate .gitattributes",
		Run:   trackCommand,
	}
)

func trackCommand(cmd *cobra.Command, args []string) {
	if lfs.LocalGitDir == "" {
		Print("Not a git repository.")
		os.Exit(128)
	}

	lfs.InstallHooks(false)
	knownPaths := findPaths()

	if len(args) == 0 {
		Print("Listing tracked paths")
		for _, t := range knownPaths {
			Print("    %s (%s)", t.Path, t.Source)
		}
		return
	}

	addTrailingLinebreak := needsTrailingLinebreak(".gitattributes")
	attributesFile, err := os.OpenFile(".gitattributes", os.O_RDWR|os.O_APPEND|os.O_CREATE, 0660)
	if err != nil {
		Print("Error opening .gitattributes file")
		return
	}
	defer attributesFile.Close()

	if addTrailingLinebreak {
		if _, err := attributesFile.WriteString("\n"); err != nil {
			Print("Error writing to .gitattributes")
		}
	}

ArgsLoop:
	for _, t := range args {
		for _, k := range knownPaths {
			if t == k.Path {
				Print("%s already supported", t)
				continue ArgsLoop
			}
		}

		encodedArg := strings.Replace(t, " ", "[[:space:]]", -1)
		_, err := attributesFile.WriteString(fmt.Sprintf("%s filter=lfs diff=lfs merge=lfs -crlf\n", encodedArg))
		if err != nil {
			Print("Error adding path %s", t)
			continue
		}
		Print("Tracking %s", t)
	}
}

type mediaPath struct {
	Path   string
	Source string
}

func findPaths() []mediaPath {
	paths := make([]mediaPath, 0)
	wd, _ := os.Getwd()

	for _, path := range findAttributeFiles() {
		attributes, err := os.Open(path)
		if err != nil {
			continue
		}

		scanner := bufio.NewScanner(attributes)
		for scanner.Scan() {
			line := scanner.Text()
			if strings.Contains(line, "filter=lfs") {
				fields := strings.Fields(line)
				relPath, _ := filepath.Rel(wd, path)
				paths = append(paths, mediaPath{Path: fields[0], Source: relPath})
			}
		}
	}

	return paths
}

func findAttributeFiles() []string {
	paths := make([]string, 0)

	repoAttributes := filepath.Join(lfs.LocalGitDir, "info", "attributes")
	if info, err := os.Stat(repoAttributes); err == nil && !info.IsDir() {
		paths = append(paths, repoAttributes)
	}

	filepath.Walk(lfs.LocalWorkingDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if !info.IsDir() && (filepath.Base(path) == ".gitattributes") {
			paths = append(paths, path)
		}
		return nil
	})

	return paths
}

func needsTrailingLinebreak(filename string) bool {
	file, err := os.Open(filename)
	if err != nil {
		return false
	}
	defer file.Close()

	buf := make([]byte, 16384)
	bytesRead := 0
	for {
		n, err := file.Read(buf)
		if err == io.EOF {
			break
		} else if err != nil {
			return false
		}
		bytesRead = n
	}

	return !strings.HasSuffix(string(buf[0:bytesRead]), "\n")
}

func init() {
	RootCmd.AddCommand(trackCmd)
}
<commit_msg>アアー アアアア アーアア<commit_after>package commands

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/github/git-lfs/lfs"
	"github.com/github/git-lfs/vendor/_nuts/github.com/spf13/cobra"
)

var (
	trackCmd = &cobra.Command{
		Use:   "track",
		Short: "Manipulate .gitattributes",
		Run:   trackCommand,
	}
)

func trackCommand(cmd *cobra.Command, args []string) {
	if lfs.LocalGitDir == "" {
		Print("Not a git repository.")
		os.Exit(128)
	}

	lfs.InstallHooks(false)
	knownPaths := findPaths()

	if len(args) == 0 {
		Print("Listing tracked paths")
		for _, t := range knownPaths {
			Print("    %s (%s)", t.Path, t.Source)
		}
		return
	}

	addTrailingLinebreak := needsTrailingLinebreak(".gitattributes")
	attributesFile, err := os.OpenFile(".gitattributes", os.O_RDWR|os.O_APPEND|os.O_CREATE, 0660)
	if err != nil {
		Print("Error opening .gitattributes file")
		return
	}
	defer attributesFile.Close()

	if addTrailingLinebreak {
		if _, err := attributesFile.WriteString("\n"); err != nil {
			Print("Error writing to .gitattributes")
		}
	}

	wd, _ := os.Getwd()

ArgsLoop:
	for _, t := range args {
		absT, _ := absRelPath(t, wd)
		for _, k := range knownPaths {
			absK, _ := absRelPath(k.Path, filepath.Join(wd, filepath.Dir(k.Source)))
			if absT == absK {
				Print("%s already supported", t)
				continue ArgsLoop
			}
		}

		encodedArg := strings.Replace(t, " ", "[[:space:]]", -1)
		_, err := attributesFile.WriteString(fmt.Sprintf("%s filter=lfs diff=lfs merge=lfs -crlf\n", encodedArg))
		if err != nil {
			Print("Error adding path %s", t)
			continue
		}
		Print("Tracking %s", t)
	}
}

type mediaPath struct {
	Path   string
	Source string
}

func findPaths() []mediaPath {
	paths := make([]mediaPath, 0)
	wd, _ := os.Getwd()

	for _, path := range findAttributeFiles() {
		attributes, err := os.Open(path)
		if err != nil {
			continue
		}

		scanner := bufio.NewScanner(attributes)
		for scanner.Scan() {
			line := scanner.Text()
			if strings.Contains(line, "filter=lfs") {
				fields := strings.Fields(line)
				relPath, _ := filepath.Rel(wd, path)
				paths = append(paths, mediaPath{Path: fields[0], Source: relPath})
			}
		}
	}

	return paths
}

func findAttributeFiles() []string {
	paths := make([]string, 0)

	repoAttributes := filepath.Join(lfs.LocalGitDir, "info", "attributes")
	if info, err := os.Stat(repoAttributes); err == nil && !info.IsDir() {
		paths = append(paths, repoAttributes)
	}

	filepath.Walk(lfs.LocalWorkingDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if !info.IsDir() && (filepath.Base(path) == ".gitattributes") {
			paths = append(paths, path)
		}
		return nil
	})

	return paths
}

func needsTrailingLinebreak(filename string) bool {
	file, err := os.Open(filename)
	if err != nil {
		return false
	}
	defer file.Close()

	buf := make([]byte, 16384)
	bytesRead := 0
	for {
		n, err := file.Read(buf)
		if err == io.EOF {
			break
		} else if err != nil {
			return false
		}
		bytesRead = n
	}

	return !strings.HasSuffix(string(buf[0:bytesRead]), "\n")
}

// absRelPath takes a path and a working directory and
// returns an absolute and a relative representation of path based on the working directory
func absRelPath(path, wd string) (string, string) {
	if filepath.IsAbs(path) {
		relPath, _ := filepath.Rel(wd, path)
		return path, relPath
	}

	absPath := filepath.Join(wd, path)
	return absPath, path
}

func init() {
	RootCmd.AddCommand(trackCmd)
}
<|endoftext|>


<commit_before><commit_msg>Also parse elements at script tag level<commit_after><|endoftext|>


<commit_before>package dht

import (
	"fmt"
	"log"
	"net"
	"net/http"
	"time"

	"github.com/hashicorp/consul/api"
	"github.com/stvp/rendezvous"
)

const (
	checkInterval = 5 * time.Second
	pollWait      = time.Second
)

func newCheckListenerAndServer() (listener net.Listener, server *http.Server, err error) {
	listener, err = net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		return listener, nil, err
	}

	server = &http.Server{
		ReadTimeout:  time.Second,
		WriteTimeout: time.Second,
		Handler: http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
			fmt.Fprintf(resp, "OK")
		}),
	}

	// When the listener is closed, this goroutine returns.
	go server.Serve(listener)

	return listener, server, err
}

// Node is a single node in a distributed hash table, coordinated using
// services registered in Consul. Key membership is determined using rendezvous
// hashing to ensure even distribution of keys and minimal key membership
// changes when a Node fails or otherwise leaves the hash table.
//
// Errors encountered when making blocking GET requests to the Consul agent API
// are logged using the log package.
type Node struct {
	// Consul
	serviceName string
	serviceID   string
	consul      *api.Client

	// HTTP health check server
	checkURL      string
	checkListener net.Listener
	checkServer   *http.Server

	// Hash table
	hashTable *rendezvous.Table
	waitIndex uint64

	// Graceful shutdown
	stop chan bool
}

// Join creates a new Node and adds it to the distributed hash table specified
// by the given name. The given id should be unique among all Nodes in the hash
// table.
func Join(name, id string) (node *Node, err error) {
	node = &Node{
		serviceName: name,
		serviceID:   id,
		stop:        make(chan bool),
	}

	node.consul, err = api.NewClient(api.DefaultConfig())
	if err != nil {
		return nil, fmt.Errorf("dht: can't create Consul API client: %s", err)
	}

	node.checkListener, node.checkServer, err = newCheckListenerAndServer()
	if err != nil {
		return nil, fmt.Errorf("dht: can't start HTTP server: %s", err)
	}

	err = node.register()
	if err != nil {
		return nil, fmt.Errorf("dht: can't register %s service: %s", node.serviceName, err)
	}

	err = node.update()
	if err != nil {
		return nil, fmt.Errorf("dht: can't fetch %s services list: %s", node.serviceName, err)
	}

	go node.poll()

	return node, nil
}

func (n *Node) register() (err error) {
	err = n.consul.Agent().ServiceRegister(&api.AgentServiceRegistration{
		Name: n.serviceName,
		ID:   n.serviceID,
		Check: &api.AgentServiceCheck{
			HTTP:     fmt.Sprintf("http://%s", n.checkListener.Addr().String()),
			Interval: checkInterval.String(),
		},
	})
	return err
}

func (n *Node) poll() {
	var err error

	for {
		select {
		case <-n.stop:
			return
		case <-time.After(pollWait):
			err = n.update()
			if err != nil {
				log.Printf("[dht %s %s] error: %s", n.serviceName, n.serviceID, err)
			}
		}
	}
}

// update blocks until the service list changes or until the Consul agent's
// timeout is reached.
func (n *Node) update() (err error) {
	opts := &api.QueryOptions{WaitIndex: n.waitIndex}
	services, meta, err := n.consul.Catalog().Service(n.serviceName, "", opts)
	if err != nil {
		return err
	}

	ids := make([]string, len(services))
	for i, service := range services {
		ids[i] = service.ServiceID
	}

	n.hashTable = rendezvous.New(ids)
	n.waitIndex = meta.LastIndex

	return nil
}

// Member returns true if the given key belongs to this Node in the distributed
// hash table.
func (n *Node) Member(key string) bool {
	return n.hashTable.Get(key) == n.serviceID
}

// Leave removes the Node from the distributed hash table by de-registering it
// from Consul. Once Leave is called, the Node should be discarded. An error is
// returned if the Node is unable to successfully deregister itself from
// Consul. In that case, Consul's health check for the Node will fail and
// require manual cleanup.
func (n *Node) Leave() (err error) {
	close(n.stop) // stop polling for state
	err = n.consul.Agent().ServiceDeregister(n.serviceID)
	n.checkListener.Close() // stop the health check http server
	return err
}
<commit_msg>Add a note about the default client timeout.<commit_after>package dht

import (
	"fmt"
	"log"
	"net"
	"net/http"
	"time"

	"github.com/hashicorp/consul/api"
	"github.com/stvp/rendezvous"
)

const (
	checkInterval = 5 * time.Second
	pollWait      = time.Second
)

func newCheckListenerAndServer() (listener net.Listener, server *http.Server, err error) {
	listener, err = net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		return listener, nil, err
	}

	server = &http.Server{
		ReadTimeout:  time.Second,
		WriteTimeout: time.Second,
		Handler: http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
			fmt.Fprintf(resp, "OK")
		}),
	}

	// When the listener is closed, this goroutine returns.
	go server.Serve(listener)

	return listener, server, err
}

// Node is a single node in a distributed hash table, coordinated using
// services registered in Consul. Key membership is determined using rendezvous
// hashing to ensure even distribution of keys and minimal key membership
// changes when a Node fails or otherwise leaves the hash table.
//
// Errors encountered when making blocking GET requests to the Consul agent API
// are logged using the log package.
type Node struct {
	// Consul
	serviceName string
	serviceID   string
	consul      *api.Client

	// HTTP health check server
	checkURL      string
	checkListener net.Listener
	checkServer   *http.Server

	// Hash table
	hashTable *rendezvous.Table
	waitIndex uint64

	// Graceful shutdown
	stop chan bool
}

// Join creates a new Node and adds it to the distributed hash table specified
// by the given name. The given id should be unique among all Nodes in the hash
// table.
func Join(name, id string) (node *Node, err error) {
	node = &Node{
		serviceName: name,
		serviceID:   id,
		stop:        make(chan bool),
	}

	node.consul, err = api.NewClient(api.DefaultConfig())
	if err != nil {
		return nil, fmt.Errorf("dht: can't create Consul API client: %s", err)
	}

	node.checkListener, node.checkServer, err = newCheckListenerAndServer()
	if err != nil {
		return nil, fmt.Errorf("dht: can't start HTTP server: %s", err)
	}

	err = node.register()
	if err != nil {
		return nil, fmt.Errorf("dht: can't register %s service: %s", node.serviceName, err)
	}

	err = node.update()
	if err != nil {
		return nil, fmt.Errorf("dht: can't fetch %s services list: %s", node.serviceName, err)
	}

	go node.poll()

	return node, nil
}

func (n *Node) register() (err error) {
	err = n.consul.Agent().ServiceRegister(&api.AgentServiceRegistration{
		Name: n.serviceName,
		ID:   n.serviceID,
		Check: &api.AgentServiceCheck{
			HTTP:     fmt.Sprintf("http://%s", n.checkListener.Addr().String()),
			Interval: checkInterval.String(),
		},
	})
	return err
}

func (n *Node) poll() {
	var err error

	for {
		select {
		case <-n.stop:
			return
		case <-time.After(pollWait):
			err = n.update()
			if err != nil {
				log.Printf("[dht %s %s] error: %s", n.serviceName, n.serviceID, err)
			}
		}
	}
}

// update blocks until the service list changes or until the Consul agent's
// timeout is reached (10 minutes by default).
func (n *Node) update() (err error) {
	opts := &api.QueryOptions{WaitIndex: n.waitIndex}
	services, meta, err := n.consul.Catalog().Service(n.serviceName, "", opts)
	if err != nil {
		return err
	}

	ids := make([]string, len(services))
	for i, service := range services {
		ids[i] = service.ServiceID
	}

	n.hashTable = rendezvous.New(ids)
	n.waitIndex = meta.LastIndex

	return nil
}

// Member returns true if the given key belongs to this Node in the distributed
// hash table.
func (n *Node) Member(key string) bool {
	return n.hashTable.Get(key) == n.serviceID
}

// Leave removes the Node from the distributed hash table by de-registering it
// from Consul. Once Leave is called, the Node should be discarded. An error is
// returned if the Node is unable to successfully deregister itself from
// Consul. In that case, Consul's health check for the Node will fail and
// require manual cleanup.
func (n *Node) Leave() (err error) {
	close(n.stop) // stop polling for state
	err = n.consul.Agent().ServiceDeregister(n.serviceID)
	n.checkListener.Close() // stop the health check http server
	return err
}
<|endoftext|>


<commit_before>package libxml2

/*
#cgo pkg-config: libxml-2.0
#include <stdbool.h>
#include "libxml/tree.h"
#include "libxml/parser.h"
#include "libxml/xpath.h"

static inline bool MY_xmlXPathNodeSetIsEmpty(xmlNodeSetPtr ptr) {
	return ptr == NULL ||
		ptr->nodeNr == 0 ||
		ptr->nodeTab == NULL;
}

static inline xmlNodePtr MY_xmlNodeSetTabAt(xmlNodePtr *nodes, int i) {
	return nodes[i];
}

static inline int MY_setXmlIndentTreeOutput(int i) {
	int old = xmlIndentTreeOutput;
	xmlIndentTreeOutput = i;
	return old;
}
*/
import "C"
import (
	"errors"
	"fmt"
	"unsafe"
)

type XmlNodeType int

const (
	ElementNode XmlNodeType = iota + 1
	AttributeNode
	TextNode
	CDataSectionNode
	EntityRefNode
	EntityNode
	PiNode
	CommentNode
	DocumentNode
	DocumentTypeNode
	DocumentFragNode
	NotationNode
	HTMLDocumentNode
	DTDNode
	ElementDecl
	AttributeDecl
	EntityDecl
	NamespaceDecl
	XIncludeStart
	XIncludeEnd
	DocbDocumentNode
)

var _XmlNodeType_index = [...]uint8{0, 11, 24, 32, 48, 61, 71, 77, 88, 100, 116, 132, 144, 160, 167, 178, 191, 201, 214, 227, 238, 254}

const _XmlNodeType_name = `ElementNodeAttributeNodeTextNodeCDataSectionNodeEntityRefNodeEntityNodePiNodeCommentNodeDocumentNodeDocumentTypeNodeDocumentFragNodeNotationNodeHTMLDocumentNodeDTDNodeElementDeclAttributeDeclEntityDeclNamespaceDeclXIncludeStartXIncludeEndDocbDocumentNode`

func (i XmlNodeType) String() string {
	i -= 1
	if i < 0 || i+1 >= XmlNodeType(len(_XmlNodeType_index)) {
		return fmt.Sprintf("XmlNodeType(%d)", i+1)
	}
	return _XmlNodeType_name[_XmlNodeType_index[i]:_XmlNodeType_index[i+1]]
}

var ErrNodeNotFound = errors.New("node not found")
var ErrInvalidArgument = errors.New("invalid argument")

// Node defines the basic DOM interface
type Node interface {
	// pointer() returns the underlying C pointer. Only we are allowed to
	// slice it, dice it, do whatever the heck with it.
	pointer() unsafe.Pointer

	ChildNodes() []Node
	OwnerDocument() *XmlDoc
	FindNodes(string) ([]Node, error)
	IsSameNode(Node) bool
	LastChild() Node
	NodeName() string
	NextSibling() Node
	ParetNode() Node
	PreviousSibling() Node
	SetNodeName(string)
	String() string
	TextContent() string
	ToString(int, bool) string
	Type() XmlNodeType
	Walk(func(Node) error)
}

type xmlNode struct {
	ptr *C.xmlNode
}

type XmlNode struct {
	*xmlNode
}

type XmlElement struct {
	*XmlNode
}

type XmlDoc struct {
	ptr  *C.xmlDoc
	root *C.xmlNode
}

type XmlText struct {
	*XmlNode
}

func wrapXmlElement(n *C.xmlElement) *XmlElement {
	return &XmlElement{wrapXmlNode((*C.xmlNode)(unsafe.Pointer(n)))}
}

func wrapXmlNode(n *C.xmlNode) *XmlNode {
	return &XmlNode{
		&xmlNode{
			ptr: (*C.xmlNode)(unsafe.Pointer(n)),
		},
	}
}

func wrapToNode(n *C.xmlNode) Node {
	switch XmlNodeType(n._type) {
	case ElementNode:
		return wrapXmlElement((*C.xmlElement)(unsafe.Pointer(n)))
	case TextNode:
		return &XmlText{&XmlNode{&xmlNode{ptr: n}}}
	default:
		return &XmlNode{&xmlNode{ptr: n}}
	}
}

func findNodes(n Node, xpath string) ([]Node, error) {
	ctx := C.xmlXPathNewContext((*C.xmlNode)(n.pointer()).doc)
	defer C.xmlXPathFreeContext(ctx)

	res := C.xmlXPathEvalExpression(stringToXmlChar(xpath), ctx)
	defer C.xmlXPathFreeObject(res)
	if C.MY_xmlXPathNodeSetIsEmpty(res.nodesetval) {
		return []Node(nil), nil
	}

	ret := make([]Node, res.nodesetval.nodeNr)
	for i := 0; i < int(res.nodesetval.nodeNr); i++ {
		ret[i] = wrapToNode(C.MY_xmlNodeSetTabAt(res.nodesetval.nodeTab, C.int(i)))
	}
	return ret, nil
}

func (n *xmlNode) pointer() unsafe.Pointer {
	return unsafe.Pointer(n.ptr)
}

func (n *xmlNode) ChildNodes() []Node {
	return childNodes(n)
}

func wrapXmlDoc(n *C.xmlDoc) *XmlDoc {
	r := C.xmlDocGetRootElement(n) // XXX Should check for n == nil
	return &XmlDoc{ptr: n, root: r}
}

func (n *xmlNode) OwnerDocument() *XmlDoc {
	return wrapXmlDoc(n.ptr.doc)
}

func (n *xmlNode) FindNodes(xpath string) ([]Node, error) {
	return findNodes(n, xpath)
}

func (n *xmlNode) IsSameNode(other Node) bool {
	return n.pointer() == other.pointer()
}

func (n *xmlNode) LastChild() Node {
	return wrapToNode(n.ptr.last)
}

func (n *xmlNode) NodeName() string {
	return xmlCharToString(n.ptr.name)
}

func (n *xmlNode) NextSibling() Node {
	return wrapToNode(n.ptr.next)
}

func (n *xmlNode) ParetNode() Node {
	return wrapToNode(n.ptr.parent)
}

func (n *xmlNode) PreviousSibling() Node {
	return wrapToNode(n.ptr.prev)
}

func (n *xmlNode) SetNodeName(name string) {
	C.xmlNodeSetName(n.ptr, stringToXmlChar(name))
}

func (n *xmlNode) String() string {
	return n.ToString(0, false)
}

func (n *xmlNode) TextContent() string {
	return xmlCharToString(C.xmlXPathCastNodeToString(n.ptr))
}

func (n *xmlNode) ToString(format int, docencoding bool) string {
	buffer := C.xmlBufferCreate()
	defer C.xmlBufferFree(buffer)
	if format <= 0 {
		C.xmlNodeDump(buffer, n.ptr.doc, n.ptr, 0, 0)
	} else {
		oIndentTreeOutput := C.MY_setXmlIndentTreeOutput(1)
		C.xmlNodeDump(buffer, n.ptr.doc, n.ptr, 0, C.int(format))
		C.MY_setXmlIndentTreeOutput(oIndentTreeOutput)
	}
	return xmlCharToString(C.xmlBufferContent(buffer))
}

func (n *xmlNode) Type() XmlNodeType {
	return XmlNodeType(n.ptr._type)
}

func (n *xmlNode) Walk(fn func(Node) error) {
	panic("should not call walk on internal struct")
}

func (n *XmlNode) Walk(fn func(Node) error) {
	walk(n, fn)
}

func walk(n Node, fn func(Node) error) {
	if err := fn(n); err != nil {
		return
	}
	for _, c := range n.ChildNodes() {
		walk(c, fn)
	}
}

func childNodes(n Node) []Node {
	ret := []Node(nil)
	for chld := ((*C.xmlNode)(n.pointer())).children; chld != nil; chld = chld.next {
		ret = append(ret, wrapToNode(chld))
	}
	return ret
}

func (d *XmlDoc) pointer() unsafe.Pointer {
	return unsafe.Pointer(d.ptr)
}

func (d *XmlDoc) DocumentElement() Node {
	if d.ptr == nil || d.root == nil {
		return nil
	}

	return wrapToNode(d.root)
}

func (d *XmlDoc) FindNodes(xpath string) ([]Node, error) {
	root := d.DocumentElement()
	if root == nil {
		return nil, ErrNodeNotFound
	}
	return root.FindNodes(xpath)
}

func (d *XmlDoc) Encoding() string {
	return xmlCharToString(d.ptr.encoding)
}

func (d *XmlDoc) Free() {
	C.xmlFreeDoc(d.ptr)
	d.ptr = nil
	d.root = nil
}

func (d *XmlDoc) String() string {
	var xc *C.xmlChar
	i := C.int(0)
	C.xmlDocDumpMemory(d.ptr, &xc, &i)
	return xmlCharToString(xc)
}

func (d *XmlDoc) Type() XmlNodeType {
	return XmlNodeType(d.ptr._type)
}

func (n *XmlDoc) Walk(fn func(Node) error) {
	walk(wrapXmlNode(n.root), fn)
}

func (n *XmlText) Data() string {
	return xmlCharToString(n.ptr.content)
}

func (n *XmlText) Walk(fn func(Node) error) {
	walk(n, fn)
}
<commit_msg>Change Type to NodeType<commit_after>package libxml2

/*
#cgo pkg-config: libxml-2.0
#include <stdbool.h>
#include "libxml/tree.h"
#include "libxml/parser.h"
#include "libxml/xpath.h"

static inline bool MY_xmlXPathNodeSetIsEmpty(xmlNodeSetPtr ptr) {
	return ptr == NULL ||
		ptr->nodeNr == 0 ||
		ptr->nodeTab == NULL;
}

static inline xmlNodePtr MY_xmlNodeSetTabAt(xmlNodePtr *nodes, int i) {
	return nodes[i];
}

static inline int MY_setXmlIndentTreeOutput(int i) {
	int old = xmlIndentTreeOutput;
	xmlIndentTreeOutput = i;
	return old;
}
*/
import "C"
import (
	"errors"
	"fmt"
	"unsafe"
)

type XmlNodeType int

const (
	ElementNode XmlNodeType = iota + 1
	AttributeNode
	TextNode
	CDataSectionNode
	EntityRefNode
	EntityNode
	PiNode
	CommentNode
	DocumentNode
	DocumentTypeNode
	DocumentFragNode
	NotationNode
	HTMLDocumentNode
	DTDNode
	ElementDecl
	AttributeDecl
	EntityDecl
	NamespaceDecl
	XIncludeStart
	XIncludeEnd
	DocbDocumentNode
)

var _XmlNodeType_index = [...]uint8{0, 11, 24, 32, 48, 61, 71, 77, 88, 100, 116, 132, 144, 160, 167, 178, 191, 201, 214, 227, 238, 254}

const _XmlNodeType_name = `ElementNodeAttributeNodeTextNodeCDataSectionNodeEntityRefNodeEntityNodePiNodeCommentNodeDocumentNodeDocumentTypeNodeDocumentFragNodeNotationNodeHTMLDocumentNodeDTDNodeElementDeclAttributeDeclEntityDeclNamespaceDeclXIncludeStartXIncludeEndDocbDocumentNode`

func (i XmlNodeType) String() string {
	i -= 1
	if i < 0 || i+1 >= XmlNodeType(len(_XmlNodeType_index)) {
		return fmt.Sprintf("XmlNodeType(%d)", i+1)
	}
	return _XmlNodeType_name[_XmlNodeType_index[i]:_XmlNodeType_index[i+1]]
}

var ErrNodeNotFound = errors.New("node not found")
var ErrInvalidArgument = errors.New("invalid argument")

// Node defines the basic DOM interface
type Node interface {
	// pointer() returns the underlying C pointer. Only we are allowed to
	// slice it, dice it, do whatever the heck with it.
	pointer() unsafe.Pointer

	ChildNodes() []Node
	OwnerDocument() *XmlDoc
	FindNodes(string) ([]Node, error)
	IsSameNode(Node) bool
	LastChild() Node
	NextSibling() Node
	NodeName() string
	NodeType() XmlNodeType
	ParetNode() Node
	PreviousSibling() Node
	SetNodeName(string)
	String() string
	TextContent() string
	ToString(int, bool) string
	Walk(func(Node) error)
}

type xmlNode struct {
	ptr *C.xmlNode
}

type XmlNode struct {
	*xmlNode
}

type XmlElement struct {
	*XmlNode
}

type XmlDoc struct {
	ptr  *C.xmlDoc
	root *C.xmlNode
}

type XmlText struct {
	*XmlNode
}

func wrapXmlElement(n *C.xmlElement) *XmlElement {
	return &XmlElement{wrapXmlNode((*C.xmlNode)(unsafe.Pointer(n)))}
}

func wrapXmlNode(n *C.xmlNode) *XmlNode {
	return &XmlNode{
		&xmlNode{
			ptr: (*C.xmlNode)(unsafe.Pointer(n)),
		},
	}
}

func wrapToNode(n *C.xmlNode) Node {
	switch XmlNodeType(n._type) {
	case ElementNode:
		return wrapXmlElement((*C.xmlElement)(unsafe.Pointer(n)))
	case TextNode:
		return &XmlText{&XmlNode{&xmlNode{ptr: n}}}
	default:
		return &XmlNode{&xmlNode{ptr: n}}
	}
}

func findNodes(n Node, xpath string) ([]Node, error) {
	ctx := C.xmlXPathNewContext((*C.xmlNode)(n.pointer()).doc)
	defer C.xmlXPathFreeContext(ctx)

	res := C.xmlXPathEvalExpression(stringToXmlChar(xpath), ctx)
	defer C.xmlXPathFreeObject(res)
	if C.MY_xmlXPathNodeSetIsEmpty(res.nodesetval) {
		return []Node(nil), nil
	}

	ret := make([]Node, res.nodesetval.nodeNr)
	for i := 0; i < int(res.nodesetval.nodeNr); i++ {
		ret[i] = wrapToNode(C.MY_xmlNodeSetTabAt(res.nodesetval.nodeTab, C.int(i)))
	}
	return ret, nil
}

func (n *xmlNode) pointer() unsafe.Pointer {
	return unsafe.Pointer(n.ptr)
}

func (n *xmlNode) ChildNodes() []Node {
	return childNodes(n)
}

func wrapXmlDoc(n *C.xmlDoc) *XmlDoc {
	r := C.xmlDocGetRootElement(n) // XXX Should check for n == nil
	return &XmlDoc{ptr: n, root: r}
}

func (n *xmlNode) OwnerDocument() *XmlDoc {
	return wrapXmlDoc(n.ptr.doc)
}

func (n *xmlNode) FindNodes(xpath string) ([]Node, error) {
	return findNodes(n, xpath)
}

func (n *xmlNode) IsSameNode(other Node) bool {
	return n.pointer() == other.pointer()
}

func (n *xmlNode) LastChild() Node {
	return wrapToNode(n.ptr.last)
}

func (n *xmlNode) NodeName() string {
	return xmlCharToString(n.ptr.name)
}

func (n *xmlNode) NextSibling() Node {
	return wrapToNode(n.ptr.next)
}

func (n *xmlNode) ParetNode() Node {
	return wrapToNode(n.ptr.parent)
}

func (n *xmlNode) PreviousSibling() Node {
	return wrapToNode(n.ptr.prev)
}

func (n *xmlNode) SetNodeName(name string) {
	C.xmlNodeSetName(n.ptr, stringToXmlChar(name))
}

func (n *xmlNode) String() string {
	return n.ToString(0, false)
}

func (n *xmlNode) TextContent() string {
	return xmlCharToString(C.xmlXPathCastNodeToString(n.ptr))
}

func (n *xmlNode) ToString(format int, docencoding bool) string {
	buffer := C.xmlBufferCreate()
	defer C.xmlBufferFree(buffer)
	if format <= 0 {
		C.xmlNodeDump(buffer, n.ptr.doc, n.ptr, 0, 0)
	} else {
		oIndentTreeOutput := C.MY_setXmlIndentTreeOutput(1)
		C.xmlNodeDump(buffer, n.ptr.doc, n.ptr, 0, C.int(format))
		C.MY_setXmlIndentTreeOutput(oIndentTreeOutput)
	}
	return xmlCharToString(C.xmlBufferContent(buffer))
}

func (n *xmlNode) NodeType() XmlNodeType {
	return XmlNodeType(n.ptr._type)
}

func (n *xmlNode) Walk(fn func(Node) error) {
	panic("should not call walk on internal struct")
}

func (n *XmlNode) Walk(fn func(Node) error) {
	walk(n, fn)
}

func walk(n Node, fn func(Node) error) {
	if err := fn(n); err != nil {
		return
	}
	for _, c := range n.ChildNodes() {
		walk(c, fn)
	}
}

func childNodes(n Node) []Node {
	ret := []Node(nil)
	for chld := ((*C.xmlNode)(n.pointer())).children; chld != nil; chld = chld.next {
		ret = append(ret, wrapToNode(chld))
	}
	return ret
}

func (d *XmlDoc) pointer() unsafe.Pointer {
	return unsafe.Pointer(d.ptr)
}

func (d *XmlDoc) DocumentElement() Node {
	if d.ptr == nil || d.root == nil {
		return nil
	}

	return wrapToNode(d.root)
}

func (d *XmlDoc) FindNodes(xpath string) ([]Node, error) {
	root := d.DocumentElement()
	if root == nil {
		return nil, ErrNodeNotFound
	}
	return root.FindNodes(xpath)
}

func (d *XmlDoc) Encoding() string {
	return xmlCharToString(d.ptr.encoding)
}

func (d *XmlDoc) Free() {
	C.xmlFreeDoc(d.ptr)
	d.ptr = nil
	d.root = nil
}

func (d *XmlDoc) String() string {
	var xc *C.xmlChar
	i := C.int(0)
	C.xmlDocDumpMemory(d.ptr, &xc, &i)
	return xmlCharToString(xc)
}

func (d *XmlDoc) NodeType() XmlNodeType {
	return XmlNodeType(d.ptr._type)
}

func (n *XmlDoc) Walk(fn func(Node) error) {
	walk(wrapXmlNode(n.root), fn)
}

func (n *XmlText) Data() string {
	return xmlCharToString(n.ptr.content)
}

func (n *XmlText) Walk(fn func(Node) error) {
	walk(n, fn)
}
<|endoftext|>


<commit_before>package flowgraph

import (
	"github.com/vectaport/fgbase"
)

// Node interface for flowgraph nodes that are connected by flowgraph edges
type Node interface {
	// Tracef for debug trace printing.  Use atomic log mechanism.
	Tracef(format string, v ...interface{})

	// LogError for logging of error messages.  Use atomic log mechanism.
	LogError(format string, v ...interface{})

	// Name returns the node name
	Name() string

	// Source returns upstream edge by index
	Source(n int) Edge

	// Destination returns downstream edge by index
	Destination(n int) Edge

	// FindSource returns upstream edge by name
	FindSource(name string) Edge

	// FindDestination returns downstream edge by name
	FindDestination(name string) Edge

	// AddSource adds a list of source edges
	AddSource(e ...Edge)

	// AddDestination adds a list of destination edges
	AddDestination(e ...Edge)

	// NumSource returns the number of upstream edges
	NumSource() int

	// NumDestination returns the number of downstream edges
	NumDestination() int

	// SetSourceNames names the sources
	SetSourceNames(nm ...string)

	// SetDestinationNames names the destinations
	SetDestinationNames(nm ...string)

	// SourceNames returns the names of the sources
	SourceNames() []string

	// DestinationNames returns the names of the destinations
	DestinationNames() []string

	// Auxiliary returns auxiliary storage used by
	// underlying implementation for storing state
	Auxiliary() interface{}

	// Base returns the value that implements this node
	// The type of this value identifies the implementation.
	Base() interface{}
}

// implementation of Node
type node struct {
	base *fgbase.Node
}

// Tracef for debug trace printing.  Uses atomic log mechanism.
func (n node) Tracef(format string, v ...interface{}) {
	n.base.Tracef(format, v)
}

// LogError for logging of error messages.  Uses atomic log mechanism.
func (n node) LogError(format string, v ...interface{}) {
	n.base.LogError(format, v)
}

// Name returns the node name
func (n node) Name() string {
	return n.base.Name
}

// Source returns upstream edge by index
func (n node) Source(i int) Edge {
	return edge{n.base.Srcs[i]}
}

// Destination returns downstream edge by index
func (n node) Destination(i int) Edge {
	return edge{n.base.Dsts[i]}
}

// FindSource returns upstream edge by name
func (n node) FindSource(name string) Edge {
	return edge{n.base.FindSrc(name)}
}

// FindDestination returns downstream edge by name
func (n node) FindDestination(name string) Edge {
	return edge{n.base.FindDst(name)}
}

// AddSource adds a list of source edges
func (n node) AddSource(e ...Edge) {
	for _, ev := range e {
		n.base.Srcs = append(n.base.Srcs, ev.Base().(*fgbase.Edge))
	}
}

// AddDestination adds a list of destination edges
func (n node) AddDestination(e ...Edge) {
	for _, ev := range e {
		n.base.Dsts = append(n.base.Dsts, ev.Base().(*fgbase.Edge))
	}
}

// NumSource returns the number of upstream edges
func (n node) NumSource() int {
	return len(n.base.Srcs)
}

// NumDestination returns the number of downstream edges
func (n node) NumDestination() int {
	return len(n.base.Dsts)
}

// SetSourceNames names the sources
func (n node) SetSourceNames(nm ...string) {
	for _, v := range nm {
		n.base.SrcNames = append(n.base.SrcNames, v)
	}
}

// SetDestinationNames names the destinations
func (n node) SetDestinationNames(nm ...string) {
	for _, v := range nm {
		n.base.DstNames = append(n.base.DstNames, v)
	}
}

// SourceNames returns the names of the sources
func (n node) SourceNames() []string {
	return n.base.SrcNames
}

// DestinationNames returns the names of the destinatiopns
func (n node) DestinationNames() []string {
	return n.base.DstNames
}

// Auxiliary returns auxiliary storage for this node used by
// the underlying implementation for storing state
func (n node) Auxiliary() interface{} {
	return n.base.Aux
}

// Base returns the value that implements this edge
// The type of this value identifies the implementation.
func (n node) Base() interface{} {
	return n.base
}
<commit_msg>working on naming sources and destinations of a node<commit_after>package flowgraph

import (
	"github.com/vectaport/fgbase"
)

// Node interface for flowgraph nodes that are connected by flowgraph edges
type Node interface {
	// Tracef for debug trace printing.  Use atomic log mechanism.
	Tracef(format string, v ...interface{})

	// LogError for logging of error messages.  Use atomic log mechanism.
	LogError(format string, v ...interface{})

	// Name returns the node name
	Name() string

	// Source returns upstream edge by index
	Source(n int) Edge

	// Destination returns downstream edge by index
	Destination(n int) Edge

	// FindSource returns upstream edge by name
	FindSource(name string) Edge

	// FindDestination returns downstream edge by name
	FindDestination(name string) Edge

	// AddSource adds a list of source edges
	AddSource(e ...Edge)

	// AddDestination adds a list of destination edges
	AddDestination(e ...Edge)

	// NumSource returns the number of upstream edges
	NumSource() int

	// NumDestination returns the number of downstream edges
	NumDestination() int

	// SetSourceNames names the sources
	SetSourceNames(nm ...string)

	// SetDestinationNames names the destinations
	SetDestinationNames(nm ...string)

	// SourceNames returns the names of the sources
	SourceNames() []string

	// DestinationNames returns the names of the destinations
	DestinationNames() []string

	// Auxiliary returns auxiliary storage used by
	// underlying implementation for storing state
	Auxiliary() interface{}

	// Base returns the value that implements this node
	// The type of this value identifies the implementation.
	Base() interface{}
}

// implementation of Node
type node struct {
	base *fgbase.Node
}

// Tracef for debug trace printing.  Uses atomic log mechanism.
func (n node) Tracef(format string, v ...interface{}) {
	n.base.Tracef(format, v)
}

// LogError for logging of error messages.  Uses atomic log mechanism.
func (n node) LogError(format string, v ...interface{}) {
	n.base.LogError(format, v)
}

// Name returns the node name
func (n node) Name() string {
	return n.base.Name
}

// Source returns upstream edge by index
func (n node) Source(i int) Edge {
	return edge{n.base.Srcs[i]}
}

// Destination returns downstream edge by index
func (n node) Destination(i int) Edge {
	return edge{n.base.Dsts[i]}
}

// FindSource returns upstream edge by name
func (n node) FindSource(name string) Edge {
	return edge{n.base.FindSrc(name)}
}

// FindDestination returns downstream edge by name
func (n node) FindDestination(name string) Edge {
	return edge{n.base.FindDst(name)}
}

// AddSource adds a list of source edges
func (n node) AddSource(e ...Edge) {
	for _, ev := range e {
		n.base.Srcs = append(n.base.Srcs, ev.Base().(*fgbase.Edge))
	}
}

// AddDestination adds a list of destination edges
func (n node) AddDestination(e ...Edge) {
	for _, ev := range e {
		n.base.Dsts = append(n.base.Dsts, ev.Base().(*fgbase.Edge))
	}
}

// NumSource returns the number of upstream edges
func (n node) NumSource() int {
	return len(n.base.Srcs)
}

// NumDestination returns the number of downstream edges
func (n node) NumDestination() int {
	return len(n.base.Dsts)
}

// SetSourceNames names the sources
func (n node) SetSourceNames(nm ...string) {
	n.base.SetSrcNames(nm...)
}

// SetDestinationNames names the destination
func (n node) SetDestinationNames(nm ...string) {
	n.base.SetDstNames(nm...)
}

// SourceNames returns the names of the sources
func (n node) SourceNames() []string {
	return n.base.SrcNames()
}

// DestinationNames returns the names of the destinatiopns
func (n node) DestinationNames() []string {
	return n.base.DstNames()
}

// Auxiliary returns auxiliary storage for this node used by
// the underlying implementation for storing state
func (n node) Auxiliary() interface{} {
	return n.base.Aux
}

// Base returns the value that implements this edge
// The type of this value identifies the implementation.
func (n node) Base() interface{} {
	return n.base
}
<|endoftext|>


<commit_before>// Copyright (c) 2013 Kelsey Hightower. All rights reserved.
// Use of this source code is governed by the Apache License, Version 2.0
// that can be found in the LICENSE file.
package etcd

import (
	"errors"
	"strings"

	goetcd "github.com/coreos/go-etcd/etcd"
)

// Client is a wrapper around the etcd client
type Client struct {
	client *goetcd.Client
}

// NewEtcdClient returns an *etcd.Client with a connection to named machines.
// It returns an error if a connection to the cluster cannot be made.
func NewEtcdClient(machines []string, cert, key string, caCert string) (*Client, error) {
	var c *goetcd.Client
	if cert != "" && key != "" {
		c, err := goetcd.NewTLSClient(machines, cert, key, caCert)
		if err != nil {
			return &Client{c}, err
		}
	} else {
		c = goetcd.NewClient(machines)
	}
	success := c.SetCluster(machines)
	if !success {
		return &Client{c}, errors.New("cannot connect to etcd cluster: " + strings.Join(machines, ","))
	}
	return &Client{c}, nil
}

// GetValues queries etcd for keys prefixed by prefix.
// Etcd paths (keys) are translated into names more suitable for use in
// templates. For example if prefix were set to '/production' and one of the
// keys were '/nginx/port'; the prefixed '/production/nginx/port' key would
// be queried for. If the value for the prefixed key where 80, the returned map
// would contain the entry vars["nginx_port"] = "80".
func (c *Client) GetValues(keys []string) (map[string]string, error) {
	vars := make(map[string]string)
	for _, key := range keys {
		resp, err := c.client.Get(key, false, true)
		if err != nil {
			return vars, err
		}
		err = nodeWalk(resp.Node, vars)
		if err != nil {
			return vars, err
		}
	}
	return vars, nil
}

// nodeWalk recursively descends nodes, updating vars.
func nodeWalk(node *goetcd.Node, vars map[string]string) error {
	if node != nil {
		key := node.Key
		if !node.Dir {
			vars[key] = node.Value
		} else {
			for _, node := range node.Nodes {
				nodeWalk(node, vars)
			}
		}
	}
	return nil
}
<commit_msg>update comment<commit_after>// Copyright (c) 2013 Kelsey Hightower. All rights reserved.
// Use of this source code is governed by the Apache License, Version 2.0
// that can be found in the LICENSE file.
package etcd

import (
	"errors"
	"strings"

	goetcd "github.com/coreos/go-etcd/etcd"
)

// Client is a wrapper around the etcd client
type Client struct {
	client *goetcd.Client
}

// NewEtcdClient returns an *etcd.Client with a connection to named machines.
// It returns an error if a connection to the cluster cannot be made.
func NewEtcdClient(machines []string, cert, key string, caCert string) (*Client, error) {
	var c *goetcd.Client
	if cert != "" && key != "" {
		c, err := goetcd.NewTLSClient(machines, cert, key, caCert)
		if err != nil {
			return &Client{c}, err
		}
	} else {
		c = goetcd.NewClient(machines)
	}
	success := c.SetCluster(machines)
	if !success {
		return &Client{c}, errors.New("cannot connect to etcd cluster: " + strings.Join(machines, ","))
	}
	return &Client{c}, nil
}

// GetValues queries etcd for keys prefixed by prefix.
func (c *Client) GetValues(keys []string) (map[string]string, error) {
	vars := make(map[string]string)
	for _, key := range keys {
		resp, err := c.client.Get(key, false, true)
		if err != nil {
			return vars, err
		}
		err = nodeWalk(resp.Node, vars)
		if err != nil {
			return vars, err
		}
	}
	return vars, nil
}

// nodeWalk recursively descends nodes, updating vars.
func nodeWalk(node *goetcd.Node, vars map[string]string) error {
	if node != nil {
		key := node.Key
		if !node.Dir {
			vars[key] = node.Value
		} else {
			for _, node := range node.Nodes {
				nodeWalk(node, vars)
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2021 Timo Savola. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build gateexecdir

package container

import (
	"os"
	"path"
)

func init() {
	if ExecDir != "" {
		if filename, err := os.Executable(); err == nil {
			ExecDir = path.Dir(filename)
		}
	}
}
<commit_msg>runtime/container: fix ExecDir configuration via linker<commit_after>// Copyright (c) 2021 Timo Savola. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build gateexecdir

package container

import (
	"os"
	"path"
)

func init() {
	if ExecDir == "" {
		if filename, err := os.Executable(); err == nil {
			ExecDir = path.Dir(filename)
		}
	}
}
<|endoftext|>