<commit_before>package kafka

import (
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/anomalyzer"
	"github.com/funkygao/gafka/cmd/kguard/monitor"
	"github.com/funkygao/gafka/telemetry"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/go-metrics"
	log "github.com/funkygao/log4go"
)

func init() {
	monitor.RegisterWatcher("kafka.topic", func() monitor.Watcher {
		return &WatchTopics{
			Tick: time.Minute,
		}
	})
}

// WatchTopics montor kafka total msg count over time.
type WatchTopics struct {
	Zkzone *zk.ZkZone
	Stop   <-chan struct{}
	Tick   time.Duration
	Wg     *sync.WaitGroup

	pubQps      map[string]metrics.Meter
	lastOffsets map[string]int64

	aggPubQpsAnomalyGauge metrics.Gauge
	aggPubQpsAnomaly      anomalyzer.Anomalyzer
}

func (this *WatchTopics) Init(ctx monitor.Context) {
	this.Zkzone = ctx.ZkZone()
	this.Stop = ctx.StopChan()
	this.Wg = ctx.Inflight()

	conf := &anomalyzer.AnomalyzerConf{
		Delay:       true,
		ActiveSize:  8,
		NSeasons:    8,
		Sensitivity: 0.3,     // magnitude
		UpperBound:  9000000, // fence
		LowerBound:  0,       // fence
		PermCount:   1000,    // diff & rank
		Methods:     []string{"diff", "fence", "highrank", "lowrank", "magnitude", "ks", "cdf"},
	}
	var err error
	this.aggPubQpsAnomaly, err = anomalyzer.NewAnomalyzer(conf, nil)
	if err != nil {
		panic(err)
	}
	this.aggPubQpsAnomalyGauge = metrics.NewRegisteredGauge("pub.qps.anomaly", nil)
}

// set?key=kta-as:4
func (this *WatchTopics) Set(key string) {
	tuples := strings.SplitN(key, ":", 2)
	if len(tuples) != 2 {
		return
	}

	switch tuples[0] {
	case "kta-as":
		if n, err := strconv.Atoi(tuples[1]); err == nil && n > 0 {
			this.aggPubQpsAnomaly.Conf.ActiveSize = n
			log.Info("ActiveSize set to %d", n)
		}

	case "kta-ns":
		if n, err := strconv.Atoi(tuples[1]); err == nil && n > 0 {
			this.aggPubQpsAnomaly.Conf.NSeasons = n
			log.Info("NSeasons set to %d", n)
		}

	case "kta-sen":
		if f, err := strconv.ParseFloat(tuples[1], 64); err == nil && f > 0.01 {
			this.aggPubQpsAnomaly.Conf.Sensitivity = f
			log.Info("Sensitivity set to %f", f)
		}

	case "kta-upper":
		if f, err := strconv.ParseFloat(tuples[1], 64); err == nil && f > 0.01 {
			this.aggPubQpsAnomaly.Conf.UpperBound = f
			log.Info("UpperBound set to %f", f)
		}

	case "kta-lower":
		if f, err := strconv.ParseFloat(tuples[1], 64); err == nil && f > 0.01 {
			this.aggPubQpsAnomaly.Conf.LowerBound = f
			log.Info("LowerBound set to %f", f)
		}
	}
}

func (this *WatchTopics) Run() {
	defer this.Wg.Done()

	ticker := time.NewTicker(this.Tick)
	defer ticker.Stop()

	this.pubQps = make(map[string]metrics.Meter, 10)
	this.lastOffsets = make(map[string]int64, 10)

	offsets := metrics.NewRegisteredGauge("msg.cum", nil)
	topics := metrics.NewRegisteredGauge("topics", nil)
	partitions := metrics.NewRegisteredGauge("partitions", nil)
	brokers := metrics.NewRegisteredGauge("brokers", nil)
	newTopics := metrics.NewRegisteredGauge("kfk.newtopic.1d", nil)
	for {

		select {
		case <-this.Stop:
			log.Info("kafka.topic stopped")
			return

		case now := <-ticker.C:
			o, t, p, b := this.report()
			offsets.Update(o)
			topics.Update(t)
			partitions.Update(p)
			brokers.Update(b)
			newTopics.Update(this.newTopicsSince(now, time.Hour*24))
		}
	}

}

func (this *WatchTopics) newTopicsSince(now time.Time, since time.Duration) (n int64) {
	excludedClusters := this.Zkzone.PublicClusters()
	this.Zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		// kateway topics excluded
		clusterExcluded := false
		for _, cluster := range excludedClusters {
			if cluster.Name() == zkcluster.Name() {
				clusterExcluded = true
				break
			}
		}
		if clusterExcluded {
			return
		}

		// find recently how many topics created
		for _, ctime := range zkcluster.TopicsCtime() {
			if now.Sub(ctime) <= since {
				n += 1
			}
		}
	})

	return
}

func (this *WatchTopics) report() (totalOffsets int64, topicsN int64,
	partitionN int64, brokersN int64) {
	var totalPubQpsRate1 float64
	this.Zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		brokerList := zkcluster.BrokerList()
		kfk, err := sarama.NewClient(brokerList, sarama.NewConfig())
		if err != nil {
			log.Error("cluster[%s] %v", zkcluster.Name(), err)
			return
		}
		defer kfk.Close()

		brokersN += int64(len(brokerList))

		topics, err := kfk.Topics()
		if err != nil {
			log.Error("cluster[%s] %v", zkcluster.Name(), err)
			return
		}

		for _, topic := range topics {
			partions, err := kfk.Partitions(topic)
			if err != nil {
				log.Error("cluster[%s] topic:%s %v", zkcluster.Name(), topic, err)
				continue
			}

			topicsN++

			offsetOfTopic := int64(0)
			for _, partitionId := range partions {
				latestOffset, err := kfk.GetOffset(topic, partitionId,
					sarama.OffsetNewest)
				if err != nil {
					log.Error("cluster[%s] topic[%s/%d]: %v",
						zkcluster.Name(), topic, partitionId, err)
					continue
				}

				partitionN++
				totalOffsets += latestOffset
				offsetOfTopic += latestOffset
			}

			// update pubQps metrics
			tag := telemetry.Tag(zkcluster.Name(), strings.Replace(topic, ".", "_", -1), "v1")
			if _, present := this.pubQps[tag]; !present {
				this.pubQps[tag] = metrics.NewRegisteredMeter(tag+"pub.qps", nil)
			}
			lastOffset := this.lastOffsets[tag]
			if lastOffset == 0 {
				// first run
				this.lastOffsets[tag] = offsetOfTopic
			} else {
				delta := offsetOfTopic - lastOffset
				if delta >= 0 {
					this.pubQps[tag].Mark(delta)
					this.lastOffsets[tag] = offsetOfTopic
				} else {
					log.Warn("cluster[%s] topic[%s] offset backwards, skipped: %d <- %d",
						zkcluster.Name(), topic, offsetOfTopic, lastOffset)
				}

			}

			totalPubQpsRate1 += this.pubQps[tag].Rate1()
		}

	})

	this.aggPubQpsAnomaly.Update([]float64{totalPubQpsRate1}) // avoid overflow
	this.aggPubQpsAnomalyGauge.Update(int64(100 * this.aggPubQpsAnomaly.Eval()))

	return
}
<commit_msg>ajust anamoly params<commit_after>package kafka

import (
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/anomalyzer"
	"github.com/funkygao/gafka/cmd/kguard/monitor"
	"github.com/funkygao/gafka/telemetry"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/go-metrics"
	log "github.com/funkygao/log4go"
)

func init() {
	monitor.RegisterWatcher("kafka.topic", func() monitor.Watcher {
		return &WatchTopics{
			Tick: time.Minute,
		}
	})
}

// WatchTopics montor kafka total msg count over time.
type WatchTopics struct {
	Zkzone *zk.ZkZone
	Stop   <-chan struct{}
	Tick   time.Duration
	Wg     *sync.WaitGroup

	pubQps      map[string]metrics.Meter
	lastOffsets map[string]int64

	aggPubQpsAnomalyGauge metrics.Gauge
	aggPubQpsAnomaly      anomalyzer.Anomalyzer
}

func (this *WatchTopics) Init(ctx monitor.Context) {
	this.Zkzone = ctx.ZkZone()
	this.Stop = ctx.StopChan()
	this.Wg = ctx.Inflight()

	conf := &anomalyzer.AnomalyzerConf{
		Delay:       true,
		ActiveSize:  5,
		NSeasons:    5,
		Sensitivity: 0.1,    // magnitude
		UpperBound:  500000, // fence
		LowerBound:  0,      // fence
		PermCount:   1000,   // diff & rank
		Methods:     []string{"diff", "fence", "highrank", "lowrank", "magnitude", "ks", "cdf"},
	}
	var err error
	this.aggPubQpsAnomaly, err = anomalyzer.NewAnomalyzer(conf, nil)
	if err != nil {
		panic(err)
	}
	this.aggPubQpsAnomalyGauge = metrics.NewRegisteredGauge("pub.qps.anomaly", nil)
}

// set?key=kta-as:4
func (this *WatchTopics) Set(key string) {
	tuples := strings.SplitN(key, ":", 2)
	if len(tuples) != 2 {
		return
	}

	switch tuples[0] {
	case "kta-as":
		if n, err := strconv.Atoi(tuples[1]); err == nil && n > 0 {
			this.aggPubQpsAnomaly.Conf.ActiveSize = n
			log.Info("ActiveSize set to %d", n)
		}

	case "kta-ns":
		if n, err := strconv.Atoi(tuples[1]); err == nil && n > 0 {
			this.aggPubQpsAnomaly.Conf.NSeasons = n
			log.Info("NSeasons set to %d", n)
		}

	case "kta-sen":
		if f, err := strconv.ParseFloat(tuples[1], 64); err == nil && f > 0.01 {
			this.aggPubQpsAnomaly.Conf.Sensitivity = f
			log.Info("Sensitivity set to %f", f)
		}

	case "kta-upper":
		if f, err := strconv.ParseFloat(tuples[1], 64); err == nil && f > 0.01 {
			this.aggPubQpsAnomaly.Conf.UpperBound = f
			log.Info("UpperBound set to %f", f)
		}

	case "kta-lower":
		if f, err := strconv.ParseFloat(tuples[1], 64); err == nil && f > 0.01 {
			this.aggPubQpsAnomaly.Conf.LowerBound = f
			log.Info("LowerBound set to %f", f)
		}
	}
}

func (this *WatchTopics) Run() {
	defer this.Wg.Done()

	ticker := time.NewTicker(this.Tick)
	defer ticker.Stop()

	this.pubQps = make(map[string]metrics.Meter, 10)
	this.lastOffsets = make(map[string]int64, 10)

	offsets := metrics.NewRegisteredGauge("msg.cum", nil)
	topics := metrics.NewRegisteredGauge("topics", nil)
	partitions := metrics.NewRegisteredGauge("partitions", nil)
	brokers := metrics.NewRegisteredGauge("brokers", nil)
	newTopics := metrics.NewRegisteredGauge("kfk.newtopic.1d", nil)
	for {

		select {
		case <-this.Stop:
			log.Info("kafka.topic stopped")
			return

		case now := <-ticker.C:
			o, t, p, b := this.report()
			offsets.Update(o)
			topics.Update(t)
			partitions.Update(p)
			brokers.Update(b)
			newTopics.Update(this.newTopicsSince(now, time.Hour*24))
		}
	}

}

func (this *WatchTopics) newTopicsSince(now time.Time, since time.Duration) (n int64) {
	excludedClusters := this.Zkzone.PublicClusters()
	this.Zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		// kateway topics excluded
		clusterExcluded := false
		for _, cluster := range excludedClusters {
			if cluster.Name() == zkcluster.Name() {
				clusterExcluded = true
				break
			}
		}
		if clusterExcluded {
			return
		}

		// find recently how many topics created
		for _, ctime := range zkcluster.TopicsCtime() {
			if now.Sub(ctime) <= since {
				n += 1
			}
		}
	})

	return
}

func (this *WatchTopics) report() (totalOffsets int64, topicsN int64,
	partitionN int64, brokersN int64) {
	var totalPubQpsRate1 float64
	this.Zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		brokerList := zkcluster.BrokerList()
		kfk, err := sarama.NewClient(brokerList, sarama.NewConfig())
		if err != nil {
			log.Error("cluster[%s] %v", zkcluster.Name(), err)
			return
		}
		defer kfk.Close()

		brokersN += int64(len(brokerList))

		topics, err := kfk.Topics()
		if err != nil {
			log.Error("cluster[%s] %v", zkcluster.Name(), err)
			return
		}

		for _, topic := range topics {
			partions, err := kfk.Partitions(topic)
			if err != nil {
				log.Error("cluster[%s] topic:%s %v", zkcluster.Name(), topic, err)
				continue
			}

			topicsN++

			offsetOfTopic := int64(0)
			for _, partitionId := range partions {
				latestOffset, err := kfk.GetOffset(topic, partitionId,
					sarama.OffsetNewest)
				if err != nil {
					log.Error("cluster[%s] topic[%s/%d]: %v",
						zkcluster.Name(), topic, partitionId, err)
					continue
				}

				partitionN++
				totalOffsets += latestOffset
				offsetOfTopic += latestOffset
			}

			// update pubQps metrics
			tag := telemetry.Tag(zkcluster.Name(), strings.Replace(topic, ".", "_", -1), "v1")
			if _, present := this.pubQps[tag]; !present {
				this.pubQps[tag] = metrics.NewRegisteredMeter(tag+"pub.qps", nil)
			}
			lastOffset := this.lastOffsets[tag]
			if lastOffset == 0 {
				// first run
				this.lastOffsets[tag] = offsetOfTopic
			} else {
				delta := offsetOfTopic - lastOffset
				if delta >= 0 {
					this.pubQps[tag].Mark(delta)
					this.lastOffsets[tag] = offsetOfTopic
				} else {
					log.Warn("cluster[%s] topic[%s] offset backwards, skipped: %d <- %d",
						zkcluster.Name(), topic, offsetOfTopic, lastOffset)
				}

			}

			totalPubQpsRate1 += this.pubQps[tag].Rate1()
		}

	})

	this.aggPubQpsAnomaly.Update([]float64{totalPubQpsRate1}) // avoid overflow
	this.aggPubQpsAnomalyGauge.Update(int64(100 * this.aggPubQpsAnomaly.Eval()))

	return
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math"

	"github.com/dustin/go-humanize"

	sessionpkg "github.com/havoc-io/mutagen/pkg/session"
	urlpkg "github.com/havoc-io/mutagen/pkg/url"
)

const (
	// maxUint64Description is a human-friendly mathematic description of
	// math.MaxUint64.
	maxUint64Description = "2⁶⁴−1"

	// emptyLabelValueDescription is a human-friendly description representing
	// an empty label value. It contains characters which are invalid for use in
	// label values, so it won't be confused for one.
	emptyLabelValueDescription = "<empty>"
)

func printEndpoint(name string, url *urlpkg.URL, configuration *sessionpkg.Configuration, version sessionpkg.Version) {
	// Print the endpoint header.
	fmt.Println(name, "configuration:")

	// Print the URL.
	fmt.Println("\tURL:", url.Format("\n\t\t"))

	// Compute and print the watch mode.
	watchModeDescription := configuration.WatchMode.Description()
	if configuration.WatchMode.IsDefault() {
		watchModeDescription += fmt.Sprintf(" (%s)", version.DefaultWatchMode().Description())
	}
	fmt.Println("\tWatch mode:", watchModeDescription)

	// Compute and print the watch polling interval, so long as we're not in
	// no-watch mode.
	if configuration.WatchMode != sessionpkg.WatchMode_WatchModeNoWatch {
		var watchPollingIntervalDescription string
		if configuration.WatchPollingInterval == 0 {
			watchPollingIntervalDescription = fmt.Sprintf("Default (%d seconds)", version.DefaultWatchPollingInterval())
		} else {
			watchPollingIntervalDescription = fmt.Sprintf("%d seconds", configuration.WatchPollingInterval)
		}
		fmt.Println("\tWatch polling interval:", watchPollingIntervalDescription)
	}

	// Compute and print the probe mode.
	probeModeDescription := configuration.ProbeMode.Description()
	if configuration.ProbeMode.IsDefault() {
		probeModeDescription += fmt.Sprintf(" (%s)", version.DefaultProbeMode().Description())
	}
	fmt.Println("\tProbe mode:", probeModeDescription)

	// Compute and print the scan mode.
	scanModeDescription := configuration.ScanMode.Description()
	if configuration.ScanMode.IsDefault() {
		scanModeDescription += fmt.Sprintf(" (%s)", version.DefaultScanMode().Description())
	}
	fmt.Println("\tScan mode:", scanModeDescription)

	// Compute and print the staging mode.
	stageModeDescription := configuration.StageMode.Description()
	if configuration.StageMode.IsDefault() {
		stageModeDescription += fmt.Sprintf(" (%s)", version.DefaultStageMode().Description())
	}
	fmt.Println("\tStage mode:", stageModeDescription)

	// Compute and print the default file mode.
	var defaultFileModeDescription string
	if configuration.DefaultFileMode == 0 {
		defaultFileModeDescription = fmt.Sprintf("Default (%#o)", version.DefaultFileMode())
	} else {
		defaultFileModeDescription = fmt.Sprintf("%#o", configuration.DefaultFileMode)
	}
	fmt.Println("\tFile mode:", defaultFileModeDescription)

	// Compute and print the default directory mode.
	var defaultDirectoryModeDescription string
	if configuration.DefaultDirectoryMode == 0 {
		defaultDirectoryModeDescription = fmt.Sprintf("Default (%#o)", version.DefaultDirectoryMode())
	} else {
		defaultDirectoryModeDescription = fmt.Sprintf("%#o", configuration.DefaultDirectoryMode)
	}
	fmt.Println("\tDirectory mode:", defaultDirectoryModeDescription)

	// Compute and print the default file/directory owner.
	defaultOwnerDescription := "Default"
	if configuration.DefaultOwner != "" {
		defaultOwnerDescription = configuration.DefaultOwner
	}
	fmt.Println("\tDefault file/directory owner:", defaultOwnerDescription)

	// Compute and print the default file/directory group.
	defaultGroupDescription := "Default"
	if configuration.DefaultGroup != "" {
		defaultGroupDescription = configuration.DefaultGroup
	}
	fmt.Println("\tDefault file/directory group:", defaultGroupDescription)
}

func printSession(state *sessionpkg.State, long bool) {
	// Print the session identifier.
	fmt.Println("Session:", state.Session.Identifier)

	// Print extended information, if desired.
	if long {
		// Print the configuration header.
		fmt.Println("Configuration:")

		// Extract configuration.
		configuration := state.Session.Configuration

		// Compute and print synchronization mode.
		synchronizationMode := configuration.SynchronizationMode.Description()
		if configuration.SynchronizationMode.IsDefault() {
			defaultSynchronizationMode := state.Session.Version.DefaultSynchronizationMode()
			synchronizationMode += fmt.Sprintf(" (%s)", defaultSynchronizationMode.Description())
		}
		fmt.Println("\tSynchronization mode:", synchronizationMode)

		// Compute and print maximum entry count.
		var maximumEntryCountDescription string
		if configuration.MaximumEntryCount == 0 {
			if m := state.Session.Version.DefaultMaximumEntryCount(); m == math.MaxUint64 {
				maximumEntryCountDescription = fmt.Sprintf("Default (%s)", maxUint64Description)
			} else {
				maximumEntryCountDescription = fmt.Sprintf("Default (%d)", m)
			}
		} else {
			maximumEntryCountDescription = fmt.Sprintf("%d", configuration.MaximumEntryCount)
		}
		fmt.Println("\tMaximum allowed entry count:", maximumEntryCountDescription)

		// Compute and print maximum staging file size.
		var maximumStagingFileSizeDescription string
		if configuration.MaximumStagingFileSize == 0 {
			maximumStagingFileSizeDescription = fmt.Sprintf(
				"Default (%s)",
				humanize.Bytes(state.Session.Version.DefaultMaximumStagingFileSize()),
			)
		} else {
			maximumStagingFileSizeDescription = fmt.Sprintf(
				"%d (%s)",
				configuration.MaximumStagingFileSize,
				humanize.Bytes(configuration.MaximumStagingFileSize),
			)
		}
		fmt.Println("\tMaximum staging file size:", maximumStagingFileSizeDescription)

		// Compute and print symlink mode.
		symlinkModeDescription := configuration.SymlinkMode.Description()
		if configuration.SymlinkMode.IsDefault() {
			defaultSymlinkMode := state.Session.Version.DefaultSymlinkMode()
			symlinkModeDescription += fmt.Sprintf(" (%s)", defaultSymlinkMode.Description())
		}
		fmt.Println("\tSymbolic link mode:", symlinkModeDescription)

		// Compute and print the VCS ignore mode.
		ignoreVCSModeDescription := configuration.IgnoreVCSMode.Description()
		if configuration.IgnoreVCSMode.IsDefault() {
			defaultIgnoreVCSMode := state.Session.Version.DefaultIgnoreVCSMode()
			ignoreVCSModeDescription += fmt.Sprintf(" (%s)", defaultIgnoreVCSMode.Description())
		}
		fmt.Println("\tIgnore VCS mode:", ignoreVCSModeDescription)

		// Print default ignores. Since this field is deprecated, we don't print
		// it if it's not set.
		if len(configuration.DefaultIgnores) > 0 {
			fmt.Println("\tDefault ignores:")
			for _, p := range configuration.DefaultIgnores {
				fmt.Printf("\t\t%s\n", p)
			}
		}

		// Print per-session ignores.
		if len(configuration.Ignores) > 0 {
			fmt.Println("\tIgnores:")
			for _, p := range configuration.Ignores {
				fmt.Printf("\t\t%s\n", p)
			}
		} else {
			fmt.Println("\tIgnores: None")
		}

		// Compute and print alpha-specific configuration.
		alphaConfigurationMerged := sessionpkg.MergeConfigurations(
			state.Session.Configuration,
			state.Session.ConfigurationAlpha,
		)
		printEndpoint("Alpha", state.Session.Alpha, alphaConfigurationMerged, state.Session.Version)

		// Compute and print beta-specific configuration.
		betaConfigurationMerged := sessionpkg.MergeConfigurations(
			state.Session.Configuration,
			state.Session.ConfigurationBeta,
		)
		printEndpoint("Beta", state.Session.Beta, betaConfigurationMerged, state.Session.Version)

		// Print labels.
		if len(state.Session.Labels) > 0 {
			fmt.Println("Labels:")
			keys := sessionpkg.ExtractAndSortLabelKeys(state.Session.Labels)
			for _, key := range keys {
				value := state.Session.Labels[key]
				if value == "" {
					value = emptyLabelValueDescription
				}
				fmt.Printf("\t%s: %s\n", key, value)
			}
		} else {
			fmt.Println("Labels: None")
		}
	}
}
<commit_msg>Relocated label printing in session printout.<commit_after>package main

import (
	"fmt"
	"math"

	"github.com/dustin/go-humanize"

	sessionpkg "github.com/havoc-io/mutagen/pkg/session"
	urlpkg "github.com/havoc-io/mutagen/pkg/url"
)

const (
	// maxUint64Description is a human-friendly mathematic description of
	// math.MaxUint64.
	maxUint64Description = "2⁶⁴−1"

	// emptyLabelValueDescription is a human-friendly description representing
	// an empty label value. It contains characters which are invalid for use in
	// label values, so it won't be confused for one.
	emptyLabelValueDescription = "<empty>"
)

func printEndpoint(name string, url *urlpkg.URL, configuration *sessionpkg.Configuration, version sessionpkg.Version) {
	// Print the endpoint header.
	fmt.Println(name, "configuration:")

	// Print the URL.
	fmt.Println("\tURL:", url.Format("\n\t\t"))

	// Compute and print the watch mode.
	watchModeDescription := configuration.WatchMode.Description()
	if configuration.WatchMode.IsDefault() {
		watchModeDescription += fmt.Sprintf(" (%s)", version.DefaultWatchMode().Description())
	}
	fmt.Println("\tWatch mode:", watchModeDescription)

	// Compute and print the watch polling interval, so long as we're not in
	// no-watch mode.
	if configuration.WatchMode != sessionpkg.WatchMode_WatchModeNoWatch {
		var watchPollingIntervalDescription string
		if configuration.WatchPollingInterval == 0 {
			watchPollingIntervalDescription = fmt.Sprintf("Default (%d seconds)", version.DefaultWatchPollingInterval())
		} else {
			watchPollingIntervalDescription = fmt.Sprintf("%d seconds", configuration.WatchPollingInterval)
		}
		fmt.Println("\tWatch polling interval:", watchPollingIntervalDescription)
	}

	// Compute and print the probe mode.
	probeModeDescription := configuration.ProbeMode.Description()
	if configuration.ProbeMode.IsDefault() {
		probeModeDescription += fmt.Sprintf(" (%s)", version.DefaultProbeMode().Description())
	}
	fmt.Println("\tProbe mode:", probeModeDescription)

	// Compute and print the scan mode.
	scanModeDescription := configuration.ScanMode.Description()
	if configuration.ScanMode.IsDefault() {
		scanModeDescription += fmt.Sprintf(" (%s)", version.DefaultScanMode().Description())
	}
	fmt.Println("\tScan mode:", scanModeDescription)

	// Compute and print the staging mode.
	stageModeDescription := configuration.StageMode.Description()
	if configuration.StageMode.IsDefault() {
		stageModeDescription += fmt.Sprintf(" (%s)", version.DefaultStageMode().Description())
	}
	fmt.Println("\tStage mode:", stageModeDescription)

	// Compute and print the default file mode.
	var defaultFileModeDescription string
	if configuration.DefaultFileMode == 0 {
		defaultFileModeDescription = fmt.Sprintf("Default (%#o)", version.DefaultFileMode())
	} else {
		defaultFileModeDescription = fmt.Sprintf("%#o", configuration.DefaultFileMode)
	}
	fmt.Println("\tFile mode:", defaultFileModeDescription)

	// Compute and print the default directory mode.
	var defaultDirectoryModeDescription string
	if configuration.DefaultDirectoryMode == 0 {
		defaultDirectoryModeDescription = fmt.Sprintf("Default (%#o)", version.DefaultDirectoryMode())
	} else {
		defaultDirectoryModeDescription = fmt.Sprintf("%#o", configuration.DefaultDirectoryMode)
	}
	fmt.Println("\tDirectory mode:", defaultDirectoryModeDescription)

	// Compute and print the default file/directory owner.
	defaultOwnerDescription := "Default"
	if configuration.DefaultOwner != "" {
		defaultOwnerDescription = configuration.DefaultOwner
	}
	fmt.Println("\tDefault file/directory owner:", defaultOwnerDescription)

	// Compute and print the default file/directory group.
	defaultGroupDescription := "Default"
	if configuration.DefaultGroup != "" {
		defaultGroupDescription = configuration.DefaultGroup
	}
	fmt.Println("\tDefault file/directory group:", defaultGroupDescription)
}

func printSession(state *sessionpkg.State, long bool) {
	// Print the session identifier.
	fmt.Println("Session:", state.Session.Identifier)

	// Print labels.
	if len(state.Session.Labels) > 0 {
		fmt.Println("Labels:")
		keys := sessionpkg.ExtractAndSortLabelKeys(state.Session.Labels)
		for _, key := range keys {
			value := state.Session.Labels[key]
			if value == "" {
				value = emptyLabelValueDescription
			}
			fmt.Printf("\t%s: %s\n", key, value)
		}
	} else {
		fmt.Println("Labels: None")
	}

	// Print extended information, if desired.
	if long {
		// Print the configuration header.
		fmt.Println("Configuration:")

		// Extract configuration.
		configuration := state.Session.Configuration

		// Compute and print synchronization mode.
		synchronizationMode := configuration.SynchronizationMode.Description()
		if configuration.SynchronizationMode.IsDefault() {
			defaultSynchronizationMode := state.Session.Version.DefaultSynchronizationMode()
			synchronizationMode += fmt.Sprintf(" (%s)", defaultSynchronizationMode.Description())
		}
		fmt.Println("\tSynchronization mode:", synchronizationMode)

		// Compute and print maximum entry count.
		var maximumEntryCountDescription string
		if configuration.MaximumEntryCount == 0 {
			if m := state.Session.Version.DefaultMaximumEntryCount(); m == math.MaxUint64 {
				maximumEntryCountDescription = fmt.Sprintf("Default (%s)", maxUint64Description)
			} else {
				maximumEntryCountDescription = fmt.Sprintf("Default (%d)", m)
			}
		} else {
			maximumEntryCountDescription = fmt.Sprintf("%d", configuration.MaximumEntryCount)
		}
		fmt.Println("\tMaximum allowed entry count:", maximumEntryCountDescription)

		// Compute and print maximum staging file size.
		var maximumStagingFileSizeDescription string
		if configuration.MaximumStagingFileSize == 0 {
			maximumStagingFileSizeDescription = fmt.Sprintf(
				"Default (%s)",
				humanize.Bytes(state.Session.Version.DefaultMaximumStagingFileSize()),
			)
		} else {
			maximumStagingFileSizeDescription = fmt.Sprintf(
				"%d (%s)",
				configuration.MaximumStagingFileSize,
				humanize.Bytes(configuration.MaximumStagingFileSize),
			)
		}
		fmt.Println("\tMaximum staging file size:", maximumStagingFileSizeDescription)

		// Compute and print symlink mode.
		symlinkModeDescription := configuration.SymlinkMode.Description()
		if configuration.SymlinkMode.IsDefault() {
			defaultSymlinkMode := state.Session.Version.DefaultSymlinkMode()
			symlinkModeDescription += fmt.Sprintf(" (%s)", defaultSymlinkMode.Description())
		}
		fmt.Println("\tSymbolic link mode:", symlinkModeDescription)

		// Compute and print the VCS ignore mode.
		ignoreVCSModeDescription := configuration.IgnoreVCSMode.Description()
		if configuration.IgnoreVCSMode.IsDefault() {
			defaultIgnoreVCSMode := state.Session.Version.DefaultIgnoreVCSMode()
			ignoreVCSModeDescription += fmt.Sprintf(" (%s)", defaultIgnoreVCSMode.Description())
		}
		fmt.Println("\tIgnore VCS mode:", ignoreVCSModeDescription)

		// Print default ignores. Since this field is deprecated, we don't print
		// it if it's not set.
		if len(configuration.DefaultIgnores) > 0 {
			fmt.Println("\tDefault ignores:")
			for _, p := range configuration.DefaultIgnores {
				fmt.Printf("\t\t%s\n", p)
			}
		}

		// Print per-session ignores.
		if len(configuration.Ignores) > 0 {
			fmt.Println("\tIgnores:")
			for _, p := range configuration.Ignores {
				fmt.Printf("\t\t%s\n", p)
			}
		} else {
			fmt.Println("\tIgnores: None")
		}

		// Compute and print alpha-specific configuration.
		alphaConfigurationMerged := sessionpkg.MergeConfigurations(
			state.Session.Configuration,
			state.Session.ConfigurationAlpha,
		)
		printEndpoint("Alpha", state.Session.Alpha, alphaConfigurationMerged, state.Session.Version)

		// Compute and print beta-specific configuration.
		betaConfigurationMerged := sessionpkg.MergeConfigurations(
			state.Session.Configuration,
			state.Session.ConfigurationBeta,
		)
		printEndpoint("Beta", state.Session.Beta, betaConfigurationMerged, state.Session.Version)
	}
}
<|endoftext|>


<commit_before>package commands

import (
	"bufio"
	"bytes"
	"encoding/hex"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/git-lfs/git-lfs/errors"
	"github.com/git-lfs/git-lfs/filepathfilter"
	"github.com/git-lfs/git-lfs/git"
	"github.com/git-lfs/git-lfs/git/githistory"
	"github.com/git-lfs/git-lfs/git/odb"
	"github.com/git-lfs/git-lfs/lfs"
	"github.com/git-lfs/git-lfs/tasklog"
	"github.com/git-lfs/git-lfs/tools"
	"github.com/spf13/cobra"
)

func migrateImportCommand(cmd *cobra.Command, args []string) {
	l := tasklog.NewLogger(os.Stderr)
	defer l.Close()

	db, err := getObjectDatabase()
	if err != nil {
		ExitWithError(err)
	}
	defer db.Close()

	if migrateNoRewrite {
		ref, err := git.CurrentRef()
		if err != nil {
			ExitWithError(errors.Wrap(err, "fatal: unable to find current reference"))
		}

		sha, _ := hex.DecodeString(ref.Sha)
		commit, err := db.Commit(sha)
		if err != nil {
			ExitWithError(errors.Wrap(err, "fatal: unable to load commit"))
		}

		root := commit.TreeID

		include, _ := getIncludeExcludeArgs(cmd)
		paths, _ := determineIncludeExcludePaths(cfg, include, nil)
		if len(paths) == 0 {
			ExitWithError(errors.New("fatal: no matching paths found"))
		}

		for _, path := range paths {
			root, err = rewriteTree(db, root, path)
			if err != nil {
				ExitWithError(errors.Wrapf(err, "fatal: could not rewrite %q", path))
			}
		}

		name, email := cfg.CurrentCommitter()
		author := fmt.Sprintf("%s <%s>", name, email)

		oid, err := db.WriteCommit(&odb.Commit{
			Author:    author,
			Committer: author,
			ParentIDs: [][]byte{sha},
			Message:   generateMigrateCommitMessage(cmd, include),
			TreeID:    root,
		})

		if err != nil {
			ExitWithError(errors.Wrap(err, "fatal: unable to write commit"))
		}

		if err := git.UpdateRef(ref, oid, "git lfs migrate import --no-rewrite"); err != nil {
			ExitWithError(errors.Wrap(err, "fatal: unable to update ref"))
		}

		if err := checkoutNonBare(l); err != nil {
			ExitWithError(errors.Wrap(err, "fatal: could not checkout"))
		}

		return
	}

	rewriter := getHistoryRewriter(cmd, db, l)

	tracked := trackedFromFilter(rewriter.Filter())
	exts := tools.NewOrderedSet()
	gitfilter := lfs.NewGitFilter(cfg)

	migrate(args, rewriter, l, &githistory.RewriteOptions{
		Verbose:           migrateVerbose,
		ObjectMapFilePath: objectMapFilePath,
		BlobFn: func(path string, b *odb.Blob) (*odb.Blob, error) {
			if filepath.Base(path) == ".gitattributes" {
				return b, nil
			}

			var buf bytes.Buffer

			if _, err := clean(gitfilter, &buf, b.Contents, path, b.Size); err != nil {
				return nil, err
			}

			if ext := filepath.Ext(path); len(ext) > 0 {
				exts.Add(fmt.Sprintf("*%s filter=lfs diff=lfs merge=lfs -text", ext))
			}

			return &odb.Blob{
				Contents: &buf, Size: int64(buf.Len()),
			}, nil
		},

		TreeCallbackFn: func(path string, t *odb.Tree) (*odb.Tree, error) {
			if path != "/" {
				// Ignore non-root trees.
				return t, nil
			}

			ours := tracked
			if ours.Cardinality() == 0 {
				// If there were no explicitly tracked
				// --include, --exclude filters, assume that the
				// include set is the wildcard filepath
				// extensions of files tracked.
				ours = exts
			}

			theirs, err := trackedFromAttrs(db, t)
			if err != nil {
				return nil, err
			}

			// Create a blob of the attributes that are optionally
			// present in the "t" tree's .gitattributes blob, and
			// union in the patterns that we've tracked.
			//
			// Perform this Union() operation each time we visit a
			// root tree such that if the underlying .gitattributes
			// is present and has a diff between commits in the
			// range of commits to migrate, those changes are
			// preserved.
			blob, err := trackedToBlob(db, theirs.Clone().Union(ours))
			if err != nil {
				return nil, err
			}

			// Finally, return a copy of the tree "t" that has the
			// new .gitattributes file included/replaced.
			return t.Merge(&odb.TreeEntry{
				Name:     ".gitattributes",
				Filemode: 0100644,
				Oid:      blob,
			}), nil
		},

		UpdateRefs: true,
	})

	if err := checkoutNonBare(l); err != nil {
		ExitWithError(errors.Wrap(err, "fatal: could not checkout"))
	}
}

// generateMigrateCommitMessage generates a commit message used with
// --no-rewrite, using --message (if given) or generating one if it isn't.
func generateMigrateCommitMessage(cmd *cobra.Command, patterns *string) string {
	if cmd.Flag("message").Changed {
		return migrateCommitMessage
	}
	if patterns != nil {
		return fmt.Sprintf("%s: convert to Git LFS", *patterns)
	}
	panic("unexpected")
}

// checkoutNonBare forces a checkout of the current reference, so long as the
// repository is non-bare.
//
// It returns nil on success, and a non-nil error on failure.
func checkoutNonBare(l *tasklog.Logger) error {
	if bare, _ := git.IsBare(); bare {
		return nil
	}

	t := l.Waiter("migrate: checkout")
	defer t.Complete()

	return git.Checkout("", nil, true)
}

// trackedFromFilter returns an ordered set of strings where each entry is a
// line in the .gitattributes file. It adds/removes the fiter/diff/merge=lfs
// attributes based on patterns included/excldued in the given filter.
func trackedFromFilter(filter *filepathfilter.Filter) *tools.OrderedSet {
	tracked := tools.NewOrderedSet()

	for _, include := range filter.Include() {
		tracked.Add(fmt.Sprintf("%s filter=lfs diff=lfs merge=lfs -text", escapeAttrPattern(include)))
	}

	for _, exclude := range filter.Exclude() {
		tracked.Add(fmt.Sprintf("%s text -filter -merge -diff", escapeAttrPattern(exclude)))
	}

	return tracked
}

var (
	// attrsCache maintains a cache from the hex-encoded SHA1 of a
	// .gitattributes blob to the set of patterns parsed from that blob.
	attrsCache = make(map[string]*tools.OrderedSet)
)

// trackedFromAttrs returns an ordered line-delimited set of the contents of a
// .gitattributes blob in a given tree "t".
//
// It returns an empty set if no attributes file could be found, or an error if
// it could not otherwise be opened.
func trackedFromAttrs(db *odb.ObjectDatabase, t *odb.Tree) (*tools.OrderedSet, error) {
	var oid []byte

	for _, e := range t.Entries {
		if strings.ToLower(e.Name) == ".gitattributes" && e.Type() == odb.BlobObjectType {
			oid = e.Oid
			break
		}
	}

	if oid == nil {
		// TODO(@ttaylorr): make (*tools.OrderedSet)(nil) a valid
		// receiver for non-mutative methods.
		return tools.NewOrderedSet(), nil
	}

	sha1 := hex.EncodeToString(oid)

	if s, ok := attrsCache[sha1]; ok {
		return s, nil
	}

	blob, err := db.Blob(oid)
	if err != nil {
		return nil, err
	}

	attrs := tools.NewOrderedSet()

	scanner := bufio.NewScanner(blob.Contents)
	for scanner.Scan() {
		attrs.Add(scanner.Text())
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}

	attrsCache[sha1] = attrs

	return attrsCache[sha1], nil
}

// trackedToBlob writes and returns the OID of a .gitattributes blob based on
// the patterns given in the ordered set of patterns, "patterns".
func trackedToBlob(db *odb.ObjectDatabase, patterns *tools.OrderedSet) ([]byte, error) {
	var attrs bytes.Buffer

	for pattern := range patterns.Iter() {
		fmt.Fprintf(&attrs, "%s\n", pattern)
	}

	return db.WriteBlob(&odb.Blob{
		Contents: &attrs,
		Size:     int64(attrs.Len()),
	})
}

func rewriteTree(db *odb.ObjectDatabase, root []byte, path string) ([]byte, error) {
	return root, nil
}
<commit_msg>commands: rewrite tree with --no-rewrite<commit_after>package commands

import (
	"bufio"
	"bytes"
	"encoding/hex"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/git-lfs/git-lfs/errors"
	"github.com/git-lfs/git-lfs/filepathfilter"
	"github.com/git-lfs/git-lfs/git"
	"github.com/git-lfs/git-lfs/git/githistory"
	"github.com/git-lfs/git-lfs/git/odb"
	"github.com/git-lfs/git-lfs/lfs"
	"github.com/git-lfs/git-lfs/tasklog"
	"github.com/git-lfs/git-lfs/tools"
	"github.com/spf13/cobra"
)

func migrateImportCommand(cmd *cobra.Command, args []string) {
	l := tasklog.NewLogger(os.Stderr)
	defer l.Close()

	db, err := getObjectDatabase()
	if err != nil {
		ExitWithError(err)
	}
	defer db.Close()

	if migrateNoRewrite {
		ref, err := git.CurrentRef()
		if err != nil {
			ExitWithError(errors.Wrap(err, "fatal: unable to find current reference"))
		}

		sha, _ := hex.DecodeString(ref.Sha)
		commit, err := db.Commit(sha)
		if err != nil {
			ExitWithError(errors.Wrap(err, "fatal: unable to load commit"))
		}

		root := commit.TreeID

		include, _ := getIncludeExcludeArgs(cmd)
		paths, _ := determineIncludeExcludePaths(cfg, include, nil)
		if len(paths) == 0 {
			ExitWithError(errors.New("fatal: no matching paths found"))
		}

		entries, err := listEntries(db, root, "")
		if err != nil {
			ExitWithError(errors.Wrap(err, "fatal: unable to list tree entries"))
		}

		filter := filepathfilter.New(paths, nil)
		for _, entry := range entries {
			if filter.Allows(entry) {
				root, err = rewriteTree(db, root, entry)
				if err != nil {
					ExitWithError(errors.Wrapf(err, "fatal: could not rewrite %q", entry))
				}
			}
		}

		name, email := cfg.CurrentCommitter()
		author := fmt.Sprintf("%s <%s>", name, email)

		oid, err := db.WriteCommit(&odb.Commit{
			Author:    author,
			Committer: author,
			ParentIDs: [][]byte{sha},
			Message:   generateMigrateCommitMessage(cmd, include),
			TreeID:    root,
		})

		if err != nil {
			ExitWithError(errors.Wrap(err, "fatal: unable to write commit"))
		}

		if err := git.UpdateRef(ref, oid, "git lfs migrate import --no-rewrite"); err != nil {
			ExitWithError(errors.Wrap(err, "fatal: unable to update ref"))
		}

		if err := checkoutNonBare(l); err != nil {
			ExitWithError(errors.Wrap(err, "fatal: could not checkout"))
		}

		return
	}

	rewriter := getHistoryRewriter(cmd, db, l)

	tracked := trackedFromFilter(rewriter.Filter())
	exts := tools.NewOrderedSet()
	gitfilter := lfs.NewGitFilter(cfg)

	migrate(args, rewriter, l, &githistory.RewriteOptions{
		Verbose:           migrateVerbose,
		ObjectMapFilePath: objectMapFilePath,
		BlobFn: func(path string, b *odb.Blob) (*odb.Blob, error) {
			if filepath.Base(path) == ".gitattributes" {
				return b, nil
			}

			var buf bytes.Buffer

			if _, err := clean(gitfilter, &buf, b.Contents, path, b.Size); err != nil {
				return nil, err
			}

			if ext := filepath.Ext(path); len(ext) > 0 {
				exts.Add(fmt.Sprintf("*%s filter=lfs diff=lfs merge=lfs -text", ext))
			}

			return &odb.Blob{
				Contents: &buf, Size: int64(buf.Len()),
			}, nil
		},

		TreeCallbackFn: func(path string, t *odb.Tree) (*odb.Tree, error) {
			if path != "/" {
				// Ignore non-root trees.
				return t, nil
			}

			ours := tracked
			if ours.Cardinality() == 0 {
				// If there were no explicitly tracked
				// --include, --exclude filters, assume that the
				// include set is the wildcard filepath
				// extensions of files tracked.
				ours = exts
			}

			theirs, err := trackedFromAttrs(db, t)
			if err != nil {
				return nil, err
			}

			// Create a blob of the attributes that are optionally
			// present in the "t" tree's .gitattributes blob, and
			// union in the patterns that we've tracked.
			//
			// Perform this Union() operation each time we visit a
			// root tree such that if the underlying .gitattributes
			// is present and has a diff between commits in the
			// range of commits to migrate, those changes are
			// preserved.
			blob, err := trackedToBlob(db, theirs.Clone().Union(ours))
			if err != nil {
				return nil, err
			}

			// Finally, return a copy of the tree "t" that has the
			// new .gitattributes file included/replaced.
			return t.Merge(&odb.TreeEntry{
				Name:     ".gitattributes",
				Filemode: 0100644,
				Oid:      blob,
			}), nil
		},

		UpdateRefs: true,
	})

	if err := checkoutNonBare(l); err != nil {
		ExitWithError(errors.Wrap(err, "fatal: could not checkout"))
	}
}

// generateMigrateCommitMessage generates a commit message used with
// --no-rewrite, using --message (if given) or generating one if it isn't.
func generateMigrateCommitMessage(cmd *cobra.Command, patterns *string) string {
	if cmd.Flag("message").Changed {
		return migrateCommitMessage
	}
	if patterns != nil {
		return fmt.Sprintf("%s: convert to Git LFS", *patterns)
	}
	panic("unexpected")
}

// checkoutNonBare forces a checkout of the current reference, so long as the
// repository is non-bare.
//
// It returns nil on success, and a non-nil error on failure.
func checkoutNonBare(l *tasklog.Logger) error {
	if bare, _ := git.IsBare(); bare {
		return nil
	}

	t := l.Waiter("migrate: checkout")
	defer t.Complete()

	return git.Checkout("", nil, true)
}

// trackedFromFilter returns an ordered set of strings where each entry is a
// line in the .gitattributes file. It adds/removes the fiter/diff/merge=lfs
// attributes based on patterns included/excldued in the given filter.
func trackedFromFilter(filter *filepathfilter.Filter) *tools.OrderedSet {
	tracked := tools.NewOrderedSet()

	for _, include := range filter.Include() {
		tracked.Add(fmt.Sprintf("%s filter=lfs diff=lfs merge=lfs -text", escapeAttrPattern(include)))
	}

	for _, exclude := range filter.Exclude() {
		tracked.Add(fmt.Sprintf("%s text -filter -merge -diff", escapeAttrPattern(exclude)))
	}

	return tracked
}

var (
	// attrsCache maintains a cache from the hex-encoded SHA1 of a
	// .gitattributes blob to the set of patterns parsed from that blob.
	attrsCache = make(map[string]*tools.OrderedSet)
)

// trackedFromAttrs returns an ordered line-delimited set of the contents of a
// .gitattributes blob in a given tree "t".
//
// It returns an empty set if no attributes file could be found, or an error if
// it could not otherwise be opened.
func trackedFromAttrs(db *odb.ObjectDatabase, t *odb.Tree) (*tools.OrderedSet, error) {
	var oid []byte

	for _, e := range t.Entries {
		if strings.ToLower(e.Name) == ".gitattributes" && e.Type() == odb.BlobObjectType {
			oid = e.Oid
			break
		}
	}

	if oid == nil {
		// TODO(@ttaylorr): make (*tools.OrderedSet)(nil) a valid
		// receiver for non-mutative methods.
		return tools.NewOrderedSet(), nil
	}

	sha1 := hex.EncodeToString(oid)

	if s, ok := attrsCache[sha1]; ok {
		return s, nil
	}

	blob, err := db.Blob(oid)
	if err != nil {
		return nil, err
	}

	attrs := tools.NewOrderedSet()

	scanner := bufio.NewScanner(blob.Contents)
	for scanner.Scan() {
		attrs.Add(scanner.Text())
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}

	attrsCache[sha1] = attrs

	return attrsCache[sha1], nil
}

// trackedToBlob writes and returns the OID of a .gitattributes blob based on
// the patterns given in the ordered set of patterns, "patterns".
func trackedToBlob(db *odb.ObjectDatabase, patterns *tools.OrderedSet) ([]byte, error) {
	var attrs bytes.Buffer

	for pattern := range patterns.Iter() {
		fmt.Fprintf(&attrs, "%s\n", pattern)
	}

	return db.WriteBlob(&odb.Blob{
		Contents: &attrs,
		Size:     int64(attrs.Len()),
	})
}

// rewriteTree replaces the blob at the provided path within the given tree with
// a git lfs pointer. It will recursively rewrite any subtrees along the path to the
// blob.
func rewriteTree(db *odb.ObjectDatabase, root []byte, path string) ([]byte, error) {
	tree, err := db.Tree(root)
	if err != nil {
		return nil, err
	}

	splits := strings.SplitN(path, "/", 2)

	switch len(splits) {
	case 1:
		// The path points to an entry at the root of this tree, so it must be a blob.
		// Try to replace this blob with a git lfs pointer.
		index := findEntry(tree, splits[0])
		if index < 0 {
			return nil, errors.Errorf("unable to find entry %s in tree", splits[0])
		}

		blobEntry := tree.Entries[index]
		blob, err := db.Blob(blobEntry.Oid)
		if err != nil {
			return nil, err
		}

		var buf bytes.Buffer

		if _, err := clean(lfs.NewGitFilter(cfg), &buf, blob.Contents, blobEntry.Name, blob.Size); err != nil {
			return nil, err
		}

		newBlob := odb.Blob{
			Contents: &buf,
			Size:     int64(buf.Len()),
		}

		newOid, err := db.WriteBlob(&newBlob)
		if err != nil {
			return nil, err
		}

		newEntry := odb.TreeEntry{
			Name:     splits[0],
			Filemode: blobEntry.Filemode,
			Oid:      newOid,
		}
		tree = tree.Merge(&newEntry)
		return db.WriteTree(tree)

	case 2:
		// The path points to an entry in a subtree contained at the root of the tree.
		// Recursively rewrite the subtree.
		head, tail := splits[0], splits[1]

		index := findEntry(tree, head)
		if index < 0 {
			return nil, errors.Errorf("unable to find entry %s in tree", head)
		}

		subtreeEntry := tree.Entries[index]
		if subtreeEntry.Type() != odb.TreeObjectType {
			return nil, errors.Errorf("migrate: expected %s to be a tree, got %s", head, subtreeEntry.Type())
		}

		rewrittenSubtree, err := rewriteTree(db, subtreeEntry.Oid, tail)
		if err != nil {
			return nil, err
		}

		tree = tree.Merge(&odb.TreeEntry{
			Filemode: subtreeEntry.Filemode,
			Name:     subtreeEntry.Name,
			Oid:      rewrittenSubtree,
		})

		return db.WriteTree(tree)

	default:
		return nil, errors.Errorf("error parsing path %s", path)
	}
}

// findEntry searches a tree for the desired entry, and returns the
// index of that entry within the tree's Entries array
func findEntry(t *odb.Tree, name string) int {
	for i, entry := range t.Entries {
		if entry.Name == name {
			return i
		}
	}

	return -1
}

// listEntries returns an array with the full paths to all blobs within the provided tree.
func listEntries(db *odb.ObjectDatabase, id []byte, parent string) ([]string, error) {
	t, err := db.Tree(id)
	if err != nil {
		return nil, err
	}

	names := make([]string, 0, len(t.Entries))

	for _, e := range t.Entries {
		switch e.Type() {
		case odb.BlobObjectType:
			names = append(names, fmt.Sprintf("%s%s", parent, e.Name))
		case odb.TreeObjectType:
			sub, err := listEntries(db, e.Oid, fmt.Sprintf("%s%s/", parent, e.Name))
			if err != nil {
				return nil, err
			}
			names = append(names, sub...)
		}
	}

	return names, nil
}
<|endoftext|>


<commit_before>package guest

const Instance = `{{define "instance"}}
  MasterInstance:
    Type: "AWS::EC2::Instance"
    Description: Master instance
    Properties:
      AvailabilityZone: {{ .MasterAZ }}
      IamInstanceProfile: !Ref MasterInstanceProfile
      ImageId: {{ .MasterImageID }}
      InstanceType: {{ .MasterInstanceType }}
      SecurityGroupIds:
      - !Ref MasterSecurityGroup
      SubnetId: !Ref PrivateSubnet
      UserData: {{ .MasterSmallCloudConfig }}
      Tags:
      - Key: Name
        Value: {{ .ClusterID }}-master
  EtcdVolume:
    Type: AWS::EC2::Volume
    Properties:
      Size: 100
      VolumeType: gp2
      AvailabilityZone: !GetAtt MasterInstance.AvailabilityZone
      Tags:
      - Key: Name
        Value: {{ .ClusterID }}-etcd
  MountPoint:
    Type: AWS::EC2::VolumeAttachment
    Properties:
      InstanceId: !Ref MasterInstance
      VolumeId: !Ref EtcdVolume
      Device: /dev/sdh
{{end}}`
<commit_msg>Make etcd volume dependent on master instance (#773)<commit_after>package guest

const Instance = `{{define "instance"}}
  MasterInstance:
    Type: "AWS::EC2::Instance"
    Description: Master instance
    Properties:
      AvailabilityZone: {{ .MasterAZ }}
      IamInstanceProfile: !Ref MasterInstanceProfile
      ImageId: {{ .MasterImageID }}
      InstanceType: {{ .MasterInstanceType }}
      SecurityGroupIds:
      - !Ref MasterSecurityGroup
      SubnetId: !Ref PrivateSubnet
      UserData: {{ .MasterSmallCloudConfig }}
      Tags:
      - Key: Name
        Value: {{ .ClusterID }}-master
  EtcdVolume:
    Type: AWS::EC2::Volume
    DependsOn:
    - MasterInstance
    Properties:
      Size: 100
      VolumeType: gp2
      AvailabilityZone: !GetAtt MasterInstance.AvailabilityZone
      Tags:
      - Key: Name
        Value: {{ .ClusterID }}-etcd
  MountPoint:
    Type: AWS::EC2::VolumeAttachment
    Properties:
      InstanceId: !Ref MasterInstance
      VolumeId: !Ref EtcdVolume
      Device: /dev/sdh
{{end}}`
<|endoftext|>


<commit_before>package main

import (
    "github.com/eaciit/knot/knot.v1"
    "eaciit/musicstream/web"
)

func main() {
    app := webapp.App()
    knot.StartApp(app, "localhost:9100")
}<commit_msg>change port<commit_after>package main

import (
    "github.com/eaciit/knot/knot.v1"
    "eaciit/musicstream/web"
)

func main() {
    app := webapp.App()
    knot.StartApp(app, "localhost:8027")
}<|endoftext|>


<commit_before>package proxmox

import (
	"errors"
	"net/url"
	"strconv"
	"strings"
	"time"

	_ "github.com/davecgh/go-spew/spew"
)

type QemuVM struct {
	Mem       float64 `json:"mem"`
	CPUs      float64 `json:"cpus"`
	NetOut    float64 `json:"netout"`
	PID       string  `json:"pid"`
	Disk      float64 `json:"disk"`
	MaxMem    float64 `json:"maxmem"`
	Status    string  `json:"status"`
	Template  float64 `json:"template"`
	NetIn     float64 `json:"netin"`
	MaxDisk   float64 `json:"maxdisk"`
	Name      string  `json:"name"`
	DiskWrite float64 `json:"diskwrite"`
	CPU       float64 `json:"cpu"`
	VMId      float64 `json:"vmid"`
	DiskRead  float64 `json:"diskread"`
	Uptime    float64 `json:"uptime"`
	Node      Node
}

type QemuList map[string]QemuVM

type QemuNet map[string]string

type QemuConfig struct {
	Bootdisk    string  `json:"bootdisk"`
	Cores       float64 `json:"cores"`
	Digest      string  `json:"digest"`
	Memory      float64 `json:"memory"`
	Net         map[string]QemuNet
	SMBios1     string            `json:"smbios1"`
	Sockets     float64           `json:"sockets"`
	Disks       map[string]string `json:"disks"`
	Description string `json:"description"`
}

type QemuStatus struct {
	CPU       float64 `json:"cpu"`
	CPUs      float64 `json:"cpus"`
	Mem       float64 `json:"mem"`
	MaxMem    float64 `json:"maxmem"`
	Disk      float64 `json:"disk"`
	MaxDisk   float64 `json:"maxdisk"`
	DiskWrite float64 `json:"diskwrite"`
	DiskRead  float64 `json:"diskread"`
	NetIn     float64 `json:"netin"`
	NetOut    float64 `json:"netout"`
	Uptime    float64 `json:"uptime"`
	QmpStatus string  `json:"qmpstatus"`
	Status    string  `json:"status"`
	Template  string  `json:"template"`
}

func (qemu QemuVM) Delete() (map[string]interface{}, error) {
	var target string
	var data map[string]interface{}
	var err error

	//fmt.Print("!QemuDelete ", qemu.VMId)

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64)
	data, err = qemu.Node.Proxmox.Delete(target)
	if err != nil {
		return nil, err
	}
	return data, nil
}

func stringToMap(data string, itemSeparator string, kvSeparator string) map[string]string {
	var result map[string]string

	result = make(map[string]string)
	list := strings.Split(data, itemSeparator)
	for _, item := range list {
		kv := strings.Split(item, kvSeparator)
		result[kv[0]] = kv[1]
	}
	return result
}

func (qemu QemuVM) Config() (QemuConfig, error) {
	var target string
	var data map[string]interface{}
	var results map[string]interface{}
	var config QemuConfig
	var err error

	//fmt.Print("!QemuConfig ", qemu.VMId)

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/config"
	data, err = qemu.Node.Proxmox.Get(target)
	results = data["data"].(map[string]interface{})
	if err != nil {
		return config, err
	}
	config = QemuConfig{
		Bootdisk: results["bootdisk"].(string),
		Cores:    results["cores"].(float64),
		Digest:   results["digest"].(string),
		Memory:   results["memory"].(float64),
		Sockets:  results["sockets"].(float64),
		SMBios1:  results["smbios1"].(string),
	}
	disktype := [3]string{"virtio", "sata", "ide"}
	disknum := [4]string{"0", "1", "2", "3"}
	config.Disks = make(map[string]string)
	for _, d := range disktype {
		for _, i := range disknum {
			id := d + i
			if disk, ok := results[id]; ok {
				config.Disks[id] = disk.(string)
			}
		}
	}
	config.Net = make(map[string]QemuNet)
	netnum := [4]string{"0", "1", "2", "3"}
	for _, n := range netnum {
		if net, ok := results["net"+n]; ok {
			config.Net["net"+n] = stringToMap(net.(string), ",", "=")
		}
	}

	return config, nil
}

func (qemu QemuVM) CurrentStatus() (QemuStatus, error) {
	var target string
	var err error
	var data map[string]interface{}
	var results map[string]interface{}
	var status QemuStatus

	//fmt.Println("!QemuStatus ", strconv.FormatFloat(qemu.VMId, 'f', 0, 64))

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/status/current"
	data, err = qemu.Node.Proxmox.Get(target)
	if err != nil {
		return status, err
	}
	results = data["data"].(map[string]interface{})
	status = QemuStatus{
		CPU:       results["cpu"].(float64),
		CPUs:      results["cpus"].(float64),
		Mem:       results["mem"].(float64),
		MaxMem:    results["maxmem"].(float64),
		Disk:      results["disk"].(float64),
		MaxDisk:   results["maxdisk"].(float64),
		DiskWrite: results["diskwrite"].(float64),
		DiskRead:  results["diskread"].(float64),
		NetIn:     results["netin"].(float64),
		NetOut:    results["netout"].(float64),
		Uptime:    results["uptime"].(float64),
		QmpStatus: results["qmpstatus"].(string),
		Status:    results["status"].(string),
		Template:  results["template"].(string),
	}
	return status, nil
}

func (qemu QemuVM) WaitForStatus(status string, timeout int) error {
	var i int
	var err error
	var qStatus QemuStatus
	for i = 0; i < timeout; i++ {
		qStatus, err = qemu.CurrentStatus()
		if err != nil {
			return err
		}

		if qStatus.Status == status {
			return nil
		}
		time.Sleep(time.Second * 1)
	}
	return errors.New("Timeout reached")
}

func (qemu QemuVM) Start() error {
	var target string
	var err error

	//fmt.Println("!QemuStart ", strconv.FormatFloat(qemu.VMId, 'f', 0, 64))

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/status/start"
	_, err = qemu.Node.Proxmox.Post(target, "")
	return err
}

func (qemu QemuVM) Stop() error {
	var target string
	var err error

	//fmt.Print("!QemuStop ", qemu.VMId)

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/status/stop"
	_, err = qemu.Node.Proxmox.Post(target, "")
	return err
}

func (qemu QemuVM) Shutdown() error {
	var target string
	var err error

	//fmt.Print("!QemuShutdown ", qemu.VMId)

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/status/shutdown"
	_, err = qemu.Node.Proxmox.Post(target, "")
	return err
}

func (qemu QemuVM) Suspend() error {
	var target string
	var err error

	//fmt.Print("!QemuSuspend ", qemu.VMId)

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/status/suspend"
	_, err = qemu.Node.Proxmox.Post(target, "")
	return err
}

func (qemu QemuVM) Resume() error {
	var target string
	var err error

	//fmt.Print("!QemuResume ", qemu.VMId)

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/status/resume"
	_, err = qemu.Node.Proxmox.Post(target, "")
	return err
}

func (qemu QemuVM) Clone(newId float64, name string, targetName string) (Task, error) {
	var target string
	var err error

	newVMID := strconv.FormatFloat(newId, 'f', 0, 64)

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/clone"

	form := url.Values{
		"newid":  {newVMID},
		"name":   {name},
		"target": {targetName},
		"full":   {"1"},
	}

	data, err := qemu.Node.Proxmox.PostForm(target, form)
	if err != err {
		return Task{}, err
	}

	t := Task{
		UPid:    data["data"].(string),
		proxmox: qemu.Node.Proxmox,
	}

	return t, nil
}
<commit_msg>set description<commit_after>package proxmox

import (
	"errors"
	"net/url"
	"strconv"
	"strings"
	"time"

	_ "github.com/davecgh/go-spew/spew"
)

type QemuVM struct {
	Mem       float64 `json:"mem"`
	CPUs      float64 `json:"cpus"`
	NetOut    float64 `json:"netout"`
	PID       string  `json:"pid"`
	Disk      float64 `json:"disk"`
	MaxMem    float64 `json:"maxmem"`
	Status    string  `json:"status"`
	Template  float64 `json:"template"`
	NetIn     float64 `json:"netin"`
	MaxDisk   float64 `json:"maxdisk"`
	Name      string  `json:"name"`
	DiskWrite float64 `json:"diskwrite"`
	CPU       float64 `json:"cpu"`
	VMId      float64 `json:"vmid"`
	DiskRead  float64 `json:"diskread"`
	Uptime    float64 `json:"uptime"`
	Node      Node
}

type QemuList map[string]QemuVM

type QemuNet map[string]string

type QemuConfig struct {
	Bootdisk    string  `json:"bootdisk"`
	Cores       float64 `json:"cores"`
	Digest      string  `json:"digest"`
	Memory      float64 `json:"memory"`
	Net         map[string]QemuNet
	SMBios1     string            `json:"smbios1"`
	Sockets     float64           `json:"sockets"`
	Disks       map[string]string `json:"disks"`
	Description string `json:"description"`
}

type QemuStatus struct {
	CPU       float64 `json:"cpu"`
	CPUs      float64 `json:"cpus"`
	Mem       float64 `json:"mem"`
	MaxMem    float64 `json:"maxmem"`
	Disk      float64 `json:"disk"`
	MaxDisk   float64 `json:"maxdisk"`
	DiskWrite float64 `json:"diskwrite"`
	DiskRead  float64 `json:"diskread"`
	NetIn     float64 `json:"netin"`
	NetOut    float64 `json:"netout"`
	Uptime    float64 `json:"uptime"`
	QmpStatus string  `json:"qmpstatus"`
	Status    string  `json:"status"`
	Template  string  `json:"template"`
}

func (qemu QemuVM) Delete() (map[string]interface{}, error) {
	var target string
	var data map[string]interface{}
	var err error

	//fmt.Print("!QemuDelete ", qemu.VMId)

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64)
	data, err = qemu.Node.Proxmox.Delete(target)
	if err != nil {
		return nil, err
	}
	return data, nil
}

func stringToMap(data string, itemSeparator string, kvSeparator string) map[string]string {
	var result map[string]string

	result = make(map[string]string)
	list := strings.Split(data, itemSeparator)
	for _, item := range list {
		kv := strings.Split(item, kvSeparator)
		result[kv[0]] = kv[1]
	}
	return result
}

func (qemu QemuVM) Config() (QemuConfig, error) {
	var target string
	var data map[string]interface{}
	var results map[string]interface{}
	var config QemuConfig
	var err error

	//fmt.Print("!QemuConfig ", qemu.VMId)

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/config"
	data, err = qemu.Node.Proxmox.Get(target)
	results = data["data"].(map[string]interface{})
	if err != nil {
		return config, err
	}
	config = QemuConfig{
		Bootdisk: results["bootdisk"].(string),
		Cores:    results["cores"].(float64),
		Digest:   results["digest"].(string),
		Memory:   results["memory"].(float64),
		Sockets:  results["sockets"].(float64),
		SMBios1:  results["smbios1"].(string),
		Description:  results["description"].(string),
	}
	disktype := [3]string{"virtio", "sata", "ide"}
	disknum := [4]string{"0", "1", "2", "3"}
	config.Disks = make(map[string]string)
	for _, d := range disktype {
		for _, i := range disknum {
			id := d + i
			if disk, ok := results[id]; ok {
				config.Disks[id] = disk.(string)
			}
		}
	}
	config.Net = make(map[string]QemuNet)
	netnum := [4]string{"0", "1", "2", "3"}
	for _, n := range netnum {
		if net, ok := results["net"+n]; ok {
			config.Net["net"+n] = stringToMap(net.(string), ",", "=")
		}
	}

	return config, nil
}

func (qemu QemuVM) CurrentStatus() (QemuStatus, error) {
	var target string
	var err error
	var data map[string]interface{}
	var results map[string]interface{}
	var status QemuStatus

	//fmt.Println("!QemuStatus ", strconv.FormatFloat(qemu.VMId, 'f', 0, 64))

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/status/current"
	data, err = qemu.Node.Proxmox.Get(target)
	if err != nil {
		return status, err
	}
	results = data["data"].(map[string]interface{})
	status = QemuStatus{
		CPU:       results["cpu"].(float64),
		CPUs:      results["cpus"].(float64),
		Mem:       results["mem"].(float64),
		MaxMem:    results["maxmem"].(float64),
		Disk:      results["disk"].(float64),
		MaxDisk:   results["maxdisk"].(float64),
		DiskWrite: results["diskwrite"].(float64),
		DiskRead:  results["diskread"].(float64),
		NetIn:     results["netin"].(float64),
		NetOut:    results["netout"].(float64),
		Uptime:    results["uptime"].(float64),
		QmpStatus: results["qmpstatus"].(string),
		Status:    results["status"].(string),
		Template:  results["template"].(string),
	}
	return status, nil
}

func (qemu QemuVM) WaitForStatus(status string, timeout int) error {
	var i int
	var err error
	var qStatus QemuStatus
	for i = 0; i < timeout; i++ {
		qStatus, err = qemu.CurrentStatus()
		if err != nil {
			return err
		}

		if qStatus.Status == status {
			return nil
		}
		time.Sleep(time.Second * 1)
	}
	return errors.New("Timeout reached")
}

func (qemu QemuVM) Start() error {
	var target string
	var err error

	//fmt.Println("!QemuStart ", strconv.FormatFloat(qemu.VMId, 'f', 0, 64))

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/status/start"
	_, err = qemu.Node.Proxmox.Post(target, "")
	return err
}

func (qemu QemuVM) Stop() error {
	var target string
	var err error

	//fmt.Print("!QemuStop ", qemu.VMId)

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/status/stop"
	_, err = qemu.Node.Proxmox.Post(target, "")
	return err
}

func (qemu QemuVM) Shutdown() error {
	var target string
	var err error

	//fmt.Print("!QemuShutdown ", qemu.VMId)

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/status/shutdown"
	_, err = qemu.Node.Proxmox.Post(target, "")
	return err
}

func (qemu QemuVM) Suspend() error {
	var target string
	var err error

	//fmt.Print("!QemuSuspend ", qemu.VMId)

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/status/suspend"
	_, err = qemu.Node.Proxmox.Post(target, "")
	return err
}

func (qemu QemuVM) Resume() error {
	var target string
	var err error

	//fmt.Print("!QemuResume ", qemu.VMId)

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/status/resume"
	_, err = qemu.Node.Proxmox.Post(target, "")
	return err
}

func (qemu QemuVM) Clone(newId float64, name string, targetName string) (Task, error) {
	var target string
	var err error

	newVMID := strconv.FormatFloat(newId, 'f', 0, 64)

	target = "nodes/" + qemu.Node.Node + "/qemu/" + strconv.FormatFloat(qemu.VMId, 'f', 0, 64) + "/clone"

	form := url.Values{
		"newid":  {newVMID},
		"name":   {name},
		"target": {targetName},
		"full":   {"1"},
	}

	data, err := qemu.Node.Proxmox.PostForm(target, form)
	if err != err {
		return Task{}, err
	}

	t := Task{
		UPid:    data["data"].(string),
		proxmox: qemu.Node.Proxmox,
	}

	return t, nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"strings"
	"sync"
	"time"

	"github.com/BurntSushi/toml"
	docker "github.com/fsouza/go-dockerclient"
)

var (
	buildVersion            string
	version                 bool
	watch                   bool
	notifyCmd               string
	notifySigHUPContainerID string
	onlyExposed             bool
	onlyPublished           bool
	configFile              string
	configs                 ConfigFile
	interval                int
	endpoint                string
	tlsCert                 string
	tlsKey                  string
	tlsCaCert               string
	tlsVerify               bool
	wg                      sync.WaitGroup
)

type Event struct {
	ContainerID string `json:"id"`
	Status      string `json:"status"`
	Image       string `json:"from"`
}

type Address struct {
	IP       string
	Port     string
	HostPort string
	Proto    string
}

type Volume struct {
	Path      string
	HostPath  string
	ReadWrite bool
}

type RuntimeContainer struct {
	ID        string
	Addresses []Address
	Gateway   string
	Name      string
	Image     DockerImage
	Env       map[string]string
	Volumes   map[string]Volume
}

type DockerImage struct {
	Registry   string
	Repository string
	Tag        string
}

func (i *DockerImage) String() string {
	ret := i.Repository
	if i.Registry != "" {
		ret = i.Registry + "/" + i.Repository
	}
	if i.Tag != "" {
		ret = ret + ":" + i.Tag
	}
	return ret
}

type Config struct {
	Template         string
	Dest             string
	Watch            bool
	NotifyCmd        string
	NotifyContainers map[string]docker.Signal
	OnlyExposed      bool
	OnlyPublished    bool
	Interval         int
}

type ConfigFile struct {
	Config []Config
}

type Context []*RuntimeContainer

func (c *Context) Env() map[string]string {

	env := make(map[string]string)
	for _, i := range os.Environ() {
		parts := strings.Split(i, "=")
		env[parts[0]] = parts[1]
	}
	return env
}

func (c *ConfigFile) filterWatches() ConfigFile {
	configWithWatches := []Config{}

	for _, config := range c.Config {
		if config.Watch {
			configWithWatches = append(configWithWatches, config)
		}
	}
	return ConfigFile{
		Config: configWithWatches,
	}
}

func (r *RuntimeContainer) Equals(o RuntimeContainer) bool {
	return r.ID == o.ID && r.Image == o.Image
}

func (r *RuntimeContainer) PublishedAddresses() []Address {
	mapped := []Address{}
	for _, address := range r.Addresses {
		if address.HostPort != "" {
			mapped = append(mapped, address)
		}
	}
	return mapped
}

func usage() {
	println("Usage: docker-gen [-config file] [-watch=false] [-notify=\"restart xyz\"] [-notify-sighup=\"container-ID\"] [-interval=0] [-endpoint tcp|unix://..] [-tlscert file] [-tlskey file] [-tlscacert file] [-tlsverify] <template> [<dest>]")
}

func NewDockerClient(endpoint string) (*docker.Client, error) {
	if strings.HasPrefix(endpoint, "unix:") {
		return docker.NewClient(endpoint)
	} else if tlsVerify || tlsCert != "" || tlsKey != "" || tlsCaCert != "" {
		if tlsVerify {
			if tlsCaCert == "" {
				return nil, errors.New("TLS verification was requested, but no -tlscacert was provided")
			}
		}

		return docker.NewTLSClient(endpoint, tlsCert, tlsKey, tlsCaCert)
	}
	return docker.NewClient(endpoint)
}

func generateFromContainers(client *docker.Client) {
	containers, err := getContainers(client)
	if err != nil {
		log.Printf("error listing containers: %s\n", err)
		return
	}
	for _, config := range configs.Config {
		changed := generateFile(config, containers)
		if !changed {
			log.Printf("Contents of %s did not change. Skipping notification '%s'", config.Dest, config.NotifyCmd)
			continue
		}
		runNotifyCmd(config)
		sendSignalToContainer(client, config)
	}
}

func runNotifyCmd(config Config) {
	if config.NotifyCmd == "" {
		return
	}

	log.Printf("Running '%s'", config.NotifyCmd)
	cmd := exec.Command("/bin/sh", "-c", config.NotifyCmd)
	out, err := cmd.CombinedOutput()
	if err != nil {
		log.Printf("Error running notify command: %s, %s\n", config.NotifyCmd, err)
		log.Print(string(out))
	}
}

func sendSignalToContainer(client *docker.Client, config Config) {
	if len(config.NotifyContainers) < 1 {
		return
	}

	for container, signal := range config.NotifyContainers {
		log.Printf("Sending container '%s' signal '%v'", container, signal)
		killOpts := docker.KillContainerOptions{
			ID:     container,
			Signal: signal,
		}
		if err := client.KillContainer(killOpts); err != nil {
			log.Printf("Error sending signal to container: %s", err)
		}
	}
}

func loadConfig(file string) error {
	_, err := toml.DecodeFile(file, &configs)
	if err != nil {
		return err
	}
	return nil
}

func generateAtInterval(client *docker.Client, configs ConfigFile) {
	for _, config := range configs.Config {

		if config.Interval == 0 {
			continue
		}

		log.Printf("Generating every %d seconds", config.Interval)
		wg.Add(1)
		ticker := time.NewTicker(time.Duration(config.Interval) * time.Second)
		quit := make(chan struct{})
		configCopy := config
		go func() {
			defer wg.Done()
			for {
				select {
				case <-ticker.C:
					containers, err := getContainers(client)
					if err != nil {
						log.Printf("Error listing containers: %s\n", err)
						continue
					}
					// ignore changed return value. always run notify command
					generateFile(configCopy, containers)
					runNotifyCmd(configCopy)
					sendSignalToContainer(client, configCopy)
				case <-quit:
					ticker.Stop()
					return
				}
			}
		}()
	}
}

func generateFromEvents(client *docker.Client, configs ConfigFile) {
	configs = configs.filterWatches()
	if len(configs.Config) == 0 {
		return
	}

	wg.Add(1)
	defer wg.Done()

	for {
		if client == nil {
			var err error
			endpoint, err := getEndpoint()
			if err != nil {
				log.Printf("Bad endpoint: %s", err)
				time.Sleep(10 * time.Second)
				continue
			}

			client, err = NewDockerClient(endpoint)
			if err != nil {
				log.Printf("Unable to connect to docker daemaon: %s", err)
				time.Sleep(10 * time.Second)
				continue
			}
			generateFromContainers(client)
		}

		eventChan := make(chan *docker.APIEvents, 100)
		defer close(eventChan)

		watching := false
		for {

			if client == nil {
				break
			}
			err := client.Ping()
			if err != nil {
				log.Printf("Unable to ping docker daemaon: %s", err)
				if watching {
					client.RemoveEventListener(eventChan)
					watching = false
					client = nil
				}
				time.Sleep(10 * time.Second)
				break

			}

			if !watching {
				err = client.AddEventListener(eventChan)
				if err != nil && err != docker.ErrListenerAlreadyExists {
					log.Printf("Error registering docker event listener: %s", err)
					time.Sleep(10 * time.Second)
					continue
				}
				watching = true
				log.Println("Watching docker events")
			}

			select {

			case event := <-eventChan:
				if event == nil {
					if watching {
						client.RemoveEventListener(eventChan)
						watching = false
						client = nil
					}
					break
				}

				if event.Status == "start" || event.Status == "stop" || event.Status == "die" {
					log.Printf("Received event %s for container %s", event.Status, event.ID[:12])
					generateFromContainers(client)
				}
			case <-time.After(10 * time.Second):
				// check for docker liveness
			}

		}
	}
}

func initFlags() {
	flag.BoolVar(&version, "version", false, "show version")
	flag.BoolVar(&watch, "watch", false, "watch for container changes")
	flag.BoolVar(&onlyExposed, "only-exposed", false, "only include containers with exposed ports")
	flag.BoolVar(&onlyPublished, "only-published", false, "only include containers with published ports (implies -only-exposed)")
	flag.StringVar(&notifyCmd, "notify", "", "run command after template is regenerated")
	flag.StringVar(&notifySigHUPContainerID, "notify-sighup", "", "send HUP signal to container.  Equivalent to `docker kill -s HUP container-ID`")
	flag.StringVar(&configFile, "config", "", "config file with template directives")
	flag.IntVar(&interval, "interval", 0, "notify command interval (s)")
	flag.StringVar(&endpoint, "endpoint", "", "docker api endpoint")
	flag.StringVar(&tlsCert, "tlscert", "", "path to TLS client certificate file")
	flag.StringVar(&tlsKey, "tlskey", "", "path to TLS client key file")
	flag.StringVar(&tlsCaCert, "tlscacert", "", "path to TLS CA certificate file")
	flag.BoolVar(&tlsVerify, "tlsverify", false, "verify docker daemon's TLS certicate")
	flag.Parse()
}

func main() {
	initFlags()

	if version {
		fmt.Println(buildVersion)
		return
	}

	if flag.NArg() < 1 && configFile == "" {
		usage()
		os.Exit(1)
	}

	if configFile != "" {
		err := loadConfig(configFile)
		if err != nil {
			log.Fatalf("error loading config %s: %s\n", configFile, err)
		}
	} else {
		config := Config{
			Template:         flag.Arg(0),
			Dest:             flag.Arg(1),
			Watch:            watch,
			NotifyCmd:        notifyCmd,
			NotifyContainers: make(map[string]docker.Signal),
			OnlyExposed:      onlyExposed,
			OnlyPublished:    onlyPublished,
			Interval:         interval,
		}
		if notifySigHUPContainerID != "" {
			config.NotifyContainers[notifySigHUPContainerID] = docker.SIGHUP
		}
		configs = ConfigFile{
			Config: []Config{config}}
	}

	endpoint, err := getEndpoint()
	if err != nil {
		log.Fatalf("Bad endpoint: %s", err)
	}

	client, err := NewDockerClient(endpoint)
	if err != nil {
		log.Fatalf("Unable to create docker client: %s", err)
	}

	generateFromContainers(client)
	generateAtInterval(client, configs)
	generateFromEvents(client, configs)
	wg.Wait()
}
<commit_msg>Fix typos in error-messages.<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"strings"
	"sync"
	"time"

	"github.com/BurntSushi/toml"
	docker "github.com/fsouza/go-dockerclient"
)

var (
	buildVersion            string
	version                 bool
	watch                   bool
	notifyCmd               string
	notifySigHUPContainerID string
	onlyExposed             bool
	onlyPublished           bool
	configFile              string
	configs                 ConfigFile
	interval                int
	endpoint                string
	tlsCert                 string
	tlsKey                  string
	tlsCaCert               string
	tlsVerify               bool
	wg                      sync.WaitGroup
)

type Event struct {
	ContainerID string `json:"id"`
	Status      string `json:"status"`
	Image       string `json:"from"`
}

type Address struct {
	IP       string
	Port     string
	HostPort string
	Proto    string
}

type Volume struct {
	Path      string
	HostPath  string
	ReadWrite bool
}

type RuntimeContainer struct {
	ID        string
	Addresses []Address
	Gateway   string
	Name      string
	Image     DockerImage
	Env       map[string]string
	Volumes   map[string]Volume
}

type DockerImage struct {
	Registry   string
	Repository string
	Tag        string
}

func (i *DockerImage) String() string {
	ret := i.Repository
	if i.Registry != "" {
		ret = i.Registry + "/" + i.Repository
	}
	if i.Tag != "" {
		ret = ret + ":" + i.Tag
	}
	return ret
}

type Config struct {
	Template         string
	Dest             string
	Watch            bool
	NotifyCmd        string
	NotifyContainers map[string]docker.Signal
	OnlyExposed      bool
	OnlyPublished    bool
	Interval         int
}

type ConfigFile struct {
	Config []Config
}

type Context []*RuntimeContainer

func (c *Context) Env() map[string]string {

	env := make(map[string]string)
	for _, i := range os.Environ() {
		parts := strings.Split(i, "=")
		env[parts[0]] = parts[1]
	}
	return env
}

func (c *ConfigFile) filterWatches() ConfigFile {
	configWithWatches := []Config{}

	for _, config := range c.Config {
		if config.Watch {
			configWithWatches = append(configWithWatches, config)
		}
	}
	return ConfigFile{
		Config: configWithWatches,
	}
}

func (r *RuntimeContainer) Equals(o RuntimeContainer) bool {
	return r.ID == o.ID && r.Image == o.Image
}

func (r *RuntimeContainer) PublishedAddresses() []Address {
	mapped := []Address{}
	for _, address := range r.Addresses {
		if address.HostPort != "" {
			mapped = append(mapped, address)
		}
	}
	return mapped
}

func usage() {
	println("Usage: docker-gen [-config file] [-watch=false] [-notify=\"restart xyz\"] [-notify-sighup=\"container-ID\"] [-interval=0] [-endpoint tcp|unix://..] [-tlscert file] [-tlskey file] [-tlscacert file] [-tlsverify] <template> [<dest>]")
}

func NewDockerClient(endpoint string) (*docker.Client, error) {
	if strings.HasPrefix(endpoint, "unix:") {
		return docker.NewClient(endpoint)
	} else if tlsVerify || tlsCert != "" || tlsKey != "" || tlsCaCert != "" {
		if tlsVerify {
			if tlsCaCert == "" {
				return nil, errors.New("TLS verification was requested, but no -tlscacert was provided")
			}
		}

		return docker.NewTLSClient(endpoint, tlsCert, tlsKey, tlsCaCert)
	}
	return docker.NewClient(endpoint)
}

func generateFromContainers(client *docker.Client) {
	containers, err := getContainers(client)
	if err != nil {
		log.Printf("error listing containers: %s\n", err)
		return
	}
	for _, config := range configs.Config {
		changed := generateFile(config, containers)
		if !changed {
			log.Printf("Contents of %s did not change. Skipping notification '%s'", config.Dest, config.NotifyCmd)
			continue
		}
		runNotifyCmd(config)
		sendSignalToContainer(client, config)
	}
}

func runNotifyCmd(config Config) {
	if config.NotifyCmd == "" {
		return
	}

	log.Printf("Running '%s'", config.NotifyCmd)
	cmd := exec.Command("/bin/sh", "-c", config.NotifyCmd)
	out, err := cmd.CombinedOutput()
	if err != nil {
		log.Printf("Error running notify command: %s, %s\n", config.NotifyCmd, err)
		log.Print(string(out))
	}
}

func sendSignalToContainer(client *docker.Client, config Config) {
	if len(config.NotifyContainers) < 1 {
		return
	}

	for container, signal := range config.NotifyContainers {
		log.Printf("Sending container '%s' signal '%v'", container, signal)
		killOpts := docker.KillContainerOptions{
			ID:     container,
			Signal: signal,
		}
		if err := client.KillContainer(killOpts); err != nil {
			log.Printf("Error sending signal to container: %s", err)
		}
	}
}

func loadConfig(file string) error {
	_, err := toml.DecodeFile(file, &configs)
	if err != nil {
		return err
	}
	return nil
}

func generateAtInterval(client *docker.Client, configs ConfigFile) {
	for _, config := range configs.Config {

		if config.Interval == 0 {
			continue
		}

		log.Printf("Generating every %d seconds", config.Interval)
		wg.Add(1)
		ticker := time.NewTicker(time.Duration(config.Interval) * time.Second)
		quit := make(chan struct{})
		configCopy := config
		go func() {
			defer wg.Done()
			for {
				select {
				case <-ticker.C:
					containers, err := getContainers(client)
					if err != nil {
						log.Printf("Error listing containers: %s\n", err)
						continue
					}
					// ignore changed return value. always run notify command
					generateFile(configCopy, containers)
					runNotifyCmd(configCopy)
					sendSignalToContainer(client, configCopy)
				case <-quit:
					ticker.Stop()
					return
				}
			}
		}()
	}
}

func generateFromEvents(client *docker.Client, configs ConfigFile) {
	configs = configs.filterWatches()
	if len(configs.Config) == 0 {
		return
	}

	wg.Add(1)
	defer wg.Done()

	for {
		if client == nil {
			var err error
			endpoint, err := getEndpoint()
			if err != nil {
				log.Printf("Bad endpoint: %s", err)
				time.Sleep(10 * time.Second)
				continue
			}

			client, err = NewDockerClient(endpoint)
			if err != nil {
				log.Printf("Unable to connect to docker daemon: %s", err)
				time.Sleep(10 * time.Second)
				continue
			}
			generateFromContainers(client)
		}

		eventChan := make(chan *docker.APIEvents, 100)
		defer close(eventChan)

		watching := false
		for {

			if client == nil {
				break
			}
			err := client.Ping()
			if err != nil {
				log.Printf("Unable to ping docker daemon: %s", err)
				if watching {
					client.RemoveEventListener(eventChan)
					watching = false
					client = nil
				}
				time.Sleep(10 * time.Second)
				break

			}

			if !watching {
				err = client.AddEventListener(eventChan)
				if err != nil && err != docker.ErrListenerAlreadyExists {
					log.Printf("Error registering docker event listener: %s", err)
					time.Sleep(10 * time.Second)
					continue
				}
				watching = true
				log.Println("Watching docker events")
			}

			select {

			case event := <-eventChan:
				if event == nil {
					if watching {
						client.RemoveEventListener(eventChan)
						watching = false
						client = nil
					}
					break
				}

				if event.Status == "start" || event.Status == "stop" || event.Status == "die" {
					log.Printf("Received event %s for container %s", event.Status, event.ID[:12])
					generateFromContainers(client)
				}
			case <-time.After(10 * time.Second):
				// check for docker liveness
			}

		}
	}
}

func initFlags() {
	flag.BoolVar(&version, "version", false, "show version")
	flag.BoolVar(&watch, "watch", false, "watch for container changes")
	flag.BoolVar(&onlyExposed, "only-exposed", false, "only include containers with exposed ports")
	flag.BoolVar(&onlyPublished, "only-published", false, "only include containers with published ports (implies -only-exposed)")
	flag.StringVar(&notifyCmd, "notify", "", "run command after template is regenerated")
	flag.StringVar(&notifySigHUPContainerID, "notify-sighup", "", "send HUP signal to container.  Equivalent to `docker kill -s HUP container-ID`")
	flag.StringVar(&configFile, "config", "", "config file with template directives")
	flag.IntVar(&interval, "interval", 0, "notify command interval (s)")
	flag.StringVar(&endpoint, "endpoint", "", "docker api endpoint")
	flag.StringVar(&tlsCert, "tlscert", "", "path to TLS client certificate file")
	flag.StringVar(&tlsKey, "tlskey", "", "path to TLS client key file")
	flag.StringVar(&tlsCaCert, "tlscacert", "", "path to TLS CA certificate file")
	flag.BoolVar(&tlsVerify, "tlsverify", false, "verify docker daemon's TLS certicate")
	flag.Parse()
}

func main() {
	initFlags()

	if version {
		fmt.Println(buildVersion)
		return
	}

	if flag.NArg() < 1 && configFile == "" {
		usage()
		os.Exit(1)
	}

	if configFile != "" {
		err := loadConfig(configFile)
		if err != nil {
			log.Fatalf("error loading config %s: %s\n", configFile, err)
		}
	} else {
		config := Config{
			Template:         flag.Arg(0),
			Dest:             flag.Arg(1),
			Watch:            watch,
			NotifyCmd:        notifyCmd,
			NotifyContainers: make(map[string]docker.Signal),
			OnlyExposed:      onlyExposed,
			OnlyPublished:    onlyPublished,
			Interval:         interval,
		}
		if notifySigHUPContainerID != "" {
			config.NotifyContainers[notifySigHUPContainerID] = docker.SIGHUP
		}
		configs = ConfigFile{
			Config: []Config{config}}
	}

	endpoint, err := getEndpoint()
	if err != nil {
		log.Fatalf("Bad endpoint: %s", err)
	}

	client, err := NewDockerClient(endpoint)
	if err != nil {
		log.Fatalf("Unable to create docker client: %s", err)
	}

	generateFromContainers(client)
	generateAtInterval(client, configs)
	generateFromEvents(client, configs)
	wg.Wait()
}
<|endoftext|>