<commit_before>package veneur

import (
	"container/ring"
	"context"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"sync"
	"time"

	"github.com/DataDog/datadog-go/statsd"
	lightstep "github.com/lightstep/lightstep-tracer-go"
	opentracing "github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/ext"
	"github.com/sirupsen/logrus"
	"github.com/stripe/veneur/ssf"
	"github.com/stripe/veneur/trace"
)

const datadogResourceKey = "resource"
const datadogNameKey = "name"

const lightStepOperationKey = "name"

const totalSpansFlushedMetricKey = "worker.spans_flushed_total"

// spanSink is a receiver of spans that handles sending those spans to some
// downstream sink. Calls to `Ingest(span)` are meant to give the sink control
// of the span, with periodic calls to flush as a signal for sinks that don't
// handle their own flushing in a separate goroutine, etc.
type spanSink interface {
	Name() string
	Ingest(ssf.SSFSpan) error
	Flush()
}

// DatadogSpanSink is a sink for sending spans to a Datadog trace agent.
type datadogSpanSink struct {
	HTTPClient   *http.Client
	buffer       *ring.Ring
	bufferSize   int
	mutex        *sync.Mutex
	stats        *statsd.Client
	commonTags   map[string]string
	traceAddress string
}

// NewDatadogSpanSink creates a new Datadog sink for trace spans.
func NewDatadogSpanSink(config *Config, stats *statsd.Client, httpClient *http.Client, commonTags map[string]string) (*datadogSpanSink, error) {
	return &datadogSpanSink{
		HTTPClient:   httpClient,
		bufferSize:   config.SsfBufferSize,
		buffer:       ring.New(config.SsfBufferSize),
		mutex:        &sync.Mutex{},
		stats:        stats,
		commonTags:   commonTags,
		traceAddress: config.DatadogTraceAPIAddress,
	}, nil
}

// Name returns the name of this sink.
func (dd *datadogSpanSink) Name() string {
	return "datadog"
}

// Ingest takes the span and adds it to the ringbuffer.
func (dd *datadogSpanSink) Ingest(span ssf.SSFSpan) error {
	dd.mutex.Lock()
	defer dd.mutex.Unlock()

	dd.buffer.Value = span
	dd.buffer = dd.buffer.Next()
	return nil
}

// Flush signals the sink to send it's spans to their destination. For this
// sync it means we'll be making an HTTP request to send them along. We assume
// it's beneficial to performance to defer these until the normal 10s flush.
func (dd *datadogSpanSink) Flush() {
	dd.mutex.Lock()

	ssfSpans := make([]ssf.SSFSpan, 0, dd.buffer.Len())

	dd.buffer.Do(func(t interface{}) {
		const tooEarly = 1497
		const tooLate = 1497629343000000

		if t != nil {
			ssfSpan, ok := t.(ssf.SSFSpan)
			if !ok {
				log.Error("Got an unknown object in tracing ring!")
				// We'll just skip this one so we don't poison pill or anything.
				return
			}

			var timeErr string
			if ssfSpan.StartTimestamp < tooEarly {
				timeErr = "type:tooEarly"
			}
			if ssfSpan.StartTimestamp > tooLate {
				timeErr = "type:tooLate"
			}
			if timeErr != "" {
				dd.stats.Incr("worker.trace.sink.timestamp_error", []string{timeErr}, 1) // TODO tag as dd?
			}

			if ssfSpan.Tags == nil {
				ssfSpan.Tags = make(map[string]string)
			}

			// Add common tags from veneur's config
			// this will overwrite tags already present on the span
			for k, v := range dd.commonTags {
				ssfSpan.Tags[k] = v
			}
			ssfSpans = append(ssfSpans, ssfSpan)
		}
	})

	// Reset the ring.
	dd.buffer = ring.New(dd.bufferSize)

	// We're done manipulating stuff, let Ingest loose again.
	dd.mutex.Unlock()

	serviceCount := make(map[string]int64)
	var finalTraces []*DatadogTraceSpan
	// Conver the SSFSpans into Datadog Spans
	for _, span := range ssfSpans {
		// -1 is a canonical way of passing in invalid info in Go
		// so we should support that too
		parentID := span.ParentId

		// check if this is the root span
		if parentID <= 0 {
			// we need parentId to be zero for json:omitempty to work
			parentID = 0
		}

		resource := span.Tags[datadogResourceKey]
		name := span.Name

		tags := map[string]string{}
		// Get the span's existing tags
		for k, v := range span.Tags {
			tags[k] = v
		}

		delete(tags, datadogResourceKey)

		// TODO implement additional metrics
		var metrics map[string]float64

		var errorCode int64
		if span.Error {
			errorCode = 2
		}

		ddspan := &DatadogTraceSpan{
			TraceID:  span.TraceId,
			SpanID:   span.Id,
			ParentID: parentID,
			Service:  span.Service,
			Name:     name,
			Resource: resource,
			Start:    span.StartTimestamp,
			Duration: span.EndTimestamp - span.StartTimestamp,
			// TODO don't hardcode
			Type:    "http",
			Error:   errorCode,
			Metrics: metrics,
			Meta:    tags,
		}
		serviceCount[span.Service]++
		finalTraces = append(finalTraces, ddspan)
	}

	if len(finalTraces) != 0 {
		// this endpoint is not documented to take an array... but it does
		// another curious constraint of this endpoint is that it does not
		// support "Content-Encoding: deflate"

		err := postHelper(context.TODO(), dd.HTTPClient, dd.stats, fmt.Sprintf("%s/spans", dd.traceAddress), finalTraces, "flush_traces", false)

		if err == nil {
			log.WithField("traces", len(finalTraces)).Info("Completed flushing traces to Datadog")
			dd.stats.Count(totalSpansFlushedMetricKey, int64(len(ssfSpans)), []string{"sink:datadog"}, 1)
			// TODO: Per service counters?
		} else {
			log.WithFields(logrus.Fields{
				"traces":        len(finalTraces),
				logrus.ErrorKey: err}).Warn("Error flushing traces to Datadog")
		}
		for service, count := range serviceCount {
			dd.stats.Count(totalSpansFlushedMetricKey, count, []string{"sink:datadog", fmt.Sprintf("service:%s", service)}, 1)
		}
	} else {
		log.Info("No traces to flush to Datadog, skipping.")
	}
}

// lightStepSpanSink is a sink for spans to be sent to the LightStep client.
type lightStepSpanSink struct {
	tracer       opentracing.Tracer
	stats        *statsd.Client
	commonTags   map[string]string
	mutex        *sync.Mutex
	serviceCount map[string]int64
}

// NewLightStepSpanSink creates a new instance of a LightStepSpanSink.
func NewLightStepSpanSink(config *Config, stats *statsd.Client, commonTags map[string]string) (*lightStepSpanSink, error) {

	var host *url.URL
	host, err := url.Parse(config.TraceLightstepCollectorHost)
	if err != nil {
		log.WithError(err).WithField(
			"host", config.TraceLightstepCollectorHost,
		).Error("Error parsing LightStep collector URL")
		return &lightStepSpanSink{}, err
	}

	port, err := strconv.Atoi(host.Port())
	if err != nil {
		log.WithError(err).WithFields(logrus.Fields{
			"port":         port,
			"default_port": lightstepDefaultPort,
		}).Warn("Error parsing LightStep port, using default")
		port = lightstepDefaultPort
	}

	reconPeriod := lightstepDefaultInterval
	if config.TraceLightstepReconnectPeriod != "" {
		reconPeriod, err = time.ParseDuration(config.TraceLightstepReconnectPeriod)
		if err != nil {
			log.WithError(err).WithFields(logrus.Fields{
				"interval":         config.TraceLightstepReconnectPeriod,
				"default_interval": lightstepDefaultInterval,
			}).Warn("Failed to parse reconnect duration, using default.")
			reconPeriod = lightstepDefaultInterval
		}
	}

	log.WithFields(logrus.Fields{
		"Host": host.Hostname(),
		"Port": port,
	}).Info("Dialing lightstep host")

	maxSpans := config.TraceLightstepMaximumSpans
	if maxSpans == 0 {
		maxSpans = config.SsfBufferSize
		log.WithField("max spans", maxSpans).Info("Using default maximum spans — ssf_buffer_size — for LightStep")
	}

	lightstepTracer := lightstep.NewTracer(lightstep.Options{
		AccessToken:     config.TraceLightstepAccessToken,
		ReconnectPeriod: reconPeriod,
		Collector: lightstep.Endpoint{
			Host:      host.Hostname(),
			Port:      port,
			Plaintext: true,
		},
		UseGRPC:          true,
		MaxBufferedSpans: maxSpans,
	})

	return &lightStepSpanSink{
		tracer:       lightstepTracer,
		stats:        stats,
		serviceCount: make(map[string]int64),
		mutex:        &sync.Mutex{},
	}, nil
}

// Name returns this sink's name.
func (ls *lightStepSpanSink) Name() string {
	return "lightstep"
}

// Ingest takes in a span and passed it along to the LS client after
// some sanity checks and improvements are made.
func (ls *lightStepSpanSink) Ingest(ssfSpan ssf.SSFSpan) error {
	parentID := ssfSpan.ParentId
	if parentID <= 0 {
		parentID = 0
	}

	var errorCode int64
	if ssfSpan.Error {
		errorCode = 1
	}

	timestamp := time.Unix(ssfSpan.StartTimestamp/1e9, ssfSpan.StartTimestamp%1e9)
	sp := ls.tracer.StartSpan(
		ssfSpan.Name,
		opentracing.StartTime(timestamp),
		lightstep.SetTraceID(uint64(ssfSpan.TraceId)),
		lightstep.SetSpanID(uint64(ssfSpan.Id)),
		lightstep.SetParentSpanID(uint64(parentID)))

	sp.SetTag(trace.ResourceKey, ssfSpan.Tags[trace.ResourceKey]) // TODO Why is this here?
	sp.SetTag(lightstep.ComponentNameKey, ssfSpan.Service)
	// TODO don't hardcode
	sp.SetTag("type", "http")
	sp.SetTag("error-code", errorCode)
	for k, v := range ssfSpan.Tags {
		sp.SetTag(k, v)
	}
	// And now set any veneur common tags
	for k, v := range ls.commonTags {
		sp.SetTag(k, v)
	}
	// TODO add metrics as tags to the span as well?

	if errorCode > 0 {
		// Note: this sets the OT-standard "error" tag, which
		// LightStep uses to flag error spans.
		ext.Error.Set(sp, true)
	}

	endTime := time.Unix(ssfSpan.EndTimestamp/1e9, ssfSpan.EndTimestamp%1e9)
	sp.FinishWithOptions(opentracing.FinishOptions{
		FinishTime: endTime,
	})

	service := ssfSpan.Service
	if service == "" {
		service = "unknown"
	}

	// Protect mutating the service count with a mutex
	ls.mutex.Lock()
	defer ls.mutex.Unlock()
	ls.serviceCount[service]++

	return nil
}

// Flush doesn't need to do anything to the LS tracer, so we emit metrics
// instead.
func (ls *lightStepSpanSink) Flush() {
	ls.mutex.Lock()
	defer ls.mutex.Unlock()

	totalCount := int64(0)
	for service, count := range ls.serviceCount {
		totalCount += count
		ls.stats.Count(totalSpansFlushedMetricKey, count, []string{"sink:lightstep", fmt.Sprintf("service:%s", service)}, 1)
	}
	ls.serviceCount = make(map[string]int64)
	log.WithField("total_spans", totalCount).Debug("Checkpointing flushed spans for Lightstep")
}
<commit_msg>Add sync poll<commit_after>package veneur

import (
	"container/ring"
	"context"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"sync"
	"time"

	"github.com/DataDog/datadog-go/statsd"
	lightstep "github.com/lightstep/lightstep-tracer-go"
	opentracing "github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/ext"
	"github.com/sirupsen/logrus"
	"github.com/stripe/veneur/ssf"
	"github.com/stripe/veneur/trace"
)

const datadogResourceKey = "resource"
const datadogNameKey = "name"

const lightStepOperationKey = "name"

const totalSpansFlushedMetricKey = "worker.spans_flushed_total"

// spanSink is a receiver of spans that handles sending those spans to some
// downstream sink. Calls to `Ingest(span)` are meant to give the sink control
// of the span, with periodic calls to flush as a signal for sinks that don't
// handle their own flushing in a separate goroutine, etc.
type spanSink interface {
	Name() string
	Ingest(ssf.SSFSpan) error
	Flush()
}

// DatadogSpanSink is a sink for sending spans to a Datadog trace agent.
type datadogSpanSink struct {
	HTTPClient   *http.Client
	buffer       *ring.Ring
	bufferSize   int
	mutex        *sync.Mutex
	stats        *statsd.Client
	commonTags   map[string]string
	traceAddress string
}

// NewDatadogSpanSink creates a new Datadog sink for trace spans.
func NewDatadogSpanSink(config *Config, stats *statsd.Client, httpClient *http.Client, commonTags map[string]string) (*datadogSpanSink, error) {
	return &datadogSpanSink{
		HTTPClient:   httpClient,
		bufferSize:   config.SsfBufferSize,
		buffer:       ring.New(config.SsfBufferSize),
		mutex:        &sync.Mutex{},
		stats:        stats,
		commonTags:   commonTags,
		traceAddress: config.DatadogTraceAPIAddress,
	}, nil
}

// Name returns the name of this sink.
func (dd *datadogSpanSink) Name() string {
	return "datadog"
}

// Ingest takes the span and adds it to the ringbuffer.
func (dd *datadogSpanSink) Ingest(span ssf.SSFSpan) error {
	dd.mutex.Lock()
	defer dd.mutex.Unlock()

	dd.buffer.Value = span
	dd.buffer = dd.buffer.Next()
	return nil
}

// Flush signals the sink to send it's spans to their destination. For this
// sync it means we'll be making an HTTP request to send them along. We assume
// it's beneficial to performance to defer these until the normal 10s flush.
func (dd *datadogSpanSink) Flush() {
	dd.mutex.Lock()

	ssfSpans := make([]ssf.SSFSpan, 0, dd.buffer.Len())

	dd.buffer.Do(func(t interface{}) {
		const tooEarly = 1497
		const tooLate = 1497629343000000

		if t != nil {
			ssfSpan, ok := t.(ssf.SSFSpan)
			if !ok {
				log.Error("Got an unknown object in tracing ring!")
				// We'll just skip this one so we don't poison pill or anything.
				return
			}

			var timeErr string
			if ssfSpan.StartTimestamp < tooEarly {
				timeErr = "type:tooEarly"
			}
			if ssfSpan.StartTimestamp > tooLate {
				timeErr = "type:tooLate"
			}
			if timeErr != "" {
				dd.stats.Incr("worker.trace.sink.timestamp_error", []string{timeErr}, 1) // TODO tag as dd?
			}

			if ssfSpan.Tags == nil {
				ssfSpan.Tags = make(map[string]string)
			}

			// Add common tags from veneur's config
			// this will overwrite tags already present on the span
			for k, v := range dd.commonTags {
				ssfSpan.Tags[k] = v
			}
			ssfSpans = append(ssfSpans, ssfSpan)
		}
	})

	// Reset the ring.
	dd.buffer = ring.New(dd.bufferSize)

	// We're done manipulating stuff, let Ingest loose again.
	dd.mutex.Unlock()

	serviceCount := make(map[string]int64)
	var finalTraces []*DatadogTraceSpan
	// Conver the SSFSpans into Datadog Spans
	for _, span := range ssfSpans {
		// -1 is a canonical way of passing in invalid info in Go
		// so we should support that too
		parentID := span.ParentId

		// check if this is the root span
		if parentID <= 0 {
			// we need parentId to be zero for json:omitempty to work
			parentID = 0
		}

		resource := span.Tags[datadogResourceKey]
		name := span.Name

		tags := map[string]string{}
		// Get the span's existing tags
		for k, v := range span.Tags {
			tags[k] = v
		}

		delete(tags, datadogResourceKey)

		// TODO implement additional metrics
		var metrics map[string]float64

		var errorCode int64
		if span.Error {
			errorCode = 2
		}

		ddspan := &DatadogTraceSpan{
			TraceID:  span.TraceId,
			SpanID:   span.Id,
			ParentID: parentID,
			Service:  span.Service,
			Name:     name,
			Resource: resource,
			Start:    span.StartTimestamp,
			Duration: span.EndTimestamp - span.StartTimestamp,
			// TODO don't hardcode
			Type:    "http",
			Error:   errorCode,
			Metrics: metrics,
			Meta:    tags,
		}
		serviceCount[span.Service]++
		finalTraces = append(finalTraces, ddspan)
	}

	if len(finalTraces) != 0 {
		// this endpoint is not documented to take an array... but it does
		// another curious constraint of this endpoint is that it does not
		// support "Content-Encoding: deflate"

		err := postHelper(context.TODO(), dd.HTTPClient, dd.stats, fmt.Sprintf("%s/spans", dd.traceAddress), finalTraces, "flush_traces", false)

		if err == nil {
			log.WithField("traces", len(finalTraces)).Info("Completed flushing traces to Datadog")
			dd.stats.Count(totalSpansFlushedMetricKey, int64(len(ssfSpans)), []string{"sink:datadog"}, 1)
			// TODO: Per service counters?
		} else {
			log.WithFields(logrus.Fields{
				"traces":        len(finalTraces),
				logrus.ErrorKey: err}).Warn("Error flushing traces to Datadog")
		}
		for service, count := range serviceCount {
			dd.stats.Count(totalSpansFlushedMetricKey, count, []string{"sink:datadog", fmt.Sprintf("service:%s", service)}, 1)
		}
	} else {
		log.Info("No traces to flush to Datadog, skipping.")
	}
}

// lightStepSpanSink is a sink for spans to be sent to the LightStep client.
type lightStepSpanSink struct {
	tracerPool   sync.Pool
	stats        *statsd.Client
	commonTags   map[string]string
	mutex        *sync.Mutex
	serviceCount map[string]int64
}

// NewLightStepSpanSink creates a new instance of a LightStepSpanSink.
func NewLightStepSpanSink(config *Config, stats *statsd.Client, commonTags map[string]string) (*lightStepSpanSink, error) {

	var host *url.URL
	host, err := url.Parse(config.TraceLightstepCollectorHost)
	if err != nil {
		log.WithError(err).WithField(
			"host", config.TraceLightstepCollectorHost,
		).Error("Error parsing LightStep collector URL")
		return &lightStepSpanSink{}, err
	}

	port, err := strconv.Atoi(host.Port())
	if err != nil {
		log.WithError(err).WithFields(logrus.Fields{
			"port":         port,
			"default_port": lightstepDefaultPort,
		}).Warn("Error parsing LightStep port, using default")
		port = lightstepDefaultPort
	}

	reconPeriod := lightstepDefaultInterval
	if config.TraceLightstepReconnectPeriod != "" {
		reconPeriod, err = time.ParseDuration(config.TraceLightstepReconnectPeriod)
		if err != nil {
			log.WithError(err).WithFields(logrus.Fields{
				"interval":         config.TraceLightstepReconnectPeriod,
				"default_interval": lightstepDefaultInterval,
			}).Warn("Failed to parse reconnect duration, using default.")
			reconPeriod = lightstepDefaultInterval
		}
	}

	log.WithFields(logrus.Fields{
		"Host": host.Hostname(),
		"Port": port,
	}).Info("Dialing lightstep host")

	maxSpans := config.TraceLightstepMaximumSpans
	if maxSpans == 0 {
		maxSpans = config.SsfBufferSize
		log.WithField("max spans", maxSpans).Info("Using default maximum spans — ssf_buffer_size — for LightStep")
	}

	p := sync.Pool{
		New: func() interface{} {
			stats.Incr("tracer.pool.new", nil, 1.0)
			return lightstep.NewTracer(lightstep.Options{
				AccessToken:     config.TraceLightstepAccessToken,
				ReconnectPeriod: reconPeriod,
				Collector: lightstep.Endpoint{
					Host:      host.Hostname(),
					Port:      port,
					Plaintext: true,
				},
				UseGRPC:          true,
				MaxBufferedSpans: maxSpans,
			})
		},
	}

	return &lightStepSpanSink{
		tracerPool:   p,
		stats:        stats,
		serviceCount: make(map[string]int64),
		mutex:        &sync.Mutex{},
	}, nil
}

// Name returns this sink's name.
func (ls *lightStepSpanSink) Name() string {
	return "lightstep"
}

// Ingest takes in a span and passed it along to the LS client after
// some sanity checks and improvements are made.
func (ls *lightStepSpanSink) Ingest(ssfSpan ssf.SSFSpan) error {
	parentID := ssfSpan.ParentId
	if parentID <= 0 {
		parentID = 0
	}

	var errorCode int64
	if ssfSpan.Error {
		errorCode = 1
	}

	timestamp := time.Unix(ssfSpan.StartTimestamp/1e9, ssfSpan.StartTimestamp%1e9)
	tracer := ls.tracerPool.Get().(opentracing.Tracer)
	defer ls.tracerPool.Put(tracer)
	sp := tracer.StartSpan(
		ssfSpan.Name,
		opentracing.StartTime(timestamp),
		lightstep.SetTraceID(uint64(ssfSpan.TraceId)),
		lightstep.SetSpanID(uint64(ssfSpan.Id)),
		lightstep.SetParentSpanID(uint64(parentID)))

	sp.SetTag(trace.ResourceKey, ssfSpan.Tags[trace.ResourceKey]) // TODO Why is this here?
	sp.SetTag(lightstep.ComponentNameKey, ssfSpan.Service)
	// TODO don't hardcode
	sp.SetTag("type", "http")
	sp.SetTag("error-code", errorCode)
	for k, v := range ssfSpan.Tags {
		sp.SetTag(k, v)
	}
	// And now set any veneur common tags
	for k, v := range ls.commonTags {
		sp.SetTag(k, v)
	}
	// TODO add metrics as tags to the span as well?

	if errorCode > 0 {
		// Note: this sets the OT-standard "error" tag, which
		// LightStep uses to flag error spans.
		ext.Error.Set(sp, true)
	}

	endTime := time.Unix(ssfSpan.EndTimestamp/1e9, ssfSpan.EndTimestamp%1e9)
	sp.FinishWithOptions(opentracing.FinishOptions{
		FinishTime: endTime,
	})

	service := ssfSpan.Service
	if service == "" {
		service = "unknown"
	}

	// Protect mutating the service count with a mutex
	ls.mutex.Lock()
	defer ls.mutex.Unlock()
	ls.serviceCount[service]++

	return nil
}

// Flush doesn't need to do anything to the LS tracer, so we emit metrics
// instead.
func (ls *lightStepSpanSink) Flush() {
	ls.mutex.Lock()
	defer ls.mutex.Unlock()

	totalCount := int64(0)
	for service, count := range ls.serviceCount {
		totalCount += count
		ls.stats.Count(totalSpansFlushedMetricKey, count, []string{"sink:lightstep", fmt.Sprintf("service:%s", service)}, 1)
	}
	ls.serviceCount = make(map[string]int64)
	log.WithField("total_spans", totalCount).Debug("Checkpointing flushed spans for Lightstep")
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"io"
	"log"
	"os/exec"
	"strings"
)

type Runner struct {
	printer     PrintResetter
	current     *exec.Cmd
	template    []string
	placeholder string
	stdoutbuf   *bytes.Buffer
	stdinbuf    *bytes.Buffer
}

func (r *Runner) runWithInput(input []byte) {
	cmd := r.cmdWithInput(string(input))

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}

	stderr, err := cmd.StderrPipe()
	if err != nil {
		log.Fatal(err)
	}

	outch := r.streamOutput(stdout)
	errch := r.streamOutput(stderr)

	if r.stdinbuf != nil {
		cmd.Stdin = bytes.NewBuffer(r.stdinbuf.Bytes())
	}

	err = cmd.Start()
	if err != nil {
		log.Fatal(err)
	}

	r.stdoutbuf = new(bytes.Buffer)
	r.current = cmd

	for str := range outch {
		r.printer.Print(str)
		r.stdoutbuf.WriteString(str)
	}

	err = cmd.Wait()
	if err != nil {
		for str := range errch {
			r.printer.Print(str)
		}
		return
	}

	r.printer.Reset()
}

func (r *Runner) cmdWithInput(input string) *exec.Cmd {
	argv := make([]string, len(r.template))
	for i := range argv {
		argv[i] = strings.Replace(r.template[i], r.placeholder, input, -1)
	}

	return exec.Command(argv[0], argv[1:]...)
}

func (r *Runner) streamOutput(stdout io.ReadCloser) <-chan string {
	ch := make(chan string)
	cmdreader := bufio.NewReader(stdout)

	go func() {
		for {
			line, err := cmdreader.ReadBytes('\n')
			if s := string(line); s != "" {
				ch <- s
			}
			if err != nil || err == io.EOF {
				break
			}
		}
		close(ch)
	}()

	return ch
}

func (r *Runner) writeCmdStdout(out io.Writer) (n int64, err error) {
	if r.stdoutbuf != nil {
		n, err = io.Copy(out, r.stdoutbuf)
	}
	return
}

func (r *Runner) killCurrent() {
	if r.current != nil {
		r.current.Process.Kill()
		r.current.Wait()

		r.current = nil
	}

	r.printer.Reset()
}
<commit_msg>Fix data races in test suite<commit_after>package main

import (
	"bufio"
	"bytes"
	"io"
	"log"
	"os/exec"
	"strings"
)

type Runner struct {
	printer     PrintResetter
	current     *exec.Cmd
	template    []string
	placeholder string
	stdoutbuf   *bytes.Buffer
	stdinbuf    *bytes.Buffer
}

func (r *Runner) runWithInput(input []byte) {
	cmd := r.cmdWithInput(string(input))

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}

	stderr, err := cmd.StderrPipe()
	if err != nil {
		log.Fatal(err)
	}

	outch := r.streamOutput(stdout)
	errch := r.streamOutput(stderr)

	if r.stdinbuf != nil {
		cmd.Stdin = bytes.NewBuffer(r.stdinbuf.Bytes())
	}

	err = cmd.Start()
	if err != nil {
		log.Fatal(err)
	}

	r.stdoutbuf = new(bytes.Buffer)
	r.current = cmd

	for str := range outch {
		r.printer.Print(str)
		r.stdoutbuf.WriteString(str)
	}

	for str := range errch {
		r.printer.Print(str)
	}

	cmd.Wait()
	r.printer.Reset()
}

func (r *Runner) cmdWithInput(input string) *exec.Cmd {
	argv := make([]string, len(r.template))
	for i := range argv {
		argv[i] = strings.Replace(r.template[i], r.placeholder, input, -1)
	}

	return exec.Command(argv[0], argv[1:]...)
}

func (r *Runner) streamOutput(stdout io.ReadCloser) <-chan string {
	ch := make(chan string)
	cmdreader := bufio.NewReader(stdout)

	go func() {
		for {
			line, err := cmdreader.ReadBytes('\n')
			if s := string(line); s != "" {
				ch <- s
			}
			if err != nil || err == io.EOF {
				break
			}
		}
		close(ch)
	}()

	return ch
}

func (r *Runner) writeCmdStdout(out io.Writer) (n int64, err error) {
	if r.stdoutbuf != nil {
		n, err = io.Copy(out, r.stdoutbuf)
	}
	return
}

func (r *Runner) killCurrent() {
	if r.current != nil {
		r.current.Process.Kill()
		r.current.Wait()

		r.current = nil
	}

	r.printer.Reset()
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"gopkg.in/alecthomas/kingpin.v2"
	"github.com/tears-of-noobs/bytefmt"
	"fmt"
	"github.com/sandreas/graft/pattern"
	"github.com/sandreas/graft/file"
	"strconv"
	"regexp"
	"path"
	"strings"
	"math"
	"time"
)

var (
	app = kingpin.New("graft", "A command-line tool to locate and transfer files")
	sourcePatternParameter = app.Arg("source-pattern", "source pattern - used to locate files (e.g. src/*)").Required().String()
	destinationPatternParameter = app.Arg("destination-pattern", "destination pattern for transfer (e.g. dst/$1)").Default("").String()

	exportTo = app.Flag("export-to", "export source listing to file, one line per found item").Default("").String()
	filesFrom = app.Flag("files-from", "import source listing from file, one line per item").Default("").String()

	minAge = app.Flag("min-age", " minimum age (e.g. -2 days, -8 weeks, 2015-10-10, etc.)").Default("").String()
	maxAge = app.Flag("max-age", "maximum age (e.g. 2 days, 8 weeks, 2015-10-10, etc.)").Default("").String()

	caseSensitive = app.Flag("case-sensitive", "be case sensitive when matching files and folders").Bool()
	dryRun = app.Flag("dry-run", "dry-run / simulation mode").Bool()
	hideMatches = app.Flag("hide-matches", "hide matches in search mode ($1: ...)").Bool()
	move = app.Flag("move", "move / rename files - do not make a copy").Bool()
	quiet = app.Flag("quiet", "quiet mode - do not show any output").Bool()
	regex = app.Flag("regex", "use a real regex instead of glob patterns (e.g. src/.*\\.jpg)").Bool()
	times = app.Flag("times", "transfer source modify times to destination").Bool()
)

var dirsToRemove = make([]string, 0)
var copyStartTime = time.Now();
var copyStartBytes = int64(0);
var nextTimeUpdateSeconds = float64(2);
var transferSpeedOutput = ""
func main() {
	kingpin.MustParse(app.Parse(os.Args[1:]))

	//firstBytes := int64(1003838300);
	//size := firstBytes * 5;
	//chunkSize:= int64(32*1024);
	//
	//handleProgress(firstBytes, size, chunkSize)
	//time.Sleep(3 * time.Second)
	//handleProgress(firstBytes*2, size, chunkSize)
	//time.Sleep(3 * time.Second)
	//handleProgress(firstBytes*3, size, chunkSize)
	//time.Sleep(3 * time.Second)
	//handleProgress(firstBytes*4, size, chunkSize)
	//time.Sleep(3 * time.Second)
	//handleProgress(firstBytes*5, size, chunkSize)
	//os.Exit(0)

	sourcePattern := *sourcePatternParameter
	destinationPattern := *destinationPatternParameter

	patternPath, pat := pattern.ParsePathPattern(sourcePattern)

	sourcePathStat, err := os.Stat(patternPath)

	if sourcePathStat.Mode().IsRegular() {
		if strings.HasSuffix(destinationPattern, "/") || strings.HasSuffix(destinationPattern, "\\") {
			destinationPattern += sourcePathStat.Name()
		}
		transferElementHandler(sourcePattern, destinationPattern)
		return
	}

	if destinationPattern == "" {
		searchIn := patternPath
		if patternPath == "" {
			searchIn = "./"
		}

		searchFor := ""
		if pat != "" {
			searchFor = pat
		}
		prntln("search in '" + searchIn + "': " + searchFor)

	} else if (*move) {
		prntln("move: " + sourcePattern + " => " + destinationPattern)
	} else {
		prntln("copy: " + sourcePattern + " => " + destinationPattern)
	}
	prntln("")

	if ! *regex {
		pat = pattern.GlobToRegex(pat)
	}

	caseInsensitiveQualifier := "(?i)"
	if *caseSensitive {
		caseInsensitiveQualifier = ""
	}

	compiledPattern, err := pattern.CompileNormalizedPathPattern(patternPath, caseInsensitiveQualifier + pat)
	if err == nil && compiledPattern.NumSubexp() == 0 && pat != "" {
		compiledPattern, err = pattern.CompileNormalizedPathPattern(patternPath, caseInsensitiveQualifier + "(" + pat + ")")
	}

	if err != nil {
		prntln("could not compile source pattern, please use slashes to qualify paths (recognized path: " + patternPath + ", pattern" + pat + ")")
		return
	}

	var matchingPaths []string

	if *filesFrom != "" {
		if ! file.Exists(*filesFrom) {
			prntln("Could not load files from " + *filesFrom)
			return
		}
		matchingPaths, err = file.ReadAllLinesFunc(*filesFrom, file.SkipEmptyLines)
	} else {
		 //matchingPaths, err = file.WalkPathByPattern(patternPath, compiledPattern, progressHandlerWalkPathByPattern)
		matchingFiles, _ := file.WalkPathFiltered(patternPath, func(f file.File, err error)(bool) {
			normalizedPath := pattern.NormalizeDirSep(f.Path)
			if ! compiledPattern.MatchString(normalizedPath) {
				return false
			}

			return minAgeFilter(f) && maxAgeFilter(f)
		}, progressHandlerWalkPathByPattern)

		for _, element := range matchingFiles {
			matchingPaths = append(matchingPaths, element.Path)
		}

		if *exportTo != "" {
			exportFile(*exportTo, matchingPaths)
		}
	}

	if err != nil {
		prntln("Could not load sources path " + patternPath + ":", err.Error())
		return
	}

	if destinationPattern == "" {
		for _, element := range matchingPaths {
			findElementHandler(element, compiledPattern)
		}
		return
	}

	dstPath, dstPatt := pattern.ParsePathPattern(destinationPattern)
	var dst string
	for _, element := range matchingPaths {
		if dstPatt == "" {
			dst = pattern.NormalizeDirSep(dstPath + element[len(patternPath)+1:])
		} else {
			dst = compiledPattern.ReplaceAllString(pattern.NormalizeDirSep(element), pattern.NormalizeDirSep(destinationPattern))
		}
		transferElementHandler(element, dst)
	}

	if *move {
		for _, dirToRemove := range dirsToRemove {
			os.Remove(dirToRemove)
		}
	}
	return
}

func minAgeFilter(f file.File)(bool) {
	if *minAge == "" {
		return true
	}

	minAgeTime, err := pattern.StrToAge(*minAge, time.Now())
	if err != nil {
		return false
	}
	return minAgeTime.UnixNano() > f.ModTime().UnixNano()
}

func maxAgeFilter(f file.File)(bool) {
	if *maxAge == "" {
		return true
	}

	maxAgeTime, err := pattern.StrToAge(*maxAge, time.Now())
	if err != nil {
		return false
	}
	return maxAgeTime.UnixNano() < f.ModTime().UnixNano()
}

func progressHandlerWalkPathByPattern(entriesWalked, entriesMatched int64, finished bool) (int64) {
	var progress string;
	if entriesMatched == 0 {
		progress = fmt.Sprintf("scanning - total: %d", entriesWalked)
	} else {
		progress = fmt.Sprintf("scanning - total: %d,  matches: %d", entriesWalked, entriesMatched)
	}
	// prnt("\x0c" + progressBar)
	prnt("\r" + progress)
	if finished {
		prntln("")
		prntln("")
	}
	if(entriesWalked > 1000) {
		return 500
	}
	return 100
}

func exportFile(file string, lines []string) {
	f, err := os.Create(*exportTo)
	if err != nil {
		prntln("could not create export file " + file + ": " + err.Error())
		return;
	}
	_, err = f.WriteString(strings.Join(lines, "\n"))
	defer f.Close()
	if err != nil {
		prntln("could not write export file " + file + ": " + err.Error())
	}

}

func appendRemoveDir(dir string) {
	if (*move) {
		dirsToRemove = append(dirsToRemove, dir)
	}
}

func handleProgress(bytesTransferred, size, chunkSize int64) (int64) {

	if size <= 0 {
		return chunkSize
	}

	bytesPerSecond := float64(0)

	if copyStartBytes > 0 {
		timeDiffNano := time.Now().UnixNano() - copyStartTime.UnixNano()
		timeDiffSecond := float64(timeDiffNano) / float64(time.Second)

		bytesDiff := bytesTransferred - copyStartBytes
		bytesPerSecond = float64(float64(bytesDiff) / float64(timeDiffSecond))
		nextTimeUpdateSeconds -= timeDiffSecond
	}

	//var nextTimeUpdateSeconds = 2;
	//var transferSpeedOutput = ""

	if transferSpeedOutput == "" || nextTimeUpdateSeconds < 0 {
		transferSpeedOutput = " - " + bytefmt.FormatBytes(bytesPerSecond, 2, true) + "/s"
		nextTimeUpdateSeconds = 2
	}

	copyStartTime = time.Now()
	copyStartBytes = bytesTransferred

	percent := float64(bytesTransferred) / float64(size)
	charCountWhenFullyTransmitted := 20
	progressChars := int(math.Floor(percent * float64(charCountWhenFullyTransmitted)))
	normalizedInt := percent * 100
	percentOutput := strconv.FormatFloat(normalizedInt, 'f', 2, 64)
	if bytesPerSecond == 0 {
		transferSpeedOutput = ""
	}
	progressBar := fmt.Sprintf("[%-" + strconv.Itoa(charCountWhenFullyTransmitted + 1)+ "s] " +percentOutput +  "%%" + transferSpeedOutput, strings.Repeat("=", progressChars) + ">")

	prnt("\r" + progressBar)
	if bytesTransferred == size {
		prntln("")
	}
	// fmt.Print("\r" + progressBar)
	return chunkSize
}

func prntln(a ...interface{}) (n int, err error) {
	if ! *quiet {
		return fmt.Println(a...)
	}
	return n, err
}

func prnt(a...interface{}) (n int, err error) {
	if ! *quiet {
		return fmt.Print(a...)
	}
	return n, err
}

func findElementHandler(element string, compiledPattern *regexp.Regexp) {
	prntln(element)
	if *hideMatches {
		return
	}
	elementMatches := pattern.BuildMatchList(compiledPattern, element)
	for i := 0; i < len(elementMatches); i++ {
		prntln("    $" + strconv.Itoa(i + 1) + ": " + elementMatches[i])
	}

}

func transferElementHandler(src, dst string) {

	prntln(src + " => " + dst)

	if *dryRun {
		return
	}

	srcStat, srcErr := os.Stat(src)

	if srcErr != nil {
		prntln("could not read source: ", srcErr)
		return
	}

	dstStat, _ := os.Stat(dst)
	dstExists := file.Exists(dst)
	if srcStat.IsDir() {
		if ! dstExists {
			if os.MkdirAll(dst, srcStat.Mode()) != nil {
				prntln("Could not create destination directory")
			}
			appendRemoveDir(dst)
			fixTimes(dst, srcStat)
			return
		}

		if dstStat.IsDir() {
			appendRemoveDir(dst)
			fixTimes(dst, srcStat)
			return
		}

		prntln("destination already exists as file, source is a directory")
		return
	}

	if dstExists && dstStat.IsDir() {
		prntln("destination already exists as directory, source is a file")
		return
	}

	srcDir := path.Dir(src)
	srcDirStat, _ := os.Stat(srcDir)

	dstDir := path.Dir(dst)
	if ! file.Exists(dstDir) {
		os.MkdirAll(dstDir, srcDirStat.Mode())
	}

	if *move {
		renameErr := os.Rename(src, dst)
		if renameErr == nil {
			appendRemoveDir(srcDir)
			fixTimes(dst, srcStat)
			return
		}
		prntln("Could not rename source")
		return
	}

	srcPointer, srcPointerErr := os.Open(src)
	if srcPointerErr != nil {
		prntln("Could not open source file")
		return
	}
	dstPointer, dstPointerErr := os.OpenFile(dst, os.O_WRONLY | os.O_CREATE, srcStat.Mode())

	if dstPointerErr != nil {
		prntln("Could not create destination file", dstPointerErr.Error())
		return
	}

	file.CopyResumed(srcPointer, dstPointer, handleProgress)
	fixTimes(dst, srcStat)
}

func fixTimes(dst string, inStats os.FileInfo) {
	if *times {
		os.Chtimes(dst, inStats.ModTime(), inStats.ModTime())
	}
}<commit_msg>Refactored progress speed, working<commit_after>package main

import (
	"os"
	"gopkg.in/alecthomas/kingpin.v2"
	"github.com/tears-of-noobs/bytefmt"
	"fmt"
	"github.com/sandreas/graft/pattern"
	"github.com/sandreas/graft/file"
	"strconv"
	"regexp"
	"path"
	"strings"
	"math"
	"time"
)

var (
	app = kingpin.New("graft", "A command-line tool to locate and transfer files")
	sourcePatternParameter = app.Arg("source-pattern", "source pattern - used to locate files (e.g. src/*)").Required().String()
	destinationPatternParameter = app.Arg("destination-pattern", "destination pattern for transfer (e.g. dst/$1)").Default("").String()

	exportTo = app.Flag("export-to", "export source listing to file, one line per found item").Default("").String()
	filesFrom = app.Flag("files-from", "import source listing from file, one line per item").Default("").String()

	minAge = app.Flag("min-age", " minimum age (e.g. -2 days, -8 weeks, 2015-10-10, etc.)").Default("").String()
	maxAge = app.Flag("max-age", "maximum age (e.g. 2 days, 8 weeks, 2015-10-10, etc.)").Default("").String()

	caseSensitive = app.Flag("case-sensitive", "be case sensitive when matching files and folders").Bool()
	dryRun = app.Flag("dry-run", "dry-run / simulation mode").Bool()
	hideMatches = app.Flag("hide-matches", "hide matches in search mode ($1: ...)").Bool()
	move = app.Flag("move", "move / rename files - do not make a copy").Bool()
	quiet = app.Flag("quiet", "quiet mode - do not show any output").Bool()
	regex = app.Flag("regex", "use a real regex instead of glob patterns (e.g. src/.*\\.jpg)").Bool()
	times = app.Flag("times", "transfer source modify times to destination").Bool()
)

var dirsToRemove = make([]string, 0)
func main() {
	kingpin.MustParse(app.Parse(os.Args[1:]))

	//firstBytes := int64(1003838300);
	//size := firstBytes * 5;
	//chunkSize:= int64(32*1024);
	//
	//handleProgress(firstBytes, size, chunkSize)
	//time.Sleep(3 * time.Second)
	//handleProgress(firstBytes*2, size, chunkSize)
	//time.Sleep(3 * time.Second)
	//handleProgress(firstBytes*3, size, chunkSize)
	//time.Sleep(3 * time.Second)
	//handleProgress(firstBytes*4, size, chunkSize)
	//time.Sleep(3 * time.Second)
	//handleProgress(firstBytes*5, size, chunkSize)
	//os.Exit(0)

	sourcePattern := *sourcePatternParameter
	destinationPattern := *destinationPatternParameter

	patternPath, pat := pattern.ParsePathPattern(sourcePattern)

	sourcePathStat, err := os.Stat(patternPath)

	if sourcePathStat.Mode().IsRegular() {
		if strings.HasSuffix(destinationPattern, "/") || strings.HasSuffix(destinationPattern, "\\") {
			destinationPattern += sourcePathStat.Name()
		}
		transferElementHandler(sourcePattern, destinationPattern)
		return
	}

	if destinationPattern == "" {
		searchIn := patternPath
		if patternPath == "" {
			searchIn = "./"
		}

		searchFor := ""
		if pat != "" {
			searchFor = pat
		}
		prntln("search in '" + searchIn + "': " + searchFor)

	} else if (*move) {
		prntln("move: " + sourcePattern + " => " + destinationPattern)
	} else {
		prntln("copy: " + sourcePattern + " => " + destinationPattern)
	}
	prntln("")

	if ! *regex {
		pat = pattern.GlobToRegex(pat)
	}

	caseInsensitiveQualifier := "(?i)"
	if *caseSensitive {
		caseInsensitiveQualifier = ""
	}

	compiledPattern, err := pattern.CompileNormalizedPathPattern(patternPath, caseInsensitiveQualifier + pat)
	if err == nil && compiledPattern.NumSubexp() == 0 && pat != "" {
		compiledPattern, err = pattern.CompileNormalizedPathPattern(patternPath, caseInsensitiveQualifier + "(" + pat + ")")
	}

	if err != nil {
		prntln("could not compile source pattern, please use slashes to qualify paths (recognized path: " + patternPath + ", pattern" + pat + ")")
		return
	}

	var matchingPaths []string

	if *filesFrom != "" {
		if ! file.Exists(*filesFrom) {
			prntln("Could not load files from " + *filesFrom)
			return
		}
		matchingPaths, err = file.ReadAllLinesFunc(*filesFrom, file.SkipEmptyLines)
	} else {
		 //matchingPaths, err = file.WalkPathByPattern(patternPath, compiledPattern, progressHandlerWalkPathByPattern)
		matchingFiles, _ := file.WalkPathFiltered(patternPath, func(f file.File, err error)(bool) {
			normalizedPath := pattern.NormalizeDirSep(f.Path)
			if ! compiledPattern.MatchString(normalizedPath) {
				return false
			}

			return minAgeFilter(f) && maxAgeFilter(f)
		}, progressHandlerWalkPathByPattern)

		for _, element := range matchingFiles {
			matchingPaths = append(matchingPaths, element.Path)
		}

		if *exportTo != "" {
			exportFile(*exportTo, matchingPaths)
		}
	}

	if err != nil {
		prntln("Could not load sources path " + patternPath + ":", err.Error())
		return
	}

	if destinationPattern == "" {
		for _, element := range matchingPaths {
			findElementHandler(element, compiledPattern)
		}
		return
	}

	dstPath, dstPatt := pattern.ParsePathPattern(destinationPattern)
	var dst string
	for _, element := range matchingPaths {
		if dstPatt == "" {
			dst = pattern.NormalizeDirSep(dstPath + element[len(patternPath)+1:])
		} else {
			dst = compiledPattern.ReplaceAllString(pattern.NormalizeDirSep(element), pattern.NormalizeDirSep(destinationPattern))
		}
		transferElementHandler(element, dst)
	}

	if *move {
		for _, dirToRemove := range dirsToRemove {
			os.Remove(dirToRemove)
		}
	}
	return
}

func minAgeFilter(f file.File)(bool) {
	if *minAge == "" {
		return true
	}

	minAgeTime, err := pattern.StrToAge(*minAge, time.Now())
	if err != nil {
		return false
	}
	return minAgeTime.UnixNano() > f.ModTime().UnixNano()
}

func maxAgeFilter(f file.File)(bool) {
	if *maxAge == "" {
		return true
	}

	maxAgeTime, err := pattern.StrToAge(*maxAge, time.Now())
	if err != nil {
		return false
	}
	return maxAgeTime.UnixNano() < f.ModTime().UnixNano()
}

func progressHandlerWalkPathByPattern(entriesWalked, entriesMatched int64, finished bool) (int64) {
	var progress string;
	if entriesMatched == 0 {
		progress = fmt.Sprintf("scanning - total: %d", entriesWalked)
	} else {
		progress = fmt.Sprintf("scanning - total: %d,  matches: %d", entriesWalked, entriesMatched)
	}
	// prnt("\x0c" + progressBar)
	prnt("\r" + progress)
	if finished {
		prntln("")
		prntln("")
	}
	if(entriesWalked > 1000) {
		return 500
	}
	return 100
}

func exportFile(file string, lines []string) {
	f, err := os.Create(*exportTo)
	if err != nil {
		prntln("could not create export file " + file + ": " + err.Error())
		return;
	}
	_, err = f.WriteString(strings.Join(lines, "\n"))
	defer f.Close()
	if err != nil {
		prntln("could not write export file " + file + ": " + err.Error())
	}

}

func appendRemoveDir(dir string) {
	if (*move) {
		dirsToRemove = append(dirsToRemove, dir)
	}
}

var timerLastUpdate time.Time
var reportInterval int64
var bytesLastUpdate int64

func startTimer(bytesTransferred, interval int64) {
	if bytesTransferred == 0 {
		timerLastUpdate = time.Now()
		reportInterval = interval
		bytesLastUpdate = 0
	}
}

func getReportStatus(bytesTransferred, size int64) (bool, float64, float64) {
	timeDiffNano := time.Now().UnixNano() - timerLastUpdate.UnixNano()
	 timeDiffSeconds := float64(timeDiffNano) / float64(time.Second)
	// if timeDiffSeconds >= float64(reportInterval) {
	if timeDiffNano >= reportInterval {
		bytesDiff := bytesTransferred - bytesLastUpdate
		bytesPerSecond := float64(float64(bytesDiff) / float64(timeDiffSeconds))
		percent := float64(bytesTransferred) / float64(size)

		bytesLastUpdate = bytesTransferred
		timerLastUpdate = time.Now()

		return true, bytesPerSecond, percent
	}

	return false, 0, 0
}

func handleProgress(bytesTransferred, size, chunkSize int64) (int64) {

	if size <= 0 {
		return chunkSize
	}

	startTimer(bytesTransferred, 1 * int64(time.Second))
	shouldReport, bytesPerSecond, percent := getReportStatus(bytesTransferred, size)
	if shouldReport {
		bandwidthOutput := " " + bytefmt.FormatBytes(bytesPerSecond, 2, true) + "/s"
		charCountWhenFullyTransmitted := 20
		progressChars := int(math.Floor(percent * float64(charCountWhenFullyTransmitted)))
		normalizedInt := percent * 100
		percentOutput := strconv.FormatFloat(normalizedInt, 'f', 2, 64)
		if bytesPerSecond == 0 {
			bandwidthOutput = ""
		}
		progressBar := fmt.Sprintf("[%-" + strconv.Itoa(charCountWhenFullyTransmitted + 1)+ "s] " +percentOutput +  "%%" + bandwidthOutput, strings.Repeat("=", progressChars) + ">")

		prnt("\r" + progressBar)
	}

	if bytesTransferred == size {
		prntln("")
	}

	return chunkSize
}

func prntln(a ...interface{}) (n int, err error) {
	if ! *quiet {
		return fmt.Println(a...)
	}
	return n, err
}

func prnt(a...interface{}) (n int, err error) {
	if ! *quiet {
		return fmt.Print(a...)
	}
	return n, err
}

func findElementHandler(element string, compiledPattern *regexp.Regexp) {
	prntln(element)
	if *hideMatches {
		return
	}
	elementMatches := pattern.BuildMatchList(compiledPattern, element)
	for i := 0; i < len(elementMatches); i++ {
		prntln("    $" + strconv.Itoa(i + 1) + ": " + elementMatches[i])
	}

}

func transferElementHandler(src, dst string) {

	prntln(src + " => " + dst)

	if *dryRun {
		return
	}

	srcStat, srcErr := os.Stat(src)

	if srcErr != nil {
		prntln("could not read source: ", srcErr)
		return
	}

	dstStat, _ := os.Stat(dst)
	dstExists := file.Exists(dst)
	if srcStat.IsDir() {
		if ! dstExists {
			if os.MkdirAll(dst, srcStat.Mode()) != nil {
				prntln("Could not create destination directory")
			}
			appendRemoveDir(dst)
			fixTimes(dst, srcStat)
			return
		}

		if dstStat.IsDir() {
			appendRemoveDir(dst)
			fixTimes(dst, srcStat)
			return
		}

		prntln("destination already exists as file, source is a directory")
		return
	}

	if dstExists && dstStat.IsDir() {
		prntln("destination already exists as directory, source is a file")
		return
	}

	srcDir := path.Dir(src)
	srcDirStat, _ := os.Stat(srcDir)

	dstDir := path.Dir(dst)
	if ! file.Exists(dstDir) {
		os.MkdirAll(dstDir, srcDirStat.Mode())
	}

	if *move {
		renameErr := os.Rename(src, dst)
		if renameErr == nil {
			appendRemoveDir(srcDir)
			fixTimes(dst, srcStat)
			return
		}
		prntln("Could not rename source")
		return
	}

	srcPointer, srcPointerErr := os.Open(src)
	if srcPointerErr != nil {
		prntln("Could not open source file")
		return
	}
	dstPointer, dstPointerErr := os.OpenFile(dst, os.O_WRONLY | os.O_CREATE, srcStat.Mode())

	if dstPointerErr != nil {
		prntln("Could not create destination file", dstPointerErr.Error())
		return
	}

	file.CopyResumed(srcPointer, dstPointer, handleProgress)
	fixTimes(dst, srcStat)
}

func fixTimes(dst string, inStats os.FileInfo) {
	if *times {
		os.Chtimes(dst, inStats.ModTime(), inStats.ModTime())
	}
}<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net"
	"net/http"
	"os"
	"strings"

	"github.com/codegangsta/cli"
)

const VERSION = "0.1"

func main() {
	app := cli.NewApp()
	app.Name = "groku"
	app.Version = VERSION
	app.Usage = "roku CLI remote"
	app.Commands = commands()
	app.Run(os.Args)
}

func findRoku() string {
	ssdp, _ := net.ResolveUDPAddr("udp", "239.255.255.250:1900")
	addr, _ := net.ResolveUDPAddr("udp", ":0")
	socket, _ := net.ListenUDP("udp", addr)

	socket.WriteToUDP([]byte("M-SEARCH * HTTP/1.1\r\n"+
		"HOST: 239.255.255.250:1900\r\n"+
		"MAN: \"ssdp:discover\"\r\n"+
		"ST: roku:ecp\r\n"+
		"MX: 3 \r\n\r\n"), ssdp)

	answerBytes := make([]byte, 1024)
	socket.ReadFromUDP(answerBytes[:])

	ret := strings.Split(string(answerBytes), "\r\n")
	return strings.TrimPrefix(ret[len(ret)-3], "LOCATION: ")
}

func commands() []cli.Command {
	cmds := []cli.Command{}
	for _, cmd := range []string{
		"Home",
		"Rev",
		"Fwd",
		"Select",
		"Left",
		"Right",
		"Down",
		"Up",
		"Back",
		"Info",
		"Backspace",
		"Search",
		"Enter",
	} {
		cmds = append(cmds, cli.Command{
			Name:  strings.ToLower(cmd),
			Usage: strings.ToLower(cmd),
			Action: func(c *cli.Context) {
				http.PostForm(fmt.Sprintf("%vkeypress/%v", findRoku(), cmd), nil)
			},
		})
	}
	cmds = append(cmds, cli.Command{
		Name:  "replay",
		Usage: "replay",
		Action: func(c *cli.Context) {
			http.PostForm(fmt.Sprintf("%vkeypress/%v", findRoku(), "InstantReplay"), nil)
		},
	})
	cmds = append(cmds, cli.Command{
		Name:  "play",
		Usage: "play/pause",
		Action: func(c *cli.Context) {
			http.PostForm(fmt.Sprintf("%vkeypress/%v", findRoku(), "Play"), nil)
		},
	})
	cmds = append(cmds, cli.Command{
		Name:  "discover",
		Usage: "discover roku on your local network",
		Action: func(c *cli.Context) {
			fmt.Println("Found roku at", findRoku())
		},
	})
	return cmds
}
<commit_msg>Actually execute the correct command<commit_after>package main

import (
	"fmt"
	"net"
	"net/http"
	"os"
	"strings"

	"github.com/codegangsta/cli"
)

const VERSION = "0.1"

func main() {
	app := cli.NewApp()
	app.Name = "groku"
	app.Version = VERSION
	app.Usage = "roku CLI remote"
	app.Commands = commands()
	app.Run(os.Args)
}

func findRoku() string {
	ssdp, _ := net.ResolveUDPAddr("udp", "239.255.255.250:1900")
	addr, _ := net.ResolveUDPAddr("udp", ":0")
	socket, _ := net.ListenUDP("udp", addr)

	socket.WriteToUDP([]byte("M-SEARCH * HTTP/1.1\r\n"+
		"HOST: 239.255.255.250:1900\r\n"+
		"MAN: \"ssdp:discover\"\r\n"+
		"ST: roku:ecp\r\n"+
		"MX: 3 \r\n\r\n"), ssdp)

	answerBytes := make([]byte, 1024)
	socket.ReadFromUDP(answerBytes[:])

	ret := strings.Split(string(answerBytes), "\r\n")
	return strings.TrimPrefix(ret[len(ret)-3], "LOCATION: ")
}

func commands() []cli.Command {
	cmds := []cli.Command{}
	for _, cmd := range []string{
		"home",
		"rev",
		"fwd",
		"select",
		"left",
		"right",
		"down",
		"up",
		"back",
		"info",
		"backspace",
		"enter",
		"search",
	} {
		cmds = append(cmds, cli.Command{
			Name:  cmd,
			Usage: cmd,
			Action: func(c *cli.Context) {
				http.PostForm(fmt.Sprintf("%vkeypress/%v", findRoku(), c.Command.Name), nil)
			},
		})
	}
	cmds = append(cmds, cli.Command{
		Name:  "replay",
		Usage: "replay",
		Action: func(c *cli.Context) {
			http.PostForm(fmt.Sprintf("%vkeypress/%v", findRoku(), "InstantReplay"), nil)
		},
	})
	cmds = append(cmds, cli.Command{
		Name:  "play",
		Usage: "play/pause",
		Action: func(c *cli.Context) {
			http.PostForm(fmt.Sprintf("%vkeypress/%v", findRoku(), "Play"), nil)
		},
	})
	cmds = append(cmds, cli.Command{
		Name:  "discover",
		Usage: "discover roku on your local network",
		Action: func(c *cli.Context) {
			fmt.Println("Found roku at", findRoku())
		},
	})
	return cmds
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/base64"
	"math"
	"runtime"
	"sync/atomic"
)

func groupGoods() {
	var mapReady, done int32
	goodIdToHashes := make(map[int32][]string)
	hashToGoodIds := make(map[string][]int32)
	go func() {
		rows, err := db.Query(`SELECT
		good_id, encode(sha512_16k, 'base64')
		FROM images
		WHERE sha512_16k IS NOT NULL
		AND (CASE WHEN length > 0 THEN length >= 50000 ELSE true END)
		`,
		)
		ce(err, "query")

		n := 0
		for rows.Next() {
			var goodId int32
			var hash string
			ce(rows.Scan(&goodId, &hash), "scan")
			goodIdToHashes[goodId] = append(goodIdToHashes[goodId], hash)
			hashToGoodIds[hash] = append(hashToGoodIds[hash], goodId)
			n++
			if n%10000 == 0 {
				pt("%d\n", n)
				if atomic.LoadInt32(&done) > 0 {
					return
				}
			}
			if n%2000000 == 0 {
				runtime.GC()
			}
		}
		ce(rows.Err(), "rows err")
		pt("images data loaded\n")
		atomic.StoreInt32(&mapReady, 1)
	}()

	goodIds := make(map[int32]struct{})
	rows, err := db.Query(`SELECT good_id
		FROM goods
		WHERE group_id IS NULL
		AND images_collected = true
		`,
	)
	ce(err, "query")
	for rows.Next() {
		var goodId int32
		ce(rows.Scan(&goodId), "scan")
		goodIds[goodId] = struct{}{}
	}
	ce(rows.Err(), "rows err")
	pt("good ids loaded\n")

	txCount := 0
	tx := db.MustBegin()
	markedCount := 0

check:
	dataIsReady := atomic.LoadInt32(&mapReady) > 0

	var goodId int32
	if len(goodIds) == 0 {
		tx.Commit()
		atomic.StoreInt32(&done, 1)
		return
	}
	for id := range goodIds {
		goodId = id
		break
	}

	var hashes []string
	if dataIsReady {
		hashes = goodIdToHashes[goodId]
	} else {
		err := db.Select(&hashes, `SELECT
			encode(sha512_16k, 'base64')
			FROM images
			WHERE good_id = $1
			`,
			goodId,
		)
		ce(err, "get hashes")
	}

	matches := make(map[int32]map[string]struct{})
	for _, hash := range hashes {
		var rightIds []int32
		if dataIsReady {
			rightIds = hashToGoodIds[hash]
		} else {
			bs, err := base64.StdEncoding.DecodeString(hash)
			ce(err, "decode hash")
			err = db.Select(&rightIds, `SELECT
				good_id
				FROM images
				WHERE
				sha512_16k = $1
				`,
				bs,
			)
			ce(err, "get ids")
		}
		for _, rightId := range rightIds {
			if _, ok := matches[rightId]; !ok {
				matches[rightId] = make(map[string]struct{})
			}
			matches[rightId][hash] = struct{}{}
		}
	}

	has := false
	for rightId, hashSet := range matches {
		//TODO 去掉第二个条件
		if len(hashSet) >= 10 && math.Abs(float64(len(hashes)-len(hashSet))) < 5 {
			pt("%d %d %d %d %d %v\n", goodId, len(hashes), rightId, len(hashSet), markedCount, dataIsReady)
			// 不到的话就算了吧
			has = true
			_, err := tx.Exec(`UPDATE goods
				SET group_id = $1
				WHERE good_id = $2
				`,
				goodId,
				rightId,
			)
			ce(err, "update goods")
			delete(goodIds, rightId)
			markedCount++
		}
	}
	if !has {
		_, err := tx.Exec(`UPDATE goods
				SET group_id = -1
				WHERE good_id = $1
				`,
			goodId,
		)
		ce(err, "update goods")
		delete(goodIds, goodId)
		markedCount++
	}

	txCount++
	if txCount >= 64 {
		ce(tx.Commit(), "commit")
		tx = db.MustBegin()
		txCount = 0
	}

	goto check
}
<commit_msg>update<commit_after>package main

import (
	"encoding/base64"
	"runtime"
	"sync/atomic"
)

func groupGoods() {
	var mapReady, done int32
	goodIdToHashes := make(map[int32][]string)
	hashToGoodIds := make(map[string][]int32)
	go func() {
		return
		rows, err := db.Query(`SELECT
		good_id, encode(sha512_16k, 'base64')
		FROM images
		WHERE sha512_16k IS NOT NULL
		AND (CASE WHEN length > 0 THEN length >= 50000 ELSE true END)
		`,
		)
		ce(err, "query")

		n := 0
		for rows.Next() {
			var goodId int32
			var hash string
			ce(rows.Scan(&goodId, &hash), "scan")
			goodIdToHashes[goodId] = append(goodIdToHashes[goodId], hash)
			hashToGoodIds[hash] = append(hashToGoodIds[hash], goodId)
			n++
			if n%10000 == 0 {
				pt("%d\n", n)
				if atomic.LoadInt32(&done) > 0 {
					return
				}
			}
			if n%2000000 == 0 {
				runtime.GC()
			}
		}
		ce(rows.Err(), "rows err")
		pt("images data loaded\n")
		atomic.StoreInt32(&mapReady, 1)
	}()

	goodIds := make(map[int32]struct{})
	rows, err := db.Query(`SELECT good_id
		FROM goods
		WHERE group_id IS NULL
		AND images_collected = true
		`,
	)
	ce(err, "query")
	for rows.Next() {
		var goodId int32
		ce(rows.Scan(&goodId), "scan")
		goodIds[goodId] = struct{}{}
	}
	ce(rows.Err(), "rows err")
	pt("%d goods to check\n", len(goodIds))

	txCount := 0
	tx := db.MustBegin()
	markedCount := 0

check:
	dataIsReady := atomic.LoadInt32(&mapReady) > 0

	var goodId int32
	if len(goodIds) == 0 {
		tx.Commit()
		atomic.StoreInt32(&done, 1)
		return
	}
	for id := range goodIds {
		goodId = id
		break
	}

	var hashes []string
	if dataIsReady {
		hashes = goodIdToHashes[goodId]
	} else {
		err := db.Select(&hashes, `SELECT
			encode(sha512_16k, 'base64')
			FROM images
			WHERE good_id = $1
			`,
			goodId,
		)
		ce(err, "get hashes")
	}

	matches := make(map[int32]map[string]struct{})
	for _, hash := range hashes {
		var rightIds []int32
		if dataIsReady {
			rightIds = hashToGoodIds[hash]
		} else {
			bs, err := base64.StdEncoding.DecodeString(hash)
			ce(err, "decode hash")
			err = db.Select(&rightIds, `SELECT
				good_id
				FROM images
				WHERE
				sha512_16k = $1
				`,
				bs,
			)
			ce(err, "get ids")
		}
		for _, rightId := range rightIds {
			if _, ok := matches[rightId]; !ok {
				matches[rightId] = make(map[string]struct{})
			}
			matches[rightId][hash] = struct{}{}
		}
	}

	has := false
	for rightId, hashSet := range matches {
		if len(hashSet) >= 10 {
			pt("%d %d %d %d %d %v\n", goodId, len(hashes), rightId, len(hashSet), markedCount, dataIsReady)
			// 不到的话就算了吧
			has = true
			_, err := tx.Exec(`UPDATE goods
				SET group_id = $1
				WHERE good_id = $2
				`,
				goodId,
				rightId,
			)
			ce(err, "update goods")
			delete(goodIds, rightId)
			markedCount++
		}
	}
	if !has {
		_, err := tx.Exec(`UPDATE goods
				SET group_id = -1
				WHERE good_id = $1
				`,
			goodId,
		)
		ce(err, "update goods")
		delete(goodIds, goodId)
		markedCount++
	}

	if markedCount%1000 == 0 {
		pt("marked %d\n", markedCount)
	}

	txCount++
	if txCount >= 64 {
		ce(tx.Commit(), "commit")
		tx = db.MustBegin()
		txCount = 0
	}

	goto check
}
<|endoftext|>


<commit_before>// Package strumt provides a way to defines scenarios for prompting
// informations on command line
package strumt

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"strings"
)

// Step represents a scenario step
type Step struct {
	prompt string
	inputs []string
	err    error
}

// NewPrompts creates a new prompt from stdin
func NewPrompts() Prompts {
	return Prompts{reader: bufio.NewReader(os.Stdin), writer: os.Stdout, prompts: map[string]Prompter{}}
}

// NewPromptsFromReaderAndWriter creates a new prompt from a given reader and writer
// , useful for testing purpose for instance by providing a buffer
func NewPromptsFromReaderAndWriter(reader io.Reader, writer io.Writer) Prompts {
	return Prompts{reader: bufio.NewReader(reader), writer: writer, prompts: map[string]Prompter{}}
}

// Prompts stores all defined prompts and current
// running prompt
type Prompts struct {
	currentPrompt Prompter
	prompts       map[string]Prompter
	reader        *bufio.Reader
	writer        io.Writer
	scenario      []Step
}

func (p *Prompts) renderPrompt(prompt Prompter) {
	switch pr := prompt.(type) {
	case PromptRenderer:
		pr.PrintPrompt(prompt.GetPromptString())
	default:
		fmt.Fprintf(p.writer, "%s : \n", prompt.GetPromptString())
	}
}

func (p *Prompts) renderError(prompt Prompter, err error) {
	switch pr := prompt.(type) {
	case ErrorRenderer:
		pr.PrintError(err)
	default:
		fmt.Fprintf(p.writer, "%s\n", err.Error())
	}
}

func (p *Prompts) isMultilineEnd() (bool, error) {
	bn, err := p.reader.ReadByte()

	if err == io.EOF {
		return true, nil
	}

	if bn == '\n' {
		return true, nil
	}

	if err := p.reader.UnreadByte(); err != nil {
		return false, err
	}

	return false, nil
}

func (p *Prompts) parseMultipleLine(prompt MultilinePrompter) ([]string, error) {
	inputs := []string{}

	for {
		input, err := p.reader.ReadString('\n')
		input = strings.TrimRight(input, "\n")

		if err != nil {
			p.currentPrompt = p.prompts[prompt.GetNextOnError(err)]

			return []string{}, err
		}

		inputs = append(inputs, input)

		end, err := p.isMultilineEnd()

		if err != nil {
			p.currentPrompt = p.prompts[prompt.GetNextOnError(err)]

			return []string{}, err
		}

		if end {
			break
		}
	}

	if err := prompt.Parse(inputs); err != nil {
		p.currentPrompt = p.prompts[prompt.GetNextOnError(err)]

		return inputs, err
	}

	if prompt.GetNextOnSuccess(inputs) == "" {
		p.currentPrompt = nil

		return inputs, nil
	}

	p.currentPrompt = p.prompts[prompt.GetNextOnSuccess(inputs)]

	return inputs, nil
}

func (p *Prompts) parseLine(prompt LinePrompter) (string, error) {
	input, err := p.reader.ReadString('\n')
	input = strings.TrimRight(input, "\n")

	if err != nil {
		p.currentPrompt = p.prompts[prompt.GetNextOnError(err)]

		return "", err
	}

	if err := prompt.Parse(input); err != nil {
		p.currentPrompt = p.prompts[prompt.GetNextOnError(err)]

		return input, err
	}

	if prompt.GetNextOnSuccess(input) == "" {
		p.currentPrompt = nil

		return input, nil
	}

	p.currentPrompt = p.prompts[prompt.GetNextOnSuccess(input)]

	return input, nil
}

// AddLinePrompter add a new LinePrompter mapped to a given id
func (p *Prompts) AddLinePrompter(id string, prompt LinePrompter) {
	p.prompts[id] = prompt
}

// AddMultilinePrompter add a new MultilinePrompter mapped to a given id
func (p *Prompts) AddMultilinePrompter(id string, prompt MultilinePrompter) {
	p.prompts[id] = prompt
}

// SetFirst defines from which prompt, the prompt sequence has to start
func (p *Prompts) SetFirst(id string) {
	p.currentPrompt = p.prompts[id]
}

// GetScenario retrieves all steps done during
// prompt
func (p *Prompts) GetScenario() []Step {
	return p.scenario
}

// Run executes prompt sequence
func (p *Prompts) Run() {
	p.scenario = []Step{}

	for {
		var err error
		inputs := []string{}

		prompt := p.currentPrompt
		p.renderPrompt(prompt)

		switch lp := prompt.(type) {
		case LinePrompter:
			var input string
			input, err = p.parseLine(lp)
			inputs = append(inputs, input)
		case MultilinePrompter:
			inputs, err = p.parseMultipleLine(lp)

		}

		if err != nil {
			p.renderError(prompt, err)
		}

		p.scenario = append(
			p.scenario,
			Step{
				prompt.GetPromptString(),
				inputs,
				err,
			},
		)

		if p.currentPrompt == nil {
			return
		}
	}
}
<commit_msg>Rewrite to simplify<commit_after>// Package strumt provides a way to defines scenarios for prompting
// informations on command line
package strumt

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"strings"
)

// Step represents a scenario step
type Step struct {
	prompt string
	inputs []string
	err    error
}

// NewPrompts creates a new prompt from stdin
func NewPrompts() Prompts {
	return Prompts{reader: bufio.NewReader(os.Stdin), writer: os.Stdout, prompts: map[string]Prompter{}}
}

// NewPromptsFromReaderAndWriter creates a new prompt from a given reader and writer
// , useful for testing purpose for instance by providing a buffer
func NewPromptsFromReaderAndWriter(reader io.Reader, writer io.Writer) Prompts {
	return Prompts{reader: bufio.NewReader(reader), writer: writer, prompts: map[string]Prompter{}}
}

// Prompts stores all defined prompts and current
// running prompt
type Prompts struct {
	currentPrompt Prompter
	prompts       map[string]Prompter
	reader        *bufio.Reader
	writer        io.Writer
	scenario      []Step
}

func (p *Prompts) parse() ([]string, Prompter, error) {
	var nextPrompt Prompter
	var inputs []string
	var err error

	switch prompt := p.currentPrompt.(type) {
	case LinePrompter:
		var input string

		input, err = parseLine(p.reader, prompt)

		if prompt.GetNextOnSuccess(input) != "" {
			nextPrompt = p.prompts[prompt.GetNextOnSuccess(input)]
		}

		inputs = append(inputs, input)
	case MultilinePrompter:
		inputs, err = parseMultipleLine(p.reader, prompt)

		if prompt.GetNextOnSuccess(inputs) != "" {
			nextPrompt = p.prompts[prompt.GetNextOnSuccess(inputs)]
		}
	}

	if err != nil {
		nextPrompt = p.prompts[p.currentPrompt.GetNextOnError(err)]
	}

	return inputs, nextPrompt, err
}

func (p *Prompts) appendScenario(promptString string, inputs []string, err error) {
	p.scenario = append(
		p.scenario,
		Step{
			promptString,
			inputs,
			err,
		},
	)
}

// AddLinePrompter add a new LinePrompter mapped to a given id
func (p *Prompts) AddLinePrompter(id string, prompt LinePrompter) {
	p.prompts[id] = prompt
}

// AddMultilinePrompter add a new MultilinePrompter mapped to a given id
func (p *Prompts) AddMultilinePrompter(id string, prompt MultilinePrompter) {
	p.prompts[id] = prompt
}

// SetFirst defines from which prompt, the prompt sequence has to start
func (p *Prompts) SetFirst(id string) {
	p.currentPrompt = p.prompts[id]
}

// GetScenario retrieves all steps done during
// prompt
func (p *Prompts) GetScenario() []Step {
	return p.scenario
}

// Run executes prompt sequence
func (p *Prompts) Run() {
	p.scenario = []Step{}

	for {
		var err error
		inputs := []string{}

		prompt := p.currentPrompt
		renderPrompt(p.writer, prompt)

		inputs, nextPrompt, err := p.parse()

		if err != nil {
			renderError(p.writer, prompt, err)
		}

		p.appendScenario(prompt.GetPromptString(), inputs, err)

		if nextPrompt == nil {
			return
		}

		p.currentPrompt = nextPrompt
	}
}

func isMultilineEnd(reader *bufio.Reader) (bool, error) {
	bn, err := reader.ReadByte()

	if err == io.EOF {
		return true, nil
	}

	if bn == '\n' {
		return true, nil
	}

	if err := reader.UnreadByte(); err != nil {
		return false, err
	}

	return false, nil
}

func parseMultipleLine(reader *bufio.Reader, prompt MultilinePrompter) ([]string, error) {
	inputs := []string{}

	for {
		input, err := reader.ReadString('\n')
		input = strings.TrimRight(input, "\n")

		if err != nil {
			return []string{}, err
		}

		inputs = append(inputs, input)

		end, err := isMultilineEnd(reader)

		if err != nil {
			return []string{}, err
		}

		if end {
			break
		}
	}

	if err := prompt.Parse(inputs); err != nil {
		return inputs, err
	}

	return inputs, nil
}

func parseLine(reader *bufio.Reader, prompt LinePrompter) (string, error) {
	input, err := reader.ReadString('\n')
	input = strings.TrimRight(input, "\n")

	if err != nil {
		return "", err
	}

	if err := prompt.Parse(input); err != nil {
		return input, err
	}

	return input, nil
}

func renderPrompt(writer io.Writer, prompt Prompter) {
	switch pr := prompt.(type) {
	case PromptRenderer:
		pr.PrintPrompt(prompt.GetPromptString())
	default:
		fmt.Fprintf(writer, "%s : \n", prompt.GetPromptString())
	}
}

func renderError(writer io.Writer, prompt Prompter, err error) {
	switch pr := prompt.(type) {
	case ErrorRenderer:
		pr.PrintError(err)
	default:
		fmt.Fprintf(writer, "%s\n", err.Error())
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net"
	"os"

	flags "github.com/jessevdk/go-flags"
)

const version = "1.0.0"

var opts struct {
	Version bool `short:"v" long:"version" description:"Show version"`
}

func main() {
	parser := flags.NewParser(&opts, flags.Default)
	parser.Usage = "HOSTNAME [OPTIONS]"
	args, _ := parser.Parse()

	if len(args) == 0 {
		if opts.Version {
			fmt.Println("ptrhost version", version)
		}
		os.Exit(1)
	}

	hostname := args[0]
	addr, _ := net.LookupHost(hostname)
	for _, v := range addr {
		resolvedHost, err := net.LookupAddr(v)
		if err == nil {
			fmt.Println(v, "->", resolvedHost[0])
		} else {
			fmt.Println(v, "->", err.(*net.DNSError).Err)
		}
	}
}
<commit_msg>Bump version to 1.0.1<commit_after>package main

import (
	"fmt"
	"net"
	"os"

	flags "github.com/jessevdk/go-flags"
)

const version = "1.0.1"

var opts struct {
	Version bool `short:"v" long:"version" description:"Show version"`
}

func main() {
	parser := flags.NewParser(&opts, flags.Default)
	parser.Usage = "HOSTNAME [OPTIONS]"
	args, _ := parser.Parse()

	if len(args) == 0 {
		if opts.Version {
			fmt.Println("ptrhost version", version)
		}
		os.Exit(1)
	}

	hostname := args[0]
	addr, _ := net.LookupHost(hostname)
	for _, v := range addr {
		resolvedHost, err := net.LookupAddr(v)
		if err == nil {
			fmt.Println(v, "->", resolvedHost[0])
		} else {
			fmt.Println(v, "->", err.(*net.DNSError).Err)
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mungers

import (
	"k8s.io/contrib/mungegithub/github"

	"github.com/golang/glog"
	"github.com/spf13/cobra"
)

// NeedsRebaseMunger will add the "needs-rebase" label to any issue which is
// unable to be automatically merged
type NeedsRebaseMunger struct{}

const needsRebase = "needs-rebase"

func init() {
	RegisterMungerOrDie(NeedsRebaseMunger{})
}

// Name is the name usable in --pr-mungers
func (NeedsRebaseMunger) Name() string { return "needs-rebase" }

// Initialize will initialize the munger
func (NeedsRebaseMunger) Initialize(config *github.Config) error { return nil }

// EachLoop is called at the start of every munge loop
func (NeedsRebaseMunger) EachLoop() error { return nil }

// AddFlags will add any request flags to the cobra `cmd`
func (NeedsRebaseMunger) AddFlags(cmd *cobra.Command, config *github.Config) {}

// Munge is the workhorse the will actually make updates to the PR
func (NeedsRebaseMunger) Munge(obj *github.MungeObject) {
	if !obj.IsPR() {
		return
	}

	mergeable, err := obj.IsMergeable()
	if err != nil {
		glog.V(2).Infof("Skipping %d - problem determining mergeable", *obj.Issue.Number)
		return
	}
	if mergeable && obj.HasLabel(needsRebase) {
		obj.RemoveLabel(needsRebase)
	}
	if !mergeable && !obj.HasLabel(needsRebase) {
		obj.AddLabels([]string{needsRebase})
	}
}
<commit_msg>Add comment when PR requires rebase<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mungers

import (
	"k8s.io/contrib/mungegithub/github"

	"github.com/golang/glog"
	"github.com/spf13/cobra"
)

// NeedsRebaseMunger will add the "needs-rebase" label to any issue which is
// unable to be automatically merged
type NeedsRebaseMunger struct{}

const needsRebase = "needs-rebase"

func init() {
	RegisterMungerOrDie(NeedsRebaseMunger{})
}

// Name is the name usable in --pr-mungers
func (NeedsRebaseMunger) Name() string { return "needs-rebase" }

// Initialize will initialize the munger
func (NeedsRebaseMunger) Initialize(config *github.Config) error { return nil }

// EachLoop is called at the start of every munge loop
func (NeedsRebaseMunger) EachLoop() error { return nil }

// AddFlags will add any request flags to the cobra `cmd`
func (NeedsRebaseMunger) AddFlags(cmd *cobra.Command, config *github.Config) {}

// Munge is the workhorse the will actually make updates to the PR
func (NeedsRebaseMunger) Munge(obj *github.MungeObject) {
	if !obj.IsPR() {
		return
	}

	mergeable, err := obj.IsMergeable()
	if err != nil {
		glog.V(2).Infof("Skipping %d - problem determining mergeable", *obj.Issue.Number)
		return
	}
	if mergeable && obj.HasLabel(needsRebase) {
		obj.RemoveLabel(needsRebase)
	}
	if !mergeable && !obj.HasLabel(needsRebase) {
		obj.AddLabels([]string{needsRebase})

		body := "PR needs rebase"
		if err := obj.WriteComment(body); err != nil {
			return
		}
	}
}
<|endoftext|>


<commit_before>/*
 Copyright 2016 Padduck, LLC

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 	http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package main

import (
	"flag"
	"io/ioutil"
	"os"
	"fmt"

	"github.com/gin-gonic/gin"
	"github.com/pufferpanel/apufferi/config"
	"github.com/pufferpanel/apufferi/logging"
	"github.com/pufferpanel/pufferd/data"
	"github.com/pufferpanel/pufferd/data/templates"
	"github.com/pufferpanel/pufferd/install"
	"github.com/pufferpanel/pufferd/migration"
	"github.com/pufferpanel/pufferd/programs"
	"github.com/pufferpanel/pufferd/shutdown"
	"runtime"
	"net/http"
	"github.com/pufferpanel/pufferd/routing"
	"path/filepath"
	"github.com/pufferpanel/pufferd/sftp"
	"github.com/braintree/manners"
	"os/signal"
	"runtime/debug"
	"github.com/pufferpanel/pufferd/commands"
	"syscall"
)

var (
	VERSION      = "nightly"
	MAJORVERSION = "nightly"
	GITHASH      = "unknown"
)

var runService = true

func main() {
	var loggingLevel string
	var authRoot string
	var authToken string
	var runInstaller bool
	var version bool
	var license bool
	var regenerate bool
	var migrate bool
	var uninstall bool
	var configPath string
	var pid int
	var runDaemon bool
	flag.StringVar(&loggingLevel, "logging", "INFO", "Lowest logging level to display")
	flag.StringVar(&authRoot, "auth", "", "Base URL to the authorization server")
	flag.StringVar(&authToken, "token", "", "Authorization token")
	flag.BoolVar(&runInstaller, "install", false, "If installing instead of running")
	flag.BoolVar(&version, "version", false, "Get the version")
	flag.BoolVar(&license, "license", false, "View license")
	flag.BoolVar(&regenerate, "regenerate", false, "Regenerate pufferd templates")
	flag.BoolVar(&migrate, "migrate", false, "Migrate Scales data to pufferd")
	flag.BoolVar(&uninstall, "uninstall", false, "Uninstall pufferd")
	flag.StringVar(&configPath, "config", "config.json", "Path to pufferd config.json")
	flag.IntVar(&pid, "shutdown", 0, "PID to shut down")
	flag.BoolVar(&runDaemon, "run", false, "Runs the daemon")
	flag.Parse()

	versionString := fmt.Sprintf("pufferd %s (%s)", VERSION, GITHASH)

	if pid != 0 {
		logging.Info("Shutting down")
		commands.Shutdown(pid)
	}

	if _, err := os.Stat(configPath); os.IsNotExist(err) && !runInstaller && !version {
		if _, err := os.Stat("/etc/pufferd/config.json"); err == nil {
			logging.Info("No config passed, defaulting to /etc/pufferd/config.json")
			configPath = "/etc/pufferd/config.json"
		} else {
			logging.Error("Cannot find a config file!")
			return
		}
	}

	if uninstall {
		commands.Uninstall(configPath)
	}

	if version || !runDaemon {
		os.Stdout.WriteString(versionString + "\r\n")
	}

	if license {
		os.Stdout.WriteString(data.LICENSE + "\r\n")
	}

	if regenerate {
		commands.Regenerate(configPath)
	}

	if migrate {
		config.Load(configPath)
		migration.MigrateFromScales()
	}

	if license || version || regenerate || migrate || pid != 0 {
		return
	}

	config.Load(configPath)

	logging.SetLevelByString(loggingLevel)
	var defaultLogFolder = "logs"
	if runtime.GOOS == "linux" {
		defaultLogFolder = "/var/log/pufferd"
	}
	var logPath = config.GetOrDefault("logPath", defaultLogFolder)
	logging.SetLogFolder(logPath)
	logging.Init()
	gin.SetMode(gin.ReleaseMode)

	logging.Info(versionString)
	logging.Info("Logging set to " + loggingLevel)

	if runInstaller {
		install.Install(configPath, authRoot, authToken)
	}

	if runInstaller || !runDaemon {
		return
	}

	programs.Initialize()

	if _, err := os.Stat(programs.TemplateFolder); os.IsNotExist(err) {
		logging.Info("No template directory found, creating")
		err = os.MkdirAll(programs.TemplateFolder, 0755)
		if err != nil {
			logging.Error("Error creating template folder", err)
		}

	}
	if files, _ := ioutil.ReadDir(programs.TemplateFolder); len(files) == 0 {
		logging.Info("Templates being copied to " + programs.TemplateFolder)
		templates.CopyTemplates()
	}

	if _, err := os.Stat(programs.ServerFolder); os.IsNotExist(err) {
		logging.Info("No server directory found, creating")
		os.MkdirAll(programs.ServerFolder, 0755)
	}

	//check if there's an update
	if config.GetOrDefault("update-check", "true") == "true" {
		go func() {
			url := "https://dl.pufferpanel.com/pufferd/" + MAJORVERSION + "/version.txt"
			logging.Debug("Checking for updates using " + url)
			resp, err := http.Get(url)
			if err != nil {
				return
			}
			defer resp.Body.Close()
			onlineVersion, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				return
			}
			if string(onlineVersion) != GITHASH {
				logging.Infof("DL server reports a different hash than this version, an update may be available")
				logging.Infof("Installed: %s", GITHASH)
				logging.Infof("Online: %s", onlineVersion)
			}
		}()
	}

	programs.LoadFromFolder()

	programs.InitService()

	for _, element := range programs.GetAll() {
		if element.IsEnabled() && element.IsAutoStart() {
			logging.Info("Queued server " + element.Id())
			programs.StartViaService(element)
		}
	}

	CreateHook()

	for runService {
		runServices()
	}

	shutdown.Shutdown()
}

func runServices() {
	r := routing.ConfigureWeb()

	useHttps := false

	dataFolder := config.GetOrDefault("datafolder", "data")
	httpsPem := filepath.Join(dataFolder, "https.pem")
	httpsKey := filepath.Join(dataFolder, "https.key")

	if _, err := os.Stat(httpsPem); os.IsNotExist(err) {
		logging.Warn("No HTTPS.PEM found in data folder, will use http instead")
	} else if _, err := os.Stat(httpsKey); os.IsNotExist(err) {
		logging.Warn("No HTTPS.KEY found in data folder, will use http instead")
	} else {
		useHttps = true
	}

	sftp.Run()

	web := config.GetOrDefault("web", config.GetOrDefault("webhost", "0.0.0.0")+":"+config.GetOrDefault("webport", "5656"))

	logging.Infof("Starting web access on %s", web)
	var err error
	if useHttps {
		err = manners.ListenAndServeTLS(web, httpsPem, httpsKey, r)
	} else {
		err = manners.ListenAndServe(web, r)
	}
	if err != nil {
		logging.Error("Error starting web service", err)
	}
}

func CreateHook() {
	c := make(chan os.Signal, 1)
	signal.Notify(c, syscall.Signal(15), syscall.Signal(1))
	go func() {
		defer func() {
			if err := recover(); err != nil {
				logging.Errorf("Error: %+v\n%s", err, debug.Stack())
			}
		}()

		var sig os.Signal

		for sig != syscall.Signal(15) {
			sig = <- c
			switch sig {
			case syscall.Signal(1):
			}
		}

		runService = false
		shutdown.CompleteShutdown()
	}()
}<commit_msg>Add reload command to interface<commit_after>/*
 Copyright 2016 Padduck, LLC

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 	http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package main

import (
	"flag"
	"io/ioutil"
	"os"
	"fmt"

	"github.com/gin-gonic/gin"
	"github.com/pufferpanel/apufferi/config"
	"github.com/pufferpanel/apufferi/logging"
	"github.com/pufferpanel/pufferd/data"
	"github.com/pufferpanel/pufferd/data/templates"
	"github.com/pufferpanel/pufferd/install"
	"github.com/pufferpanel/pufferd/migration"
	"github.com/pufferpanel/pufferd/programs"
	"github.com/pufferpanel/pufferd/shutdown"
	"runtime"
	"net/http"
	"github.com/pufferpanel/pufferd/routing"
	"path/filepath"
	"github.com/pufferpanel/pufferd/sftp"
	"github.com/braintree/manners"
	"os/signal"
	"runtime/debug"
	"github.com/pufferpanel/pufferd/commands"
	"syscall"
)

var (
	VERSION      = "nightly"
	MAJORVERSION = "nightly"
	GITHASH      = "unknown"
)

var runService = true

func main() {
	var loggingLevel string
	var authRoot string
	var authToken string
	var runInstaller bool
	var version bool
	var license bool
	var regenerate bool
	var migrate bool
	var uninstall bool
	var configPath string
	var shutdownPid int
	var runDaemon bool
	var reloadPid int
	flag.StringVar(&loggingLevel, "logging", "INFO", "Lowest logging level to display")
	flag.StringVar(&authRoot, "auth", "", "Base URL to the authorization server")
	flag.StringVar(&authToken, "token", "", "Authorization token")
	flag.BoolVar(&runInstaller, "install", false, "If installing instead of running")
	flag.BoolVar(&version, "version", false, "Get the version")
	flag.BoolVar(&license, "license", false, "View license")
	flag.BoolVar(&regenerate, "regenerate", false, "Regenerate pufferd templates")
	flag.BoolVar(&migrate, "migrate", false, "Migrate Scales data to pufferd")
	flag.BoolVar(&uninstall, "uninstall", false, "Uninstall pufferd")
	flag.StringVar(&configPath, "config", "config.json", "Path to pufferd config.json")
	flag.IntVar(&shutdownPid, "shutdown", 0, "PID to shut down")
	flag.IntVar(&reloadPid, "shutdown", 0, "PID to shut down")
	flag.BoolVar(&runDaemon, "run", false, "Runs the daemon")
	flag.Parse()

	versionString := fmt.Sprintf("pufferd %s (%s)", VERSION, GITHASH)

	if shutdownPid != 0 {
		logging.Info("Shutting down")
		commands.Shutdown(shutdownPid)
	}

	if reloadPid != 0 {
		logging.Info("Reloading")
		commands.Reload(reloadPid)
	}

	if _, err := os.Stat(configPath); os.IsNotExist(err) && !runInstaller && !version {
		if _, err := os.Stat("/etc/pufferd/config.json"); err == nil {
			logging.Info("No config passed, defaulting to /etc/pufferd/config.json")
			configPath = "/etc/pufferd/config.json"
		} else {
			logging.Error("Cannot find a config file!")
			return
		}
	}

	if uninstall {
		commands.Uninstall(configPath)
	}

	if version || !runDaemon {
		os.Stdout.WriteString(versionString + "\r\n")
	}

	if license {
		os.Stdout.WriteString(data.LICENSE + "\r\n")
	}

	if regenerate {
		commands.Regenerate(configPath)
	}

	if migrate {
		config.Load(configPath)
		migration.MigrateFromScales()
	}

	if license || version || regenerate || migrate || shutdownPid != 0 || reloadPid != 0{
		return
	}

	config.Load(configPath)

	logging.SetLevelByString(loggingLevel)
	var defaultLogFolder = "logs"
	if runtime.GOOS == "linux" {
		defaultLogFolder = "/var/log/pufferd"
	}
	var logPath = config.GetOrDefault("logPath", defaultLogFolder)
	logging.SetLogFolder(logPath)
	logging.Init()
	gin.SetMode(gin.ReleaseMode)

	logging.Info(versionString)
	logging.Info("Logging set to " + loggingLevel)

	if runInstaller {
		install.Install(configPath, authRoot, authToken)
	}

	if runInstaller || !runDaemon {
		return
	}

	programs.Initialize()

	if _, err := os.Stat(programs.TemplateFolder); os.IsNotExist(err) {
		logging.Info("No template directory found, creating")
		err = os.MkdirAll(programs.TemplateFolder, 0755)
		if err != nil {
			logging.Error("Error creating template folder", err)
		}

	}
	if files, _ := ioutil.ReadDir(programs.TemplateFolder); len(files) == 0 {
		logging.Info("Templates being copied to " + programs.TemplateFolder)
		templates.CopyTemplates()
	}

	if _, err := os.Stat(programs.ServerFolder); os.IsNotExist(err) {
		logging.Info("No server directory found, creating")
		os.MkdirAll(programs.ServerFolder, 0755)
	}

	//check if there's an update
	if config.GetOrDefault("update-check", "true") == "true" {
		go func() {
			url := "https://dl.pufferpanel.com/pufferd/" + MAJORVERSION + "/version.txt"
			logging.Debug("Checking for updates using " + url)
			resp, err := http.Get(url)
			if err != nil {
				return
			}
			defer resp.Body.Close()
			onlineVersion, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				return
			}
			if string(onlineVersion) != GITHASH {
				logging.Infof("DL server reports a different hash than this version, an update may be available")
				logging.Infof("Installed: %s", GITHASH)
				logging.Infof("Online: %s", onlineVersion)
			}
		}()
	}

	programs.LoadFromFolder()

	programs.InitService()

	for _, element := range programs.GetAll() {
		if element.IsEnabled() && element.IsAutoStart() {
			logging.Info("Queued server " + element.Id())
			programs.StartViaService(element)
		}
	}

	CreateHook()

	for runService {
		runServices()
	}

	shutdown.Shutdown()
}

func runServices() {
	r := routing.ConfigureWeb()

	useHttps := false

	dataFolder := config.GetOrDefault("datafolder", "data")
	httpsPem := filepath.Join(dataFolder, "https.pem")
	httpsKey := filepath.Join(dataFolder, "https.key")

	if _, err := os.Stat(httpsPem); os.IsNotExist(err) {
		logging.Warn("No HTTPS.PEM found in data folder, will use http instead")
	} else if _, err := os.Stat(httpsKey); os.IsNotExist(err) {
		logging.Warn("No HTTPS.KEY found in data folder, will use http instead")
	} else {
		useHttps = true
	}

	sftp.Run()

	web := config.GetOrDefault("web", config.GetOrDefault("webhost", "0.0.0.0")+":"+config.GetOrDefault("webport", "5656"))

	logging.Infof("Starting web access on %s", web)
	var err error
	if useHttps {
		err = manners.ListenAndServeTLS(web, httpsPem, httpsKey, r)
	} else {
		err = manners.ListenAndServe(web, r)
	}
	if err != nil {
		logging.Error("Error starting web service", err)
	}
}

func CreateHook() {
	c := make(chan os.Signal, 1)
	signal.Notify(c, syscall.Signal(15), syscall.Signal(1))
	go func() {
		defer func() {
			if err := recover(); err != nil {
				logging.Errorf("Error: %+v\n%s", err, debug.Stack())
			}
		}()

		var sig os.Signal

		for sig != syscall.Signal(15) {
			sig = <- c
			switch sig {
			case syscall.Signal(1):
			}
		}

		runService = false
		shutdown.CompleteShutdown()
	}()
}<|endoftext|>


<commit_before>/*
 Copyright 2016 Padduck, LLC

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 	http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package main

import (
	"flag"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"

	"fmt"

	"net/http"
	"strings"

	"github.com/braintree/manners"
	"github.com/gin-gonic/gin"
	"github.com/pufferpanel/pufferd/config"
	"github.com/pufferpanel/pufferd/data"
	"github.com/pufferpanel/pufferd/data/templates"
	"github.com/pufferpanel/pufferd/install"
	"github.com/pufferpanel/pufferd/logging"
	"github.com/pufferpanel/pufferd/migration"
	"github.com/pufferpanel/pufferd/programs"
	"github.com/pufferpanel/pufferd/routing"
	"github.com/pufferpanel/pufferd/sftp"
	"github.com/pufferpanel/pufferd/shutdown"
	"github.com/pufferpanel/pufferd/uninstaller"
)

var (
	VERSION      = "nightly"
	MAJORVERSION = "nightly"
	BUILDDATE    = "unknown"
	GITHASH      = "unknown"
)

func main() {
	var loggingLevel string
	var authRoot string
	var authToken string
	var runInstaller bool
	var version bool
	var license bool
	var regenerate bool
	var migrate bool
	var uninstall bool
	var configPath string
	var pid int
	var installService bool
	flag.StringVar(&loggingLevel, "logging", "INFO", "Lowest logging level to display")
	flag.StringVar(&authRoot, "auth", "", "Base URL to the authorization server")
	flag.StringVar(&authToken, "token", "", "Authorization token")
	flag.BoolVar(&runInstaller, "install", false, "If installing instead of running")
	flag.BoolVar(&version, "version", false, "Get the version")
	flag.BoolVar(&license, "license", false, "View license")
	flag.BoolVar(&regenerate, "regenerate", false, "Regenerate pufferd templates")
	flag.BoolVar(&migrate, "migrate", false, "Migrate Scales data to pufferd")
	flag.BoolVar(&uninstall, "uninstall", false, "Uninstall pufferd")
	flag.StringVar(&configPath, "config", "config.json", "Path to pufferd config.json")
	flag.IntVar(&pid, "shutdown", 0, "PID to shut down")
	flag.BoolVar(&installService, "installService", false, "Installs the pufferd service file")
	flag.Parse()

	versionString := fmt.Sprintf("pufferd %s (%s %s)", VERSION, BUILDDATE, GITHASH)

	if pid != 0 {
		logging.Info("Shutting down")
		shutdown.Command(pid)
	}

	if _, err := os.Stat(configPath); os.IsNotExist(err) {
		if _, err := os.Stat("/etc/pufferd/config.json"); err == nil {
			logging.Info("No config passed, defaulting to /etc/pufferd/config.json")
			configPath = "/etc/pufferd/config.json"
		} else {
			logging.Error("Cannot find a config file!")
			logging.Warn("pufferd could be unstable")
		}
	}

	if uninstall {
		fmt.Println("This option will UNINSTALL pufferd, are you sure? [no]")
		var response string
		fmt.Scanln(&response)
		if strings.ToLower(response) == "yes" || strings.ToLower(response) == "y" {
			if os.Geteuid() != 0 {
				logging.Error("To uninstall pufferd you need to have sudo or root privileges")
			} else {
				config.Load(configPath)
				uninstaller.StartProcess()
				logging.Info("pufferd is now uninstalled.")
			}
		} else {
			logging.Info("Uninstall process aborted")
			logging.Info("Exiting")
		}
		return
	}

	if version {
		os.Stdout.WriteString(versionString + "\r\n")
	}

	if license {
		os.Stdout.WriteString(data.LICENSE + "\r\n")
	}

	if regenerate {
		config.Load(configPath)
		programs.Initialize()

		if _, err := os.Stat(programs.TemplateFolder); os.IsNotExist(err) {
			logging.Info("No template directory found, creating")
			err = os.MkdirAll(programs.TemplateFolder, 0755)
			if err != nil {
				logging.Error("Error creating template folder", err)
			}
		}
		// Overwrite existing templates
		templates.CopyTemplates()
		logging.Info("Templates regenerated")
	}

	if migrate {
		config.Load(configPath)
		migration.MigrateFromScales()
	}

	if installService {
		install.InstallService()
	}

	if license || version || regenerate || migrate || pid != 0 {
		return
	}

	config.Load(configPath)

	logging.SetLevelByString(loggingLevel)
	logging.Init()
	gin.SetMode(gin.ReleaseMode)

	logging.Info(versionString)
	logging.Info("Logging set to " + loggingLevel)

	if runInstaller {
		install.Install(configPath, authRoot, authToken)
	}

	if runInstaller || installService {
		return
	}

	programs.Initialize()

	if _, err := os.Stat(programs.TemplateFolder); os.IsNotExist(err) {
		logging.Info("No template directory found, creating")
		err = os.MkdirAll(programs.TemplateFolder, 0755)
		if err != nil {
			logging.Error("Error creating template folder", err)
		}

	}
	if files, _ := ioutil.ReadDir(programs.TemplateFolder); len(files) == 0 {
		logging.Info("Templates being copied to " + programs.TemplateFolder)
		templates.CopyTemplates()
	}

	if _, err := os.Stat(programs.ServerFolder); os.IsNotExist(err) {
		logging.Info("No server directory found, creating")
		os.MkdirAll(programs.ServerFolder, 0755)
	}

	programs.LoadFromFolder()

	for _, element := range programs.GetAll() {
		if element.IsEnabled() {
			logging.Info("Starting server " + element.Id())
			element.Start()
			err := programs.Save(element.Id())
			if err != nil {
				logging.Error("Error saving server file", err)
			}
		}
	}

	r := routing.ConfigureWeb()

	var useHttps bool
	useHttps = false

	dataFolder := config.GetOrDefault("datafolder", "data")
	httpsPem := filepath.Join(dataFolder, "https.pem")
	httpsKey := filepath.Join(dataFolder, "https.key")

	if _, err := os.Stat(httpsPem); os.IsNotExist(err) {
		logging.Warn("No HTTPS.PEM found in data folder, will use http instead")
	} else if _, err := os.Stat(httpsKey); os.IsNotExist(err) {
		logging.Warn("No HTTPS.KEY found in data folder, will use http instead")
	} else {
		useHttps = true
	}

	sftp.Run()

	//check if there's an update
	if config.GetOrDefault("update-check", "true") == "true" {
		go func() {
			url := "https://dl.pufferpanel.com/pufferd/" + MAJORVERSION + "/version.txt"
			logging.Debug("Checking for updates using " + url)
			resp, err := http.Get(url)
			if err != nil {
				return
			}
			defer resp.Body.Close()
			onlineVersion, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				return
			}
			if string(onlineVersion) != GITHASH {
				logging.Infof("DL server reports a different hash than this version, an update may be available")
				logging.Infof("Installed: %s", GITHASH)
				logging.Infof("Online: %s", onlineVersion)
			}
		}()
	}

	web := config.GetOrDefault("web", config.GetOrDefault("webhost", "0.0.0.0")+":"+config.GetOrDefault("webport", "5656"))

	shutdown.CreateHook()

	logging.Infof("Starting web access on %s", web)
	var err error
	if useHttps {
		err = manners.ListenAndServeTLS(web, httpsPem, httpsKey, r)
	} else {
		err = manners.ListenAndServe(web, r)
	}
	if err != nil {
		logging.Error("Error starting web service", err)
	}
	shutdown.Shutdown()
}
<commit_msg>Remove unused import<commit_after>/*
 Copyright 2016 Padduck, LLC

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 	http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package main

import (
	"flag"
	"io/ioutil"
	"os"
	"path/filepath"

	"fmt"

	"net/http"
	"strings"

	"github.com/braintree/manners"
	"github.com/gin-gonic/gin"
	"github.com/pufferpanel/pufferd/config"
	"github.com/pufferpanel/pufferd/data"
	"github.com/pufferpanel/pufferd/data/templates"
	"github.com/pufferpanel/pufferd/install"
	"github.com/pufferpanel/pufferd/logging"
	"github.com/pufferpanel/pufferd/migration"
	"github.com/pufferpanel/pufferd/programs"
	"github.com/pufferpanel/pufferd/routing"
	"github.com/pufferpanel/pufferd/sftp"
	"github.com/pufferpanel/pufferd/shutdown"
	"github.com/pufferpanel/pufferd/uninstaller"
)

var (
	VERSION      = "nightly"
	MAJORVERSION = "nightly"
	BUILDDATE    = "unknown"
	GITHASH      = "unknown"
)

func main() {
	var loggingLevel string
	var authRoot string
	var authToken string
	var runInstaller bool
	var version bool
	var license bool
	var regenerate bool
	var migrate bool
	var uninstall bool
	var configPath string
	var pid int
	var installService bool
	flag.StringVar(&loggingLevel, "logging", "INFO", "Lowest logging level to display")
	flag.StringVar(&authRoot, "auth", "", "Base URL to the authorization server")
	flag.StringVar(&authToken, "token", "", "Authorization token")
	flag.BoolVar(&runInstaller, "install", false, "If installing instead of running")
	flag.BoolVar(&version, "version", false, "Get the version")
	flag.BoolVar(&license, "license", false, "View license")
	flag.BoolVar(&regenerate, "regenerate", false, "Regenerate pufferd templates")
	flag.BoolVar(&migrate, "migrate", false, "Migrate Scales data to pufferd")
	flag.BoolVar(&uninstall, "uninstall", false, "Uninstall pufferd")
	flag.StringVar(&configPath, "config", "config.json", "Path to pufferd config.json")
	flag.IntVar(&pid, "shutdown", 0, "PID to shut down")
	flag.BoolVar(&installService, "installService", false, "Installs the pufferd service file")
	flag.Parse()

	versionString := fmt.Sprintf("pufferd %s (%s %s)", VERSION, BUILDDATE, GITHASH)

	if pid != 0 {
		logging.Info("Shutting down")
		shutdown.Command(pid)
	}

	if _, err := os.Stat(configPath); os.IsNotExist(err) {
		if _, err := os.Stat("/etc/pufferd/config.json"); err == nil {
			logging.Info("No config passed, defaulting to /etc/pufferd/config.json")
			configPath = "/etc/pufferd/config.json"
		} else {
			logging.Error("Cannot find a config file!")
			logging.Warn("pufferd could be unstable")
		}
	}

	if uninstall {
		fmt.Println("This option will UNINSTALL pufferd, are you sure? [no]")
		var response string
		fmt.Scanln(&response)
		if strings.ToLower(response) == "yes" || strings.ToLower(response) == "y" {
			if os.Geteuid() != 0 {
				logging.Error("To uninstall pufferd you need to have sudo or root privileges")
			} else {
				config.Load(configPath)
				uninstaller.StartProcess()
				logging.Info("pufferd is now uninstalled.")
			}
		} else {
			logging.Info("Uninstall process aborted")
			logging.Info("Exiting")
		}
		return
	}

	if version {
		os.Stdout.WriteString(versionString + "\r\n")
	}

	if license {
		os.Stdout.WriteString(data.LICENSE + "\r\n")
	}

	if regenerate {
		config.Load(configPath)
		programs.Initialize()

		if _, err := os.Stat(programs.TemplateFolder); os.IsNotExist(err) {
			logging.Info("No template directory found, creating")
			err = os.MkdirAll(programs.TemplateFolder, 0755)
			if err != nil {
				logging.Error("Error creating template folder", err)
			}
		}
		// Overwrite existing templates
		templates.CopyTemplates()
		logging.Info("Templates regenerated")
	}

	if migrate {
		config.Load(configPath)
		migration.MigrateFromScales()
	}

	if installService {
		install.InstallService()
	}

	if license || version || regenerate || migrate || pid != 0 {
		return
	}

	config.Load(configPath)

	logging.SetLevelByString(loggingLevel)
	logging.Init()
	gin.SetMode(gin.ReleaseMode)

	logging.Info(versionString)
	logging.Info("Logging set to " + loggingLevel)

	if runInstaller {
		install.Install(configPath, authRoot, authToken)
	}

	if runInstaller || installService {
		return
	}

	programs.Initialize()

	if _, err := os.Stat(programs.TemplateFolder); os.IsNotExist(err) {
		logging.Info("No template directory found, creating")
		err = os.MkdirAll(programs.TemplateFolder, 0755)
		if err != nil {
			logging.Error("Error creating template folder", err)
		}

	}
	if files, _ := ioutil.ReadDir(programs.TemplateFolder); len(files) == 0 {
		logging.Info("Templates being copied to " + programs.TemplateFolder)
		templates.CopyTemplates()
	}

	if _, err := os.Stat(programs.ServerFolder); os.IsNotExist(err) {
		logging.Info("No server directory found, creating")
		os.MkdirAll(programs.ServerFolder, 0755)
	}

	programs.LoadFromFolder()

	for _, element := range programs.GetAll() {
		if element.IsEnabled() {
			logging.Info("Starting server " + element.Id())
			element.Start()
			err := programs.Save(element.Id())
			if err != nil {
				logging.Error("Error saving server file", err)
			}
		}
	}

	r := routing.ConfigureWeb()

	var useHttps bool
	useHttps = false

	dataFolder := config.GetOrDefault("datafolder", "data")
	httpsPem := filepath.Join(dataFolder, "https.pem")
	httpsKey := filepath.Join(dataFolder, "https.key")

	if _, err := os.Stat(httpsPem); os.IsNotExist(err) {
		logging.Warn("No HTTPS.PEM found in data folder, will use http instead")
	} else if _, err := os.Stat(httpsKey); os.IsNotExist(err) {
		logging.Warn("No HTTPS.KEY found in data folder, will use http instead")
	} else {
		useHttps = true
	}

	sftp.Run()

	//check if there's an update
	if config.GetOrDefault("update-check", "true") == "true" {
		go func() {
			url := "https://dl.pufferpanel.com/pufferd/" + MAJORVERSION + "/version.txt"
			logging.Debug("Checking for updates using " + url)
			resp, err := http.Get(url)
			if err != nil {
				return
			}
			defer resp.Body.Close()
			onlineVersion, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				return
			}
			if string(onlineVersion) != GITHASH {
				logging.Infof("DL server reports a different hash than this version, an update may be available")
				logging.Infof("Installed: %s", GITHASH)
				logging.Infof("Online: %s", onlineVersion)
			}
		}()
	}

	web := config.GetOrDefault("web", config.GetOrDefault("webhost", "0.0.0.0")+":"+config.GetOrDefault("webport", "5656"))

	shutdown.CreateHook()

	logging.Infof("Starting web access on %s", web)
	var err error
	if useHttps {
		err = manners.ListenAndServeTLS(web, httpsPem, httpsKey, r)
	} else {
		err = manners.ListenAndServe(web, r)
	}
	if err != nil {
		logging.Error("Error starting web service", err)
	}
	shutdown.Shutdown()
}
<|endoftext|>


<commit_before>package main

import (
  "fmt"
  "time"
  )

func main() {
  fmt.Println("Hello World!")
  fmt.Println("This is my Go Playground")
  fmt.Println("The time is", time.Now())
}
<commit_msg>added random number line<commit_after>package main

import (
  "fmt"
  "time"
  )

func main() {
  fmt.Println("Hello World!")
  fmt.Println("This is my Go Playground!")
  fmt.Println("The time is: ", time.Now())
  fmt.Println("MA random number is: ", rand.Intn(10))
}
<|endoftext|>