<commit_before>//
// Copyright 2016 Rackspace
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS-IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package poller

import (
	"context"
	"math/rand"
	"time"

	"errors"
	"fmt"
	log "github.com/Sirupsen/logrus"
	set "github.com/deckarep/golang-set"
	"github.com/racker/rackspace-monitoring-poller/check"
)

const (
	checkPreparationBufferSize = 10
)

// EleScheduler implements Scheduler interface.
// See Scheduler for more information.
type EleScheduler struct {
	ctx    context.Context
	cancel context.CancelFunc

	zoneID       string
	checks       map[string]check.Check
	preparations chan ChecksPrepared

	stream ConnectionStream

	scheduler CheckScheduler
	executor  CheckExecutor
}

// NewScheduler instantiates a new Scheduler with standard scheduling and executor behaviors.
// It sets up checks, context, and passed in zoneid
func NewScheduler(zoneID string, stream ConnectionStream) Scheduler {
	return NewCustomScheduler(zoneID, stream, nil, nil)
}

// NewCustomScheduler instantiates a new Scheduler using NewScheduler but allows for more customization.
// Nil can be passed to either checkScheduler and/or checkExecutor to enable the default behavior.
func NewCustomScheduler(zoneID string, stream ConnectionStream, checkScheduler CheckScheduler, checkExecutor CheckExecutor) Scheduler {
	s := &EleScheduler{
		checks:       make(map[string]check.Check),
		preparations: make(chan ChecksPrepared, checkPreparationBufferSize),
		stream:       stream,
		zoneID:       zoneID,
		scheduler:    checkScheduler,
		executor:     checkExecutor,
	}

	// by default we are our own scheduler/executor of checks
	if s.scheduler == nil {
		s.scheduler = s
	}
	if s.executor == nil {
		s.executor = s
	}

	s.ctx, s.cancel = context.WithCancel(context.Background())

	go s.runReconciler()

	return s

}

// GetZoneID retrieves zone id
func (s *EleScheduler) GetZoneID() string {
	return s.zoneID
}

// GetContext retrieves cancelable context
func (s *EleScheduler) GetContext() (ctx context.Context, cancel context.CancelFunc) {
	return s.ctx, s.cancel
}

// Close cancels the context and closes the connection
func (s *EleScheduler) Close() {
	s.cancel()
}

func (s *EleScheduler) GetScheduledChecks() []check.Check {
	checks := make([]check.Check, 0, len(s.checks))

	for _, entry := range s.checks {
		checks = append(checks, entry)
	}

	return checks
}

func (s *EleScheduler) ReconcileChecks(cp ChecksPrepared) {
	s.preparations <- cp
}

func (s *EleScheduler) runReconciler() {
	for {
		select {
		case cp := <-s.preparations:
			s.reconcile(cp)

		case <-s.ctx.Done():
			return
		}
	}
}

func (s *EleScheduler) ValidateChecks(cp ChecksPreparing) error {
	actionableChecks := cp.GetActionableChecks()
	for _, ac := range actionableChecks {

		switch ac.Action {
		case ActionTypeStart:
			_, exists := s.checks[ac.Id]
			if exists {
				return errors.New(fmt.Sprintf("Reconciling was told to start a check, but it already existed: %v", ac.Id))
			}
		case ActionTypeRestart:
			_, exists := s.checks[ac.Id]
			if !exists {
				return errors.New(fmt.Sprintf("Reconciling was told to restart a check, but it does not exist: %v", ac.Id))
			}
		case ActionTypeContinue:
			_, exists := s.checks[ac.Id]
			if !exists {
				return errors.New(fmt.Sprintf("Reconciling was told to continue a check, but it does not exist: %v", ac.Id))
			}
		}
	}

	return nil
}

func (s *EleScheduler) reconcile(cp ChecksPrepared) {
	log.WithField("cp", cp).Debug("Reconciling prepared checks")

	// remainder will be used at the end to find ones were implicitly removed and need to be canceled out
	remainder := set.NewThreadUnsafeSet()
	for id, _ := range s.checks {
		remainder.Add(id)
	}

	actionableChecks := cp.GetActionableChecks()
	for _, ac := range actionableChecks {
		remainder.Remove(ac.Id)

		switch ac.Action {
		case ActionTypeStart:
			existingCheck, exists := s.checks[ac.Id]
			if exists {
				log.WithField("checkId", ac.Id).Warn("Reconciling was told to start a check, but it already existed.")
				existingCheck.Cancel()
			}
			err := s.initiateCheck(*ac)
			if err != nil {
				log.WithField("details", string(*ac.RawDetails)).Warn("Unable to initiate check")
			}

		case ActionTypeRestart:
			existingCheck, exists := s.checks[ac.Id]
			if exists {
				existingCheck.Cancel()
			} else {
				log.WithField("checkId", ac.Id).Warn("Reconciling was told to restart a check, but it does not exist.")
			}
			err := s.initiateCheck(*ac)
			if err != nil {
				log.WithField("details", string(*ac.RawDetails)).Warn("Unable to initiate check")
			}

		case ActionTypeContinue:
			_, exists := s.checks[ac.Id]
			if !exists {
				log.WithField("checkId", ac.Id).Warn("Reconciling was told to continue a check, but it does not exist.")
			}
		}
	}

	for checkIdToRemove := range remainder.Iter() {
		log.WithField("checkId", checkIdToRemove).Debug("Removing check implicitly due to absence in check preparation")

		checkIdToRemoveStr := checkIdToRemove.(string)
		checkToRemove := s.checks[checkIdToRemoveStr]
		delete(s.checks, checkIdToRemoveStr)
		checkToRemove.Cancel()
	}

	if log.GetLevel() >= log.DebugLevel {
		log.WithFields(log.Fields{
			"prefix": "scheduler",
			"checks": s.checks,
		}).Debug("Reconciled and scheduled")
	}
}

func (s *EleScheduler) initiateCheck(ac ActionableCheck) error {
	newCheck, err := check.NewCheckParsed(s.ctx, ac.CheckIn)
	if err != nil {
		return err
	}
	s.checks[newCheck.GetID()] = newCheck
	s.scheduler.Schedule(newCheck)

	return nil
}

// Schedule is the default implementation of CheckScheduler that kicks off a go routine to run a check's timer.
func (s *EleScheduler) Schedule(ch check.Check) {
	if !ch.IsDisabled() {
		go s.runCheckTimerLoop(ch)
	}
}

func (s *EleScheduler) runCheckTimerLoop(ch check.Check) {
	// Spread the checks out over 30 seconds
	jitter := rand.Intn(CheckSpreadInMilliseconds) + 1

	log.WithFields(log.Fields{
		"checkId":    ch.GetID(),
		"jitterMs":   jitter,
		"waitPeriod": ch.GetWaitPeriod(),
	}).Info("Starting check")

	time.Sleep(time.Duration(jitter) * time.Millisecond)
	for {
		select {
		case <-time.After(ch.GetWaitPeriod()):
			s.executor.Execute(ch)

		case <-ch.Done(): // session cancellation is propagated since check context is child of session context
			log.WithField("check", ch.GetID()).Info("Check or session has been cancelled")
			return
		}
	}
}

// Execute perform the default CheckExecutor behavior by running the check and sending its results via SendMetrics.
func (s *EleScheduler) Execute(ch check.Check) {
	crs, err := ch.Run()
	if err != nil {
		log.Errorf("Error running check: %v", err)
	} else {
		s.SendMetrics(crs)
	}

}

// SendMetrics sends metrics passed in crs parameter via the stream
func (s *EleScheduler) SendMetrics(crs *check.ResultSet) {
	s.stream.SendMetrics(crs)
}
<commit_msg>Add periodic info logging of scheduled checks (#118)<commit_after>//
// Copyright 2016 Rackspace
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS-IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package poller

import (
	"context"
	"math/rand"
	"time"

	"errors"
	"fmt"
	log "github.com/Sirupsen/logrus"
	set "github.com/deckarep/golang-set"
	"github.com/racker/rackspace-monitoring-poller/check"
)

const (
	checkPreparationBufferSize = 10
	checkLoggerDuration        = 5 * time.Minute
)

// EleScheduler implements Scheduler interface.
// See Scheduler for more information.
type EleScheduler struct {
	ctx    context.Context
	cancel context.CancelFunc

	zoneID       string
	checks       map[string]check.Check
	preparations chan ChecksPrepared

	stream ConnectionStream

	scheduler CheckScheduler
	executor  CheckExecutor
}

// NewScheduler instantiates a new Scheduler with standard scheduling and executor behaviors.
// It sets up checks, context, and passed in zoneid
func NewScheduler(zoneID string, stream ConnectionStream) Scheduler {
	return NewCustomScheduler(zoneID, stream, nil, nil)
}

// NewCustomScheduler instantiates a new Scheduler using NewScheduler but allows for more customization.
// Nil can be passed to either checkScheduler and/or checkExecutor to enable the default behavior.
func NewCustomScheduler(zoneID string, stream ConnectionStream, checkScheduler CheckScheduler, checkExecutor CheckExecutor) Scheduler {
	s := &EleScheduler{
		checks:       make(map[string]check.Check),
		preparations: make(chan ChecksPrepared, checkPreparationBufferSize),
		stream:       stream,
		zoneID:       zoneID,
		scheduler:    checkScheduler,
		executor:     checkExecutor,
	}

	// by default we are our own scheduler/executor of checks
	if s.scheduler == nil {
		s.scheduler = s
	}
	if s.executor == nil {
		s.executor = s
	}

	s.ctx, s.cancel = context.WithCancel(context.Background())

	go s.runReconciler()

	return s

}

// GetZoneID retrieves zone id
func (s *EleScheduler) GetZoneID() string {
	return s.zoneID
}

// GetContext retrieves cancelable context
func (s *EleScheduler) GetContext() (ctx context.Context, cancel context.CancelFunc) {
	return s.ctx, s.cancel
}

// Close cancels the context and closes the connection
func (s *EleScheduler) Close() {
	s.cancel()
}

func (s *EleScheduler) GetScheduledChecks() []check.Check {
	checks := make([]check.Check, 0, len(s.checks))

	for _, entry := range s.checks {
		checks = append(checks, entry)
	}

	return checks
}

func (s *EleScheduler) ReconcileChecks(cp ChecksPrepared) {
	s.preparations <- cp
}

func (s *EleScheduler) runReconciler() {

	logTicker := time.NewTicker(checkLoggerDuration)
	defer logTicker.Stop()

	for {
		select {
		case cp := <-s.preparations:
			s.reconcile(cp)
			s.logScheduledChecks()

		case <-s.ctx.Done():
			return

		case <-logTicker.C:
			s.logScheduledChecks()
		}
	}
}

func (s *EleScheduler) logScheduledChecks() {
	if len(s.checks) > 0 {
		typeCounts := log.Fields{}
		for _, ch := range s.checks {
			count, _ := typeCounts[ch.GetCheckType()].(int)
			count++
			typeCounts[ch.GetCheckType()] = count
		}
		log.WithFields(typeCounts).Info("Checks scheduled to run")
	} else {
		log.Info("No checks are scheduled to run")
	}
}

func (s *EleScheduler) ValidateChecks(cp ChecksPreparing) error {
	actionableChecks := cp.GetActionableChecks()
	for _, ac := range actionableChecks {

		switch ac.Action {
		case ActionTypeStart:
			_, exists := s.checks[ac.Id]
			if exists {
				return errors.New(fmt.Sprintf("Reconciling was told to start a check, but it already existed: %v", ac.Id))
			}
		case ActionTypeRestart:
			_, exists := s.checks[ac.Id]
			if !exists {
				return errors.New(fmt.Sprintf("Reconciling was told to restart a check, but it does not exist: %v", ac.Id))
			}
		case ActionTypeContinue:
			_, exists := s.checks[ac.Id]
			if !exists {
				return errors.New(fmt.Sprintf("Reconciling was told to continue a check, but it does not exist: %v", ac.Id))
			}
		}
	}

	return nil
}

func (s *EleScheduler) reconcile(cp ChecksPrepared) {
	log.WithField("cp", cp).Debug("Reconciling prepared checks")

	// remainder will be used at the end to find ones were implicitly removed and need to be canceled out
	remainder := set.NewThreadUnsafeSet()
	for id, _ := range s.checks {
		remainder.Add(id)
	}

	actionableChecks := cp.GetActionableChecks()
	for _, ac := range actionableChecks {
		remainder.Remove(ac.Id)

		switch ac.Action {
		case ActionTypeStart:
			existingCheck, exists := s.checks[ac.Id]
			if exists {
				log.WithField("checkId", ac.Id).Warn("Reconciling was told to start a check, but it already existed.")
				existingCheck.Cancel()
			}
			err := s.initiateCheck(*ac)
			if err != nil {
				log.WithField("details", string(*ac.RawDetails)).Warn("Unable to initiate check")
			}

		case ActionTypeRestart:
			existingCheck, exists := s.checks[ac.Id]
			if exists {
				existingCheck.Cancel()
			} else {
				log.WithField("checkId", ac.Id).Warn("Reconciling was told to restart a check, but it does not exist.")
			}
			err := s.initiateCheck(*ac)
			if err != nil {
				log.WithField("details", string(*ac.RawDetails)).Warn("Unable to initiate check")
			}

		case ActionTypeContinue:
			_, exists := s.checks[ac.Id]
			if !exists {
				log.WithField("checkId", ac.Id).Warn("Reconciling was told to continue a check, but it does not exist.")
			}
		}
	}

	for checkIdToRemove := range remainder.Iter() {
		log.WithField("checkId", checkIdToRemove).Debug("Removing check implicitly due to absence in check preparation")

		checkIdToRemoveStr := checkIdToRemove.(string)
		checkToRemove := s.checks[checkIdToRemoveStr]
		delete(s.checks, checkIdToRemoveStr)
		checkToRemove.Cancel()
	}

	if log.GetLevel() >= log.DebugLevel {
		log.WithFields(log.Fields{
			"prefix": "scheduler",
			"checks": s.checks,
		}).Debug("Reconciled and scheduled")
	}
}

func (s *EleScheduler) initiateCheck(ac ActionableCheck) error {
	newCheck, err := check.NewCheckParsed(s.ctx, ac.CheckIn)
	if err != nil {
		return err
	}
	s.checks[newCheck.GetID()] = newCheck
	s.scheduler.Schedule(newCheck)

	return nil
}

// Schedule is the default implementation of CheckScheduler that kicks off a go routine to run a check's timer.
func (s *EleScheduler) Schedule(ch check.Check) {
	if !ch.IsDisabled() {
		go s.runCheckTimerLoop(ch)
	}
}

func (s *EleScheduler) runCheckTimerLoop(ch check.Check) {
	// Spread the checks out over 30 seconds
	jitter := rand.Intn(CheckSpreadInMilliseconds) + 1

	log.WithFields(log.Fields{
		"checkId":    ch.GetID(),
		"jitterMs":   jitter,
		"waitPeriod": ch.GetWaitPeriod(),
	}).Debug("Starting check")

	time.Sleep(time.Duration(jitter) * time.Millisecond)
	for {
		select {
		case <-time.After(ch.GetWaitPeriod()):
			s.executor.Execute(ch)

		case <-ch.Done(): // session cancellation is propagated since check context is child of session context
			log.WithField("check", ch.GetID()).Info("Check or session has been cancelled")
			return
		}
	}
}

// Execute perform the default CheckExecutor behavior by running the check and sending its results via SendMetrics.
func (s *EleScheduler) Execute(ch check.Check) {
	log.WithFields(log.Fields{
		"id":     ch.GetID(),
		"type":   ch.GetCheckType(),
		"period": ch.GetPeriod(),
	}).Debug("Running check")

	crs, err := ch.Run()
	if err != nil {
		log.Errorf("Error running check: %v", err)
	} else {
		s.SendMetrics(crs)
	}

}

// SendMetrics sends metrics passed in crs parameter via the stream
func (s *EleScheduler) SendMetrics(crs *check.ResultSet) {
	s.stream.SendMetrics(crs)
}
<|endoftext|>


<commit_before>package pool_test

import (
	"context"
	"testing"
	"time"

	"github.com/jackc/pgconn"
	"github.com/jackc/pgx/v4"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// Conn.Release is an asynchronous process that returns immediately. There is no signal when the actual work is
// completed. To test something that relies on the actual work for Conn.Release being completed we must simply wait.
// This function wraps the sleep so there is more meaning for the callers.
func waitForReleaseToComplete() {
	time.Sleep(5 * time.Millisecond)
}

type execer interface {
	Exec(ctx context.Context, sql string, arguments ...interface{}) (pgconn.CommandTag, error)
}

func testExec(t *testing.T, db execer) {
	results, err := db.Exec(context.Background(), "set time zone 'America/Chicago'")
	require.NoError(t, err)
	assert.EqualValues(t, "SET", results)
}

type queryer interface {
	Query(ctx context.Context, sql string, args ...interface{}) (pgx.Rows, error)
}

func testQuery(t *testing.T, db queryer) {
	var sum, rowCount int32

	rows, err := db.Query(context.Background(), "select generate_series(1,$1)", 10)
	require.NoError(t, err)

	for rows.Next() {
		var n int32
		rows.Scan(&n)
		sum += n
		rowCount++
	}

	assert.NoError(t, rows.Err())
	assert.Equal(t, int32(10), rowCount)
	assert.Equal(t, int32(55), sum)
}

type queryRower interface {
	QueryRow(ctx context.Context, sql string, args ...interface{}) pgx.Row
}

func testQueryRow(t *testing.T, db queryRower) {
	var what, who string
	err := db.QueryRow(context.Background(), "select 'hello', $1", "world").Scan(&what, &who)
	assert.NoError(t, err)
	assert.Equal(t, "hello", what)
	assert.Equal(t, "world", who)
}

type sendBatcher interface {
	SendBatch(context.Context, *pgx.Batch) pgx.BatchResults
}

func testSendBatch(t *testing.T, db sendBatcher) {
	batch := &pgx.Batch{}
	batch.Queue("select 1", nil, nil, nil)
	batch.Queue("select 2", nil, nil, nil)

	br := db.SendBatch(context.Background(), batch)

	var err error
	var n int32
	err = br.QueryRowResults().Scan(&n)
	assert.NoError(t, err)
	assert.EqualValues(t, 1, n)

	err = br.QueryRowResults().Scan(&n)
	assert.NoError(t, err)
	assert.EqualValues(t, 2, n)

	err = br.Close()
	assert.NoError(t, err)
}

type copyFromer interface {
	CopyFrom(context.Context, pgx.Identifier, []string, pgx.CopyFromSource) (int64, error)
}

func testCopyFrom(t *testing.T, db interface {
	execer
	queryer
	copyFromer
}) {
	_, err := db.Exec(context.Background(), `create temporary table foo(a int2, b int4, c int8, d varchar, e text, f date, g timestamptz)`)
	require.NoError(t, err)

	tzedTime := time.Date(2010, 2, 3, 4, 5, 6, 0, time.Local)

	inputRows := [][]interface{}{
		{int16(0), int32(1), int64(2), "abc", "efg", time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC), tzedTime},
		{nil, nil, nil, nil, nil, nil, nil},
	}

	copyCount, err := db.CopyFrom(context.Background(), pgx.Identifier{"foo"}, []string{"a", "b", "c", "d", "e", "f", "g"}, pgx.CopyFromRows(inputRows))
	assert.NoError(t, err)
	assert.EqualValues(t, len(inputRows), copyCount)

	rows, err := db.Query(context.Background(), "select * from foo")
	assert.NoError(t, err)

	var outputRows [][]interface{}
	for rows.Next() {
		row, err := rows.Values()
		if err != nil {
			t.Errorf("Unexpected error for rows.Values(): %v", err)
		}
		outputRows = append(outputRows, row)
	}

	assert.NoError(t, rows.Err())
	assert.Equal(t, inputRows, outputRows)
}
<commit_msg>Provide type for extended protocol query<commit_after>package pool_test

import (
	"context"
	"testing"
	"time"

	"github.com/jackc/pgconn"
	"github.com/jackc/pgx/v4"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// Conn.Release is an asynchronous process that returns immediately. There is no signal when the actual work is
// completed. To test something that relies on the actual work for Conn.Release being completed we must simply wait.
// This function wraps the sleep so there is more meaning for the callers.
func waitForReleaseToComplete() {
	time.Sleep(5 * time.Millisecond)
}

type execer interface {
	Exec(ctx context.Context, sql string, arguments ...interface{}) (pgconn.CommandTag, error)
}

func testExec(t *testing.T, db execer) {
	results, err := db.Exec(context.Background(), "set time zone 'America/Chicago'")
	require.NoError(t, err)
	assert.EqualValues(t, "SET", results)
}

type queryer interface {
	Query(ctx context.Context, sql string, args ...interface{}) (pgx.Rows, error)
}

func testQuery(t *testing.T, db queryer) {
	var sum, rowCount int32

	rows, err := db.Query(context.Background(), "select generate_series(1,$1)", 10)
	require.NoError(t, err)

	for rows.Next() {
		var n int32
		rows.Scan(&n)
		sum += n
		rowCount++
	}

	assert.NoError(t, rows.Err())
	assert.Equal(t, int32(10), rowCount)
	assert.Equal(t, int32(55), sum)
}

type queryRower interface {
	QueryRow(ctx context.Context, sql string, args ...interface{}) pgx.Row
}

func testQueryRow(t *testing.T, db queryRower) {
	var what, who string
	err := db.QueryRow(context.Background(), "select 'hello', $1::text", "world").Scan(&what, &who)
	assert.NoError(t, err)
	assert.Equal(t, "hello", what)
	assert.Equal(t, "world", who)
}

type sendBatcher interface {
	SendBatch(context.Context, *pgx.Batch) pgx.BatchResults
}

func testSendBatch(t *testing.T, db sendBatcher) {
	batch := &pgx.Batch{}
	batch.Queue("select 1", nil, nil, nil)
	batch.Queue("select 2", nil, nil, nil)

	br := db.SendBatch(context.Background(), batch)

	var err error
	var n int32
	err = br.QueryRowResults().Scan(&n)
	assert.NoError(t, err)
	assert.EqualValues(t, 1, n)

	err = br.QueryRowResults().Scan(&n)
	assert.NoError(t, err)
	assert.EqualValues(t, 2, n)

	err = br.Close()
	assert.NoError(t, err)
}

type copyFromer interface {
	CopyFrom(context.Context, pgx.Identifier, []string, pgx.CopyFromSource) (int64, error)
}

func testCopyFrom(t *testing.T, db interface {
	execer
	queryer
	copyFromer
}) {
	_, err := db.Exec(context.Background(), `create temporary table foo(a int2, b int4, c int8, d varchar, e text, f date, g timestamptz)`)
	require.NoError(t, err)

	tzedTime := time.Date(2010, 2, 3, 4, 5, 6, 0, time.Local)

	inputRows := [][]interface{}{
		{int16(0), int32(1), int64(2), "abc", "efg", time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC), tzedTime},
		{nil, nil, nil, nil, nil, nil, nil},
	}

	copyCount, err := db.CopyFrom(context.Background(), pgx.Identifier{"foo"}, []string{"a", "b", "c", "d", "e", "f", "g"}, pgx.CopyFromRows(inputRows))
	assert.NoError(t, err)
	assert.EqualValues(t, len(inputRows), copyCount)

	rows, err := db.Query(context.Background(), "select * from foo")
	assert.NoError(t, err)

	var outputRows [][]interface{}
	for rows.Next() {
		row, err := rows.Values()
		if err != nil {
			t.Errorf("Unexpected error for rows.Values(): %v", err)
		}
		outputRows = append(outputRows, row)
	}

	assert.NoError(t, rows.Err())
	assert.Equal(t, inputRows, outputRows)
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package etcdmain

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"path"
	"reflect"
	"strings"
	"time"

	"github.com/coreos/etcd/discovery"
	"github.com/coreos/etcd/etcdserver"
	"github.com/coreos/etcd/etcdserver/etcdhttp"
	"github.com/coreos/etcd/pkg/cors"
	"github.com/coreos/etcd/pkg/osutil"
	"github.com/coreos/etcd/pkg/transport"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/proxy"
	"github.com/coreos/etcd/rafthttp"
)

const (
	// the owner can make/remove files inside the directory
	privateDirMode = 0700
)

func Main() {
	cfg := NewConfig()
	err := cfg.Parse(os.Args[1:])
	if err != nil {
		log.Printf("etcd: error verifying flags, %v", err)
		os.Exit(2)
	}

	var stopped <-chan struct{}

	shouldProxy := cfg.isProxy()
	if !shouldProxy {
		stopped, err = startEtcd(cfg)
		if err == discovery.ErrFullCluster && cfg.shouldFallbackToProxy() {
			log.Printf("etcd: discovery cluster full, falling back to %s", fallbackFlagProxy)
			shouldProxy = true
		}
	}
	if shouldProxy {
		err = startProxy(cfg)
	}
	if err != nil {
		switch err {
		case discovery.ErrDuplicateID:
			log.Fatalf("etcd: member %s has previously registered with discovery service (%s), but the data-dir (%s) on disk cannot be found.",
				cfg.name, cfg.durl, cfg.dir)
		default:
			log.Fatalf("etcd: %v", err)
		}
	}

	osutil.HandleInterrupts()

	<-stopped
	osutil.Exit(0)
}

// startEtcd launches the etcd server and HTTP handlers for client/server communication.
func startEtcd(cfg *config) (<-chan struct{}, error) {
	cls, err := setupCluster(cfg)
	if err != nil {
		return nil, fmt.Errorf("error setting up initial cluster: %v", err)
	}

	if cfg.dir == "" {
		cfg.dir = fmt.Sprintf("%v.etcd", cfg.name)
		log.Printf("no data-dir provided, using default data-dir ./%s", cfg.dir)
	}

	pt, err := transport.NewTimeoutTransport(cfg.peerTLSInfo, rafthttp.DialTimeout, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout)
	if err != nil {
		return nil, err
	}

	if !cfg.peerTLSInfo.Empty() {
		log.Printf("etcd: peerTLS: %s", cfg.peerTLSInfo)
	}
	plns := make([]net.Listener, 0)
	for _, u := range cfg.lpurls {
		var l net.Listener
		l, err = transport.NewTimeoutListener(u.Host, u.Scheme, cfg.peerTLSInfo, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout)
		if err != nil {
			return nil, err
		}

		urlStr := u.String()
		log.Print("etcd: listening for peers on ", urlStr)
		defer func() {
			if err != nil {
				l.Close()
				log.Print("etcd: stopping listening for peers on ", urlStr)
			}
		}()
		plns = append(plns, l)
	}

	if !cfg.clientTLSInfo.Empty() {
		log.Printf("etcd: clientTLS: %s", cfg.clientTLSInfo)
	}
	clns := make([]net.Listener, 0)
	for _, u := range cfg.lcurls {
		var l net.Listener
		l, err = transport.NewKeepAliveListener(u.Host, u.Scheme, cfg.clientTLSInfo)
		if err != nil {
			return nil, err
		}

		urlStr := u.String()
		log.Print("etcd: listening for client requests on ", urlStr)
		defer func() {
			if err != nil {
				l.Close()
				log.Print("etcd: stopping listening for client requests on ", urlStr)
			}
		}()
		clns = append(clns, l)
	}

	srvcfg := &etcdserver.ServerConfig{
		Name:            cfg.name,
		ClientURLs:      cfg.acurls,
		PeerURLs:        cfg.apurls,
		DataDir:         cfg.dir,
		SnapCount:       cfg.snapCount,
		MaxSnapFiles:    cfg.maxSnapFiles,
		MaxWALFiles:     cfg.maxWalFiles,
		Cluster:         cls,
		DiscoveryURL:    cfg.durl,
		DiscoveryProxy:  cfg.dproxy,
		NewCluster:      cfg.isNewCluster(),
		ForceNewCluster: cfg.forceNewCluster,
		Transport:       pt,
		TickMs:          cfg.TickMs,
		ElectionTicks:   cfg.electionTicks(),
	}
	var s *etcdserver.EtcdServer
	s, err = etcdserver.NewServer(srvcfg)
	if err != nil {
		return nil, err
	}
	s.Start()
	osutil.RegisterInterruptHandler(s.Stop)

	if cfg.corsInfo.String() != "" {
		log.Printf("etcd: cors = %s", cfg.corsInfo)
	}
	ch := &cors.CORSHandler{
		Handler: etcdhttp.NewClientHandler(s),
		Info:    cfg.corsInfo,
	}
	ph := etcdhttp.NewPeerHandler(s.Cluster, etcdserver.RaftTimer(s), s.RaftHandler())
	// Start the peer server in a goroutine
	for _, l := range plns {
		go func(l net.Listener) {
			log.Fatal(serveHTTP(l, ph, 5*time.Minute))
		}(l)
	}
	// Start a client server goroutine for each listen address
	for _, l := range clns {
		go func(l net.Listener) {
			// read timeout does not work with http close notify
			// TODO: https://github.com/golang/go/issues/9524
			log.Fatal(serveHTTP(l, ch, 0))
		}(l)
	}
	return s.StopNotify(), nil
}

// startProxy launches an HTTP proxy for client communication which proxies to other etcd nodes.
func startProxy(cfg *config) error {
	cls, err := setupCluster(cfg)
	if err != nil {
		return fmt.Errorf("error setting up initial cluster: %v", err)
	}

	if cfg.durl != "" {
		s, err := discovery.GetCluster(cfg.durl, cfg.dproxy)
		if err != nil {
			return err
		}
		if cls, err = etcdserver.NewClusterFromString(cfg.durl, s); err != nil {
			return err
		}
	}

	pt, err := transport.NewTransport(cfg.clientTLSInfo)
	if err != nil {
		return err
	}

	tr, err := transport.NewTransport(cfg.peerTLSInfo)
	if err != nil {
		return err
	}

	if cfg.dir == "" {
		cfg.dir = fmt.Sprintf("%v.etcd", cfg.name)
		log.Printf("no proxy data-dir provided, using default proxy data-dir ./%s", cfg.dir)
	}
	cfg.dir = path.Join(cfg.dir, "proxy")
	err = os.MkdirAll(cfg.dir, 0700)
	if err != nil {
		return err
	}

	var peerURLs []string
	clusterfile := path.Join(cfg.dir, "cluster")

	b, err := ioutil.ReadFile(clusterfile)
	switch {
	case err == nil:
		urls := struct{ PeerURLs []string }{}
		err := json.Unmarshal(b, &urls)
		if err != nil {
			return err
		}
		peerURLs = urls.PeerURLs
		log.Printf("proxy: using peer urls %v from cluster file ./%s", peerURLs, clusterfile)
	case os.IsNotExist(err):
		peerURLs = cls.PeerURLs()
		log.Printf("proxy: using peer urls %v ", peerURLs)
	default:
		return err
	}

	uf := func() []string {
		gcls, err := etcdserver.GetClusterFromRemotePeers(peerURLs, tr)
		// TODO: remove the 2nd check when we fix GetClusterFromPeers
		// GetClusterFromPeers should not return nil error with an invaild empty cluster
		if err != nil {
			log.Printf("proxy: %v", err)
			return []string{}
		}
		if len(gcls.Members()) == 0 {
			return cls.ClientURLs()
		}
		cls = gcls

		urls := struct{ PeerURLs []string }{cls.PeerURLs()}
		b, err := json.Marshal(urls)
		if err != nil {
			log.Printf("proxy: error on marshal peer urls %s", err)
			return cls.ClientURLs()
		}

		err = ioutil.WriteFile(clusterfile+".bak", b, 0600)
		if err != nil {
			log.Printf("proxy: error on writing urls %s", err)
			return cls.ClientURLs()
		}
		err = os.Rename(clusterfile+".bak", clusterfile)
		if err != nil {
			log.Printf("proxy: error on updating clusterfile %s", err)
			return cls.ClientURLs()
		}
		if !reflect.DeepEqual(cls.PeerURLs(), peerURLs) {
			log.Printf("proxy: updated peer urls in cluster file from %v to %v", peerURLs, cls.PeerURLs())
		}
		peerURLs = cls.PeerURLs()

		return cls.ClientURLs()
	}
	ph := proxy.NewHandler(pt, uf)
	ph = &cors.CORSHandler{
		Handler: ph,
		Info:    cfg.corsInfo,
	}

	if cfg.isReadonlyProxy() {
		ph = proxy.NewReadonlyHandler(ph)
	}
	// Start a proxy server goroutine for each listen address
	for _, u := range cfg.lcurls {
		l, err := transport.NewListener(u.Host, u.Scheme, cfg.clientTLSInfo)
		if err != nil {
			return err
		}

		host := u.Host
		go func() {
			log.Print("proxy: listening for client requests on ", host)
			log.Fatal(http.Serve(l, ph))
		}()
	}
	return nil
}

// setupCluster sets up an initial cluster definition for bootstrap or discovery.
func setupCluster(cfg *config) (*etcdserver.Cluster, error) {
	var cls *etcdserver.Cluster
	var err error
	switch {
	case cfg.durl != "":
		// If using discovery, generate a temporary cluster based on
		// self's advertised peer URLs
		clusterStr := genClusterString(cfg.name, cfg.apurls)
		cls, err = etcdserver.NewClusterFromString(cfg.durl, clusterStr)
	case cfg.dnsCluster != "":
		clusterStr, clusterToken, err := discovery.SRVGetCluster(cfg.name, cfg.dnsCluster, cfg.initialClusterToken, cfg.apurls)
		if err != nil {
			return nil, err
		}
		cls, err = etcdserver.NewClusterFromString(clusterToken, clusterStr)
	default:
		// We're statically configured, and cluster has appropriately been set.
		cls, err = etcdserver.NewClusterFromString(cfg.initialClusterToken, cfg.initialCluster)
	}
	return cls, err
}

func genClusterString(name string, urls types.URLs) string {
	addrs := make([]string, 0)
	for _, u := range urls {
		addrs = append(addrs, fmt.Sprintf("%v=%v", name, u.String()))
	}
	return strings.Join(addrs, ",")
}
<commit_msg>etcdmain: identify data dir type<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package etcdmain

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"path"
	"reflect"
	"strings"
	"time"

	"github.com/coreos/etcd/discovery"
	"github.com/coreos/etcd/etcdserver"
	"github.com/coreos/etcd/etcdserver/etcdhttp"
	"github.com/coreos/etcd/pkg/cors"
	"github.com/coreos/etcd/pkg/fileutil"
	"github.com/coreos/etcd/pkg/osutil"
	"github.com/coreos/etcd/pkg/transport"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/proxy"
	"github.com/coreos/etcd/rafthttp"
)

type dirType string

const (
	// the owner can make/remove files inside the directory
	privateDirMode = 0700
)

var (
	dirMember = dirType("member")
	dirProxy  = dirType("proxy")
	dirEmpty  = dirType("empty")
)

func Main() {
	cfg := NewConfig()
	err := cfg.Parse(os.Args[1:])
	if err != nil {
		log.Printf("etcd: error verifying flags, %v", err)
		os.Exit(2)
	}

	var stopped <-chan struct{}

	if cfg.dir == "" {
		cfg.dir = fmt.Sprintf("%v.etcd", cfg.name)
		log.Printf("etcd: no data-dir provided, using default data-dir ./%s", cfg.dir)
	}

	which := identifyDataDirOrDie(cfg.dir)
	if which != dirEmpty {
		log.Printf("etcd: already initialized as %v before, starting as etcd %v...", which, which)
	}

	shouldProxy := cfg.isProxy() || which == dirProxy
	if !shouldProxy {
		stopped, err = startEtcd(cfg)
		if err == discovery.ErrFullCluster && cfg.shouldFallbackToProxy() {
			log.Printf("etcd: discovery cluster full, falling back to %s", fallbackFlagProxy)
			shouldProxy = true
		}
	}
	if shouldProxy {
		err = startProxy(cfg)
	}
	if err != nil {
		switch err {
		case discovery.ErrDuplicateID:
			log.Fatalf("etcd: member %s has previously registered with discovery service (%s), but the data-dir (%s) on disk cannot be found.",
				cfg.name, cfg.durl, cfg.dir)
		default:
			log.Fatalf("etcd: %v", err)
		}
	}

	osutil.HandleInterrupts()

	<-stopped
	osutil.Exit(0)
}

// startEtcd launches the etcd server and HTTP handlers for client/server communication.
func startEtcd(cfg *config) (<-chan struct{}, error) {
	cls, err := setupCluster(cfg)
	if err != nil {
		return nil, fmt.Errorf("error setting up initial cluster: %v", err)
	}

	pt, err := transport.NewTimeoutTransport(cfg.peerTLSInfo, rafthttp.DialTimeout, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout)
	if err != nil {
		return nil, err
	}

	if !cfg.peerTLSInfo.Empty() {
		log.Printf("etcd: peerTLS: %s", cfg.peerTLSInfo)
	}
	plns := make([]net.Listener, 0)
	for _, u := range cfg.lpurls {
		var l net.Listener
		l, err = transport.NewTimeoutListener(u.Host, u.Scheme, cfg.peerTLSInfo, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout)
		if err != nil {
			return nil, err
		}

		urlStr := u.String()
		log.Print("etcd: listening for peers on ", urlStr)
		defer func() {
			if err != nil {
				l.Close()
				log.Print("etcd: stopping listening for peers on ", urlStr)
			}
		}()
		plns = append(plns, l)
	}

	if !cfg.clientTLSInfo.Empty() {
		log.Printf("etcd: clientTLS: %s", cfg.clientTLSInfo)
	}
	clns := make([]net.Listener, 0)
	for _, u := range cfg.lcurls {
		var l net.Listener
		l, err = transport.NewKeepAliveListener(u.Host, u.Scheme, cfg.clientTLSInfo)
		if err != nil {
			return nil, err
		}

		urlStr := u.String()
		log.Print("etcd: listening for client requests on ", urlStr)
		defer func() {
			if err != nil {
				l.Close()
				log.Print("etcd: stopping listening for client requests on ", urlStr)
			}
		}()
		clns = append(clns, l)
	}

	srvcfg := &etcdserver.ServerConfig{
		Name:            cfg.name,
		ClientURLs:      cfg.acurls,
		PeerURLs:        cfg.apurls,
		DataDir:         cfg.dir,
		SnapCount:       cfg.snapCount,
		MaxSnapFiles:    cfg.maxSnapFiles,
		MaxWALFiles:     cfg.maxWalFiles,
		Cluster:         cls,
		DiscoveryURL:    cfg.durl,
		DiscoveryProxy:  cfg.dproxy,
		NewCluster:      cfg.isNewCluster(),
		ForceNewCluster: cfg.forceNewCluster,
		Transport:       pt,
		TickMs:          cfg.TickMs,
		ElectionTicks:   cfg.electionTicks(),
	}
	var s *etcdserver.EtcdServer
	s, err = etcdserver.NewServer(srvcfg)
	if err != nil {
		return nil, err
	}
	s.Start()
	osutil.RegisterInterruptHandler(s.Stop)

	if cfg.corsInfo.String() != "" {
		log.Printf("etcd: cors = %s", cfg.corsInfo)
	}
	ch := &cors.CORSHandler{
		Handler: etcdhttp.NewClientHandler(s),
		Info:    cfg.corsInfo,
	}
	ph := etcdhttp.NewPeerHandler(s.Cluster, etcdserver.RaftTimer(s), s.RaftHandler())
	// Start the peer server in a goroutine
	for _, l := range plns {
		go func(l net.Listener) {
			log.Fatal(serveHTTP(l, ph, 5*time.Minute))
		}(l)
	}
	// Start a client server goroutine for each listen address
	for _, l := range clns {
		go func(l net.Listener) {
			// read timeout does not work with http close notify
			// TODO: https://github.com/golang/go/issues/9524
			log.Fatal(serveHTTP(l, ch, 0))
		}(l)
	}
	return s.StopNotify(), nil
}

// startProxy launches an HTTP proxy for client communication which proxies to other etcd nodes.
func startProxy(cfg *config) error {
	cls, err := setupCluster(cfg)
	if err != nil {
		return fmt.Errorf("error setting up initial cluster: %v", err)
	}

	if cfg.durl != "" {
		s, err := discovery.GetCluster(cfg.durl, cfg.dproxy)
		if err != nil {
			return err
		}
		if cls, err = etcdserver.NewClusterFromString(cfg.durl, s); err != nil {
			return err
		}
	}

	pt, err := transport.NewTransport(cfg.clientTLSInfo)
	if err != nil {
		return err
	}

	tr, err := transport.NewTransport(cfg.peerTLSInfo)
	if err != nil {
		return err
	}

	cfg.dir = path.Join(cfg.dir, "proxy")
	err = os.MkdirAll(cfg.dir, 0700)
	if err != nil {
		return err
	}

	var peerURLs []string
	clusterfile := path.Join(cfg.dir, "cluster")

	b, err := ioutil.ReadFile(clusterfile)
	switch {
	case err == nil:
		urls := struct{ PeerURLs []string }{}
		err := json.Unmarshal(b, &urls)
		if err != nil {
			return err
		}
		peerURLs = urls.PeerURLs
		log.Printf("proxy: using peer urls %v from cluster file ./%s", peerURLs, clusterfile)
	case os.IsNotExist(err):
		peerURLs = cls.PeerURLs()
		log.Printf("proxy: using peer urls %v ", peerURLs)
	default:
		return err
	}

	uf := func() []string {
		gcls, err := etcdserver.GetClusterFromRemotePeers(peerURLs, tr)
		// TODO: remove the 2nd check when we fix GetClusterFromPeers
		// GetClusterFromPeers should not return nil error with an invaild empty cluster
		if err != nil {
			log.Printf("proxy: %v", err)
			return []string{}
		}
		if len(gcls.Members()) == 0 {
			return cls.ClientURLs()
		}
		cls = gcls

		urls := struct{ PeerURLs []string }{cls.PeerURLs()}
		b, err := json.Marshal(urls)
		if err != nil {
			log.Printf("proxy: error on marshal peer urls %s", err)
			return cls.ClientURLs()
		}

		err = ioutil.WriteFile(clusterfile+".bak", b, 0600)
		if err != nil {
			log.Printf("proxy: error on writing urls %s", err)
			return cls.ClientURLs()
		}
		err = os.Rename(clusterfile+".bak", clusterfile)
		if err != nil {
			log.Printf("proxy: error on updating clusterfile %s", err)
			return cls.ClientURLs()
		}
		if !reflect.DeepEqual(cls.PeerURLs(), peerURLs) {
			log.Printf("proxy: updated peer urls in cluster file from %v to %v", peerURLs, cls.PeerURLs())
		}
		peerURLs = cls.PeerURLs()

		return cls.ClientURLs()
	}
	ph := proxy.NewHandler(pt, uf)
	ph = &cors.CORSHandler{
		Handler: ph,
		Info:    cfg.corsInfo,
	}

	if cfg.isReadonlyProxy() {
		ph = proxy.NewReadonlyHandler(ph)
	}
	// Start a proxy server goroutine for each listen address
	for _, u := range cfg.lcurls {
		l, err := transport.NewListener(u.Host, u.Scheme, cfg.clientTLSInfo)
		if err != nil {
			return err
		}

		host := u.Host
		go func() {
			log.Print("proxy: listening for client requests on ", host)
			log.Fatal(http.Serve(l, ph))
		}()
	}
	return nil
}

// setupCluster sets up an initial cluster definition for bootstrap or discovery.
func setupCluster(cfg *config) (*etcdserver.Cluster, error) {
	var cls *etcdserver.Cluster
	var err error
	switch {
	case cfg.durl != "":
		// If using discovery, generate a temporary cluster based on
		// self's advertised peer URLs
		clusterStr := genClusterString(cfg.name, cfg.apurls)
		cls, err = etcdserver.NewClusterFromString(cfg.durl, clusterStr)
	case cfg.dnsCluster != "":
		clusterStr, clusterToken, err := discovery.SRVGetCluster(cfg.name, cfg.dnsCluster, cfg.initialClusterToken, cfg.apurls)
		if err != nil {
			return nil, err
		}
		cls, err = etcdserver.NewClusterFromString(clusterToken, clusterStr)
	default:
		// We're statically configured, and cluster has appropriately been set.
		cls, err = etcdserver.NewClusterFromString(cfg.initialClusterToken, cfg.initialCluster)
	}
	return cls, err
}

func genClusterString(name string, urls types.URLs) string {
	addrs := make([]string, 0)
	for _, u := range urls {
		addrs = append(addrs, fmt.Sprintf("%v=%v", name, u.String()))
	}
	return strings.Join(addrs, ",")
}

// identifyDataDirOrDie returns the type of the data dir.
// Dies if the datadir is invalid.
func identifyDataDirOrDie(dir string) dirType {
	names, err := fileutil.ReadDir(dir)
	if err != nil {
		if os.IsNotExist(err) {
			return dirEmpty
		}
		log.Fatalf("etcd: error listing data dir: %s", dir)
	}

	var m, p bool
	for _, name := range names {
		switch dirType(name) {
		case dirMember:
			m = true
		case dirProxy:
			p = true
		default:
			log.Printf("etcd: found invalid file/dir %s under data dir %s (Ignore this if you are upgrading etcd)", name, dir)
		}
	}

	if m && p {
		log.Fatal("etcd: invalid datadir. Both member and proxy directories exist.")
	}
	if m {
		return dirMember
	}
	if p {
		return dirProxy
	}
	return dirEmpty
}
<|endoftext|>


<commit_before>// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package etcdmain

import (
	"fmt"
	"os"

	"github.com/coreos/go-systemd/daemon"
	systemdutil "github.com/coreos/go-systemd/util"
)

func Main() {
	checkSupportArch()

	if len(os.Args) > 1 {
		switch os.Args[1] {
		case "gateway", "grpc-proxy":
			if err := rootCmd.Execute(); err != nil {
				fmt.Fprint(os.Stderr, err)
				os.Exit(1)
			}
			return
		}
	}

	startEtcdOrProxyV2()
}

func notifySystemd() {
	if !systemdutil.IsRunningSystemd() {
		return
	}
	sent, err := daemon.SdNotify(false, "READY=1")
	if err != nil {
		plog.Errorf("failed to notify systemd for readiness: %v", err)
	}
	if !sent {
		plog.Errorf("forgot to set Type=notify in systemd service file?")
	}
}
<commit_msg>etcdmain: support grpc-proxy/gateway compiled with -tags cov<commit_after>// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package etcdmain

import (
	"fmt"
	"os"
	"strings"

	"github.com/coreos/go-systemd/daemon"
	systemdutil "github.com/coreos/go-systemd/util"
)

func Main() {
	checkSupportArch()

	if len(os.Args) > 1 {
		cmd := os.Args[1]
		if covArgs := os.Getenv("ETCDCOV_ARGS"); len(covArgs) > 0 {
			args := strings.Split(os.Getenv("ETCDCOV_ARGS"), "\xe7\xcd")[1:]
			rootCmd.SetArgs(args)
			cmd = "grpc-proxy"
		}
		switch cmd {
		case "gateway", "grpc-proxy":
			if err := rootCmd.Execute(); err != nil {
				fmt.Fprint(os.Stderr, err)
				os.Exit(1)
			}
			return
		}
	}

	startEtcdOrProxyV2()
}

func notifySystemd() {
	if !systemdutil.IsRunningSystemd() {
		return
	}
	sent, err := daemon.SdNotify(false, "READY=1")
	if err != nil {
		plog.Errorf("failed to notify systemd for readiness: %v", err)
	}
	if !sent {
		plog.Errorf("forgot to set Type=notify in systemd service file?")
	}
}
<|endoftext|>


<commit_before>package markdown

import (
	"fmt"

	g "github.com/SimonRichardson/butler/generic"
)

type headerType string

var (
	H1 headerType = "#"
	H2 headerType = "##"
	H3 headerType = "###"
	H4 headerType = "####"
	H5 headerType = "#####"
	H6 headerType = "######"
)

func (h headerType) IsInline() bool {
	return false
}

func (h headerType) Children() g.Option {
	return g.Option_.Empty()
}

func (h headerType) String() string {
	return string(h)
}

type header struct {
	Type  headerType
	Value marks
}

func (h header) IsInline() bool {
	return false
}

func (h header) Children() g.Option {
	return g.Option_.Of(g.List_.Of(h.Value))
}

func (h header) String() string {
	return fmt.Sprintf("%s", h.Type.String())
}

func h1(val marks) header {
	return header{
		Type:  H1,
		Value: val,
	}
}

func h2(val marks) header {
	return header{
		Type:  H2,
		Value: val,
	}
}

func h3(val marks) header {
	return header{
		Type:  H3,
		Value: val,
	}
}

func h4(val marks) header {
	return header{
		Type:  H4,
		Value: val,
	}
}

func h5(val marks) header {
	return header{
		Type:  H5,
		Value: val,
	}
}

func h6(val marks) header {
	return header{
		Type:  H6,
		Value: val,
	}
}
<commit_msg>Starting to write headers correctly<commit_after>package markdown

import (
	"fmt"

	g "github.com/SimonRichardson/butler/generic"
)

type headerType string

var (
	H1 headerType = "#"
	H2 headerType = "##"
	H3 headerType = "###"
	H4 headerType = "####"
	H5 headerType = "#####"
	H6 headerType = "######"
)

func (h headerType) IsInline() bool {
	return true
}

func (h headerType) Children() g.Option {
	return g.Option_.Empty()
}

func (h headerType) String() string {
	return fmt.Sprintf("%s ", string(h))
}

type header struct {
	nodes g.List
}

func (h header) IsInline() bool {
	return false
}

func (h header) Children() g.Option {
	return g.Option_.Of(h.nodes)
}

func (h header) String() string {
	return ""
}

func h1(val marks) header {
	return header{
		nodes: fromMarks(append([]marks{H1}, val)),
	}
}

func h2(val marks) header {
	return header{
		nodes: fromMarks(append([]marks{H2}, val)),
	}
}

func h3(val marks) header {
	return header{
		nodes: fromMarks(append([]marks{H3}, val)),
	}
}

func h4(val marks) header {
	return header{
		nodes: fromMarks(append([]marks{H4}, val)),
	}
}

func h5(val marks) header {
	return header{
		nodes: fromMarks(append([]marks{H5}, val)),
	}
}

func h6(val marks) header {
	return header{
		nodes: fromMarks(append([]marks{H6}, val)),
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"os"
	"strings"
	"text/tabwriter"
	"text/template"
)

var helpCmd = &Command{
	Name:    "help",
	Usage:   "[<command>]",
	Summary: "Help on the rx command and subcommands.",
}

var helpDump = helpCmd.Flag.Bool("godoc", false, "Dump the godoc output for the command(s)")

func helpFunc(cmd *Command, args ...string) {
	var selected []*Command

	if len(args) > 0 {
		want := strings.ToLower(args[0])
		for _, cmd := range commands {
			if cmd.Name == want {
				selected = append(selected, cmd)
			}
		}
	}

	switch {
	case *helpDump:
		render(stdout, docTemplate, commands)
	case len(selected) < len(args):
		fmt.Fprintf(stdout, "error: unknown command %q\n", args[0])
		render(stdout, helpTemplate, helpCmd)
	case len(selected) == 0:
		render(stdout, usageTemplate, commands)
	case len(selected) == 1:
		render(stdout, helpTemplate, selected[0])
	}
}

func init() {
	helpCmd.Run = helpFunc
}

func tabify(w io.Writer) *tabwriter.Writer {
	return tabwriter.NewWriter(w, 0, 0, 1, ' ', 0)
}

var templateFuncs = template.FuncMap{
	"flags": func(indent int, args ...interface{}) string {
		b := new(bytes.Buffer)
		prefix := strings.Repeat(" ", indent)
		w := tabify(b)
		visit := func(f *flag.Flag) {
			dash := "--"
			if len(f.Name) == 1 {
				dash = "-"
			}
			eq := "= " + f.DefValue
			switch typeName := fmt.Sprintf("%T", f.Value); {
			case typeName == "*flag.stringValue":
				// TODO(kevlar): make my own stringValue type so as to not depend on this?
				eq = fmt.Sprintf("= %q", f.DefValue)
			case f.DefValue == "":
				eq = ""
			}
			fmt.Fprintf(w, "%s%s%s\t%s\t   %s\n", prefix, dash, f.Name, eq, f.Usage)
		}
		if len(args) == 0 {
			flag.VisitAll(visit)
		} else {
			args[0].(*Command).Flag.VisitAll(visit)
		}
		w.Flush()
		if b.Len() == 0 {
			return ""
		}
		return fmt.Sprintf("\nOptions:\n%s", b)
	},
	"title": func(s string) string {
		return strings.Title(s + " command")
	},
	"trim": func(s string) string {
		return strings.TrimSpace(s)
	},
}

var stdout io.Writer = tabConverter{os.Stdout}

type tabConverter struct{ io.Writer }

func (t tabConverter) Write(p []byte) (int, error) {
	p = bytes.Replace(p, []byte{'\t'}, []byte{' ', ' ', ' ', ' '}, -1)
	return t.Writer.Write(p)
}

func render(w io.Writer, tpl string, data interface{}) {
	t := template.New("help")
	t.Funcs(templateFuncs)
	if err := template.Must(t.Parse(tpl)).Execute(w, data); err != nil {
		panic(err)
	}
}

var generalHelp = `	rx [<options>] [<subcommand> [<suboptions>] [<arguments> ...]]
{{flags 2}}
Commands:{{range .}}
	{{.Name | printf "%-10s"}} {{.Summary}}{{end}}

Use "rx help <command>" for more help with a command.
`

var usageTemplate = `rx is a command-line dependency management tool for Go projects.

Usage:
` + generalHelp

var helpTemplate = `Usage: rx {{.Name}} [options]{{with .Usage}} {{.}}{{end}}{{if .Abbrev}}
       rx {{.Abbrev}} [options]{{with .Usage}} {{.}}{{end}}
{{end}}{{flags 2 .}}
{{.Summary}}
{{if .Help}}
{{.Help | trim}}{{end}}
`

var docTemplate = `/*
The rx command is a dependency and version management system for Go projects.
It is built on top of the go tool and utilizes the $GOPATH convention.

Installation

As usual, the rx tool can be installed or upgraded via the "go" tool:
	go get -u kylelemons.net/go/rx

General Usage

The rx command is composed of numerous sub-commands.
Sub-commands can be abbreviated to any unique prefix on the command-line.
The general usage is:

` + generalHelp + `

See below for a description of the various sub-commands understood by rx.
{{range .}}
{{.Name | title}}

{{.Summary | trim}}

Usage:
	rx {{.Name}} {{.Usage | trim}}
{{flags 2 .}}
{{.Help | trim}}
{{end}}
*/
package main
`
<commit_msg>add note not to depend on options<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"os"
	"strings"
	"text/tabwriter"
	"text/template"
)

var helpCmd = &Command{
	Name:    "help",
	Usage:   "[<command>]",
	Summary: "Help on the rx command and subcommands.",
}

var helpDump = helpCmd.Flag.Bool("godoc", false, "Dump the godoc output for the command(s)")

func helpFunc(cmd *Command, args ...string) {
	var selected []*Command

	if len(args) > 0 {
		want := strings.ToLower(args[0])
		for _, cmd := range commands {
			if cmd.Name == want {
				selected = append(selected, cmd)
			}
		}
	}

	switch {
	case *helpDump:
		render(stdout, docTemplate, commands)
	case len(selected) < len(args):
		fmt.Fprintf(stdout, "error: unknown command %q\n", args[0])
		render(stdout, helpTemplate, helpCmd)
	case len(selected) == 0:
		render(stdout, usageTemplate, commands)
	case len(selected) == 1:
		render(stdout, helpTemplate, selected[0])
	}
}

func init() {
	helpCmd.Run = helpFunc
}

func tabify(w io.Writer) *tabwriter.Writer {
	return tabwriter.NewWriter(w, 0, 0, 1, ' ', 0)
}

var templateFuncs = template.FuncMap{
	"flags": func(indent int, args ...interface{}) string {
		b := new(bytes.Buffer)
		prefix := strings.Repeat(" ", indent)
		w := tabify(b)
		visit := func(f *flag.Flag) {
			dash := "--"
			if len(f.Name) == 1 {
				dash = "-"
			}
			eq := "= " + f.DefValue
			switch typeName := fmt.Sprintf("%T", f.Value); {
			case typeName == "*flag.stringValue":
				// TODO(kevlar): make my own stringValue type so as to not depend on this?
				eq = fmt.Sprintf("= %q", f.DefValue)
			case f.DefValue == "":
				eq = ""
			}
			fmt.Fprintf(w, "%s%s%s\t%s\t   %s\n", prefix, dash, f.Name, eq, f.Usage)
		}
		if len(args) == 0 {
			flag.VisitAll(visit)
		} else {
			args[0].(*Command).Flag.VisitAll(visit)
		}
		w.Flush()
		if b.Len() == 0 {
			return ""
		}
		return fmt.Sprintf("\nOptions:\n%s", b)
	},
	"title": func(s string) string {
		return strings.Title(s + " command")
	},
	"trim": func(s string) string {
		return strings.TrimSpace(s)
	},
}

var stdout io.Writer = tabConverter{os.Stdout}

type tabConverter struct{ io.Writer }

func (t tabConverter) Write(p []byte) (int, error) {
	p = bytes.Replace(p, []byte{'\t'}, []byte{' ', ' ', ' ', ' '}, -1)
	return t.Writer.Write(p)
}

func render(w io.Writer, tpl string, data interface{}) {
	t := template.New("help")
	t.Funcs(templateFuncs)
	if err := template.Must(t.Parse(tpl)).Execute(w, data); err != nil {
		panic(err)
	}
}

var generalHelp = `	rx [<options>] [<subcommand> [<suboptions>] [<arguments> ...]]
{{flags 2}}
Commands:{{range .}}
	{{.Name | printf "%-10s"}} {{.Summary}}{{end}}

Use "rx help <command>" for more help with a command.
`

var usageTemplate = `rx is a command-line dependency management tool for Go projects.

Usage:
` + generalHelp

var helpTemplate = `Usage: rx {{.Name}} [options]{{with .Usage}} {{.}}{{end}}{{if .Abbrev}}
       rx {{.Abbrev}} [options]{{with .Usage}} {{.}}{{end}}
{{end}}{{flags 2 .}}
{{.Summary}}
{{if .Help}}
{{.Help | trim}}{{end}}
`

var docTemplate = `/*
The rx command is a dependency and version management system for Go projects.
It is built on top of the go tool and utilizes the $GOPATH convention.

Warning

This tool is very much in flux.  Don't depend on commands, options, or
pretty much anything else being stable yet.

Installation

As usual, the rx tool can be installed or upgraded via the "go" tool:
	go get -u kylelemons.net/go/rx

General Usage

The rx command is composed of numerous sub-commands.
Sub-commands can be abbreviated to any unique prefix on the command-line.
The general usage is:

` + generalHelp + `

See below for a description of the various sub-commands understood by rx.
{{range .}}
{{.Name | title}}

{{.Summary | trim}}

Usage:
	rx {{.Name}} {{.Usage | trim}}
{{flags 2 .}}
{{.Help | trim}}
{{end}}
*/
package main
`
<|endoftext|>


<commit_before>package task

import (
	"fmt"
	"sort"
	"text/tabwriter"
)

// PrintTasksHelp prints help os tasks that have a description
func (e *Executor) PrintTasksHelp() {
	tasks := e.tasksWithDesc()
	if len(tasks) == 0 {
		return
	}
	e.outf("Available tasks for this project:")

	// Format in tab-separated columns with a tab stop of 8.
	w := tabwriter.NewWriter(e.Stdout, 0, 8, 0, '\t', 0)
	for _, task := range tasks {
		fmt.Fprintln(w, fmt.Sprintf("* %s: \t%s", task, e.Tasks[task].Desc))
	}
	w.Flush()
}

func (e *Executor) tasksWithDesc() (tasks []string) {
	for name, task := range e.Tasks {
		if task.Desc != "" {
			tasks = append(tasks, name)
		}
	}
	sort.Strings(tasks)
	return
}
<commit_msg>list: print message with there's no task with description<commit_after>package task

import (
	"fmt"
	"sort"
	"text/tabwriter"
)

// PrintTasksHelp prints help os tasks that have a description
func (e *Executor) PrintTasksHelp() {
	tasks := e.tasksWithDesc()
	if len(tasks) == 0 {
		e.outf("task: No tasks with description available")
		return
	}
	e.outf("task: Available tasks for this project:")

	// Format in tab-separated columns with a tab stop of 8.
	w := tabwriter.NewWriter(e.Stdout, 0, 8, 0, '\t', 0)
	for _, task := range tasks {
		fmt.Fprintln(w, fmt.Sprintf("* %s: \t%s", task, e.Tasks[task].Desc))
	}
	w.Flush()
}

func (e *Executor) tasksWithDesc() (tasks []string) {
	for name, task := range e.Tasks {
		if task.Desc != "" {
			tasks = append(tasks, name)
		}
	}
	sort.Strings(tasks)
	return
}
<|endoftext|>


<commit_before>package urknall

import (
	"fmt"
	"github.com/dynport/urknall/fw"
	"strings"
)

// The host type. Use the "NewHost" function to create the basic value.
//
// Please note that you need to set the primary interface (the one the host is accessible on) name, if that is not
// "eth0". That should only be necessary on rare circumstances.
//
// A host is added a set of packages, that are provisioned on request.
//
//	TODO(gfrey): Add better support for interfaces and IPs.
//	TODO(gfrey): Add handling and support for IPv6 (currently the firewall will block everything).
type Host struct {
	IP        string // Host's IP address used to provision the system.
	User      string // User used to log in.
	Password  string // SSH password to be used (besides ssh-agent)
	Hostname  string // Hostname used on the system.
	Interface string // Primary network interface of the host.
	Timezone  string // Local Timezone to be set

	Tags []string
	Env  []string // custom env settings to be used for all sessions

	Paranoid bool // Make the firewall as restrictive as possible.
	WithVPN  bool // Connect host to a VPN. Assumes "tun0" as interface.

	BuildHost               bool   // Whether this host should be used to build urknall binary packages.
	BinaryPackageRepository string // Where should urknall binary packages be searched?

	Firewall fw.Firewall // List of rules used for the firewall.
	IPSets   []*fw.IPSet // List of ipsets for the firewall.

	packageNames   []string
	userRunlists   []*Runlist
	systemRunlists []*Runlist
}

// Get the user used to access the host. If none is given the "root" account is as default.
func (h *Host) user() string {
	if h.User == "" {
		return "root"
	}
	return h.User
}

// Get the host's primary interface. If none is given "eth0" is returned as default.
func (h *Host) publicInterface() string {
	if h.Interface == "" {
		return "eth0"
	}
	return h.Interface
}

// Alias for the AddCommands methods.
func (h *Host) Add(name string, cmd interface{}, cmds ...interface{}) {
	h.AddCommands(name, cmd, cmds...)
}

// Register the list of given commands (either of the cmd.Command type or as string) as a package (without
// configuration) with the given name.
func (h *Host) AddCommands(name string, cmd interface{}, cmds ...interface{}) {
	cmdList := append([]interface{}{cmd}, cmds...)
	h.AddPackage(name, NewPackage(cmdList...))
}

// Add the given package with the given name to the host.
//
// The name is used as reference during provisioning and allows for provisioning the very same package in different
// configuration (with different version for example). Package names must be unique and the "uk." prefix is reserved for
// urknall internal packages.
func (h *Host) AddPackage(name string, pkg Package) {
	if strings.HasPrefix(name, "uk.") {
		panic(fmt.Sprintf(`package name prefix "uk." reserved (in %q)`, name))
	}

	if strings.Contains(name, " ") {
		panic(fmt.Sprintf(`package names must not contain spaces (%q does)`, name))
	}

	for i := range h.packageNames {
		if h.packageNames[i] == name {
			panic(fmt.Sprintf("package with name %q exists already", name))
		}
	}

	h.packageNames = append(h.packageNames, name)
	h.userRunlists = append(h.userRunlists, newRunlist(name, pkg, h))
}

// Add the given package with the given name to the host.
func (h *Host) addSystemPackage(name string, pkg Package) (e error) {
	name = "uk." + name
	for i := range h.packageNames {
		if h.packageNames[i] == name {
			return fmt.Errorf("package with name %q exists already", name)
		}
	}

	h.packageNames = append(h.packageNames, name)
	h.systemRunlists = append(h.systemRunlists, newRunlist(name, pkg, h))
	return nil
}

// Provision the host, i.e. execute all the commands contained in the packages registered with this host.
func (h *Host) Provision(opts *ProvisionOptions) (e error) {
	sc := newSSHClient(h, opts)
	return sc.provision()
}

// Create a binary package from the given package. This is an optimization for packages that download, compile and
// install sources. As compilation might take its time this step can be done once and the effort be reused. Keep in mind
// that:
//
//	* Packages to be precompiled should only contain the steps necessary to compile and install the sources
//	  (configuration will only be executed during the package creation).
//	* Packages must implement the extend BinaryPackage interface.
//	* There must be a binary package repository reachable and configured for all hosts that should use the binary
//	  packages.
//	* The host to build binary packages on must have the BuildHost flag set. This is to make sure that the side effects
//	  (like installed compilers, extracted sources, etc) are known to the administration and don't happen by accident.
func (h *Host) CreateUrknallImage(pkg BinaryPackage) (e error) {
	if !h.BuildHost {
		return fmt.Errorf("Host %q is not a build host.", h.Hostname)
	}
	sc := newSSHClient(h, nil)
	return sc.buildBinaryPackage(pkg)
}

// Predicate to test whether sudo is required (user for the host is not "root").
func (h *Host) isSudoRequired() bool {
	if h.User != "" && h.User != "root" {
		return true
	}
	return false
}

func (h *Host) runlists() (r []*Runlist) {
	if h.systemRunlists == nil {
		h.buildSystemRunlists()
	}

	r = make([]*Runlist, 0, len(h.systemRunlists)+len(h.userRunlists))
	r = append(r, h.systemRunlists...)
	r = append(r, h.userRunlists...)
	return r
}

func (h *Host) precompileRunlists() (e error) {
	for _, runlist := range h.runlists() {
		if len(runlist.commands) > 0 {
			return fmt.Errorf("pkg %q seems to be packaged already", runlist.name)
		}

		if e = runlist.compileWithBinaryPackages(); e != nil {
			return e
		}
	}

	return nil
}
<commit_msg>renamed host method to build binary package<commit_after>package urknall

import (
	"fmt"
	"github.com/dynport/urknall/fw"
	"strings"
)

// The host type. Use the "NewHost" function to create the basic value.
//
// Please note that you need to set the primary interface (the one the host is accessible on) name, if that is not
// "eth0". That should only be necessary on rare circumstances.
//
// A host is added a set of packages, that are provisioned on request.
//
//	TODO(gfrey): Add better support for interfaces and IPs.
//	TODO(gfrey): Add handling and support for IPv6 (currently the firewall will block everything).
type Host struct {
	IP        string // Host's IP address used to provision the system.
	User      string // User used to log in.
	Password  string // SSH password to be used (besides ssh-agent)
	Hostname  string // Hostname used on the system.
	Interface string // Primary network interface of the host.
	Timezone  string // Local Timezone to be set

	Tags []string
	Env  []string // custom env settings to be used for all sessions

	Paranoid bool // Make the firewall as restrictive as possible.
	WithVPN  bool // Connect host to a VPN. Assumes "tun0" as interface.

	BuildHost               bool   // Whether this host should be used to build urknall binary packages.
	BinaryPackageRepository string // Where should urknall binary packages be searched?

	Firewall fw.Firewall // List of rules used for the firewall.
	IPSets   []*fw.IPSet // List of ipsets for the firewall.

	packageNames   []string
	userRunlists   []*Runlist
	systemRunlists []*Runlist
}

// Get the user used to access the host. If none is given the "root" account is as default.
func (h *Host) user() string {
	if h.User == "" {
		return "root"
	}
	return h.User
}

// Get the host's primary interface. If none is given "eth0" is returned as default.
func (h *Host) publicInterface() string {
	if h.Interface == "" {
		return "eth0"
	}
	return h.Interface
}

// Alias for the AddCommands methods.
func (h *Host) Add(name string, cmd interface{}, cmds ...interface{}) {
	h.AddCommands(name, cmd, cmds...)
}

// Register the list of given commands (either of the cmd.Command type or as string) as a package (without
// configuration) with the given name.
func (h *Host) AddCommands(name string, cmd interface{}, cmds ...interface{}) {
	cmdList := append([]interface{}{cmd}, cmds...)
	h.AddPackage(name, NewPackage(cmdList...))
}

// Add the given package with the given name to the host.
//
// The name is used as reference during provisioning and allows for provisioning the very same package in different
// configuration (with different version for example). Package names must be unique and the "uk." prefix is reserved for
// urknall internal packages.
func (h *Host) AddPackage(name string, pkg Package) {
	if strings.HasPrefix(name, "uk.") {
		panic(fmt.Sprintf(`package name prefix "uk." reserved (in %q)`, name))
	}

	if strings.Contains(name, " ") {
		panic(fmt.Sprintf(`package names must not contain spaces (%q does)`, name))
	}

	for i := range h.packageNames {
		if h.packageNames[i] == name {
			panic(fmt.Sprintf("package with name %q exists already", name))
		}
	}

	h.packageNames = append(h.packageNames, name)
	h.userRunlists = append(h.userRunlists, newRunlist(name, pkg, h))
}

// Add the given package with the given name to the host.
func (h *Host) addSystemPackage(name string, pkg Package) (e error) {
	name = "uk." + name
	for i := range h.packageNames {
		if h.packageNames[i] == name {
			return fmt.Errorf("package with name %q exists already", name)
		}
	}

	h.packageNames = append(h.packageNames, name)
	h.systemRunlists = append(h.systemRunlists, newRunlist(name, pkg, h))
	return nil
}

// Provision the host, i.e. execute all the commands contained in the packages registered with this host.
func (h *Host) Provision(opts *ProvisionOptions) (e error) {
	sc := newSSHClient(h, opts)
	return sc.provision()
}

// Create a binary package from the given package. This is an optimization for packages that download, compile and
// install sources. As compilation might take its time this step can be done once and the effort be reused. Keep in mind
// that:
//
//	* Packages to be precompiled should only contain the steps necessary to compile and install the sources
//	  (configuration will only be executed during the package creation).
//	* Packages must implement the extend BinaryPackage interface.
//	* There must be a binary package repository reachable and configured for all hosts that should use the binary
//	  packages.
//	* The host to build binary packages on must have the BuildHost flag set. This is to make sure that the side effects
//	  (like installed compilers, extracted sources, etc) are known to the administration and don't happen by accident.
func (h *Host) BuildBinaryPackage(pkg BinaryPackage) (e error) {
	if !h.BuildHost {
		return fmt.Errorf("Host %q is not a build host.", h.Hostname)
	}
	sc := newSSHClient(h, nil)
	return sc.buildBinaryPackage(pkg)
}

// Predicate to test whether sudo is required (user for the host is not "root").
func (h *Host) isSudoRequired() bool {
	if h.User != "" && h.User != "root" {
		return true
	}
	return false
}

func (h *Host) runlists() (r []*Runlist) {
	if h.systemRunlists == nil {
		h.buildSystemRunlists()
	}

	r = make([]*Runlist, 0, len(h.systemRunlists)+len(h.userRunlists))
	r = append(r, h.systemRunlists...)
	r = append(r, h.userRunlists...)
	return r
}

func (h *Host) precompileRunlists() (e error) {
	for _, runlist := range h.runlists() {
		if len(runlist.commands) > 0 {
			return fmt.Errorf("pkg %q seems to be packaged already", runlist.name)
		}

		if e = runlist.compileWithBinaryPackages(); e != nil {
			return e
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package app

import (
	"bytes"
	"text/template"

	"fmt"

	"github.com/murlokswarm/uid"
)

const (
	htmlContextTmpl = `
<!DOCTYPE html>

    <meta charset="UTF-8">

    {{range .CSS}}
    <link type="text/css" rel="stylesheet" href="css/{{.}}" />{{end}}

<title>{{.Title}}</title>

    {{range .JS}}
    {{end}}

    `
)

// HTMLContext contains the data required to generate the minimum HTML to
// setup a webview based context.
// Should be used only in drivers implementations.
type HTMLContext struct {
	ID       uid.ID
	Title    string
	Lang     string
	MurlokJS string
	JS       []string
	CSS      []string
}

// HTML generate the HTML based on the data of c.
func (c HTMLContext) HTML() string {
	var b bytes.Buffer
	t := template.Must(template.New("").Parse(htmlContextTmpl))
	t.Execute(&b, c)

	fmt.Println(b.String())

	return b.String()
}
<commit_msg>clean<commit_after>package app

import (
	"bytes"
	"text/template"

	"github.com/murlokswarm/uid"
)

const (
	htmlContextTmpl = `
<!DOCTYPE html>

    <meta charset="UTF-8">

    {{range .CSS}}
    <link type="text/css" rel="stylesheet" href="css/{{.}}" />{{end}}

<title>{{.Title}}</title>

    {{range .JS}}
    {{end}}

    `
)

// HTMLContext contains the data required to generate the minimum HTML to
// setup a webview based context.
// Should be used only in drivers implementations.
type HTMLContext struct {
	ID       uid.ID
	Title    string
	Lang     string
	MurlokJS string
	JS       []string
	CSS      []string
}

// HTML generate the HTML based on the data of c.
func (c HTMLContext) HTML() string {
	var b bytes.Buffer
	t := template.Must(template.New("").Parse(htmlContextTmpl))
	t.Execute(&b, c)
	return b.String()
}
<|endoftext|>


<commit_before>package udf

import (
	"fmt"
	"pfi/sensorbee/sensorbee/core"
	"pfi/sensorbee/sensorbee/data"
	"sync"
)

// UDF is an interface having a user defined function.
type UDF interface {
	// Call calls the UDF.
	Call(*core.Context, ...data.Value) (data.Value, error)

	// Accept checks if the function accepts the given number of arguments
	// excluding core.Context.
	Accept(arity int) bool
}

type function struct {
	f     func(*core.Context, ...data.Value) (data.Value, error)
	arity int
}

func (f *function) Call(ctx *core.Context, args ...data.Value) (data.Value, error) {
	return f.f(ctx, args...)
}

func (f *function) Accept(arity int) bool {
	if f.arity < 0 {
		return true
	}
	return arity == f.arity
}

// VariadicFunc creates a UDF based on a function receiving the variadic number
// of data.Values.
func VariadicFunc(f func(*core.Context, ...data.Value) (data.Value, error)) UDF {
	return &function{
		f:     f,
		arity: -1,
	}
}

func Func(f func(*core.Context, ...data.Value) (data.Value, error), arity int) UDF {
	return &function{
		f:     f,
		arity: arity,
	}
}

func NullaryFunc(f func(*core.Context) (data.Value, error)) UDF {
	genFunc := func(ctx *core.Context, vs ...data.Value) (data.Value, error) {
		if len(vs) != 0 {
			return nil, fmt.Errorf("the function should be used as nullary")
		}
		return f(ctx)
	}
	return Func(genFunc, 0)
}

func UnaryFunc(f func(*core.Context, data.Value) (data.Value, error)) UDF {
	genFunc := func(ctx *core.Context, vs ...data.Value) (data.Value, error) {
		if len(vs) != 1 {
			return nil, fmt.Errorf("the function should be used as unary")
		}
		return f(ctx, vs[0])
	}
	return Func(genFunc, 1)
}

func BinaryFunc(f func(*core.Context, data.Value, data.Value) (data.Value, error)) UDF {
	genFunc := func(ctx *core.Context, vs ...data.Value) (data.Value, error) {
		if len(vs) != 2 {
			return nil, fmt.Errorf("the function should be used as binary")
		}
		return f(ctx, vs[0], vs[1])
	}
	return Func(genFunc, 2)
}

func TernaryFunc(f func(*core.Context, data.Value, data.Value, data.Value) (data.Value, error)) UDF {
	genFunc := func(ctx *core.Context, vs ...data.Value) (data.Value, error) {
		if len(vs) != 3 {
			return nil, fmt.Errorf("the function should be used as ternary")
		}
		return f(ctx, vs[0], vs[1], vs[2])
	}
	return Func(genFunc, 3)
}

// TODO: Add magic UDF generator func NewUDF(f interface{}) (UDF, error)
//       It accepts any function whose arguments are convertible to data.Value.
//       For example, NewUDF(func(*core.Context, a, b int) (int, error) {return a + b}).
//       Even NewUDF(func(a, b int) int {return a + b}) could be valid
//       (i.e. Context and error can be optional)

// FunctionRegistry is an interface to lookup functions for use in BQL
// statements by their name.
type FunctionRegistry interface {
	// Context returns a core.Context associated with the registry.
	Context() *core.Context

	// Lookup will return a function with the given name and arity
	// or an error if there is none. Note that this interface allows
	// multiple functions with the same name but different arity,
	// and it also allows functions with an arbitrary number of
	// parameters. However, a function returned must never be used
	// with a different arity than the one given in the Lookup call.
	Lookup(name string, arity int) (UDF, error)
}

// FunctionManager is a FunctionRegistry that allows to register
// additional functions.
type FunctionManager interface {
	FunctionRegistry

	// Register allows to add a function.
	Register(name string, f UDF) error
}

type defaultFunctionRegistry struct {
	ctx   *core.Context
	m     sync.RWMutex
	funcs map[string]UDF
}

func NewDefaultFunctionRegistry(ctx *core.Context) FunctionManager {
	reg := &defaultFunctionRegistry{
		ctx:   ctx,
		funcs: map[string]UDF{},
	}
	return reg
}

func (fr *defaultFunctionRegistry) Context() *core.Context {
	return fr.ctx
}

func (fr *defaultFunctionRegistry) Lookup(name string, arity int) (UDF, error) {
	fr.m.RLock()
	defer fr.m.RUnlock()
	if f, exists := fr.funcs[name]; exists {
		if f.Accept(arity) {
			return f, nil
		}
		return nil, fmt.Errorf("function '%s' is not %d-ary", name, arity)
	}
	return nil, fmt.Errorf("function '%s' is unknown", name)
}

func (fr *defaultFunctionRegistry) Register(name string, f UDF) error {
	fr.m.Lock()
	defer fr.m.Unlock()
	if _, exists := fr.funcs[name]; exists {
		return fmt.Errorf("there is already a function named '%s'", name)
	}
	fr.funcs[name] = f
	return nil
}

var (
	// globalUDFRegistry has UDFs visible to all topologies. Do NOT use this
	// instance in running topologies because it doesn't have an actual
	// Context. Call CopyGlobalUDSRegistry to retrieve a new valid instance.
	globalUDFRegistry = NewDefaultFunctionRegistry(nil).(*defaultFunctionRegistry)
)

// RegisterGlobalUDF adds a UDF which is visible to all topologies. UDFs
// registered after running topologies might not be seen by those topologies.
// Call it from init functions to avoid such conditions.
func RegisterGlobalUDF(name string, f UDF) error {
	return globalUDFRegistry.Register(name, f)
}

// CopyGlobalUDFRegistry creates a new FunctionManager which has all UDFs
// registered to the global function manager.
func CopyGlobalUDFRegistry(ctx *core.Context) FunctionManager {
	globalUDFRegistry.m.RLock()
	defer globalUDFRegistry.m.RUnlock()
	reg := &defaultFunctionRegistry{
		ctx:   ctx,
		funcs: make(map[string]UDF, len(globalUDFRegistry.funcs)),
	}

	for n, f := range globalUDFRegistry.funcs {
		reg.funcs[n] = f
	}
	return reg
}

func init() {
	// register some standard functions
	toString := func(ctx *core.Context, v data.Value) (data.Value, error) {
		return data.String(v.String()), nil
	}
	globalUDFRegistry.Register("str", UnaryFunc(toString))
}
<commit_msg>Add udf.MustRegisterGlobalUDF<commit_after>package udf

import (
	"fmt"
	"pfi/sensorbee/sensorbee/core"
	"pfi/sensorbee/sensorbee/data"
	"sync"
)

// UDF is an interface having a user defined function.
type UDF interface {
	// Call calls the UDF.
	Call(*core.Context, ...data.Value) (data.Value, error)

	// Accept checks if the function accepts the given number of arguments
	// excluding core.Context.
	Accept(arity int) bool
}

type function struct {
	f     func(*core.Context, ...data.Value) (data.Value, error)
	arity int
}

func (f *function) Call(ctx *core.Context, args ...data.Value) (data.Value, error) {
	return f.f(ctx, args...)
}

func (f *function) Accept(arity int) bool {
	if f.arity < 0 {
		return true
	}
	return arity == f.arity
}

// VariadicFunc creates a UDF based on a function receiving the variadic number
// of data.Values.
func VariadicFunc(f func(*core.Context, ...data.Value) (data.Value, error)) UDF {
	return &function{
		f:     f,
		arity: -1,
	}
}

func Func(f func(*core.Context, ...data.Value) (data.Value, error), arity int) UDF {
	return &function{
		f:     f,
		arity: arity,
	}
}

func NullaryFunc(f func(*core.Context) (data.Value, error)) UDF {
	genFunc := func(ctx *core.Context, vs ...data.Value) (data.Value, error) {
		if len(vs) != 0 {
			return nil, fmt.Errorf("the function should be used as nullary")
		}
		return f(ctx)
	}
	return Func(genFunc, 0)
}

func UnaryFunc(f func(*core.Context, data.Value) (data.Value, error)) UDF {
	genFunc := func(ctx *core.Context, vs ...data.Value) (data.Value, error) {
		if len(vs) != 1 {
			return nil, fmt.Errorf("the function should be used as unary")
		}
		return f(ctx, vs[0])
	}
	return Func(genFunc, 1)
}

func BinaryFunc(f func(*core.Context, data.Value, data.Value) (data.Value, error)) UDF {
	genFunc := func(ctx *core.Context, vs ...data.Value) (data.Value, error) {
		if len(vs) != 2 {
			return nil, fmt.Errorf("the function should be used as binary")
		}
		return f(ctx, vs[0], vs[1])
	}
	return Func(genFunc, 2)
}

func TernaryFunc(f func(*core.Context, data.Value, data.Value, data.Value) (data.Value, error)) UDF {
	genFunc := func(ctx *core.Context, vs ...data.Value) (data.Value, error) {
		if len(vs) != 3 {
			return nil, fmt.Errorf("the function should be used as ternary")
		}
		return f(ctx, vs[0], vs[1], vs[2])
	}
	return Func(genFunc, 3)
}

// TODO: Add magic UDF generator func NewUDF(f interface{}) (UDF, error)
//       It accepts any function whose arguments are convertible to data.Value.
//       For example, NewUDF(func(*core.Context, a, b int) (int, error) {return a + b}).
//       Even NewUDF(func(a, b int) int {return a + b}) could be valid
//       (i.e. Context and error can be optional)

// FunctionRegistry is an interface to lookup functions for use in BQL
// statements by their name.
type FunctionRegistry interface {
	// Context returns a core.Context associated with the registry.
	Context() *core.Context

	// Lookup will return a function with the given name and arity
	// or an error if there is none. Note that this interface allows
	// multiple functions with the same name but different arity,
	// and it also allows functions with an arbitrary number of
	// parameters. However, a function returned must never be used
	// with a different arity than the one given in the Lookup call.
	Lookup(name string, arity int) (UDF, error)
}

// FunctionManager is a FunctionRegistry that allows to register
// additional functions.
type FunctionManager interface {
	FunctionRegistry

	// Register allows to add a function.
	Register(name string, f UDF) error
}

type defaultFunctionRegistry struct {
	ctx   *core.Context
	m     sync.RWMutex
	funcs map[string]UDF
}

func NewDefaultFunctionRegistry(ctx *core.Context) FunctionManager {
	reg := &defaultFunctionRegistry{
		ctx:   ctx,
		funcs: map[string]UDF{},
	}
	return reg
}

func (fr *defaultFunctionRegistry) Context() *core.Context {
	return fr.ctx
}

func (fr *defaultFunctionRegistry) Lookup(name string, arity int) (UDF, error) {
	fr.m.RLock()
	defer fr.m.RUnlock()
	if f, exists := fr.funcs[name]; exists {
		if f.Accept(arity) {
			return f, nil
		}
		return nil, fmt.Errorf("function '%s' is not %d-ary", name, arity)
	}
	return nil, fmt.Errorf("function '%s' is unknown", name)
}

func (fr *defaultFunctionRegistry) Register(name string, f UDF) error {
	fr.m.Lock()
	defer fr.m.Unlock()
	if _, exists := fr.funcs[name]; exists {
		return fmt.Errorf("there is already a function named '%s'", name)
	}
	fr.funcs[name] = f
	return nil
}

var (
	// globalUDFRegistry has UDFs visible to all topologies. Do NOT use this
	// instance in running topologies because it doesn't have an actual
	// Context. Call CopyGlobalUDSRegistry to retrieve a new valid instance.
	globalUDFRegistry = NewDefaultFunctionRegistry(nil).(*defaultFunctionRegistry)
)

// RegisterGlobalUDF adds a UDF which is visible to all topologies. UDFs
// registered after running topologies might not be seen by those topologies.
// Call it from init functions to avoid such conditions.
func RegisterGlobalUDF(name string, f UDF) error {
	return globalUDFRegistry.Register(name, f)
}

// MustRegisterGlobalUDF is like RegisterGlobalUDF but
// panics if the name is already registered.
func MustRegisterGlobalUDF(name string, f UDF) {
	if err := globalUDFRegistry.Register(name, f); err != nil {
		panic(fmt.Errorf("udf.MustRegisterGlobalUDF: cannot register '%v': %v", name, err))
	}
}

// CopyGlobalUDFRegistry creates a new FunctionManager which has all UDFs
// registered to the global function manager.
func CopyGlobalUDFRegistry(ctx *core.Context) FunctionManager {
	globalUDFRegistry.m.RLock()
	defer globalUDFRegistry.m.RUnlock()
	reg := &defaultFunctionRegistry{
		ctx:   ctx,
		funcs: make(map[string]UDF, len(globalUDFRegistry.funcs)),
	}

	for n, f := range globalUDFRegistry.funcs {
		reg.funcs[n] = f
	}
	return reg
}

func init() {
	// register some standard functions
	toString := func(ctx *core.Context, v data.Value) (data.Value, error) {
		return data.String(v.String()), nil
	}
	globalUDFRegistry.Register("str", UnaryFunc(toString))
}
<|endoftext|>


<commit_before>package main

/*
This could probably be smarter, but it'll do for now.
*/

const HTML_STOPPED = `

Your service is currently powered down

Click here to start.

	`

const HTML_STARTING = `

Your service is starting, please wait.

Your site will be loaded once startup is complete.

	`

const HTML_STOPPING = `

Your service is being powered down.

Please wait for shutdown to complete before restarting.

	`

const HTML_UNHEALTHY = `

Your service appears to be in an unhealthy or inconsistent state

This may be a temporary error, or may require manual intervention.

	`

const HTML_ERROR = `

An error occured processing your request

%v

	`
<commit_msg>Tweak html again<commit_after>package main

/*
This could probably be smarter, but it'll do for now.
*/

const HTML_STOPPED = `

Your service is currently powered down

Click here to start.

	`

const HTML_STARTING = `

Your service is starting, please wait.

Your site will be loaded once startup is complete.

	`

const HTML_STOPPING = `

Your service is being powered down.

Please wait for shutdown to complete before restarting.

	`

const HTML_UNHEALTHY = `

Your service appears to be in an unhealthy or inconsistent state

This may be a temporary error, or may require manual intervention.

	`

const HTML_ERROR = `

An error occured processing your request

%v

	`
<|endoftext|>