<commit_before>// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
)

// Repo represents the repository under test
// For more information about this structure take a look to the README
type Repo struct {
	// URL is the url of the repository
	URL string

	// MasterBranch is the master branch of this repository
	MasterBranch string

	// PR is the pull request number
	PR int

	// RefreshTime is the time to wait for checking if a pull request needs to be tested
	RefreshTime string

	// Toke is the repository access token
	Token string

	// Setup contains the conmmands needed to setup the environment
	Setup []string

	// Run contains the commands to run the test
	Run []string

	// Teardown contains the commands to be executed once Run ends
	Teardown []string

	// OnSuccess contains the commands to be executed if Setup, Run and Teardown finished correctly
	OnSuccess []string

	// OnFailure contains the commands to be executed if any of Setup, Run or Teardown fail
	OnFailure []string

	// TTY specify whether a tty must be allocate to run the stages
	TTY bool

	// PostOnSuccess is the comment to be posted if the test finished correctly
	PostOnSuccess string

	// PostOnFailure is the comment to be posted if the test fails
	PostOnFailure string

	// LogDir is the logs directory
	LogDir string

	// Language is the language of the repository
	Language RepoLanguage

	// CommentTrigger is the comment that must be present to trigger the test
	CommentTrigger RepoComment

	// LogServer contains the information of the server where the logs must be placed
	LogServer LogServer

	// Whitelist is the list of users whose pull request can be tested
	Whitelist string

	// cvr control version repository
	cvr CVR

	// refresh is RefreshTime once parsed
	refresh time.Duration

	// env contains the environment variables to be used in each stage
	env []string

	// whitelistUsers is the whitelist once parsed
	whitelistUsers []string

	// logger of the repository
	logger *logrus.Entry

	// prConfig is the configuration used to create pull request objects
	prConfig pullRequestConfig
}

const (
	logDirMode    = 0755
	logFileMode   = 0664
	logServerUser = "root"
)

var defaultEnv = []string{"CI=true", "LOCALCI=true"}

var runTestsInParallel bool

var testLock sync.Mutex

func (r *Repo) setupCvr() error {
	var err error

	// validate url
	r.URL = strings.TrimSpace(r.URL)
	if len(r.URL) == 0 {
		return fmt.Errorf("missing repository url")
	}

	// set repository logger
	r.logger = ciLog.WithFields(logrus.Fields{
		"Repo": r.URL,
	})

	// get the control version repository
	r.cvr, err = newCVR(r.URL, r.Token)
	r.logger.Debugf("control version repository: %#v", r.cvr)

	return err
}

func (r *Repo) setupLogServer() error {
	if reflect.DeepEqual(r.LogServer, LogServer{}) {
		return nil
	}

	if len(r.LogServer.IP) == 0 {
		return fmt.Errorf("missing server ip")
	}

	if len(r.LogServer.User) == 0 {
		r.LogServer.User = logServerUser
	}

	if len(r.LogServer.Dir) == 0 {
		r.LogServer.Dir = defaultLogDir
	}

	return nil
}

func (r *Repo) setupLogDir() error {
	// create log directory
	if err := os.MkdirAll(r.LogDir, logDirMode); err != nil {
		return err
	}

	return nil
}

func (r *Repo) setupRefreshTime() error {
	var err error

	// validate refresh time
	r.refresh, err = time.ParseDuration(r.RefreshTime)
	if err != nil {
		return fmt.Errorf("failed to parse refresh time '%s' %s", r.RefreshTime, err)
	}

	return nil
}

func (r *Repo) setupCommentTrigger() error {
	if reflect.DeepEqual(r.CommentTrigger, RepoComment{}) {
		return nil
	}

	if len(r.CommentTrigger.Comment) == 0 {
		return fmt.Errorf("missing comment trigger")
	}

	return nil
}

func (r *Repo) setupLanguage() error {
	return r.Language.setup()
}

func (r *Repo) setupStages() error {
	if len(r.Run) == 0 {
		return fmt.Errorf("missing run commands")
	}

	return nil
}

func (r *Repo) setupWhitelist() error {
	// get the list of users
	r.whitelistUsers = strings.Split(r.Whitelist, ",")
	return nil
}

func (r *Repo) setupEnvars() error {
	// add environment variables
	r.env = os.Environ()
	r.env = append(r.env, defaultEnv...)
	repoSlug := fmt.Sprintf("LOCALCI_REPO_SLUG=%s", r.cvr.getRepoSlug())
	r.env = append(r.env, repoSlug)

	return nil
}

// setup the repository. This method MUST BE called before use any other
func (r *Repo) setup() error {
	var err error

	setupFuncs := []func() error{
		r.setupCvr,
		r.setupRefreshTime,
		r.setupLogDir,
		r.setupLogServer,
		r.setupCommentTrigger,
		r.setupLanguage,
		r.setupStages,
		r.setupWhitelist,
		r.setupEnvars,
	}

	for _, setupFunc := range setupFuncs {
		if err = setupFunc(); err != nil {
			return err
		}
	}

	r.prConfig = pullRequestConfig{
		cvr:            r.cvr,
		logger:         r.logger,
		commentTrigger: r.CommentTrigger,
		postOnFailure:  r.PostOnFailure,
		postOnSuccess:  r.PostOnSuccess,
	}

	r.logger.Debugf("control version repository: %#v", r.cvr)

	return nil
}

// loop to monitor the repository
func (r *Repo) loop() {
	revisionsTested := make(map[string]revision)

	r.logger.Debugf("monitoring in a loop the repository: %+v", *r)

	appendPullRequests := func(revisions *[]revision, prs []int) error {
		for _, prNumber := range prs {
			r.logger.Debugf("requesting pull request %d", prNumber)
			pr, err := newPullRequest(prNumber, r.prConfig)
			if err != nil {
				return fmt.Errorf("failed to get pull request '%d' %s", prNumber, err)
			}
			*revisions = append(*revisions, pr)
		}
		return nil
	}

	for {
		var revisionsToTest []revision

		// append master branch
		r.logger.Debugf("requesting master branch: %s", r.MasterBranch)
		branch, err := newRepoBranch(r.MasterBranch, r.cvr, r.logger)
		if err != nil {
			r.logger.Warnf("failed to get master branch %s: %s", r.MasterBranch, err)
		} else {
			revisionsToTest = append(revisionsToTest, branch)
		}

		// append pull requests
		if r.PR != 0 {
			// if PR is not 0 then we have to monitor just one PR
			if err = appendPullRequests(&revisionsToTest, []int{r.PR}); err != nil {
				r.logger.Warnf("failed to append pull request %d", r.PR, err)
			}
		} else {
			// append open pull request
			r.logger.Debugf("requesting open pull requests")
			prs, err := r.cvr.getOpenPullRequests()
			if err != nil {
				r.logger.Warnf("failed to get open pull requests: %s", err)
			} else if err = appendPullRequests(&revisionsToTest, prs); err != nil {
				r.logger.Warnf("failed to append pull requests %+v: %s", prs, err)
			}
		}

		r.logger.Debugf("testing revisions: %#v", revisionsToTest)
		r.testRevisions(revisionsToTest, &revisionsTested)

		r.logger.Debugf("going to sleep: %s", r.RefreshTime)
		time.Sleep(r.refresh)
	}
}

func (r *Repo) testRevisions(revisions []revision, revisionsTested *map[string]revision) {
	// remove revisions that are not in the list of open pull request and already tested
	for k, v := range *revisionsTested {
		found := false
		// iterate over open pull requests and master branch
		for _, r := range revisions {
			if r.id() == k {
				found = true
				break
			}
		}

		if !found && !v.isBeingTested() {
			delete((*revisionsTested), k)
		}
	}

	for _, revision := range revisions {
		tested, ok := (*revisionsTested)[revision.id()]
		if ok {
			// checking if the old version of the PR is being tested
			if tested.isBeingTested() {
				r.logger.Debugf("revision is being tested: %#v", tested)
				continue
			}
			if revision.equal(tested) {
				r.logger.Debugf("revision was already tested: %#v", revision)
				continue
			}
		}

		// setup revision
		langEnv, err := r.setupRevision(revision)
		if err != nil {
			r.logger.Errorf("failed to setup revision %#v: %s", revision, err)
			continue
		}

		// cleanup revision
		defer func() {
			err = langEnv.cleanup()
			if err != nil {
				r.logger.Error(err)
			}
		}()

		// test revision
		if runTestsInParallel {
			go func() {
				if err := r.testRevision(revision, langEnv); err != nil {
					r.logger.Errorf("failed to test revision %#v %s", revision, err)
				}
			}()
		} else {
			testLock.Lock()
			if err := r.testRevision(revision, langEnv); err != nil {
				r.logger.Errorf("failed to test revision %#v %s", revision, err)
			}
			testLock.Unlock()
		}

		// copy the PR that was tested
		(*revisionsTested)[revision.id()] = revision
	}
}

// test the pull request specified in the configuration file
// if pr does not exist an error is returned
func (r *Repo) test() error {
	if r.PR == 0 {
		return fmt.Errorf("Missing pull request number in configuration file")
	}

	rev, err := newPullRequest(r.PR, r.prConfig)
	if err != nil {
		return fmt.Errorf("failed to get pull request %d %s", r.PR, err)
	}

	// run tests in parallel does not make sense when
	// we are just testing one pull request
	runTestsInParallel = false

	// setup revision
	langEnv, err := r.setupRevision(rev)
	if err != nil {
		return fmt.Errorf("failed to setup revision %#v: %s", rev, err)
	}

	// cleanup revision
	defer func() {
		err = langEnv.cleanup()
		if err != nil {
			r.logger.Error(err)
		}
	}()

	// test revision
	return r.testRevision(rev, langEnv)
}

// setupRevision generates a language environment, downloads the revision
// and creates the logs directory
func (r *Repo) setupRevision(rev revision) (languageConfig, error) {
	// generate a new environment to run the stages
	langEnv, err := r.Language.generateEnvironment(r.cvr.getProjectSlug())
	if err != nil {
		return languageConfig{}, err
	}

	// download the revision
	if err = rev.download(langEnv.workingDir); err != nil {
		return languageConfig{}, err
	}

	// cleanup and set the log directory of the pull request
	logDir := filepath.Join(r.LogDir, rev.logDirName())
	_ = os.RemoveAll(logDir)
	if err = os.MkdirAll(logDir, logDirMode); err != nil {
		return languageConfig{}, err
	}

	return langEnv, nil
}

// testRevision tests a specific revision
// returns an error if the test fail
func (r *Repo) testRevision(rev revision, langEnv languageConfig) error {
	config := stageConfig{
		logger:     r.logger,
		workingDir: langEnv.workingDir,
		tty:        r.TTY,
	}

	// set environment variables
	config.env = r.env

	// appends language environment variables
	if len(langEnv.env) > 0 {
		config.env = append(config.env, langEnv.env...)
	}

	// copy logs to server if we have an IP address
	if len(r.LogServer.IP) != 0 {
		defer func() {
			if err := r.LogServer.copy(config.logDir); err != nil {
				r.logger.Errorf("failed to copy log dir %s to server %+v", config.logDir, r.LogServer)
			}
		}()
	}

	r.logger.Debugf("stage config: %+v", config)

	stages := map[string]stage{
		"setup":     stage{name: "setup", commands: r.Setup},
		"run":       stage{name: "run", commands: r.Run},
		"teardown":  stage{name: "teardown", commands: r.Teardown},
		"onSuccess": stage{name: "onSuccess", commands: r.OnSuccess},
		"onFailure": stage{name: "onFailure", commands: r.OnFailure},
	}

	// run test
	return rev.test(config, stages)
}
<commit_msg>localCI: fix running as a systemd service<commit_after>// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
)

// Repo represents the repository under test
// For more information about this structure take a look to the README
type Repo struct {
	// URL is the url of the repository
	URL string

	// MasterBranch is the master branch of this repository
	MasterBranch string

	// PR is the pull request number
	PR int

	// RefreshTime is the time to wait for checking if a pull request needs to be tested
	RefreshTime string

	// Toke is the repository access token
	Token string

	// Setup contains the conmmands needed to setup the environment
	Setup []string

	// Run contains the commands to run the test
	Run []string

	// Teardown contains the commands to be executed once Run ends
	Teardown []string

	// OnSuccess contains the commands to be executed if Setup, Run and Teardown finished correctly
	OnSuccess []string

	// OnFailure contains the commands to be executed if any of Setup, Run or Teardown fail
	OnFailure []string

	// TTY specify whether a tty must be allocate to run the stages
	TTY bool

	// PostOnSuccess is the comment to be posted if the test finished correctly
	PostOnSuccess string

	// PostOnFailure is the comment to be posted if the test fails
	PostOnFailure string

	// LogDir is the logs directory
	LogDir string

	// Language is the language of the repository
	Language RepoLanguage

	// CommentTrigger is the comment that must be present to trigger the test
	CommentTrigger RepoComment

	// LogServer contains the information of the server where the logs must be placed
	LogServer LogServer

	// Whitelist is the list of users whose pull request can be tested
	Whitelist string

	// cvr control version repository
	cvr CVR

	// refresh is RefreshTime once parsed
	refresh time.Duration

	// env contains the environment variables to be used in each stage
	env []string

	// whitelistUsers is the whitelist once parsed
	whitelistUsers []string

	// logger of the repository
	logger *logrus.Entry

	// prConfig is the configuration used to create pull request objects
	prConfig pullRequestConfig
}

const (
	logDirMode    = 0755
	logFileMode   = 0664
	logServerUser = "root"
)

var defaultEnv = []string{"CI=true", "LOCALCI=true"}

var runTestsInParallel bool

var testLock sync.Mutex

func (r *Repo) setupCvr() error {
	var err error

	// validate url
	r.URL = strings.TrimSpace(r.URL)
	if len(r.URL) == 0 {
		return fmt.Errorf("missing repository url")
	}

	// set repository logger
	r.logger = ciLog.WithFields(logrus.Fields{
		"Repo": r.URL,
	})

	// get the control version repository
	r.cvr, err = newCVR(r.URL, r.Token)
	r.logger.Debugf("control version repository: %#v", r.cvr)

	return err
}

func (r *Repo) setupLogServer() error {
	if reflect.DeepEqual(r.LogServer, LogServer{}) {
		return nil
	}

	if len(r.LogServer.IP) == 0 {
		return fmt.Errorf("missing server ip")
	}

	if len(r.LogServer.User) == 0 {
		r.LogServer.User = logServerUser
	}

	if len(r.LogServer.Dir) == 0 {
		r.LogServer.Dir = defaultLogDir
	}

	return nil
}

func (r *Repo) setupLogDir() error {
	// create log directory
	if err := os.MkdirAll(r.LogDir, logDirMode); err != nil {
		return err
	}

	return nil
}

func (r *Repo) setupRefreshTime() error {
	var err error

	// validate refresh time
	r.refresh, err = time.ParseDuration(r.RefreshTime)
	if err != nil {
		return fmt.Errorf("failed to parse refresh time '%s' %s", r.RefreshTime, err)
	}

	return nil
}

func (r *Repo) setupCommentTrigger() error {
	if reflect.DeepEqual(r.CommentTrigger, RepoComment{}) {
		return nil
	}

	if len(r.CommentTrigger.Comment) == 0 {
		return fmt.Errorf("missing comment trigger")
	}

	return nil
}

func (r *Repo) setupLanguage() error {
	return r.Language.setup()
}

func (r *Repo) setupStages() error {
	if len(r.Run) == 0 {
		return fmt.Errorf("missing run commands")
	}

	return nil
}

func (r *Repo) setupWhitelist() error {
	// get the list of users
	r.whitelistUsers = strings.Split(r.Whitelist, ",")
	return nil
}

func (r *Repo) setupEnvars() error {
	// add environment variables
	r.env = os.Environ()
	r.env = append(r.env, defaultEnv...)
	repoSlug := fmt.Sprintf("LOCALCI_REPO_SLUG=%s", r.cvr.getRepoSlug())
	r.env = append(r.env, repoSlug)

	return nil
}

// setup the repository. This method MUST BE called before use any other
func (r *Repo) setup() error {
	var err error

	setupFuncs := []func() error{
		r.setupCvr,
		r.setupRefreshTime,
		r.setupLogDir,
		r.setupLogServer,
		r.setupCommentTrigger,
		r.setupLanguage,
		r.setupStages,
		r.setupWhitelist,
		r.setupEnvars,
	}

	for _, setupFunc := range setupFuncs {
		if err = setupFunc(); err != nil {
			return err
		}
	}

	r.prConfig = pullRequestConfig{
		cvr:            r.cvr,
		logger:         r.logger,
		commentTrigger: r.CommentTrigger,
		postOnFailure:  r.PostOnFailure,
		postOnSuccess:  r.PostOnSuccess,
	}

	r.logger.Debugf("control version repository: %#v", r.cvr)

	return nil
}

// loop to monitor the repository
func (r *Repo) loop() {
	revisionsTested := make(map[string]revision)

	r.logger.Debugf("monitoring in a loop the repository: %+v", *r)

	appendPullRequests := func(revisions *[]revision, prs []int) error {
		for _, prNumber := range prs {
			r.logger.Debugf("requesting pull request %d", prNumber)
			pr, err := newPullRequest(prNumber, r.prConfig)
			if err != nil {
				return fmt.Errorf("failed to get pull request '%d' %s", prNumber, err)
			}
			*revisions = append(*revisions, pr)
		}
		return nil
	}

	for {
		var revisionsToTest []revision

		// append master branch
		r.logger.Debugf("requesting master branch: %s", r.MasterBranch)
		branch, err := newRepoBranch(r.MasterBranch, r.cvr, r.logger)
		if err != nil {
			r.logger.Warnf("failed to get master branch %s: %s", r.MasterBranch, err)
		} else {
			revisionsToTest = append(revisionsToTest, branch)
		}

		// append pull requests
		if r.PR != 0 {
			// if PR is not 0 then we have to monitor just one PR
			if err = appendPullRequests(&revisionsToTest, []int{r.PR}); err != nil {
				r.logger.Warnf("failed to append pull request %d", r.PR, err)
			}
		} else {
			// append open pull request
			r.logger.Debugf("requesting open pull requests")
			prs, err := r.cvr.getOpenPullRequests()
			if err != nil {
				r.logger.Warnf("failed to get open pull requests: %s", err)
			} else if err = appendPullRequests(&revisionsToTest, prs); err != nil {
				r.logger.Warnf("failed to append pull requests %+v: %s", prs, err)
			}
		}

		r.logger.Debugf("testing revisions: %#v", revisionsToTest)
		r.testRevisions(revisionsToTest, &revisionsTested)

		r.logger.Debugf("going to sleep: %s", r.RefreshTime)
		time.Sleep(r.refresh)
	}
}

func (r *Repo) testRevisions(revisions []revision, revisionsTested *map[string]revision) {
	// remove revisions that are not in the list of open pull request and already tested
	for k, v := range *revisionsTested {
		found := false
		// iterate over open pull requests and master branch
		for _, r := range revisions {
			if r.id() == k {
				found = true
				break
			}
		}

		if !found && !v.isBeingTested() {
			delete((*revisionsTested), k)
		}
	}

	for _, revision := range revisions {
		tested, ok := (*revisionsTested)[revision.id()]
		if ok {
			// checking if the old version of the PR is being tested
			if tested.isBeingTested() {
				r.logger.Debugf("revision is being tested: %#v", tested)
				continue
			}
			if revision.equal(tested) {
				r.logger.Debugf("revision was already tested: %#v", revision)
				continue
			}
		}

		// setup revision
		langEnv, err := r.setupRevision(revision)
		if err != nil {
			r.logger.Errorf("failed to setup revision %#v: %s", revision, err)
			continue
		}

		// cleanup revision
		defer func() {
			err = langEnv.cleanup()
			if err != nil {
				r.logger.Error(err)
			}
		}()

		// test revision
		if runTestsInParallel {
			go func() {
				if err := r.testRevision(revision, langEnv); err != nil {
					r.logger.Errorf("failed to test revision %#v %s", revision, err)
				}
			}()
		} else {
			testLock.Lock()
			if err := r.testRevision(revision, langEnv); err != nil {
				r.logger.Errorf("failed to test revision %#v %s", revision, err)
			}
			testLock.Unlock()
		}

		// copy the PR that was tested
		(*revisionsTested)[revision.id()] = revision
	}
}

// test the pull request specified in the configuration file
// if pr does not exist an error is returned
func (r *Repo) test() error {
	if r.PR == 0 {
		return fmt.Errorf("Missing pull request number in configuration file")
	}

	rev, err := newPullRequest(r.PR, r.prConfig)
	if err != nil {
		return fmt.Errorf("failed to get pull request %d %s", r.PR, err)
	}

	// run tests in parallel does not make sense when
	// we are just testing one pull request
	runTestsInParallel = false

	// setup revision
	langEnv, err := r.setupRevision(rev)
	if err != nil {
		return fmt.Errorf("failed to setup revision %#v: %s", rev, err)
	}

	// cleanup revision
	defer func() {
		err = langEnv.cleanup()
		if err != nil {
			r.logger.Error(err)
		}
	}()

	// test revision
	return r.testRevision(rev, langEnv)
}

// setupRevision generates a language environment, downloads the revision
// and creates the logs directory
func (r *Repo) setupRevision(rev revision) (languageConfig, error) {
	// generate a new environment to run the stages
	langEnv, err := r.Language.generateEnvironment(r.cvr.getProjectSlug())
	if err != nil {
		return languageConfig{}, err
	}

	// download the revision
	if err = rev.download(langEnv.workingDir); err != nil {
		return languageConfig{}, err
	}

	// cleanup and set the log directory of the pull request
	logDir := filepath.Join(r.LogDir, rev.logDirName())
	_ = os.RemoveAll(logDir)
	if err = os.MkdirAll(logDir, logDirMode); err != nil {
		return languageConfig{}, err
	}

	return langEnv, nil
}

// testRevision tests a specific revision
// returns an error if the test fail
func (r *Repo) testRevision(rev revision, langEnv languageConfig) error {
	config := stageConfig{
		logger:     r.logger,
		workingDir: langEnv.workingDir,
		tty:        r.TTY,
		logDir:     filepath.Join(r.LogDir, rev.logDirName()),
	}

	// set environment variables
	config.env = r.env

	// appends language environment variables
	if len(langEnv.env) > 0 {
		config.env = append(config.env, langEnv.env...)
	}

	// copy logs to server if we have an IP address
	if len(r.LogServer.IP) != 0 {
		defer func() {
			if err := r.LogServer.copy(config.logDir); err != nil {
				r.logger.Errorf("failed to copy log dir %s to server %+v", config.logDir, r.LogServer)
			}
		}()
	}

	r.logger.Debugf("stage config: %+v", config)

	stages := map[string]stage{
		"setup":     stage{name: "setup", commands: r.Setup},
		"run":       stage{name: "run", commands: r.Run},
		"teardown":  stage{name: "teardown", commands: r.Teardown},
		"onSuccess": stage{name: "onSuccess", commands: r.OnSuccess},
		"onFailure": stage{name: "onFailure", commands: r.OnFailure},
	}

	// run test
	return rev.test(config, stages)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strings"

	"github.com/pkg/errors"

	"github.com/havoc-io/mutagen/cmd"
	"github.com/havoc-io/mutagen/daemon"
	"github.com/havoc-io/mutagen/rpc"
	sessionpkg "github.com/havoc-io/mutagen/session"
	"github.com/havoc-io/mutagen/sync"
)

var listUsage = `usage: mutagen list [-h|--help] [-m|--monitor] [<session>]

Lists existing synchronization sessions and their statuses. A specific session
identifier can be specified to show information for only that session. If
coupled with the -m/--monitor flag, the list command will show a dynamic display
of synchronization status for the specified session.
`

func printSession(monitor bool, state sessionpkg.SessionState) {
	// Print the session identifier.
	fmt.Println("Session:", state.Session.Identifier)

	// If we're in monitor mode, that's all the information we print.
	if monitor {
		return
	}

	// Print status.
	statusString := state.State.Status.String()
	if state.Session.Paused {
		statusString = "Paused"
	}
	fmt.Println("Status:", statusString)

	// Print the last error, if any.
	if state.State.LastError != "" {
		fmt.Println("Last error:", state.State.LastError)
	}
}

func formatConnectionStatus(connected bool) string {
	if connected {
		return "Connected"
	}
	return "Disconnected"
}

func printEndpoint(monitor, alpha bool, state sessionpkg.SessionState) {
	// Print the header and URL. We combine them in monitoring mode.
	header := "Alpha:"
	url := state.Session.Alpha
	if !alpha {
		header = "Beta:"
		url = state.Session.Beta
	}
	if monitor {
		fmt.Println(header, url.Format())
	} else {
		fmt.Println(header)
		fmt.Println("\tURL:", url.Format())
	}

	// If we're in mointor mode, that's all the information we print.
	if monitor {
		return
	}

	// Print status.
	connected := state.State.AlphaConnected
	if !alpha {
		connected = state.State.BetaConnected
	}
	fmt.Println("\tStatus:", formatConnectionStatus(connected))

	// Print problems, if any.
	problems := state.State.AlphaProblems
	if !alpha {
		problems = state.State.BetaProblems
	}
	if len(problems) > 0 {
		fmt.Println("\tProblems:")
		for _, p := range problems {
			fmt.Printf("\t\t%s: %v\n", p.Path, p.Error)
		}
	}
}

func formatEntryKind(entry *sync.Entry) string {
	if entry == nil {
		return "<non-existent>"
	} else if entry.Kind == sync.EntryKind_Directory {
		return "Directory"
	} else if entry.Kind == sync.EntryKind_File {
		return "File"
	} else {
		return "<unknown>"
	}
}

func printConflicts(conflicts []sync.Conflict) {
	// Print the header.
	fmt.Println("Conflicts:")

	// Print conflicts.
	for i, c := range conflicts {
		// Print the alpha changes.
		for _, a := range c.AlphaChanges {
			fmt.Printf(
				"\t(α) %s (%s -> %s)\n",
				a.Path,
				formatEntryKind(a.Old),
				formatEntryKind(a.New),
			)
		}

		// Print the beta changes.
		for _, b := range c.BetaChanges {
			fmt.Printf(
				"\t(β) %s (%s -> %s)\n",
				b.Path,
				formatEntryKind(b.Old),
				formatEntryKind(b.New),
			)
		}

		// If we're not on the last conflict, print a newline.
		if i < len(conflicts)-1 {
			fmt.Println()
		}
	}
}

type connectionState struct {
	alphaConnected bool
	betaConnected  bool
}

var connectionStatePrefixes = map[connectionState]string{
	{false, false}: "XX",
	{true, false}:  "-X",
	{false, true}:  "X-",
	{true, true}:   "--",
}

func monitorPrefix(state sessionpkg.SessionState) string {
	switch state.State.Status {
	case sessionpkg.SynchronizationStatusDisconnected:
		fallthrough
	case sessionpkg.SynchronizationStatusConnecting:
		return connectionStatePrefixes[connectionState{
			state.State.AlphaConnected,
			state.State.BetaConnected,
		}]
	case sessionpkg.SynchronizationStatusInitializing:
		return "**"
	case sessionpkg.SynchronizationStatusScanning:
		return "--"
	case sessionpkg.SynchronizationStatusReconciling:
		return "~~"
	case sessionpkg.SynchronizationStatusStaging:
		return "><"
	case sessionpkg.SynchronizationStatusTransitioning:
		return "<>"
	case sessionpkg.SynchronizationStatusSaving:
		return "[]"
	default:
		return "  "
	}
}

func monitorConflictSummary(conflicts []sync.Conflict) string {
	if len(conflicts) > 0 {
		return "X"
	}
	return "-"
}

func monitorProblemSummary(problems []sync.Problem) string {
	if len(problems) > 0 {
		return "X"
	}
	return "-"
}

const monitorStatusBarInnerWidth = 31

func monitorStatusBar(status sessionpkg.StagingStatus) string {
	// If there is no staging going on, then return empty spaces.
	if status.Total == 0 {
		return fmt.Sprintf("[%s]", strings.Repeat(" ", monitorStatusBarInnerWidth))
	}

	// Watch for invalid or easy status cases.
	if status.Index >= status.Total {
		return fmt.Sprintf("[%s]", strings.Repeat("#", monitorStatusBarInnerWidth))
	}

	// Compute the number of spaces meant to be occupied by completed blocks.
	fractionCompleted := float32(status.Index) / float32(status.Total)
	completedSpaces := int(fractionCompleted * monitorStatusBarInnerWidth)

	// Compute the resultant bar.
	return fmt.Sprintf(
		"[%s%s]",
		strings.Repeat("#", completedSpaces),
		strings.Repeat("-", monitorStatusBarInnerWidth-completedSpaces),
	)
}

func printMonitorLine(state sessionpkg.SessionState) {
	// Print out a carriage return to wipe out the previous line.
	fmt.Print("\r")

	// Print the state prefix and a trailing space.
	fmt.Printf("%s ", monitorPrefix(state))

	// Print the conflict status and a trailing space.
	fmt.Printf("%s ", monitorConflictSummary(state.State.Conflicts))

	// Print the alpha status bar and a trailing space.
	fmt.Printf(
		"α(%s)%s ",
		monitorProblemSummary(state.State.AlphaProblems),
		monitorStatusBar(state.State.AlphaStaging),
	)

	// Print the beta status bar.
	fmt.Printf(
		"β(%s)%s",
		monitorProblemSummary(state.State.BetaProblems),
		monitorStatusBar(state.State.BetaStaging),
	)
}

func listMain(arguments []string) error {
	// Parse flags.
	var session string
	var monitor bool
	flagSet := cmd.NewFlagSet("list", listUsage, []int{0, 1})
	flagSet.BoolVarP(&monitor, "monitor", "m", false, "continuously monitor session")
	sessionArguments := flagSet.ParseOrDie(arguments)
	if len(sessionArguments) == 1 {
		session = sessionArguments[0]
	}

	// Check that options are sane.
	if monitor && session == "" {
		return errors.New("-m/--monitor only supported with single session")
	}

	// Create a daemon client.
	daemonClient := rpc.NewClient(daemon.NewOpener())

	// Invoke the session list method and ensure the resulting stream is closed
	// when we're done.
	stream, err := daemonClient.Invoke(sessionpkg.MethodList)
	if err != nil {
		return errors.Wrap(err, "unable to invoke session listing")
	}
	defer stream.Close()

	// Send the list request.
	if err := stream.Send(sessionpkg.ListRequest{
		Session: session,
		Monitor: monitor,
	}); err != nil {
		return errors.Wrap(err, "unable to send listing request")
	}

	// Loop indefinitely. We'll bail after a single response if monitoring
	// wasn't requested.
	printSessionInformation := true
	monitorLinePrinted := false
	for {
		// Receive the next response. If there's an error, clear the monitor
		// line (if any) before returning for better error legibility.
		var response sessionpkg.ListResponse
		if err := stream.Receive(&response); err != nil {
			if monitorLinePrinted {
				fmt.Println()
			}
			return errors.Wrap(err, "unable to receive listing response")
		}

		// The first time through the loop (which will be the only time if not
		// monitoring), we print the session state.
		if printSessionInformation {
			// Loop through and print sessions.
			for i, s := range response.Sessions {
				// Print the session information.
				printSession(monitor, s)

				// Print alpha information.
				printEndpoint(monitor, true, s)

				// Print beta information.
				printEndpoint(monitor, false, s)

				// Print conflicts (if any) if we're not in monitor mode.
				if !monitor && len(s.State.Conflicts) > 0 {
					printConflicts(s.State.Conflicts)
				}

				// If we're not in monitor mode and this isn't the last session,
				// print a newline. We don't really need the monitor check since
				// there should only be one session in monitor mode, but it is
				// safer in case the daemon has sent something weird back.
				if !monitor && i < len(response.Sessions)-1 {
					fmt.Println()
				}
			}

			// Mark the session information as printed.
			printSessionInformation = false
		}

		// If we're not monitoring, we're done, otherwise print the monitoring
		// line.
		if !monitor {
			return nil
		}

		// Validate the response for monitoring. If there's an error, clear the
		// monitor line (if any) before returning for better error legibility.
		if len(response.Sessions) != 1 {
			err = errors.New("invalid listing response")
		} else if response.Sessions[0].Session.Identifier != session {
			err = errors.New("listing response returned invalid session")
		}
		if err != nil {
			if monitorLinePrinted {
				fmt.Println()
			}
			return err
		}

		// Print the monitoring line and record that we've done so.
		printMonitorLine(response.Sessions[0])
		monitorLinePrinted = true

		// Send another (empty) request to let the daemon know that we're ready
		// for another response. This is a backpressure mechanism to keep the
		// daemon from sending more requests than we can handle in monitor mode.
		// If there's an error, clear the monitor line (if any) before returning
		// for better error legibility.
		if err := stream.Send(sessionpkg.ListRequest{}); err != nil {
			if monitorLinePrinted {
				fmt.Println()
			}
			return errors.Wrap(err, "unable to send ready request")
		}
	}
}
<commit_msg>Made monitor status bar one column narrower each.<commit_after>package main

import (
	"fmt"
	"strings"

	"github.com/pkg/errors"

	"github.com/havoc-io/mutagen/cmd"
	"github.com/havoc-io/mutagen/daemon"
	"github.com/havoc-io/mutagen/rpc"
	sessionpkg "github.com/havoc-io/mutagen/session"
	"github.com/havoc-io/mutagen/sync"
)

var listUsage = `usage: mutagen list [-h|--help] [-m|--monitor] [<session>]

Lists existing synchronization sessions and their statuses. A specific session
identifier can be specified to show information for only that session. If
coupled with the -m/--monitor flag, the list command will show a dynamic display
of synchronization status for the specified session.
`

func printSession(monitor bool, state sessionpkg.SessionState) {
	// Print the session identifier.
	fmt.Println("Session:", state.Session.Identifier)

	// If we're in monitor mode, that's all the information we print.
	if monitor {
		return
	}

	// Print status.
	statusString := state.State.Status.String()
	if state.Session.Paused {
		statusString = "Paused"
	}
	fmt.Println("Status:", statusString)

	// Print the last error, if any.
	if state.State.LastError != "" {
		fmt.Println("Last error:", state.State.LastError)
	}
}

func formatConnectionStatus(connected bool) string {
	if connected {
		return "Connected"
	}
	return "Disconnected"
}

func printEndpoint(monitor, alpha bool, state sessionpkg.SessionState) {
	// Print the header and URL. We combine them in monitoring mode.
	header := "Alpha:"
	url := state.Session.Alpha
	if !alpha {
		header = "Beta:"
		url = state.Session.Beta
	}
	if monitor {
		fmt.Println(header, url.Format())
	} else {
		fmt.Println(header)
		fmt.Println("\tURL:", url.Format())
	}

	// If we're in mointor mode, that's all the information we print.
	if monitor {
		return
	}

	// Print status.
	connected := state.State.AlphaConnected
	if !alpha {
		connected = state.State.BetaConnected
	}
	fmt.Println("\tStatus:", formatConnectionStatus(connected))

	// Print problems, if any.
	problems := state.State.AlphaProblems
	if !alpha {
		problems = state.State.BetaProblems
	}
	if len(problems) > 0 {
		fmt.Println("\tProblems:")
		for _, p := range problems {
			fmt.Printf("\t\t%s: %v\n", p.Path, p.Error)
		}
	}
}

func formatEntryKind(entry *sync.Entry) string {
	if entry == nil {
		return "<non-existent>"
	} else if entry.Kind == sync.EntryKind_Directory {
		return "Directory"
	} else if entry.Kind == sync.EntryKind_File {
		return "File"
	} else {
		return "<unknown>"
	}
}

func printConflicts(conflicts []sync.Conflict) {
	// Print the header.
	fmt.Println("Conflicts:")

	// Print conflicts.
	for i, c := range conflicts {
		// Print the alpha changes.
		for _, a := range c.AlphaChanges {
			fmt.Printf(
				"\t(α) %s (%s -> %s)\n",
				a.Path,
				formatEntryKind(a.Old),
				formatEntryKind(a.New),
			)
		}

		// Print the beta changes.
		for _, b := range c.BetaChanges {
			fmt.Printf(
				"\t(β) %s (%s -> %s)\n",
				b.Path,
				formatEntryKind(b.Old),
				formatEntryKind(b.New),
			)
		}

		// If we're not on the last conflict, print a newline.
		if i < len(conflicts)-1 {
			fmt.Println()
		}
	}
}

type connectionState struct {
	alphaConnected bool
	betaConnected  bool
}

var connectionStatePrefixes = map[connectionState]string{
	{false, false}: "XX",
	{true, false}:  "-X",
	{false, true}:  "X-",
	{true, true}:   "--",
}

func monitorPrefix(state sessionpkg.SessionState) string {
	switch state.State.Status {
	case sessionpkg.SynchronizationStatusDisconnected:
		fallthrough
	case sessionpkg.SynchronizationStatusConnecting:
		return connectionStatePrefixes[connectionState{
			state.State.AlphaConnected,
			state.State.BetaConnected,
		}]
	case sessionpkg.SynchronizationStatusInitializing:
		return "**"
	case sessionpkg.SynchronizationStatusScanning:
		return "--"
	case sessionpkg.SynchronizationStatusReconciling:
		return "~~"
	case sessionpkg.SynchronizationStatusStaging:
		return "><"
	case sessionpkg.SynchronizationStatusTransitioning:
		return "<>"
	case sessionpkg.SynchronizationStatusSaving:
		return "[]"
	default:
		return "  "
	}
}

func monitorConflictSummary(conflicts []sync.Conflict) string {
	if len(conflicts) > 0 {
		return "X"
	}
	return "-"
}

func monitorProblemSummary(problems []sync.Problem) string {
	if len(problems) > 0 {
		return "X"
	}
	return "-"
}

// TODO: If this has a value of 31, then the monitor line will have a width of
// exactly 80 columns. But on cmd.exe consoles, the line needs to be narrower
// than the console (which is 80 columns by default) in order to process '\r'
// properly, otherwise it'll just move to the next line when it receives '\r'.
// So for now, we've set it to 30, giving the monitor line a width of 78
// columns, but in the future we might just be better off switching to some
// curses-like interface to dynamically set the monitor line width based on the
// console width.
const monitorStatusBarInnerWidth = 30

func monitorStatusBar(status sessionpkg.StagingStatus) string {
	// If there is no staging going on, then return empty spaces.
	if status.Total == 0 {
		return fmt.Sprintf("[%s]", strings.Repeat(" ", monitorStatusBarInnerWidth))
	}

	// Watch for invalid or easy status cases.
	if status.Index >= status.Total {
		return fmt.Sprintf("[%s]", strings.Repeat("#", monitorStatusBarInnerWidth))
	}

	// Compute the number of spaces meant to be occupied by completed blocks.
	fractionCompleted := float32(status.Index) / float32(status.Total)
	completedSpaces := int(fractionCompleted * monitorStatusBarInnerWidth)

	// Compute the resultant bar.
	return fmt.Sprintf(
		"[%s%s]",
		strings.Repeat("#", completedSpaces),
		strings.Repeat("-", monitorStatusBarInnerWidth-completedSpaces),
	)
}

func printMonitorLine(state sessionpkg.SessionState) {
	// Print out a carriage return to wipe out the previous line.
	fmt.Print("\r")

	// Print the state prefix and a trailing space.
	fmt.Printf("%s ", monitorPrefix(state))

	// Print the conflict status and a trailing space.
	fmt.Printf("%s ", monitorConflictSummary(state.State.Conflicts))

	// Print the alpha status bar and a trailing space.
	fmt.Printf(
		"α(%s)%s ",
		monitorProblemSummary(state.State.AlphaProblems),
		monitorStatusBar(state.State.AlphaStaging),
	)

	// Print the beta status bar.
	fmt.Printf(
		"β(%s)%s",
		monitorProblemSummary(state.State.BetaProblems),
		monitorStatusBar(state.State.BetaStaging),
	)
}

func listMain(arguments []string) error {
	// Parse flags.
	var session string
	var monitor bool
	flagSet := cmd.NewFlagSet("list", listUsage, []int{0, 1})
	flagSet.BoolVarP(&monitor, "monitor", "m", false, "continuously monitor session")
	sessionArguments := flagSet.ParseOrDie(arguments)
	if len(sessionArguments) == 1 {
		session = sessionArguments[0]
	}

	// Check that options are sane.
	if monitor && session == "" {
		return errors.New("-m/--monitor only supported with single session")
	}

	// Create a daemon client.
	daemonClient := rpc.NewClient(daemon.NewOpener())

	// Invoke the session list method and ensure the resulting stream is closed
	// when we're done.
	stream, err := daemonClient.Invoke(sessionpkg.MethodList)
	if err != nil {
		return errors.Wrap(err, "unable to invoke session listing")
	}
	defer stream.Close()

	// Send the list request.
	if err := stream.Send(sessionpkg.ListRequest{
		Session: session,
		Monitor: monitor,
	}); err != nil {
		return errors.Wrap(err, "unable to send listing request")
	}

	// Loop indefinitely. We'll bail after a single response if monitoring
	// wasn't requested.
	printSessionInformation := true
	monitorLinePrinted := false
	for {
		// Receive the next response. If there's an error, clear the monitor
		// line (if any) before returning for better error legibility.
		var response sessionpkg.ListResponse
		if err := stream.Receive(&response); err != nil {
			if monitorLinePrinted {
				fmt.Println()
			}
			return errors.Wrap(err, "unable to receive listing response")
		}

		// The first time through the loop (which will be the only time if not
		// monitoring), we print the session state.
		if printSessionInformation {
			// Loop through and print sessions.
			for i, s := range response.Sessions {
				// Print the session information.
				printSession(monitor, s)

				// Print alpha information.
				printEndpoint(monitor, true, s)

				// Print beta information.
				printEndpoint(monitor, false, s)

				// Print conflicts (if any) if we're not in monitor mode.
				if !monitor && len(s.State.Conflicts) > 0 {
					printConflicts(s.State.Conflicts)
				}

				// If we're not in monitor mode and this isn't the last session,
				// print a newline. We don't really need the monitor check since
				// there should only be one session in monitor mode, but it is
				// safer in case the daemon has sent something weird back.
				if !monitor && i < len(response.Sessions)-1 {
					fmt.Println()
				}
			}

			// Mark the session information as printed.
			printSessionInformation = false
		}

		// If we're not monitoring, we're done, otherwise print the monitoring
		// line.
		if !monitor {
			return nil
		}

		// Validate the response for monitoring. If there's an error, clear the
		// monitor line (if any) before returning for better error legibility.
		if len(response.Sessions) != 1 {
			err = errors.New("invalid listing response")
		} else if response.Sessions[0].Session.Identifier != session {
			err = errors.New("listing response returned invalid session")
		}
		if err != nil {
			if monitorLinePrinted {
				fmt.Println()
			}
			return err
		}

		// Print the monitoring line and record that we've done so.
		printMonitorLine(response.Sessions[0])
		monitorLinePrinted = true

		// Send another (empty) request to let the daemon know that we're ready
		// for another response. This is a backpressure mechanism to keep the
		// daemon from sending more requests than we can handle in monitor mode.
		// If there's an error, clear the monitor line (if any) before returning
		// for better error legibility.
		if err := stream.Send(sessionpkg.ListRequest{}); err != nil {
			if monitorLinePrinted {
				fmt.Println()
			}
			return errors.Wrap(err, "unable to send ready request")
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"

	executorinit "github.com/cloudfoundry-incubator/executor/initializer"
)

var gardenNetwork = flag.String(
	"gardenNetwork",
	executorinit.DefaultConfiguration.GardenNetwork,
	"network mode for garden server (tcp, unix)",
)

var gardenAddr = flag.String(
	"gardenAddr",
	executorinit.DefaultConfiguration.GardenAddr,
	"network address for garden server",
)

var memoryMBFlag = flag.String(
	"memoryMB",
	executorinit.DefaultConfiguration.MemoryMB,
	"the amount of memory the executor has available in megabytes",
)

var diskMBFlag = flag.String(
	"diskMB",
	executorinit.DefaultConfiguration.DiskMB,
	"the amount of disk the executor has available in megabytes",
)

var tempDir = flag.String(
	"tempDir",
	executorinit.DefaultConfiguration.TempDir,
	"location to store temporary assets",
)

var registryPruningInterval = flag.Duration(
	"pruneInterval",
	executorinit.DefaultConfiguration.RegistryPruningInterval,
	"amount of time during which a container can remain in the allocated state",
)

var containerInodeLimit = flag.Uint64(
	"containerInodeLimit",
	executorinit.DefaultConfiguration.ContainerInodeLimit,
	"max number of inodes per container",
)

var containerMaxCpuShares = flag.Uint64(
	"containerMaxCpuShares",
	executorinit.DefaultConfiguration.ContainerMaxCpuShares,
	"cpu shares allocatable to a container",
)

var cachePath = flag.String(
	"cachePath",
	executorinit.DefaultConfiguration.CachePath,
	"location to cache assets",
)

var maxCacheSizeInBytes = flag.Uint64(
	"maxCacheSizeInBytes",
	executorinit.DefaultConfiguration.MaxCacheSizeInBytes,
	"maximum size of the cache (in bytes) - you should include a healthy amount of overhead",
)

var skipCertVerify = flag.Bool(
	"skipCertVerify",
	executorinit.DefaultConfiguration.SkipCertVerify,
	"skip SSL certificate verification",
)

var healthyMonitoringInterval = flag.Duration(
	"healthyMonitoringInterval",
	executorinit.DefaultConfiguration.HealthyMonitoringInterval,
	"interval on which to check healthy containers",
)

var unhealthyMonitoringInterval = flag.Duration(
	"unhealthyMonitoringInterval",
	executorinit.DefaultConfiguration.UnhealthyMonitoringInterval,
	"interval on which to check unhealthy containers",
)

var exportNetworkEnvVars = flag.Bool(
	"exportNetworkEnvVars",
	executorinit.DefaultConfiguration.ExportNetworkEnvVars,
	"export network environment variables into container (e.g. CF_INSTANCE_IP, CF_INSTANCE_PORT)",
)

var containerOwnerName = flag.String(
	"containerOwnerName",
	executorinit.DefaultConfiguration.ContainerOwnerName,
	"owner name with which to tag containers",
)

var createWorkPoolSize = flag.Int(
	"createWorkPoolSize",
	executorinit.DefaultConfiguration.CreateWorkPoolSize,
	"Number of concurrent create operations in garden",
)

var deleteWorkPoolSize = flag.Int(
	"deleteWorkPoolSize",
	executorinit.DefaultConfiguration.DeleteWorkPoolSize,
	"Number of concurrent delete operations in garden",
)

var readWorkPoolSize = flag.Int(
	"readWorkPoolSize",
	executorinit.DefaultConfiguration.ReadWorkPoolSize,
	"Number of concurrent read operations in garden",
)

var metricsWorkPoolSize = flag.Int(
	"metricsWorkPoolSize",
	executorinit.DefaultConfiguration.MetricsWorkPoolSize,
	"Number of concurrent metrics operations in garden",
)

var healthCheckWorkPoolSize = flag.Int(
	"healthCheckWorkPoolSize",
	executorinit.DefaultConfiguration.HealthCheckWorkPoolSize,
	"Number of concurrent ping operations in garden",
)

var maxConcurrentDownloads = flag.Int(
	"maxConcurrentDownloads",
	executorinit.DefaultConfiguration.MaxConcurrentDownloads,
	"Number of concurrent download steps",
)

var gardenHealthcheckInterval = flag.Duration(
	"gardenHealthcheckInterval",
	executorinit.DefaultConfiguration.GardenHealthcheckInterval,
	"Frequency for healthchecking garden",
)

var gardenHealthcheckTimeout = flag.Duration(
	"gardenHealthcheckTimeout",
	executorinit.DefaultConfiguration.GardenHealthcheckTimeout,
	"Maximum allowed time for garden healthcheck",
)

var gardenHealthcheckCommandRetryPause = flag.Duration(
	"gardenHealthcheckCommandRetryPause",
	executorinit.DefaultConfiguration.GardenHealthcheckCommandRetryPause,
	"Time to wait between retrying garden commands",
)

var gardenHealthcheckProcessPath = flag.String(
	"gardenHealthcheckProcessPath",
	executorinit.DefaultConfiguration.GardenHealthcheckProcessPath,
	"Path of the command to run to perform a container healthcheck",
)

var gardenHealthcheckProcessUser = flag.String(
	"gardenHealthcheckProcessUser",
	executorinit.DefaultConfiguration.GardenHealthcheckProcessUser,
	"User to use while performing a container healthcheck",
)

var gardenHealthcheckProcessDir = flag.String(
	"gardenHealthcheckProcessDir",
	executorinit.DefaultConfiguration.GardenHealthcheckProcessDir,
	"Directory to run the healthcheck process from",
)

func executorConfig(gardenHealthcheckRootFS string, gardenHealthcheckArgs, gardenHealthcheckEnv []string) executorinit.Configuration {
	return executorinit.Configuration{
		GardenNetwork:                      *gardenNetwork,
		GardenAddr:                         *gardenAddr,
		ContainerOwnerName:                 *containerOwnerName,
		TempDir:                            *tempDir,
		CachePath:                          *cachePath,
		MaxCacheSizeInBytes:                *maxCacheSizeInBytes,
		SkipCertVerify:                     *skipCertVerify,
		ExportNetworkEnvVars:               *exportNetworkEnvVars,
		ContainerMaxCpuShares:              *containerMaxCpuShares,
		ContainerInodeLimit:                *containerInodeLimit,
		HealthyMonitoringInterval:          *healthyMonitoringInterval,
		UnhealthyMonitoringInterval:        *unhealthyMonitoringInterval,
		HealthCheckWorkPoolSize:            *healthCheckWorkPoolSize,
		CreateWorkPoolSize:                 *createWorkPoolSize,
		DeleteWorkPoolSize:                 *deleteWorkPoolSize,
		ReadWorkPoolSize:                   *readWorkPoolSize,
		MetricsWorkPoolSize:                *metricsWorkPoolSize,
		RegistryPruningInterval:            *registryPruningInterval,
		MemoryMB:                           *memoryMBFlag,
		DiskMB:                             *diskMBFlag,
		MaxConcurrentDownloads:             *maxConcurrentDownloads,
		GardenHealthcheckInterval:          *gardenHealthcheckInterval,
		GardenHealthcheckTimeout:           *gardenHealthcheckTimeout,
		GardenHealthcheckCommandRetryPause: *gardenHealthcheckCommandRetryPause,
		GardenHealthcheckRootFS:            gardenHealthcheckRootFS,
		GardenHealthcheckProcessPath:       *gardenHealthcheckProcessPath,
		GardenHealthcheckProcessUser:       *gardenHealthcheckProcessUser,
		GardenHealthcheckProcessDir:        *gardenHealthcheckProcessDir,
		GardenHealthcheckProcessArgs:       gardenHealthcheckArgs,
		GardenHealthcheckProcessEnv:        gardenHealthcheckEnv,
	}
}
<commit_msg>Update reservedExpirationTime and containerReapInterval flags<commit_after>package main

import (
	"flag"

	executorinit "github.com/cloudfoundry-incubator/executor/initializer"
)

var gardenNetwork = flag.String(
	"gardenNetwork",
	executorinit.DefaultConfiguration.GardenNetwork,
	"network mode for garden server (tcp, unix)",
)

var gardenAddr = flag.String(
	"gardenAddr",
	executorinit.DefaultConfiguration.GardenAddr,
	"network address for garden server",
)

var memoryMBFlag = flag.String(
	"memoryMB",
	executorinit.DefaultConfiguration.MemoryMB,
	"the amount of memory the executor has available in megabytes",
)

var diskMBFlag = flag.String(
	"diskMB",
	executorinit.DefaultConfiguration.DiskMB,
	"the amount of disk the executor has available in megabytes",
)

var tempDir = flag.String(
	"tempDir",
	executorinit.DefaultConfiguration.TempDir,
	"location to store temporary assets",
)

var reservedExpirationTime = flag.Duration(
	"reservedExpirationTime",
	executorinit.DefaultConfiguration.ReservedExpirationTime,
	"amount of time during which a container can remain in the allocated state",
)

var containerReapInterval = flag.Duration(
	"containerReapInterval",
	executorinit.DefaultConfiguration.ContainerReapInterval,
	"interval at which the executor reaps extra/missing containers",
)

var containerInodeLimit = flag.Uint64(
	"containerInodeLimit",
	executorinit.DefaultConfiguration.ContainerInodeLimit,
	"max number of inodes per container",
)

var containerMaxCpuShares = flag.Uint64(
	"containerMaxCpuShares",
	executorinit.DefaultConfiguration.ContainerMaxCpuShares,
	"cpu shares allocatable to a container",
)

var cachePath = flag.String(
	"cachePath",
	executorinit.DefaultConfiguration.CachePath,
	"location to cache assets",
)

var maxCacheSizeInBytes = flag.Uint64(
	"maxCacheSizeInBytes",
	executorinit.DefaultConfiguration.MaxCacheSizeInBytes,
	"maximum size of the cache (in bytes) - you should include a healthy amount of overhead",
)

var skipCertVerify = flag.Bool(
	"skipCertVerify",
	executorinit.DefaultConfiguration.SkipCertVerify,
	"skip SSL certificate verification",
)

var healthyMonitoringInterval = flag.Duration(
	"healthyMonitoringInterval",
	executorinit.DefaultConfiguration.HealthyMonitoringInterval,
	"interval on which to check healthy containers",
)

var unhealthyMonitoringInterval = flag.Duration(
	"unhealthyMonitoringInterval",
	executorinit.DefaultConfiguration.UnhealthyMonitoringInterval,
	"interval on which to check unhealthy containers",
)

var exportNetworkEnvVars = flag.Bool(
	"exportNetworkEnvVars",
	executorinit.DefaultConfiguration.ExportNetworkEnvVars,
	"export network environment variables into container (e.g. CF_INSTANCE_IP, CF_INSTANCE_PORT)",
)

var containerOwnerName = flag.String(
	"containerOwnerName",
	executorinit.DefaultConfiguration.ContainerOwnerName,
	"owner name with which to tag containers",
)

var createWorkPoolSize = flag.Int(
	"createWorkPoolSize",
	executorinit.DefaultConfiguration.CreateWorkPoolSize,
	"Number of concurrent create operations in garden",
)

var deleteWorkPoolSize = flag.Int(
	"deleteWorkPoolSize",
	executorinit.DefaultConfiguration.DeleteWorkPoolSize,
	"Number of concurrent delete operations in garden",
)

var readWorkPoolSize = flag.Int(
	"readWorkPoolSize",
	executorinit.DefaultConfiguration.ReadWorkPoolSize,
	"Number of concurrent read operations in garden",
)

var metricsWorkPoolSize = flag.Int(
	"metricsWorkPoolSize",
	executorinit.DefaultConfiguration.MetricsWorkPoolSize,
	"Number of concurrent metrics operations in garden",
)

var healthCheckWorkPoolSize = flag.Int(
	"healthCheckWorkPoolSize",
	executorinit.DefaultConfiguration.HealthCheckWorkPoolSize,
	"Number of concurrent ping operations in garden",
)

var maxConcurrentDownloads = flag.Int(
	"maxConcurrentDownloads",
	executorinit.DefaultConfiguration.MaxConcurrentDownloads,
	"Number of concurrent download steps",
)

var gardenHealthcheckInterval = flag.Duration(
	"gardenHealthcheckInterval",
	executorinit.DefaultConfiguration.GardenHealthcheckInterval,
	"Frequency for healthchecking garden",
)

var gardenHealthcheckTimeout = flag.Duration(
	"gardenHealthcheckTimeout",
	executorinit.DefaultConfiguration.GardenHealthcheckTimeout,
	"Maximum allowed time for garden healthcheck",
)

var gardenHealthcheckCommandRetryPause = flag.Duration(
	"gardenHealthcheckCommandRetryPause",
	executorinit.DefaultConfiguration.GardenHealthcheckCommandRetryPause,
	"Time to wait between retrying garden commands",
)

var gardenHealthcheckProcessPath = flag.String(
	"gardenHealthcheckProcessPath",
	executorinit.DefaultConfiguration.GardenHealthcheckProcessPath,
	"Path of the command to run to perform a container healthcheck",
)

var gardenHealthcheckProcessUser = flag.String(
	"gardenHealthcheckProcessUser",
	executorinit.DefaultConfiguration.GardenHealthcheckProcessUser,
	"User to use while performing a container healthcheck",
)

var gardenHealthcheckProcessDir = flag.String(
	"gardenHealthcheckProcessDir",
	executorinit.DefaultConfiguration.GardenHealthcheckProcessDir,
	"Directory to run the healthcheck process from",
)

func executorConfig(gardenHealthcheckRootFS string, gardenHealthcheckArgs, gardenHealthcheckEnv []string) executorinit.Configuration {
	return executorinit.Configuration{
		GardenNetwork:                      *gardenNetwork,
		GardenAddr:                         *gardenAddr,
		ContainerOwnerName:                 *containerOwnerName,
		TempDir:                            *tempDir,
		CachePath:                          *cachePath,
		MaxCacheSizeInBytes:                *maxCacheSizeInBytes,
		SkipCertVerify:                     *skipCertVerify,
		ExportNetworkEnvVars:               *exportNetworkEnvVars,
		ContainerMaxCpuShares:              *containerMaxCpuShares,
		ContainerInodeLimit:                *containerInodeLimit,
		HealthyMonitoringInterval:          *healthyMonitoringInterval,
		UnhealthyMonitoringInterval:        *unhealthyMonitoringInterval,
		HealthCheckWorkPoolSize:            *healthCheckWorkPoolSize,
		CreateWorkPoolSize:                 *createWorkPoolSize,
		DeleteWorkPoolSize:                 *deleteWorkPoolSize,
		ReadWorkPoolSize:                   *readWorkPoolSize,
		MetricsWorkPoolSize:                *metricsWorkPoolSize,
		ReservedExpirationTime:             *reservedExpirationTime,
		ContainerReapInterval:              *containerReapInterval,
		MemoryMB:                           *memoryMBFlag,
		DiskMB:                             *diskMBFlag,
		MaxConcurrentDownloads:             *maxConcurrentDownloads,
		GardenHealthcheckInterval:          *gardenHealthcheckInterval,
		GardenHealthcheckTimeout:           *gardenHealthcheckTimeout,
		GardenHealthcheckCommandRetryPause: *gardenHealthcheckCommandRetryPause,
		GardenHealthcheckRootFS:            gardenHealthcheckRootFS,
		GardenHealthcheckProcessPath:       *gardenHealthcheckProcessPath,
		GardenHealthcheckProcessUser:       *gardenHealthcheckProcessUser,
		GardenHealthcheckProcessDir:        *gardenHealthcheckProcessDir,
		GardenHealthcheckProcessArgs:       gardenHealthcheckArgs,
		GardenHealthcheckProcessEnv:        gardenHealthcheckEnv,
	}
}
<|endoftext|>


<commit_before>/*
rqlite -- replicating SQLite via the Raft consensus protocol..

rqlite is a distributed system that provides a replicated relational database,
using SQLite as the storage engine.

rqlite is written in Go and uses Raft to achieve consensus across all the
instances of the SQLite databases. rqlite ensures that every change made to
the database is made to a majority of underlying SQLite files, or none-at-all.
*/

package main

import (
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"os/signal"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"time"

	"github.com/rqlite/rqlite/auth"
	"github.com/rqlite/rqlite/cluster"
	"github.com/rqlite/rqlite/disco"
	httpd "github.com/rqlite/rqlite/http"
	"github.com/rqlite/rqlite/store"
	"github.com/rqlite/rqlite/tcp"
)

const sqliteDSN = "db.sqlite"

const logo = `
            _ _ _           _
           | (_) |         | |
  _ __ __ _| |_| |_ ___  __| |
 | '__/ _  | | | __/ _ \/ _  |  The lightweight, distributed
 | | | (_| | | | ||  __/ (_| |  relational database.
 |_|  \__, |_|_|\__\___|\__,_|
         | |
         |_|
`

// These variables are populated via the Go linker.
var (
	version   = "3"
	commit    = "unknown"
	branch    = "unknown"
	buildtime = "unknown"
)

const (
	muxRaftHeader = 1 // Raft consensus communications
	muxMetaHeader = 2 // Cluster meta communications
)

const (
	publishPeerDelay   = 1 * time.Second
	publishPeerTimeout = 30 * time.Second
)

var httpAddr string
var httpAdv string
var authFile string
var x509Cert string
var x509Key string
var raftAddr string
var raftAdv string
var joinAddr string
var noVerify bool
var discoURL string
var discoID string
var expvar bool
var pprofEnabled bool
var dsn string
var onDisk bool
var raftSnapThreshold uint64
var raftHeartbeatTimeout string
var raftApplyTimeout string
var raftOpenTimeout string
var showVersion bool
var cpuProfile string
var memProfile string

const desc = `rqlite is a distributed system that provides a replicated relational database.`

func init() {
	flag.StringVar(&httpAddr, "http", "localhost:4001", "HTTP server bind address. For HTTPS, set X.509 cert and key")
	flag.StringVar(&httpAdv, "httpadv", "", "Advertised HTTP address. If not set, same as HTTP server")
	flag.StringVar(&x509Cert, "x509cert", "", "Path to X.509 certificate")
	flag.StringVar(&x509Key, "x509key", "", "Path to X.509 private key for certificate")
	flag.StringVar(&authFile, "auth", "", "Path to authentication and authorization file. If not set, not enabled")
	flag.StringVar(&raftAddr, "raft", "localhost:4002", "Raft communication bind address")
	flag.StringVar(&raftAdv, "raftadv", "", "Advertised Raft communication address. If not set, same as Raft bind")
	flag.StringVar(&joinAddr, "join", "", "Join a cluster via node at protocol://host:port")
	flag.BoolVar(&noVerify, "noverify", false, "Skip verification of remote HTTPS cert when joining cluster")
	flag.StringVar(&discoURL, "disco", "http://discovery.rqlite.com", "Set Discovery Service URL")
	flag.StringVar(&discoID, "discoid", "", "Set Discovery ID. If not set, Discovery Service not used")
	flag.BoolVar(&expvar, "expvar", true, "Serve expvar data on HTTP server")
	flag.BoolVar(&pprofEnabled, "pprof", true, "Serve pprof data on HTTP server")
	flag.StringVar(&dsn, "dsn", "", `SQLite DSN parameters. E.g. "cache=shared&mode=memory"`)
	flag.BoolVar(&onDisk, "ondisk", false, "Use an on-disk SQLite database")
	flag.BoolVar(&showVersion, "version", false, "Show version information and exit")
	flag.StringVar(&raftHeartbeatTimeout, "rafttimeout", "1s", "Raft heartbeat timeout")
	flag.StringVar(&raftApplyTimeout, "raftapplytimeout", "10s", "Raft apply timeout")
	flag.StringVar(&raftOpenTimeout, "raftopentimeout", "120s", "Time for initial Raft logs to be applied. Use 0s duration to skip wait")
	flag.Uint64Var(&raftSnapThreshold, "raftsnap", 8192, "Number of outstanding log entries that trigger snapshot")
	flag.StringVar(&cpuProfile, "cpuprofile", "", "Path to file for CPU profiling information")
	flag.StringVar(&memProfile, "memprofile", "", "Path to file for memory profiling information")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "\n%s\n\n", desc)
		fmt.Fprintf(os.Stderr, "Usage: %s [arguments] <data directory>\n", os.Args[0])
		flag.PrintDefaults()
	}
}

func main() {
	flag.Parse()

	if showVersion {
		fmt.Printf("rqlited %s %s %s (commit %s, branch %s)\n",
			version, runtime.GOOS, runtime.GOARCH, commit, branch)
		os.Exit(0)
	}

	// Ensure the data path is set.
	if flag.NArg() == 0 {
		flag.Usage()
		os.Exit(1)
	}

	dataPath := flag.Arg(0)

	// Display logo.
	fmt.Println(logo)

	// Configure logging and pump out initial message.
	log.SetFlags(log.LstdFlags)
	log.SetOutput(os.Stderr)
	log.SetPrefix("[rqlited] ")
	log.Printf("rqlited starting, version %s, commit %s, branch %s", version, commit, branch)
	log.Printf("target architecture is %s, operating system target is %s", runtime.GOARCH, runtime.GOOS)

	// Start requested profiling.
	startProfile(cpuProfile, memProfile)

	// Set up TCP communication between nodes.
	ln, err := net.Listen("tcp", raftAddr)
	if err != nil {
		log.Fatalf("failed to listen on %s: %s", raftAddr, err.Error())
	}
	var adv net.Addr
	if raftAdv != "" {
		adv, err = net.ResolveTCPAddr("tcp", raftAdv)
		if err != nil {
			log.Fatalf("failed to resolve advertise address %s: %s", raftAdv, err.Error())
		}
	}
	mux := tcp.NewMux(ln, adv)
	go mux.Serve()

	// Start up mux and get transports for cluster.
	raftTn := mux.Listen(muxRaftHeader)

	// Create and open the store.
	dataPath, err = filepath.Abs(dataPath)
	if err != nil {
		log.Fatalf("failed to determine absolute data path: %s", err.Error())
	}
	dbConf := store.NewDBConfig(dsn, !onDisk)

	str := store.New(&store.StoreConfig{
		DBConf: dbConf,
		Dir:    dataPath,
		Tn:     raftTn,
	})

	// Set optional parameters on store.
	str.SnapshotThreshold = raftSnapThreshold
	str.HeartbeatTimeout, err = time.ParseDuration(raftHeartbeatTimeout)
	if err != nil {
		log.Fatalf("failed to parse Raft heartbeat timeout %s: %s", raftHeartbeatTimeout, err.Error())
	}
	str.ApplyTimeout, err = time.ParseDuration(raftApplyTimeout)
	if err != nil {
		log.Fatalf("failed to parse Raft apply timeout %s: %s", raftApplyTimeout, err.Error())
	}
	str.OpenTimeout, err = time.ParseDuration(raftOpenTimeout)
	if err != nil {
		log.Fatalf("failed to parse Raft open timeout %s: %s", raftOpenTimeout, err.Error())
	}

	// Determine join addresses, if necessary.
	ja, err := store.JoinAllowed(dataPath)
	if err != nil {
		log.Fatalf("unable to determine if join permitted: %s", err.Error())
	}

	var joins []string
	if ja {
		joins, err = determineJoinAddresses()
		if err != nil {
			log.Fatalf("unable to determine join addresses: %s", err.Error())
		}
	} else {
		log.Println("node is already member of cluster, skip determining join addresses")
	}

	// Now, open it.
	if err := str.Open(len(joins) == 0); err != nil {
		log.Fatalf("failed to open store: %s", err.Error())
	}

	// Create and configure cluster service.
	tn := mux.Listen(muxMetaHeader)
	cs := cluster.NewService(tn, str)
	if err := cs.Open(); err != nil {
		log.Fatalf("failed to open cluster service: %s", err.Error())
	}

	// Execute any requested join operation.
	if len(joins) > 0 {
		log.Println("join addresses are:", joins)
		advAddr := raftAddr
		if raftAdv != "" {
			advAddr = raftAdv
		}
		if j, err := cluster.Join(joins, advAddr, noVerify); err != nil {
			log.Fatalf("failed to join cluster at %s: %s", joins, err.Error())
		} else {
			log.Println("successfully joined cluster at", j)
		}

	} else {
		log.Println("no join addresses set")
	}

	// Publish to the cluster the mapping between this Raft address and API address.
	// The Raft layer broadcasts the resolved address, so use that as the key. But
	// only set different HTTP advertise address if set.
	apiAdv := httpAddr
	if httpAdv != "" {
		apiAdv = httpAdv
	}

	if err := publishAPIAddr(cs, raftTn.Addr().String(), apiAdv, publishPeerTimeout); err != nil {
		log.Fatalf("failed to set peer for %s to %s: %s", raftAddr, httpAddr, err.Error())
	}
	log.Printf("set peer for %s to %s", raftTn.Addr().String(), apiAdv)

	// Create HTTP server and load authentication information, if supplied.
	var s *httpd.Service
	if authFile != "" {
		f, err := os.Open(authFile)
		if err != nil {
			log.Fatalf("failed to open authentication file %s: %s", authFile, err.Error())
		}
		credentialStore := auth.NewCredentialsStore()
		if err := credentialStore.Load(f); err != nil {
			log.Fatalf("failed to load authentication file: %s", err.Error())
		}
		s = httpd.New(httpAddr, str, credentialStore)
	} else {
		s = httpd.New(httpAddr, str, nil)
	}

	s.CertFile = x509Cert
	s.KeyFile = x509Key
	s.Expvar = expvar
	s.Pprof = pprofEnabled
	s.BuildInfo = map[string]interface{}{
		"commit":     commit,
		"branch":     branch,
		"version":    version,
		"build_time": buildtime,
	}
	if err := s.Start(); err != nil {
		log.Fatalf("failed to start HTTP server: %s", err.Error())
	}

	terminate := make(chan os.Signal, 1)
	signal.Notify(terminate, os.Interrupt)
	<-terminate
	if err := str.Close(true); err != nil {
		log.Printf("failed to close store: %s", err.Error())
	}
	stopProfile()
	log.Println("rqlite server stopped")
}

func determineJoinAddresses() ([]string, error) {
	apiAdv := httpAddr
	if httpAdv != "" {
		apiAdv = httpAdv
	}

	var addrs []string
	if joinAddr != "" {
		// An explicit join address is first priority.
		addrs = append(addrs, joinAddr)
	}

	if discoID != "" {
		log.Printf("registering with Discovery Service at %s with ID %s", discoURL, discoID)
		c := disco.New(discoURL)
		r, err := c.Register(discoID, apiAdv)
		if err != nil {
			return nil, err
		}
		log.Println("Discovery Service responded with nodes:", r.Nodes)
		for _, a := range r.Nodes {
			if a != apiAdv {
				// Only other nodes can be joined.
				addrs = append(addrs, a)
			}
		}
	}

	return addrs, nil
}

func publishAPIAddr(c *cluster.Service, raftAddr, apiAddr string, t time.Duration) error {
	tck := time.NewTicker(publishPeerDelay)
	defer tck.Stop()
	tmr := time.NewTimer(t)
	defer tmr.Stop()

	for {
		select {
		case <-tck.C:
			if err := c.SetPeer(raftAddr, apiAddr); err != nil {
				log.Printf("failed to set peer for %s to %s: %s (retrying)",
					raftAddr, apiAddr, err.Error())
				continue
			}
			return nil
		case <-tmr.C:
			return fmt.Errorf("set peer timeout expired")
		}
	}
}

// prof stores the file locations of active profiles.
var prof struct {
	cpu *os.File
	mem *os.File
}

// startProfile initializes the CPU and memory profile, if specified.
func startProfile(cpuprofile, memprofile string) {
	if cpuprofile != "" {
		f, err := os.Create(cpuprofile)
		if err != nil {
			log.Fatalf("failed to create CPU profile file at %s: %s", cpuprofile, err.Error())
		}
		log.Printf("writing CPU profile to: %s\n", cpuprofile)
		prof.cpu = f
		pprof.StartCPUProfile(prof.cpu)
	}

	if memprofile != "" {
		f, err := os.Create(memprofile)
		if err != nil {
			log.Fatalf("failed to create memory profile file at %s: %s", cpuprofile, err.Error())
		}
		log.Printf("writing memory profile to: %s\n", memprofile)
		prof.mem = f
		runtime.MemProfileRate = 4096
	}
}

// stopProfile closes the CPU and memory profiles if they are running.
func stopProfile() {
	if prof.cpu != nil {
		pprof.StopCPUProfile()
		prof.cpu.Close()
		log.Println("CPU profiling stopped")
	}
	if prof.mem != nil {
		pprof.Lookup("heap").WriteTo(prof.mem, 0)
		prof.mem.Close()
		log.Println("memory profiling stopped")
	}
}
<commit_msg>const out the name 'rqlited'<commit_after>/*
rqlite -- replicating SQLite via the Raft consensus protocol..

rqlite is a distributed system that provides a replicated relational database,
using SQLite as the storage engine.

rqlite is written in Go and uses Raft to achieve consensus across all the
instances of the SQLite databases. rqlite ensures that every change made to
the database is made to a majority of underlying SQLite files, or none-at-all.
*/

package main

import (
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"os/signal"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"time"

	"github.com/rqlite/rqlite/auth"
	"github.com/rqlite/rqlite/cluster"
	"github.com/rqlite/rqlite/disco"
	httpd "github.com/rqlite/rqlite/http"
	"github.com/rqlite/rqlite/store"
	"github.com/rqlite/rqlite/tcp"
)

const sqliteDSN = "db.sqlite"

const logo = `
            _ _ _           _
           | (_) |         | |
  _ __ __ _| |_| |_ ___  __| |
 | '__/ _  | | | __/ _ \/ _  |  The lightweight, distributed
 | | | (_| | | | ||  __/ (_| |  relational database.
 |_|  \__, |_|_|\__\___|\__,_|
         | |
         |_|
`

// These variables are populated via the Go linker.
var (
	version   = "3"
	commit    = "unknown"
	branch    = "unknown"
	buildtime = "unknown"
)

const (
	muxRaftHeader = 1 // Raft consensus communications
	muxMetaHeader = 2 // Cluster meta communications
)

const (
	publishPeerDelay   = 1 * time.Second
	publishPeerTimeout = 30 * time.Second
)

var httpAddr string
var httpAdv string
var authFile string
var x509Cert string
var x509Key string
var raftAddr string
var raftAdv string
var joinAddr string
var noVerify bool
var discoURL string
var discoID string
var expvar bool
var pprofEnabled bool
var dsn string
var onDisk bool
var raftSnapThreshold uint64
var raftHeartbeatTimeout string
var raftApplyTimeout string
var raftOpenTimeout string
var showVersion bool
var cpuProfile string
var memProfile string

const name = `rqlited`
const desc = `rqlite is a distributed system that provides a replicated relational database.`

func init() {
	flag.StringVar(&httpAddr, "http", "localhost:4001", "HTTP server bind address. For HTTPS, set X.509 cert and key")
	flag.StringVar(&httpAdv, "httpadv", "", "Advertised HTTP address. If not set, same as HTTP server")
	flag.StringVar(&x509Cert, "x509cert", "", "Path to X.509 certificate")
	flag.StringVar(&x509Key, "x509key", "", "Path to X.509 private key for certificate")
	flag.StringVar(&authFile, "auth", "", "Path to authentication and authorization file. If not set, not enabled")
	flag.StringVar(&raftAddr, "raft", "localhost:4002", "Raft communication bind address")
	flag.StringVar(&raftAdv, "raftadv", "", "Advertised Raft communication address. If not set, same as Raft bind")
	flag.StringVar(&joinAddr, "join", "", "Join a cluster via node at protocol://host:port")
	flag.BoolVar(&noVerify, "noverify", false, "Skip verification of remote HTTPS cert when joining cluster")
	flag.StringVar(&discoURL, "disco", "http://discovery.rqlite.com", "Set Discovery Service URL")
	flag.StringVar(&discoID, "discoid", "", "Set Discovery ID. If not set, Discovery Service not used")
	flag.BoolVar(&expvar, "expvar", true, "Serve expvar data on HTTP server")
	flag.BoolVar(&pprofEnabled, "pprof", true, "Serve pprof data on HTTP server")
	flag.StringVar(&dsn, "dsn", "", `SQLite DSN parameters. E.g. "cache=shared&mode=memory"`)
	flag.BoolVar(&onDisk, "ondisk", false, "Use an on-disk SQLite database")
	flag.BoolVar(&showVersion, "version", false, "Show version information and exit")
	flag.StringVar(&raftHeartbeatTimeout, "rafttimeout", "1s", "Raft heartbeat timeout")
	flag.StringVar(&raftApplyTimeout, "raftapplytimeout", "10s", "Raft apply timeout")
	flag.StringVar(&raftOpenTimeout, "raftopentimeout", "120s", "Time for initial Raft logs to be applied. Use 0s duration to skip wait")
	flag.Uint64Var(&raftSnapThreshold, "raftsnap", 8192, "Number of outstanding log entries that trigger snapshot")
	flag.StringVar(&cpuProfile, "cpuprofile", "", "Path to file for CPU profiling information")
	flag.StringVar(&memProfile, "memprofile", "", "Path to file for memory profiling information")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "\n%s\n\n", desc)
		fmt.Fprintf(os.Stderr, "Usage: %s [arguments] <data directory>\n", name)
		flag.PrintDefaults()
	}
}

func main() {
	flag.Parse()

	if showVersion {
		fmt.Printf("%s %s %s %s (commit %s, branch %s)\n",
			name, version, runtime.GOOS, runtime.GOARCH, commit, branch)
		os.Exit(0)
	}

	// Ensure the data path is set.
	if flag.NArg() == 0 {
		flag.Usage()
		os.Exit(1)
	}

	dataPath := flag.Arg(0)

	// Display logo.
	fmt.Println(logo)

	// Configure logging and pump out initial message.
	log.SetFlags(log.LstdFlags)
	log.SetOutput(os.Stderr)
	log.SetPrefix(fmt.Sprintf("[%s] ", name))
	log.Printf("%s starting, version %s, commit %s, branch %s", name, version, commit, branch)
	log.Printf("target architecture is %s, operating system target is %s", runtime.GOARCH, runtime.GOOS)

	// Start requested profiling.
	startProfile(cpuProfile, memProfile)

	// Set up TCP communication between nodes.
	ln, err := net.Listen("tcp", raftAddr)
	if err != nil {
		log.Fatalf("failed to listen on %s: %s", raftAddr, err.Error())
	}
	var adv net.Addr
	if raftAdv != "" {
		adv, err = net.ResolveTCPAddr("tcp", raftAdv)
		if err != nil {
			log.Fatalf("failed to resolve advertise address %s: %s", raftAdv, err.Error())
		}
	}
	mux := tcp.NewMux(ln, adv)
	go mux.Serve()

	// Start up mux and get transports for cluster.
	raftTn := mux.Listen(muxRaftHeader)

	// Create and open the store.
	dataPath, err = filepath.Abs(dataPath)
	if err != nil {
		log.Fatalf("failed to determine absolute data path: %s", err.Error())
	}
	dbConf := store.NewDBConfig(dsn, !onDisk)

	str := store.New(&store.StoreConfig{
		DBConf: dbConf,
		Dir:    dataPath,
		Tn:     raftTn,
	})

	// Set optional parameters on store.
	str.SnapshotThreshold = raftSnapThreshold
	str.HeartbeatTimeout, err = time.ParseDuration(raftHeartbeatTimeout)
	if err != nil {
		log.Fatalf("failed to parse Raft heartbeat timeout %s: %s", raftHeartbeatTimeout, err.Error())
	}
	str.ApplyTimeout, err = time.ParseDuration(raftApplyTimeout)
	if err != nil {
		log.Fatalf("failed to parse Raft apply timeout %s: %s", raftApplyTimeout, err.Error())
	}
	str.OpenTimeout, err = time.ParseDuration(raftOpenTimeout)
	if err != nil {
		log.Fatalf("failed to parse Raft open timeout %s: %s", raftOpenTimeout, err.Error())
	}

	// Determine join addresses, if necessary.
	ja, err := store.JoinAllowed(dataPath)
	if err != nil {
		log.Fatalf("unable to determine if join permitted: %s", err.Error())
	}

	var joins []string
	if ja {
		joins, err = determineJoinAddresses()
		if err != nil {
			log.Fatalf("unable to determine join addresses: %s", err.Error())
		}
	} else {
		log.Println("node is already member of cluster, skip determining join addresses")
	}

	// Now, open it.
	if err := str.Open(len(joins) == 0); err != nil {
		log.Fatalf("failed to open store: %s", err.Error())
	}

	// Create and configure cluster service.
	tn := mux.Listen(muxMetaHeader)
	cs := cluster.NewService(tn, str)
	if err := cs.Open(); err != nil {
		log.Fatalf("failed to open cluster service: %s", err.Error())
	}

	// Execute any requested join operation.
	if len(joins) > 0 {
		log.Println("join addresses are:", joins)
		advAddr := raftAddr
		if raftAdv != "" {
			advAddr = raftAdv
		}
		if j, err := cluster.Join(joins, advAddr, noVerify); err != nil {
			log.Fatalf("failed to join cluster at %s: %s", joins, err.Error())
		} else {
			log.Println("successfully joined cluster at", j)
		}

	} else {
		log.Println("no join addresses set")
	}

	// Publish to the cluster the mapping between this Raft address and API address.
	// The Raft layer broadcasts the resolved address, so use that as the key. But
	// only set different HTTP advertise address if set.
	apiAdv := httpAddr
	if httpAdv != "" {
		apiAdv = httpAdv
	}

	if err := publishAPIAddr(cs, raftTn.Addr().String(), apiAdv, publishPeerTimeout); err != nil {
		log.Fatalf("failed to set peer for %s to %s: %s", raftAddr, httpAddr, err.Error())
	}
	log.Printf("set peer for %s to %s", raftTn.Addr().String(), apiAdv)

	// Create HTTP server and load authentication information, if supplied.
	var s *httpd.Service
	if authFile != "" {
		f, err := os.Open(authFile)
		if err != nil {
			log.Fatalf("failed to open authentication file %s: %s", authFile, err.Error())
		}
		credentialStore := auth.NewCredentialsStore()
		if err := credentialStore.Load(f); err != nil {
			log.Fatalf("failed to load authentication file: %s", err.Error())
		}
		s = httpd.New(httpAddr, str, credentialStore)
	} else {
		s = httpd.New(httpAddr, str, nil)
	}

	s.CertFile = x509Cert
	s.KeyFile = x509Key
	s.Expvar = expvar
	s.Pprof = pprofEnabled
	s.BuildInfo = map[string]interface{}{
		"commit":     commit,
		"branch":     branch,
		"version":    version,
		"build_time": buildtime,
	}
	if err := s.Start(); err != nil {
		log.Fatalf("failed to start HTTP server: %s", err.Error())
	}

	terminate := make(chan os.Signal, 1)
	signal.Notify(terminate, os.Interrupt)
	<-terminate
	if err := str.Close(true); err != nil {
		log.Printf("failed to close store: %s", err.Error())
	}
	stopProfile()
	log.Println("rqlite server stopped")
}

func determineJoinAddresses() ([]string, error) {
	apiAdv := httpAddr
	if httpAdv != "" {
		apiAdv = httpAdv
	}

	var addrs []string
	if joinAddr != "" {
		// An explicit join address is first priority.
		addrs = append(addrs, joinAddr)
	}

	if discoID != "" {
		log.Printf("registering with Discovery Service at %s with ID %s", discoURL, discoID)
		c := disco.New(discoURL)
		r, err := c.Register(discoID, apiAdv)
		if err != nil {
			return nil, err
		}
		log.Println("Discovery Service responded with nodes:", r.Nodes)
		for _, a := range r.Nodes {
			if a != apiAdv {
				// Only other nodes can be joined.
				addrs = append(addrs, a)
			}
		}
	}

	return addrs, nil
}

func publishAPIAddr(c *cluster.Service, raftAddr, apiAddr string, t time.Duration) error {
	tck := time.NewTicker(publishPeerDelay)
	defer tck.Stop()
	tmr := time.NewTimer(t)
	defer tmr.Stop()

	for {
		select {
		case <-tck.C:
			if err := c.SetPeer(raftAddr, apiAddr); err != nil {
				log.Printf("failed to set peer for %s to %s: %s (retrying)",
					raftAddr, apiAddr, err.Error())
				continue
			}
			return nil
		case <-tmr.C:
			return fmt.Errorf("set peer timeout expired")
		}
	}
}

// prof stores the file locations of active profiles.
var prof struct {
	cpu *os.File
	mem *os.File
}

// startProfile initializes the CPU and memory profile, if specified.
func startProfile(cpuprofile, memprofile string) {
	if cpuprofile != "" {
		f, err := os.Create(cpuprofile)
		if err != nil {
			log.Fatalf("failed to create CPU profile file at %s: %s", cpuprofile, err.Error())
		}
		log.Printf("writing CPU profile to: %s\n", cpuprofile)
		prof.cpu = f
		pprof.StartCPUProfile(prof.cpu)
	}

	if memprofile != "" {
		f, err := os.Create(memprofile)
		if err != nil {
			log.Fatalf("failed to create memory profile file at %s: %s", cpuprofile, err.Error())
		}
		log.Printf("writing memory profile to: %s\n", memprofile)
		prof.mem = f
		runtime.MemProfileRate = 4096
	}
}

// stopProfile closes the CPU and memory profiles if they are running.
func stopProfile() {
	if prof.cpu != nil {
		pprof.StopCPUProfile()
		prof.cpu.Close()
		log.Println("CPU profiling stopped")
	}
	if prof.mem != nil {
		pprof.Lookup("heap").WriteTo(prof.mem, 0)
		prof.mem.Close()
		log.Println("memory profiling stopped")
	}
}
<|endoftext|>