<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package engine

import (
	"fmt"
	"github.com/keybase/client/go/libkb"
	keybase1 "github.com/keybase/client/go/protocol/keybase1"
	"time"
)

type PGPPullEngineArg struct {
	UserAsserts []string
}

type PGPPullEngine struct {
	listTrackingEngine *ListTrackingEngine
	userAsserts        []string
	gpgClient          *libkb.GpgCLI
	libkb.Contextified
}

func NewPGPPullEngine(g *libkb.GlobalContext, arg *PGPPullEngineArg) *PGPPullEngine {
	return &PGPPullEngine{
		listTrackingEngine: NewListTrackingEngine(g, &ListTrackingEngineArg{}),
		userAsserts:        arg.UserAsserts,
		Contextified:       libkb.NewContextified(g),
	}
}

func (e *PGPPullEngine) Name() string {
	return "PGPPull"
}

func (e *PGPPullEngine) Prereqs() Prereqs {
	return Prereqs{}
}

func (e *PGPPullEngine) RequiredUIs() []libkb.UIKind {
	return []libkb.UIKind{
		libkb.LogUIKind,
	}
}

func (e *PGPPullEngine) SubConsumers() []libkb.UIConsumer {
	return []libkb.UIConsumer{e.listTrackingEngine}
}

func proofSetFromUserSummary(summary keybase1.UserSummary) *libkb.ProofSet {
	proofs := []libkb.Proof{
		{Key: "keybase", Value: summary.Username},
		{Key: "uid", Value: summary.Uid.String()},
	}
	for _, socialProof := range summary.Proofs.Social {
		proofs = append(proofs, libkb.Proof{
			Key:   socialProof.ProofType,
			Value: socialProof.ProofName,
		})
	}
	for _, webProof := range summary.Proofs.Web {
		for _, protocol := range webProof.Protocols {
			proofs = append(proofs, libkb.Proof{
				Key:   protocol,
				Value: webProof.Hostname,
			})
		}
	}
	return libkb.NewProofSet(proofs)
}

func (e *PGPPullEngine) getTrackedUserSummaries(m libkb.MetaContext) ([]keybase1.UserSummary, []string, error) {
	err := RunEngine2(m, e.listTrackingEngine)
	if err != nil {
		return nil, nil, err
	}
	allTrackedSummaries := e.listTrackingEngine.TableResult()

	// Without any userAsserts specified, just all summaries and no leftovers.
	if e.userAsserts == nil || len(e.userAsserts) == 0 {
		return allTrackedSummaries, nil, nil
	}

	// With userAsserts specified, return only those summaries. If an assert
	// doesn't match any tracked users, that's an error. If an assert matches
	// more than one tracked user, that is also an error. If multiple
	// assertions match the same user, that's fine.

	// First parse all the assertion expressions.
	parsedAsserts := make(map[string]libkb.AssertionExpression)
	for _, assertString := range e.userAsserts {
		assertExpr, err := libkb.AssertionParseAndOnly(e.G().MakeAssertionContext(), assertString)
		if err != nil {
			return nil, nil, err
		}
		parsedAsserts[assertString] = assertExpr
	}

	// Then loop over all the tracked users, keeping track of which expressions
	// have matched before.
	matchedSummaries := make(map[string]keybase1.UserSummary)
	assertionsUsed := make(map[string]bool)
	for _, summary := range allTrackedSummaries {
		proofSet := proofSetFromUserSummary(summary)
		for assertStr, parsedAssert := range parsedAsserts {
			if parsedAssert.MatchSet(*proofSet) {
				if assertionsUsed[assertStr] {
					return nil, nil, fmt.Errorf("Assertion \"%s\" matched more than one tracked user.", assertStr)
				}
				assertionsUsed[assertStr] = true
				matchedSummaries[summary.Username] = summary
			}
		}
	}

	var leftovers []string
	// Make sure every assertion found a match.
	for _, assertString := range e.userAsserts {
		if !assertionsUsed[assertString] {
			m.Info("Assertion \"%s\" did not match any tracked users.", assertString)
			leftovers = append(leftovers, assertString)
		}
	}

	matchedList := []keybase1.UserSummary{}
	for _, summary := range matchedSummaries {
		matchedList = append(matchedList, summary)
	}
	return matchedList, leftovers, nil
}

func (e *PGPPullEngine) runLoggedOut(m libkb.MetaContext) error {
	if len(e.userAsserts) == 0 {
		return libkb.PGPPullLoggedOutError{}
	}
	t := time.Now()
	for i, assertString := range e.userAsserts {
		t = e.rateLimit(t, i)
		if err := e.processUserWithIdentify(m, assertString); err != nil {
			return err
		}
	}
	return nil
}

func (e *PGPPullEngine) processUserWithIdentify(m libkb.MetaContext, u string) error {
	m.Debug("Processing with identify: %s", u)

	iarg := keybase1.Identify2Arg{
		UserAssertion:    u,
		ForceRemoteCheck: true,
		AlwaysBlock:      true,
		NeedProofSet:     true, // forces prompt even if we declined before
	}
	topts := keybase1.TrackOptions{
		LocalOnly:  true,
		ForPGPPull: true,
	}
	ieng := NewResolveThenIdentify2WithTrack(m.G(), &iarg, topts)
	if err := RunEngine2(m, ieng); err != nil {
		m.Info("identify run err: %s", err)
		return err
	}

	// prompt if the identify is correct
	result := ieng.ConfirmResult()
	if !result.IdentityConfirmed {
		m.Warning("Not confirmed; skipping key import")
		return nil
	}

	idRes, err := ieng.Result(m)
	if err != nil {
		return err
	}
	// with more plumbing, there is likely a more efficient way to get this identified user out
	// of the identify2 engine, but `pgp pull` is not likely to be called often.
	arg := libkb.NewLoadUserArgWithMetaContext(m).WithUID(idRes.Upk.GetUID())
	user, err := libkb.LoadUser(arg)
	if err != nil {
		return err
	}
	return e.exportKeysToGPG(m, user, nil)
}

func (e *PGPPullEngine) Run(m libkb.MetaContext) error {

	e.gpgClient = libkb.NewGpgCLI(m.G(), m.UIs().LogUI)
	err := e.gpgClient.Configure()
	if err != nil {
		return err
	}

	if ok, _ := isLoggedIn(m); !ok {
		return e.runLoggedOut(m)
	}

	return e.runLoggedIn(m)
}

func (e *PGPPullEngine) runLoggedIn(m libkb.MetaContext) error {
	summaries, leftovers, err := e.getTrackedUserSummaries(m)
	// leftovers contains unmatched assertions, likely users
	// we want to pull but we do not track.
	if err != nil {
		return err
	}

	// Loop over the list of all users we track.
	t := time.Now()
	for i, userSummary := range summaries {
		t = e.rateLimit(t, i)
		// Compute the set of tracked pgp fingerprints. LoadUser will fetch key
		// data from the server, and we will compare it against this.
		trackedFingerprints := make(map[string]bool)
		for _, pubKey := range userSummary.Proofs.PublicKeys {
			if pubKey.PGPFingerprint != "" {
				trackedFingerprints[pubKey.PGPFingerprint] = true
			}
		}

		// Get user data from the server.
		user, err := libkb.LoadUser(
			libkb.NewLoadUserByNameArg(e.G(), userSummary.Username).
				WithPublicKeyOptional())
		if err != nil {
			m.Error("Failed to load user %s: %s", userSummary.Username, err)
			continue
		}
		if user.GetStatus() == keybase1.StatusCode_SCDeleted {
			m.Debug("User %q is deleted, skipping", userSummary.Username)
			continue
		}

		if err = e.exportKeysToGPG(m, user, trackedFingerprints); err != nil {
			return err
		}
	}

	// Loop over unmatched list and process with identify prompts.
	for i, assertString := range leftovers {
		t = e.rateLimit(t, i)
		if err := e.processUserWithIdentify(m, assertString); err != nil {
			return err
		}
	}

	return nil
}

func (e *PGPPullEngine) exportKeysToGPG(m libkb.MetaContext, user *libkb.User, tfp map[string]bool) error {
	for _, bundle := range user.GetActivePGPKeys(false) {
		// Check each key against the tracked set.
		if tfp != nil && !tfp[bundle.GetFingerprint().String()] {
			m.Warning("Keybase says that %s owns key %s, but you have not tracked this fingerprint before.", user.GetName(), bundle.GetFingerprint())
			continue
		}

		if err := e.gpgClient.ExportKey(*bundle, false /* export public key only */, false /* no batch */); err != nil {
			return err
		}

		m.Info("Imported key for %s.", user.GetName())
	}
	return nil
}

func (e *PGPPullEngine) rateLimit(start time.Time, index int) time.Time {
	// server currently limiting to 32 req/s, but there can be 4 requests for each loaduser call.
	const loadUserPerSec = 4
	if index == 0 {
		return start
	}
	if index%loadUserPerSec != 0 {
		return start
	}
	d := time.Second - time.Since(start)
	if d > 0 {
		e.G().Log.Debug("sleeping for %s to slow down api requests", d)
		time.Sleep(d)
	}
	return time.Now()
}
<commit_msg>reduce severity of gpg2 import fails during keybase pgp pull (#16531)<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package engine

import (
	"fmt"
	"time"

	"github.com/keybase/client/go/libkb"
	keybase1 "github.com/keybase/client/go/protocol/keybase1"
)

type PGPPullEngineArg struct {
	UserAsserts []string
}

type PGPPullEngine struct {
	listTrackingEngine *ListTrackingEngine
	userAsserts        []string
	gpgClient          *libkb.GpgCLI
	libkb.Contextified
}

func NewPGPPullEngine(g *libkb.GlobalContext, arg *PGPPullEngineArg) *PGPPullEngine {
	return &PGPPullEngine{
		listTrackingEngine: NewListTrackingEngine(g, &ListTrackingEngineArg{}),
		userAsserts:        arg.UserAsserts,
		Contextified:       libkb.NewContextified(g),
	}
}

func (e *PGPPullEngine) Name() string {
	return "PGPPull"
}

func (e *PGPPullEngine) Prereqs() Prereqs {
	return Prereqs{}
}

func (e *PGPPullEngine) RequiredUIs() []libkb.UIKind {
	return []libkb.UIKind{
		libkb.LogUIKind,
	}
}

func (e *PGPPullEngine) SubConsumers() []libkb.UIConsumer {
	return []libkb.UIConsumer{e.listTrackingEngine}
}

func proofSetFromUserSummary(summary keybase1.UserSummary) *libkb.ProofSet {
	proofs := []libkb.Proof{
		{Key: "keybase", Value: summary.Username},
		{Key: "uid", Value: summary.Uid.String()},
	}
	for _, socialProof := range summary.Proofs.Social {
		proofs = append(proofs, libkb.Proof{
			Key:   socialProof.ProofType,
			Value: socialProof.ProofName,
		})
	}
	for _, webProof := range summary.Proofs.Web {
		for _, protocol := range webProof.Protocols {
			proofs = append(proofs, libkb.Proof{
				Key:   protocol,
				Value: webProof.Hostname,
			})
		}
	}
	return libkb.NewProofSet(proofs)
}

func (e *PGPPullEngine) getTrackedUserSummaries(m libkb.MetaContext) ([]keybase1.UserSummary, []string, error) {
	err := RunEngine2(m, e.listTrackingEngine)
	if err != nil {
		return nil, nil, err
	}
	allTrackedSummaries := e.listTrackingEngine.TableResult()

	// Without any userAsserts specified, just all summaries and no leftovers.
	if e.userAsserts == nil || len(e.userAsserts) == 0 {
		return allTrackedSummaries, nil, nil
	}

	// With userAsserts specified, return only those summaries. If an assert
	// doesn't match any tracked users, that's an error. If an assert matches
	// more than one tracked user, that is also an error. If multiple
	// assertions match the same user, that's fine.

	// First parse all the assertion expressions.
	parsedAsserts := make(map[string]libkb.AssertionExpression)
	for _, assertString := range e.userAsserts {
		assertExpr, err := libkb.AssertionParseAndOnly(e.G().MakeAssertionContext(), assertString)
		if err != nil {
			return nil, nil, err
		}
		parsedAsserts[assertString] = assertExpr
	}

	// Then loop over all the tracked users, keeping track of which expressions
	// have matched before.
	matchedSummaries := make(map[string]keybase1.UserSummary)
	assertionsUsed := make(map[string]bool)
	for _, summary := range allTrackedSummaries {
		proofSet := proofSetFromUserSummary(summary)
		for assertStr, parsedAssert := range parsedAsserts {
			if parsedAssert.MatchSet(*proofSet) {
				if assertionsUsed[assertStr] {
					return nil, nil, fmt.Errorf("Assertion \"%s\" matched more than one tracked user.", assertStr)
				}
				assertionsUsed[assertStr] = true
				matchedSummaries[summary.Username] = summary
			}
		}
	}

	var leftovers []string
	// Make sure every assertion found a match.
	for _, assertString := range e.userAsserts {
		if !assertionsUsed[assertString] {
			m.Info("Assertion \"%s\" did not match any tracked users.", assertString)
			leftovers = append(leftovers, assertString)
		}
	}

	matchedList := []keybase1.UserSummary{}
	for _, summary := range matchedSummaries {
		matchedList = append(matchedList, summary)
	}
	return matchedList, leftovers, nil
}

func (e *PGPPullEngine) runLoggedOut(m libkb.MetaContext) error {
	if len(e.userAsserts) == 0 {
		return libkb.PGPPullLoggedOutError{}
	}
	t := time.Now()
	for i, assertString := range e.userAsserts {
		t = e.rateLimit(t, i)
		if err := e.processUserWithIdentify(m, assertString); err != nil {
			return err
		}
	}
	return nil
}

func (e *PGPPullEngine) processUserWithIdentify(m libkb.MetaContext, u string) error {
	m.Debug("Processing with identify: %s", u)

	iarg := keybase1.Identify2Arg{
		UserAssertion:    u,
		ForceRemoteCheck: true,
		AlwaysBlock:      true,
		NeedProofSet:     true, // forces prompt even if we declined before
	}
	topts := keybase1.TrackOptions{
		LocalOnly:  true,
		ForPGPPull: true,
	}
	ieng := NewResolveThenIdentify2WithTrack(m.G(), &iarg, topts)
	if err := RunEngine2(m, ieng); err != nil {
		m.Info("identify run err: %s", err)
		return err
	}

	// prompt if the identify is correct
	result := ieng.ConfirmResult()
	if !result.IdentityConfirmed {
		m.Warning("Not confirmed; skipping key import")
		return nil
	}

	idRes, err := ieng.Result(m)
	if err != nil {
		return err
	}
	// with more plumbing, there is likely a more efficient way to get this identified user out
	// of the identify2 engine, but `pgp pull` is not likely to be called often.
	arg := libkb.NewLoadUserArgWithMetaContext(m).WithUID(idRes.Upk.GetUID())
	user, err := libkb.LoadUser(arg)
	if err != nil {
		return err
	}
	return e.exportKeysToGPG(m, user, nil)
}

func (e *PGPPullEngine) Run(m libkb.MetaContext) error {

	e.gpgClient = libkb.NewGpgCLI(m.G(), m.UIs().LogUI)
	err := e.gpgClient.Configure()
	if err != nil {
		return err
	}

	if ok, _ := isLoggedIn(m); !ok {
		return e.runLoggedOut(m)
	}

	return e.runLoggedIn(m)
}

func (e *PGPPullEngine) runLoggedIn(m libkb.MetaContext) error {
	summaries, leftovers, err := e.getTrackedUserSummaries(m)
	// leftovers contains unmatched assertions, likely users
	// we want to pull but we do not track.
	if err != nil {
		return err
	}

	// Loop over the list of all users we track.
	t := time.Now()
	for i, userSummary := range summaries {
		t = e.rateLimit(t, i)
		// Compute the set of tracked pgp fingerprints. LoadUser will fetch key
		// data from the server, and we will compare it against this.
		trackedFingerprints := make(map[string]bool)
		for _, pubKey := range userSummary.Proofs.PublicKeys {
			if pubKey.PGPFingerprint != "" {
				trackedFingerprints[pubKey.PGPFingerprint] = true
			}
		}

		// Get user data from the server.
		user, err := libkb.LoadUser(
			libkb.NewLoadUserByNameArg(e.G(), userSummary.Username).
				WithPublicKeyOptional())
		if err != nil {
			m.Error("Failed to load user %s: %s", userSummary.Username, err)
			continue
		}
		if user.GetStatus() == keybase1.StatusCode_SCDeleted {
			m.Debug("User %q is deleted, skipping", userSummary.Username)
			continue
		}

		if err = e.exportKeysToGPG(m, user, trackedFingerprints); err != nil {
			return err
		}
	}

	// Loop over unmatched list and process with identify prompts.
	for i, assertString := range leftovers {
		t = e.rateLimit(t, i)
		if err := e.processUserWithIdentify(m, assertString); err != nil {
			return err
		}
	}

	return nil
}

func (e *PGPPullEngine) exportKeysToGPG(m libkb.MetaContext, user *libkb.User, tfp map[string]bool) error {
	for _, bundle := range user.GetActivePGPKeys(false) {
		// Check each key against the tracked set.
		if tfp != nil && !tfp[bundle.GetFingerprint().String()] {
			m.Warning("Keybase says that %s owns key %s, but you have not tracked this fingerprint before.", user.GetName(), bundle.GetFingerprint())
			continue
		}

		if err := e.gpgClient.ExportKey(*bundle, false /* export public key only */, false /* no batch */); err != nil {
			m.Warning("Failed to import %'s public key %s: %s", user.GetName(), bundle.GetFingerprint(), err.Error())
			continue
		}

		m.Info("Imported key for %s.", user.GetName())
	}
	return nil
}

func (e *PGPPullEngine) rateLimit(start time.Time, index int) time.Time {
	// server currently limiting to 32 req/s, but there can be 4 requests for each loaduser call.
	const loadUserPerSec = 4
	if index == 0 {
		return start
	}
	if index%loadUserPerSec != 0 {
		return start
	}
	d := time.Second - time.Since(start)
	if d > 0 {
		e.G().Log.Debug("sleeping for %s to slow down api requests", d)
		time.Sleep(d)
	}
	return time.Now()
}
<|endoftext|>


<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package libkb

import (
	"fmt"

	keybase1 "github.com/keybase/client/go/protocol/keybase1"
	jsonw "github.com/keybase/go-jsonw"
)

type SigHint struct {
	sigID     keybase1.SigID
	remoteID  string
	apiURL    string
	humanURL  string
	checkText string
	// `isVerified` indicates if the client generated the values or they were
	// received from the server and are trusted but not verified.
	isVerified bool
}

func (sh SigHint) GetHumanURL() string  { return sh.humanURL }
func (sh SigHint) GetAPIURL() string    { return sh.apiURL }
func (sh SigHint) GetCheckText() string { return sh.checkText }

func NewSigHint(jw *jsonw.Wrapper) (sh *SigHint, err error) {
	sh = &SigHint{}
	sh.sigID, err = GetSigID(jw.AtKey("sig_id"), true)
	sh.remoteID, _ = jw.AtKey("remote_id").GetString()
	sh.apiURL, _ = jw.AtKey("api_url").GetString()
	sh.humanURL, _ = jw.AtKey("human_url").GetString()
	sh.checkText, _ = jw.AtKey("proof_text_check").GetString()
	sh.isVerified, _ = jw.AtKey("isVerified").GetBool()
	return sh, err
}

func NewVerifiedSigHint(sigID keybase1.SigID, remoteID, apiURL, humanURL, checkText string) *SigHint {
	return &SigHint{
		sigID:      sigID,
		remoteID:   remoteID,
		apiURL:     apiURL,
		humanURL:   humanURL,
		checkText:  checkText,
		isVerified: true,
	}
}

func (sh SigHint) MarshalToJSON() *jsonw.Wrapper {
	ret := jsonw.NewDictionary()
	ret.SetKey("sig_id", jsonw.NewString(sh.sigID.ToString(true)))
	ret.SetKey("remote_id", jsonw.NewString(sh.remoteID))
	ret.SetKey("api_url", jsonw.NewString(sh.apiURL))
	ret.SetKey("human_url", jsonw.NewString(sh.humanURL))
	ret.SetKey("proof_text_check", jsonw.NewString(sh.checkText))
	ret.SetKey("is_verified", jsonw.NewBool(sh.isVerified))
	return ret
}

type SigHints struct {
	Contextified
	uid     keybase1.UID
	version int
	hints   map[keybase1.SigID]*SigHint
	dirty   bool
}

func NewSigHints(jw *jsonw.Wrapper, uid keybase1.UID, dirty bool, g *GlobalContext) (sh *SigHints, err error) {
	sh = &SigHints{
		uid:          uid,
		dirty:        dirty,
		version:      0,
		Contextified: NewContextified(g),
	}
	err = sh.PopulateWith(jw)
	if err != nil {
		sh = nil
	}
	return
}

func (sh SigHints) Lookup(i keybase1.SigID) *SigHint {
	obj := sh.hints[i]
	return obj
}

func (sh *SigHints) PopulateWith(jw *jsonw.Wrapper) (err error) {

	if jw == nil || jw.IsNil() {
		return
	}

	jw.AtKey("version").GetIntVoid(&sh.version, &err)
	if err != nil {
		return
	}

	sh.hints = make(map[keybase1.SigID]*SigHint)
	var n int
	n, err = jw.AtKey("hints").Len()
	if err != nil {
		return
	}

	for i := 0; i < n; i++ {
		hint, tmpe := NewSigHint(jw.AtKey("hints").AtIndex(i))
		if tmpe != nil {
			sh.G().Log.Warning("Bad SigHint Loaded: %s", tmpe)
		} else {
			sh.hints[hint.sigID] = hint
		}
	}
	return
}

func (sh SigHints) MarshalToJSON() *jsonw.Wrapper {
	ret := jsonw.NewDictionary()
	ret.SetKey("version", jsonw.NewInt(sh.version))
	ret.SetKey("hints", jsonw.NewArray(len(sh.hints)))
	i := 0
	for _, v := range sh.hints {
		ret.AtKey("hints").SetIndex(i, v.MarshalToJSON())
		i++
	}
	return ret
}

func (sh *SigHints) Store(m MetaContext) (err error) {
	m.CDebugf("+ SigHints.Store() for uid=%s", sh.uid)
	if sh.dirty {
		err = sh.G().LocalDb.Put(DbKeyUID(DBSigHints, sh.uid), []DbKey{}, sh.MarshalToJSON())
		sh.dirty = false
	} else {
		m.CDebugf("| SigHints.Store() skipped; wasn't dirty")
	}
	m.CDebugf("- SigHints.Store() for uid=%s -> %v", sh.uid, ErrToOk(err))
	return err
}

func LoadSigHints(m MetaContext, uid keybase1.UID) (sh *SigHints, err error) {
	defer m.CTrace(fmt.Sprintf("+ LoadSigHints(%s)", uid), func() error { return err })()
	var jw *jsonw.Wrapper
	jw, err = m.G().LocalDb.Get(DbKeyUID(DBSigHints, uid))
	if err != nil {
		jw = nil
		m.CDebugf("| SigHints failed to access local storage: %s", err)
	}
	// jw might be nil here, but that's allowed.
	sh, err = NewSigHints(jw, uid, false, m.G())
	if err == nil {
		m.CDebugf("| SigHints loaded @v%d", sh.version)
	}
	m.CDebugf("- LoadSigHints(%s)", uid)
	return
}

func (sh *SigHints) Refresh(m MetaContext) (err error) {
	defer m.CTrace(fmt.Sprintf("Refresh SigHints for uid=%s", sh.uid), func() error { return err })()
	res, err := m.G().API.Get(APIArg{
		Endpoint:    "sig/hints",
		SessionType: APISessionTypeNONE,
		Args: HTTPArgs{
			"uid": UIDArg(sh.uid),
			"low": I{sh.version},
		},
		MetaContext: m,
	})
	if err != nil {
		return err
	}

	return sh.RefreshWith(m, res.Body)
}

func (sh *SigHints) RefreshWith(m MetaContext, jw *jsonw.Wrapper) (err error) {
	defer m.CTrace("RefreshWith", func() error { return err })()

	n, err := jw.AtKey("hints").Len()
	if err != nil {
		return err
	}
	if n == 0 {
		m.CDebugf("| No changes; version %d was up-to-date", sh.version)
	} else if err = sh.PopulateWith(jw); err != nil {
		return err
	} else {
		sh.dirty = true
	}
	return nil
}

func LoadAndRefreshSigHints(m MetaContext, uid keybase1.UID) (*SigHints, error) {
	sh, err := LoadSigHints(m, uid)
	if err != nil {
		return nil, err
	}
	if err = sh.Refresh(m); err != nil {
		return nil, err
	}
	return sh, nil
}
<commit_msg>dont' log error for invalid sig hint (#14057)<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package libkb

import (
	"fmt"

	keybase1 "github.com/keybase/client/go/protocol/keybase1"
	jsonw "github.com/keybase/go-jsonw"
)

type SigHint struct {
	sigID     keybase1.SigID
	remoteID  string
	apiURL    string
	humanURL  string
	checkText string
	// `isVerified` indicates if the client generated the values or they were
	// received from the server and are trusted but not verified.
	isVerified bool
}

func (sh SigHint) GetHumanURL() string  { return sh.humanURL }
func (sh SigHint) GetAPIURL() string    { return sh.apiURL }
func (sh SigHint) GetCheckText() string { return sh.checkText }

func NewSigHint(jw *jsonw.Wrapper) (sh *SigHint, err error) {
	if jw == nil {
		return nil, nil
	}
	sh = &SigHint{}
	sh.sigID, err = GetSigID(jw.AtKey("sig_id"), true)
	sh.remoteID, _ = jw.AtKey("remote_id").GetString()
	sh.apiURL, _ = jw.AtKey("api_url").GetString()
	sh.humanURL, _ = jw.AtKey("human_url").GetString()
	sh.checkText, _ = jw.AtKey("proof_text_check").GetString()
	sh.isVerified, _ = jw.AtKey("isVerified").GetBool()
	return sh, err
}

func NewVerifiedSigHint(sigID keybase1.SigID, remoteID, apiURL, humanURL, checkText string) *SigHint {
	return &SigHint{
		sigID:      sigID,
		remoteID:   remoteID,
		apiURL:     apiURL,
		humanURL:   humanURL,
		checkText:  checkText,
		isVerified: true,
	}
}

func (sh SigHint) MarshalToJSON() *jsonw.Wrapper {
	ret := jsonw.NewDictionary()
	ret.SetKey("sig_id", jsonw.NewString(sh.sigID.ToString(true)))
	ret.SetKey("remote_id", jsonw.NewString(sh.remoteID))
	ret.SetKey("api_url", jsonw.NewString(sh.apiURL))
	ret.SetKey("human_url", jsonw.NewString(sh.humanURL))
	ret.SetKey("proof_text_check", jsonw.NewString(sh.checkText))
	ret.SetKey("is_verified", jsonw.NewBool(sh.isVerified))
	return ret
}

type SigHints struct {
	Contextified
	uid     keybase1.UID
	version int
	hints   map[keybase1.SigID]*SigHint
	dirty   bool
}

func NewSigHints(jw *jsonw.Wrapper, uid keybase1.UID, dirty bool, g *GlobalContext) (sh *SigHints, err error) {
	sh = &SigHints{
		uid:          uid,
		dirty:        dirty,
		version:      0,
		Contextified: NewContextified(g),
	}
	err = sh.PopulateWith(jw)
	if err != nil {
		sh = nil
	}
	return
}

func (sh SigHints) Lookup(i keybase1.SigID) *SigHint {
	obj := sh.hints[i]
	return obj
}

func (sh *SigHints) PopulateWith(jw *jsonw.Wrapper) (err error) {

	if jw == nil || jw.IsNil() {
		return
	}

	jw.AtKey("version").GetIntVoid(&sh.version, &err)
	if err != nil {
		return
	}

	sh.hints = make(map[keybase1.SigID]*SigHint)
	var n int
	n, err = jw.AtKey("hints").Len()
	if err != nil {
		return
	}

	for i := 0; i < n; i++ {
		hint, tmpe := NewSigHint(jw.AtKey("hints").AtIndex(i))
		if tmpe != nil {
			sh.G().Log.Warning("Bad SigHint Loaded: %s", tmpe)
		} else {
			sh.hints[hint.sigID] = hint
		}
	}
	return
}

func (sh SigHints) MarshalToJSON() *jsonw.Wrapper {
	ret := jsonw.NewDictionary()
	ret.SetKey("version", jsonw.NewInt(sh.version))
	ret.SetKey("hints", jsonw.NewArray(len(sh.hints)))
	i := 0
	for _, v := range sh.hints {
		ret.AtKey("hints").SetIndex(i, v.MarshalToJSON())
		i++
	}
	return ret
}

func (sh *SigHints) Store(m MetaContext) (err error) {
	m.CDebugf("+ SigHints.Store() for uid=%s", sh.uid)
	if sh.dirty {
		err = sh.G().LocalDb.Put(DbKeyUID(DBSigHints, sh.uid), []DbKey{}, sh.MarshalToJSON())
		sh.dirty = false
	} else {
		m.CDebugf("| SigHints.Store() skipped; wasn't dirty")
	}
	m.CDebugf("- SigHints.Store() for uid=%s -> %v", sh.uid, ErrToOk(err))
	return err
}

func LoadSigHints(m MetaContext, uid keybase1.UID) (sh *SigHints, err error) {
	defer m.CTrace(fmt.Sprintf("+ LoadSigHints(%s)", uid), func() error { return err })()
	var jw *jsonw.Wrapper
	jw, err = m.G().LocalDb.Get(DbKeyUID(DBSigHints, uid))
	if err != nil {
		jw = nil
		m.CDebugf("| SigHints failed to access local storage: %s", err)
	}
	// jw might be nil here, but that's allowed.
	sh, err = NewSigHints(jw, uid, false, m.G())
	if err == nil {
		m.CDebugf("| SigHints loaded @v%d", sh.version)
	}
	m.CDebugf("- LoadSigHints(%s)", uid)
	return
}

func (sh *SigHints) Refresh(m MetaContext) (err error) {
	defer m.CTrace(fmt.Sprintf("Refresh SigHints for uid=%s", sh.uid), func() error { return err })()
	res, err := m.G().API.Get(APIArg{
		Endpoint:    "sig/hints",
		SessionType: APISessionTypeNONE,
		Args: HTTPArgs{
			"uid": UIDArg(sh.uid),
			"low": I{sh.version},
		},
		MetaContext: m,
	})
	if err != nil {
		return err
	}

	return sh.RefreshWith(m, res.Body)
}

func (sh *SigHints) RefreshWith(m MetaContext, jw *jsonw.Wrapper) (err error) {
	defer m.CTrace("RefreshWith", func() error { return err })()

	n, err := jw.AtKey("hints").Len()
	if err != nil {
		return err
	}
	if n == 0 {
		m.CDebugf("| No changes; version %d was up-to-date", sh.version)
	} else if err = sh.PopulateWith(jw); err != nil {
		return err
	} else {
		sh.dirty = true
	}
	return nil
}

func LoadAndRefreshSigHints(m MetaContext, uid keybase1.UID) (*SigHints, error) {
	sh, err := LoadSigHints(m, uid)
	if err != nil {
		return nil, err
	}
	if err = sh.Refresh(m); err != nil {
		return nil, err
	}
	return sh, nil
}
<|endoftext|>


<commit_before>package goakit

import (
	"fmt"
	"path/filepath"

	"goa.design/goa/codegen"
	httpcodegen "goa.design/goa/http/codegen"
	httpdesign "goa.design/goa/http/design"
)

// MountFiles produces the files containing the HTTP handler mount functions
// that configure the mux to serve the requests.
func MountFiles(root *httpdesign.RootExpr) []*codegen.File {
	fw := make([]*codegen.File, len(root.HTTPServices))
	for i, svc := range root.HTTPServices {
		fw[i] = mountFile(svc)
	}
	return fw
}

// mountFile returns the file defining the mount handler functions for the given
// service.
func mountFile(svc *httpdesign.ServiceExpr) *codegen.File {
	path := filepath.Join(codegen.Gendir, "http", codegen.SnakeCase(svc.Name()), "kitserver", "mount.go")
	data := httpcodegen.HTTPServices.Get(svc.Name())
	title := fmt.Sprintf("%s go-kit HTTP server encoders and decoders", svc.Name())
	sections := []*codegen.SectionTemplate{
		codegen.Header(title, "server", []*codegen.ImportSpec{
			{Path: "net/http"},
			{Path: "goa.design/goa/http", Name: "goahttp"},
		}),
	}
	for _, e := range data.Endpoints {
		sections = append(sections, &codegen.SectionTemplate{
			Name:   "goakit-mount-handler",
			Source: mountHandlerT,
			Data:   e,
		})
	}

	return &codegen.File{Path: path, SectionTemplates: sections}
}

// input: EndpointData
const mountHandlerT = `{{ printf "%s configures the mux to serve the \"%s\" service \"%s\" endpoint." .MountHandler .ServiceName .Method.Name | comment }}
func {{ .MountHandler }}(mux goahttp.Muxer, h http.Handler) {
	f, ok := h.(http.HandlerFunc)
	if !ok {
		f = func(w http.ResponseWriter, r *http.Request) {
			h.ServeHTTP(w, r)
		}
	}
	{{- range .Routes }}
	mux.Handle("{{ .Verb }}", "{{ .Path }}", f)
	{{- end }}
}
`
<commit_msg>Handle file servers in goakit plugin<commit_after>package goakit

import (
	"fmt"
	"path/filepath"

	"goa.design/goa/codegen"
	httpcodegen "goa.design/goa/http/codegen"
	httpdesign "goa.design/goa/http/design"
)

// MountFiles produces the files containing the HTTP handler mount functions
// that configure the mux to serve the requests.
func MountFiles(root *httpdesign.RootExpr) []*codegen.File {
	fw := make([]*codegen.File, len(root.HTTPServices))
	for i, svc := range root.HTTPServices {
		fw[i] = mountFile(svc)
	}
	return fw
}

// mountFile returns the file defining the mount handler functions for the given
// service.
func mountFile(svc *httpdesign.ServiceExpr) *codegen.File {
	path := filepath.Join(codegen.Gendir, "http", codegen.SnakeCase(svc.Name()), "kitserver", "mount.go")
	data := httpcodegen.HTTPServices.Get(svc.Name())
	title := fmt.Sprintf("%s go-kit HTTP server encoders and decoders", svc.Name())
	sections := []*codegen.SectionTemplate{
		codegen.Header(title, "server", []*codegen.ImportSpec{
			{Path: "net/http"},
			{Path: "goa.design/goa/http", Name: "goahttp"},
		}),
	}
	for _, e := range data.Endpoints {
		sections = append(sections, &codegen.SectionTemplate{
			Name:   "goakit-mount-handler",
			Source: mountHandlerT,
			Data:   e,
		})
	}

	return &codegen.File{Path: path, SectionTemplates: sections}
}

// input: EndpointData
const mountHandlerT = `{{ printf "%s configures the mux to serve the \"%s\" service \"%s\" endpoint." .MountHandler .ServiceName .Method.Name | comment }}
func {{ .MountHandler }}(mux goahttp.Muxer, h http.Handler) {
	f, ok := h.(http.HandlerFunc)
	if !ok {
		f = func(w http.ResponseWriter, r *http.Request) {
			h.ServeHTTP(w, r)
		}
	}
	{{- range .Routes }}
	mux.Handle("{{ .Verb }}", "{{ .Path }}", f)
	{{- end }}
	{{- range .FileServers }}
		{{- if .IsDir }}
	mux.Handle("GET", "{{ .RequestPath }}", http.FileServer(http.Dir({{ printf "%q" .FilePath }})))
		{{- else }}
	mux.Handle("GET", "{{ .RequestPath }}", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			http.ServeFile(w, r, {{ printf "%q" .FilePath }})
		}))
		{{- end }}
	{{- end }}
}
`
<|endoftext|>


<commit_before>/*
Copyright IBM Corp. All Rights Reserved.

SPDX-License-Identifier: Apache-2.0
*/

package comm

import (
	"crypto/ecdsa"
	"crypto/elliptic"
	"crypto/rand"
	"crypto/tls"
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"math/big"
	"os"

	"github.com/hyperledger/fabric/common/util"
	gutil "github.com/hyperledger/fabric/gossip/util"
	"golang.org/x/net/context"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/peer"
)

func writeFile(filename string, keyType string, data []byte) error {
	f, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer f.Close()
	return pem.Encode(f, &pem.Block{Type: keyType, Bytes: data})
}

// GenerateCertificatesOrPanic generates a a random pair of public and private keys
// and return TLS certificate
func GenerateCertificatesOrPanic() tls.Certificate {
	privKeyFile := fmt.Sprintf("key.%d.priv", gutil.RandomUInt64())
	certKeyFile := fmt.Sprintf("cert.%d.pub", gutil.RandomUInt64())

	defer os.Remove(privKeyFile)
	defer os.Remove(certKeyFile)
	privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
	if err != nil {
		panic(err)
	}

	sn, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
	if err != nil {
		panic(err)
	}
	template := x509.Certificate{
		KeyUsage:     x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		SerialNumber: sn,
		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
	}
	rawBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey)
	if err != nil {
		panic(err)
	}
	err = writeFile(certKeyFile, "CERTIFICATE", rawBytes)
	if err != nil {
		panic(err)
	}
	privBytes, err := x509.MarshalECPrivateKey(privateKey)
	if err != nil {
		panic(err)
	}
	err = writeFile(privKeyFile, "EC PRIVATE KEY", privBytes)
	if err != nil {
		panic(err)
	}
	cert, err := tls.LoadX509KeyPair(certKeyFile, privKeyFile)
	if err != nil {
		panic(err)
	}
	if len(cert.Certificate) == 0 {
		panic("Certificate chain is empty")
	}
	return cert
}

func certHashFromRawCert(rawCert []byte) []byte {
	if len(rawCert) == 0 {
		return nil
	}
	return util.ComputeSHA256(rawCert)
}

// ExtractCertificateHash extracts the hash of the certificate from the stream
func extractCertificateHashFromContext(ctx context.Context) []byte {
	pr, extracted := peer.FromContext(ctx)
	if !extracted {
		return nil
	}

	authInfo := pr.AuthInfo
	if authInfo == nil {
		return nil
	}

	tlsInfo, isTLSConn := authInfo.(credentials.TLSInfo)
	if !isTLSConn {
		return nil
	}
	certs := tlsInfo.State.PeerCertificates
	if len(certs) == 0 {
		return nil
	}
	raw := certs[0].Raw
	return certHashFromRawCert(raw)
}
<commit_msg>[FAB-7674] use buffers instead of temporary files<commit_after>/*
Copyright IBM Corp. All Rights Reserved.

SPDX-License-Identifier: Apache-2.0
*/

package comm

import (
	"crypto/ecdsa"
	"crypto/elliptic"
	"crypto/rand"
	"crypto/tls"
	"crypto/x509"
	"encoding/pem"
	"math/big"

	"github.com/hyperledger/fabric/common/util"
	"golang.org/x/net/context"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/peer"
)

// GenerateCertificatesOrPanic generates a a random pair of public and private keys
// and return TLS certificate
func GenerateCertificatesOrPanic() tls.Certificate {
	privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
	if err != nil {
		panic(err)
	}
	sn, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
	if err != nil {
		panic(err)
	}
	template := x509.Certificate{
		KeyUsage:     x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		SerialNumber: sn,
		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
	}
	rawBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey)
	if err != nil {
		panic(err)
	}
	privBytes, err := x509.MarshalECPrivateKey(privateKey)
	if err != nil {
		panic(err)
	}
	encodedCert := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: rawBytes})
	encodedKey := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: privBytes})
	cert, err := tls.X509KeyPair(encodedCert, encodedKey)
	if err != nil {
		panic(err)
	}
	if len(cert.Certificate) == 0 {
		panic("Certificate chain is empty")
	}
	return cert
}

func certHashFromRawCert(rawCert []byte) []byte {
	if len(rawCert) == 0 {
		return nil
	}
	return util.ComputeSHA256(rawCert)
}

// ExtractCertificateHash extracts the hash of the certificate from the stream
func extractCertificateHashFromContext(ctx context.Context) []byte {
	pr, extracted := peer.FromContext(ctx)
	if !extracted {
		return nil
	}

	authInfo := pr.AuthInfo
	if authInfo == nil {
		return nil
	}

	tlsInfo, isTLSConn := authInfo.(credentials.TLSInfo)
	if !isTLSConn {
		return nil
	}
	certs := tlsInfo.State.PeerCertificates
	if len(certs) == 0 {
		return nil
	}
	raw := certs[0].Raw
	return certHashFromRawCert(raw)
}
<|endoftext|>


<commit_before>// +build linux

package main

import (
	"errors"
	"flag"
	"fmt"
	"github.com/caglar10ur/lxc"
	"koding/newkite/kite"
	"koding/newkite/protocol"
	"strings"
)

type Supervisor struct{}

var port = flag.String("port", "4005", "port to bind itself")

func main() {
	flag.Parse()
	options := &protocol.Options{
		PublicIP: "localhost",
		Kitename: "supervisor",
		Version:  "0.0.1",
		Port:     *port,
	}

	methods := map[string]string{
		"vm.create":  "Create",
		"vm.destroy": "Destroy",
		"vm.start":   "Start",
		"vm.stop":    "Stop",
		"vm.run":     "Run",
	}

	k := kite.New(options)
	k.AddMethods(new(Supervisor), methods)
	k.Start()
}

func (s *Supervisor) Create(r *protocol.KiteDnodeRequest, result *bool) error {
	var params struct {
		ContainerName string
		Template      string
	}

	if r.Args.Unmarshal(&params) != nil || params.ContainerName == "" || params.Template == "" {
		return errors.New("{ containerName: [string], template: [string] }")
	}

	err := s.lxcCreate(params.ContainerName, params.Template)
	if err != nil {
		return err
	}

	*result = true
	return nil
}

func (s *Supervisor) Destroy(r *protocol.KiteDnodeRequest, result *bool) error {
	var params struct {
		ContainerName string
	}

	if r.Args.Unmarshal(&params) != nil || params.ContainerName == "" {
		return errors.New("{ containerName: [string] }")
	}

	err := s.lxcDestroy(params.ContainerName)
	if err != nil {
		return err
	}

	*result = true
	return nil
}

func (s *Supervisor) Start(r *protocol.KiteDnodeRequest, result *bool) error {
	var params struct {
		ContainerName string
	}

	if r.Args.Unmarshal(&params) != nil || params.ContainerName == "" {
		return errors.New("{ containerName: [string] }")
	}

	err := s.lxcStart(params.ContainerName)
	if err != nil {
		return err
	}

	*result = true
	return nil
}

func (s *Supervisor) Stop(r *protocol.KiteDnodeRequest, result *bool) error {
	var params struct {
		ContainerName string
	}

	if r.Args.Unmarshal(&params) != nil || params.ContainerName == "" {
		return errors.New("{ containerName: [string] }")
	}

	err := s.lxcStop(params.ContainerName)
	if err != nil {
		return err
	}

	*result = true
	return nil
}

func (s *Supervisor) Run(r *protocol.KiteDnodeRequest, result *bool) error {
	var params struct {
		ContainerName string
		Command       string
	}

	if r.Args.Unmarshal(&params) != nil || params.ContainerName == "" || params.Command == "" {
		return errors.New("{ containerName: [string], command : [string]}")
	}

	err := s.lxcRun(params.ContainerName, params.Command)
	if err != nil {
		return err
	}

	return nil
}

func (s *Supervisor) lxcRun(containerName, command string) error {
	fmt.Printf("running '%s' on '%s'\n", command, containerName)

	c := lxc.NewContainer(containerName)
	defer lxc.PutContainer(c)

	args := strings.Split(strings.TrimSpace(command), " ")

	if err := c.AttachRunCommand(args...); err != nil {
		fmt.Errorf("ERROR: %s\n", err.Error())
	}
	return nil
}

func (s *Supervisor) lxcStart(containerName string) error {
	fmt.Println("starting ", containerName)

	c := lxc.NewContainer(containerName)
	defer lxc.PutContainer(c)

	err := c.SetDaemonize()
	if err != nil {
		return fmt.Errorf("ERROR: %s\n", err)
	}

	err = c.Start(false)
	if err != nil {
		return fmt.Errorf("ERROR: %s\n", err)
	}

	return nil
}

func (s *Supervisor) lxcStop(containerName string) error {
	fmt.Println("stopping ", containerName)

	c := lxc.NewContainer(containerName)
	defer lxc.PutContainer(c)

	err := c.Stop()
	if err != nil {
		return fmt.Errorf("ERROR: %s\n", err)
	}

	return nil
}

func (s *Supervisor) lxcCreate(containerName, template string) error {
	fmt.Printf("creating vm '%s' with template '%s'\n", containerName, template)

	c := lxc.NewContainer(containerName)
	defer lxc.PutContainer(c)
	return c.Create(template)
}

func (s *Supervisor) lxcDestroy(containerName string) error {
	fmt.Println("destroying ", containerName)

	c := lxc.NewContainer(containerName)
	defer lxc.PutContainer(c)
	return c.Destroy()
}
<commit_msg>kite/supervisor: fix running commands<commit_after>// +build linux

package main

import (
	"errors"
	"flag"
	"fmt"
	"github.com/caglar10ur/lxc"
	"koding/newkite/kite"
	"koding/newkite/protocol"
	"strings"
)

type Supervisor struct{}

var port = flag.String("port", "4005", "port to bind itself")

func main() {
	flag.Parse()
	options := &protocol.Options{
		PublicIP: "localhost",
		Kitename: "supervisor",
		Version:  "0.0.1",
		Port:     *port,
	}

	methods := map[string]string{
		"vm.create":  "Create",
		"vm.destroy": "Destroy",
		"vm.start":   "Start",
		"vm.stop":    "Stop",
		"vm.run":     "Run",
	}

	k := kite.New(options)
	k.AddMethods(new(Supervisor), methods)
	k.Start()
}

func (s *Supervisor) Create(r *protocol.KiteDnodeRequest, result *bool) error {
	var params struct {
		ContainerName string
		Template      string
	}

	if r.Args.Unmarshal(&params) != nil || params.ContainerName == "" || params.Template == "" {
		return errors.New("{ containerName: [string], template: [string] }")
	}

	err := s.lxcCreate(params.ContainerName, params.Template)
	if err != nil {
		return err
	}

	*result = true
	return nil
}

func (s *Supervisor) Destroy(r *protocol.KiteDnodeRequest, result *bool) error {
	var params struct {
		ContainerName string
	}

	if r.Args.Unmarshal(&params) != nil || params.ContainerName == "" {
		return errors.New("{ containerName: [string] }")
	}

	err := s.lxcDestroy(params.ContainerName)
	if err != nil {
		return err
	}

	*result = true
	return nil
}

func (s *Supervisor) Start(r *protocol.KiteDnodeRequest, result *bool) error {
	var params struct {
		ContainerName string
	}

	if r.Args.Unmarshal(&params) != nil || params.ContainerName == "" {
		return errors.New("{ containerName: [string] }")
	}

	err := s.lxcStart(params.ContainerName)
	if err != nil {
		return err
	}

	*result = true
	return nil
}

func (s *Supervisor) Stop(r *protocol.KiteDnodeRequest, result *bool) error {
	var params struct {
		ContainerName string
	}

	if r.Args.Unmarshal(&params) != nil || params.ContainerName == "" {
		return errors.New("{ containerName: [string] }")
	}

	err := s.lxcStop(params.ContainerName)
	if err != nil {
		return err
	}

	*result = true
	return nil
}

func (s *Supervisor) Run(r *protocol.KiteDnodeRequest, result *bool) error {
	var params struct {
		ContainerName string
		Command       string
	}

	if r.Args.Unmarshal(&params) != nil || params.ContainerName == "" || params.Command == "" {
		return errors.New("{ containerName: [string], command : [string]}")
	}

	err := s.lxcRun(params.ContainerName, params.Command)
	if err != nil {
		return err
	}

	return nil
}

func (s *Supervisor) lxcRun(containerName, command string) error {
	fmt.Printf("running '%s' on '%s'\n", command, containerName)

	c := lxc.NewContainer(containerName)
	defer lxc.PutContainer(c)

	args := strings.Split(strings.TrimSpace(command), " ")

	if err := c.AttachRunCommand(args...); err != nil {
		return fmt.Errorf("ERROR: %s\n", err.Error())
	}

	return nil
}

func (s *Supervisor) lxcStart(containerName string) error {
	fmt.Println("starting ", containerName)

	c := lxc.NewContainer(containerName)
	defer lxc.PutContainer(c)

	err := c.SetDaemonize()
	if err != nil {
		return fmt.Errorf("ERROR: %s\n", err)
	}

	err = c.Start(false)
	if err != nil {
		return fmt.Errorf("ERROR: %s\n", err)
	}

	return nil
}

func (s *Supervisor) lxcStop(containerName string) error {
	fmt.Println("stopping ", containerName)

	c := lxc.NewContainer(containerName)
	defer lxc.PutContainer(c)

	err := c.Stop()
	if err != nil {
		return fmt.Errorf("ERROR: %s\n", err)
	}

	return nil
}

func (s *Supervisor) lxcCreate(containerName, template string) error {
	fmt.Printf("creating vm '%s' with template '%s'\n", containerName, template)

	c := lxc.NewContainer(containerName)
	defer lxc.PutContainer(c)
	return c.Create(template)
}

func (s *Supervisor) lxcDestroy(containerName string) error {
	fmt.Println("destroying ", containerName)

	c := lxc.NewContainer(containerName)
	defer lxc.PutContainer(c)
	return c.Destroy()
}
<|endoftext|>


<commit_before>/*
Copyright 2021 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package docker

import (
	"context"
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/google/go-cmp/cmp"

	"vitess.io/vitess/go/sqltypes"

	"vitess.io/vitess/go/mysql"

	"github.com/stretchr/testify/require"
)

func TestMain(m *testing.M) {
	exitCode := func() int {
		err := makeVttestserverDockerImages()
		if err != nil {
			return 1
		}
		return m.Run()
	}()
	os.Exit(exitCode)
}

func TestUnsharded(t *testing.T) {
	dockerImages := []string{vttestserverMysql57image, vttestserverMysql80image}
	for _, image := range dockerImages {
		t.Run(image, func(t *testing.T) {
			vtest := newVttestserver(image, []string{"unsharded_ks"}, []int{1}, 1000, 33577)
			err := vtest.startDockerImage()
			require.NoError(t, err)
			defer vtest.teardown()

			// wait for the docker to be setup
			time.Sleep(10 * time.Second)

			ctx := context.Background()
			vttestParams := mysql.ConnParams{
				Host: "localhost",
				Port: vtest.port,
			}
			conn, err := mysql.Connect(ctx, &vttestParams)
			require.NoError(t, err)
			defer conn.Close()
			assertMatches(t, conn, "show databases", `[[VARCHAR("unsharded_ks")] [VARCHAR("information_schema")] [VARCHAR("mysql")] [VARCHAR("sys")] [VARCHAR("performance_schema")]]`)
			_, err = execute(t, conn, "create table unsharded_ks.t1(id int)")
			require.NoError(t, err)
			_, err = execute(t, conn, "insert into unsharded_ks.t1(id) values (10),(20),(30)")
			require.NoError(t, err)
			assertMatches(t, conn, "select * from unsharded_ks.t1", `[[INT32(10)] [INT32(20)] [INT32(30)]]`)
		})
	}
}

func TestSharded(t *testing.T) {
	dockerImages := []string{vttestserverMysql57image, vttestserverMysql80image}
	for _, image := range dockerImages {
		t.Run(image, func(t *testing.T) {
			vtest := newVttestserver(image, []string{"ks"}, []int{2}, 1000, 33577)
			err := vtest.startDockerImage()
			require.NoError(t, err)
			defer vtest.teardown()

			// wait for the docker to be setup
			time.Sleep(10 * time.Second)

			ctx := context.Background()
			vttestParams := mysql.ConnParams{
				Host: "localhost",
				Port: vtest.port,
			}
			conn, err := mysql.Connect(ctx, &vttestParams)
			require.NoError(t, err)
			defer conn.Close()
			assertMatches(t, conn, "show databases", `[[VARCHAR("ks")] [VARCHAR("information_schema")] [VARCHAR("mysql")] [VARCHAR("sys")] [VARCHAR("performance_schema")]]`)
			_, err = execute(t, conn, "create table ks.t1(id int)")
			require.NoError(t, err)
			_, err = execute(t, conn, "alter vschema on ks.t1 add vindex `binary_md5`(id) using `binary_md5`")
			require.NoError(t, err)
			_, err = execute(t, conn, "insert into ks.t1(id) values (10),(20),(30)")
			require.NoError(t, err)
			assertMatches(t, conn, "select id from ks.t1 order by id", `[[INT32(10)] [INT32(20)] [INT32(30)]]`)
		})
	}
}

func execute(t *testing.T, conn *mysql.Conn, query string) (*sqltypes.Result, error) {
	t.Helper()
	return conn.ExecuteFetch(query, 1000, true)
}

func checkedExec(t *testing.T, conn *mysql.Conn, query string) *sqltypes.Result {
	t.Helper()
	qr, err := conn.ExecuteFetch(query, 1000, true)
	require.NoError(t, err)
	return qr
}

func assertMatches(t *testing.T, conn *mysql.Conn, query, expected string) {
	t.Helper()
	qr := checkedExec(t, conn, query)
	got := fmt.Sprintf("%v", qr.Rows)
	diff := cmp.Diff(expected, got)
	if diff != "" {
		t.Errorf("Query: %s (-want +got):\n%s", query, diff)
	}
}
<commit_msg>added test for mysql max connections for vttestserver image<commit_after>/*
Copyright 2021 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package docker

import (
	"context"
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/google/go-cmp/cmp"

	"vitess.io/vitess/go/sqltypes"

	"vitess.io/vitess/go/mysql"

	"github.com/stretchr/testify/require"
)

func TestMain(m *testing.M) {
	exitCode := func() int {
		err := makeVttestserverDockerImages()
		if err != nil {
			return 1
		}
		return m.Run()
	}()
	os.Exit(exitCode)
}

func TestUnsharded(t *testing.T) {
	dockerImages := []string{vttestserverMysql57image, vttestserverMysql80image}
	for _, image := range dockerImages {
		t.Run(image, func(t *testing.T) {
			vtest := newVttestserver(image, []string{"unsharded_ks"}, []int{1}, 1000, 33577)
			err := vtest.startDockerImage()
			require.NoError(t, err)
			defer vtest.teardown()

			// wait for the docker to be setup
			time.Sleep(10 * time.Second)

			ctx := context.Background()
			vttestParams := mysql.ConnParams{
				Host: "localhost",
				Port: vtest.port,
			}
			conn, err := mysql.Connect(ctx, &vttestParams)
			require.NoError(t, err)
			defer conn.Close()
			assertMatches(t, conn, "show databases", `[[VARCHAR("unsharded_ks")] [VARCHAR("information_schema")] [VARCHAR("mysql")] [VARCHAR("sys")] [VARCHAR("performance_schema")]]`)
			_, err = execute(t, conn, "create table unsharded_ks.t1(id int)")
			require.NoError(t, err)
			_, err = execute(t, conn, "insert into unsharded_ks.t1(id) values (10),(20),(30)")
			require.NoError(t, err)
			assertMatches(t, conn, "select * from unsharded_ks.t1", `[[INT32(10)] [INT32(20)] [INT32(30)]]`)
		})
	}
}

func TestSharded(t *testing.T) {
	dockerImages := []string{vttestserverMysql57image, vttestserverMysql80image}
	for _, image := range dockerImages {
		t.Run(image, func(t *testing.T) {
			vtest := newVttestserver(image, []string{"ks"}, []int{2}, 1000, 33577)
			err := vtest.startDockerImage()
			require.NoError(t, err)
			defer vtest.teardown()

			// wait for the docker to be setup
			time.Sleep(10 * time.Second)

			ctx := context.Background()
			vttestParams := mysql.ConnParams{
				Host: "localhost",
				Port: vtest.port,
			}
			conn, err := mysql.Connect(ctx, &vttestParams)
			require.NoError(t, err)
			defer conn.Close()
			assertMatches(t, conn, "show databases", `[[VARCHAR("ks")] [VARCHAR("information_schema")] [VARCHAR("mysql")] [VARCHAR("sys")] [VARCHAR("performance_schema")]]`)
			_, err = execute(t, conn, "create table ks.t1(id int)")
			require.NoError(t, err)
			_, err = execute(t, conn, "alter vschema on ks.t1 add vindex `binary_md5`(id) using `binary_md5`")
			require.NoError(t, err)
			_, err = execute(t, conn, "insert into ks.t1(id) values (10),(20),(30)")
			require.NoError(t, err)
			assertMatches(t, conn, "select id from ks.t1 order by id", `[[INT32(10)] [INT32(20)] [INT32(30)]]`)
		})
	}
}

func TestMysqlMaxCons(t *testing.T) {
	dockerImages := []string{vttestserverMysql57image, vttestserverMysql80image}
	for _, image := range dockerImages {
		t.Run(image, func(t *testing.T) {
			vtest := newVttestserver(image, []string{"ks"}, []int{2}, 100000, 33577)
			err := vtest.startDockerImage()
			require.NoError(t, err)
			defer vtest.teardown()

			// wait for the docker to be setup
			time.Sleep(10 * time.Second)

			ctx := context.Background()
			vttestParams := mysql.ConnParams{
				Host: "localhost",
				Port: vtest.port,
			}
			conn, err := mysql.Connect(ctx, &vttestParams)
			require.NoError(t, err)
			defer conn.Close()
			assertMatches(t, conn, "select @@max_connections", `[[UINT64(100000)]]`)
		})
	}
}

func execute(t *testing.T, conn *mysql.Conn, query string) (*sqltypes.Result, error) {
	t.Helper()
	return conn.ExecuteFetch(query, 1000, true)
}

func checkedExec(t *testing.T, conn *mysql.Conn, query string) *sqltypes.Result {
	t.Helper()
	qr, err := conn.ExecuteFetch(query, 1000, true)
	require.NoError(t, err)
	return qr
}

func assertMatches(t *testing.T, conn *mysql.Conn, query, expected string) {
	t.Helper()
	qr := checkedExec(t, conn, query)
	got := fmt.Sprintf("%v", qr.Rows)
	diff := cmp.Diff(expected, got)
	if diff != "" {
		t.Errorf("Query: %s (-want +got):\n%s", query, diff)
	}
}
<|endoftext|>


<commit_before>package grafana

import (
	"bytes"
	"encoding/json"
	"testing"
)

func TestStackVal_UnmarshalJSON_GotTrue(t *testing.T) {
	var sampleOut struct {
		Val BoolString `json:"val"`
	}
	var sampleIn = []byte(`{"val":true}`)

	json.Unmarshal(sampleIn, &sampleOut)

	if !sampleOut.Val.Flag {
		t.Errorf("should be true but got false")
	}
	if sampleOut.Val.Value != "" {
		t.Error("string value should be empty")
	}
}

func TestStackVal_UnmarshalJSON_GotFalse(t *testing.T) {
	var sampleOut struct {
		Val BoolString `json:"val"`
	}
	var sampleIn = []byte(`{"val":false}`)

	json.Unmarshal(sampleIn, &sampleOut)

	if sampleOut.Val.Flag {
		t.Errorf("should be false but got true")
	}
	if sampleOut.Val.Value != "" {
		t.Error("string value should be empty")
	}
}

func TestStackVal_UnmarshalJSON_GotString(t *testing.T) {
	var sampleOut struct {
		Val BoolString `json:"val"`
	}
	var sampleIn = []byte(`{"val":"A"}`)

	json.Unmarshal(sampleIn, &sampleOut)

	if sampleOut.Val.Flag {
		t.Error("should be false but got true")
	}
	if sampleOut.Val.Value != "A" {
		t.Errorf("should be 'A' but got '%s'", sampleOut.Val.Value)
	}
}

func TestStackVal_MarshalJSON_GotTrue(t *testing.T) {
	var sampleInp struct {
		Val BoolString `json:"val"`
	}
	sampleInp.Val.Flag = true
	var sampleOut = []byte(`{"val":true}`)

	data, _ := json.Marshal(sampleInp)

	if bytes.Compare(data, sampleOut) != 0 {
		t.Errorf("should be %s but got %s", sampleOut, data)
	}
}

func TestStackVal_MarshalJSON_GotFalse(t *testing.T) {
	var sampleInp struct {
		Val BoolString `json:"val"`
	}
	sampleInp.Val.Flag = false
	var sampleOut = []byte(`{"val":false}`)

	data, _ := json.Marshal(sampleInp)

	if bytes.Compare(data, sampleOut) != 0 {
		t.Errorf("should be %s but got %s", sampleOut, data)
	}
}

func TestStackVal_MarshalJSON_GotString(t *testing.T) {
	var sampleInp struct {
		Val BoolString `json:"val"`
	}
	sampleInp.Val.Value = "A"
	var sampleOut = []byte(`{"val":"A"}`)

	data, _ := json.Marshal(sampleInp)

	if bytes.Compare(data, sampleOut) != 0 {
		t.Errorf("should be %s but got %s", sampleOut, data)
	}
}

func TestNewGraph(t *testing.T) {
	var title = "Sample Title"

	graph := NewGraph(title)

	if graph.GraphPanel == nil {
		t.Error("should be not nil")
	}
	if graph.TextPanel != nil {
		t.Error("should be nil")
	}
	if graph.DashlistPanel != nil {
		t.Error("should be nil")
	}
	if graph.SinglestatPanel != nil {
		t.Error("should be nil")
	}
	if graph.Title != title {
		t.Errorf("title should be %s but %s", title, graph.Title)
	}
}

func TestGraph_AddTarget(t *testing.T) {
	var target = Target{
		RefID:      "A",
		Datasource: "Sample Source",
		Expr:       "sample request"}
	graph := NewGraph("")

	graph.AddTarget(&target)

	if len(graph.GraphPanel.Targets) != 1 {
		t.Errorf("should be 1 but %d", len(graph.GraphPanel.Targets))
	}
	if graph.GraphPanel.Targets[0].RefID != "A" {
		t.Errorf("should be equal A but %s", graph.GraphPanel.Targets[0].RefID)
	}
}

func TestGraph_SetTargetNew(t *testing.T) {
	var (
		target1 = Target{
			RefID:      "A",
			Datasource: "Sample Source 1",
			Expr:       "sample request 1"}
		target2 = Target{
			RefID:      "B",
			Datasource: "Sample Source 2",
			Expr:       "sample request 2"}
	)
	graph := NewGraph("")
	graph.AddTarget(&target1)

	graph.SetTarget(&target2)

	if len(graph.GraphPanel.Targets) != 2 {
		t.Errorf("should be 2 but %d", len(graph.GraphPanel.Targets))
	}
	if graph.GraphPanel.Targets[0].RefID != "A" {
		t.Errorf("should be equal A but %s", graph.GraphPanel.Targets[0].RefID)
	}
	if graph.GraphPanel.Targets[1].RefID != "B" {
		t.Errorf("should be equal B but %s", graph.GraphPanel.Targets[1].RefID)
	}
}

func TestGraph_SetTargetUpdate(t *testing.T) {
	var (
		target1 = Target{
			RefID:      "A",
			Datasource: "Sample Source 1",
			Expr:       "sample request 1"}
		target2 = Target{
			RefID:      "A",
			Datasource: "Sample Source 2",
			Expr:       "sample request 2"}
	)
	graph := NewGraph("")
	graph.AddTarget(&target1)

	graph.SetTarget(&target2)

	if len(graph.GraphPanel.Targets) != 1 {
		t.Errorf("should be 1 but %d", len(graph.GraphPanel.Targets))
	}
	if graph.GraphPanel.Targets[0].RefID != "A" {
		t.Errorf("should be equal A but %s", graph.GraphPanel.Targets[0].RefID)
	}
}
<commit_msg>Add unit test for Elasticsearch datasource parsing<commit_after>package grafana

import (
	"bytes"
	"encoding/json"
	"testing"
)

func TestStackVal_UnmarshalJSON_GotTrue(t *testing.T) {
	var sampleOut struct {
		Val BoolString `json:"val"`
	}
	var sampleIn = []byte(`{"val":true}`)

	json.Unmarshal(sampleIn, &sampleOut)

	if !sampleOut.Val.Flag {
		t.Errorf("should be true but got false")
	}
	if sampleOut.Val.Value != "" {
		t.Error("string value should be empty")
	}
}

func TestStackVal_UnmarshalJSON_GotFalse(t *testing.T) {
	var sampleOut struct {
		Val BoolString `json:"val"`
	}
	var sampleIn = []byte(`{"val":false}`)

	json.Unmarshal(sampleIn, &sampleOut)

	if sampleOut.Val.Flag {
		t.Errorf("should be false but got true")
	}
	if sampleOut.Val.Value != "" {
		t.Error("string value should be empty")
	}
}

func TestStackVal_UnmarshalJSON_GotString(t *testing.T) {
	var sampleOut struct {
		Val BoolString `json:"val"`
	}
	var sampleIn = []byte(`{"val":"A"}`)

	json.Unmarshal(sampleIn, &sampleOut)

	if sampleOut.Val.Flag {
		t.Error("should be false but got true")
	}
	if sampleOut.Val.Value != "A" {
		t.Errorf("should be 'A' but got '%s'", sampleOut.Val.Value)
	}
}

func TestStackVal_MarshalJSON_GotTrue(t *testing.T) {
	var sampleInp struct {
		Val BoolString `json:"val"`
	}
	sampleInp.Val.Flag = true
	var sampleOut = []byte(`{"val":true}`)

	data, _ := json.Marshal(sampleInp)

	if bytes.Compare(data, sampleOut) != 0 {
		t.Errorf("should be %s but got %s", sampleOut, data)
	}
}

func TestStackVal_MarshalJSON_GotFalse(t *testing.T) {
	var sampleInp struct {
		Val BoolString `json:"val"`
	}
	sampleInp.Val.Flag = false
	var sampleOut = []byte(`{"val":false}`)

	data, _ := json.Marshal(sampleInp)

	if bytes.Compare(data, sampleOut) != 0 {
		t.Errorf("should be %s but got %s", sampleOut, data)
	}
}

func TestStackVal_MarshalJSON_GotString(t *testing.T) {
	var sampleInp struct {
		Val BoolString `json:"val"`
	}
	sampleInp.Val.Value = "A"
	var sampleOut = []byte(`{"val":"A"}`)

	data, _ := json.Marshal(sampleInp)

	if bytes.Compare(data, sampleOut) != 0 {
		t.Errorf("should be %s but got %s", sampleOut, data)
	}
}

func TestNewGraph(t *testing.T) {
	var title = "Sample Title"

	graph := NewGraph(title)

	if graph.GraphPanel == nil {
		t.Error("should be not nil")
	}
	if graph.TextPanel != nil {
		t.Error("should be nil")
	}
	if graph.DashlistPanel != nil {
		t.Error("should be nil")
	}
	if graph.SinglestatPanel != nil {
		t.Error("should be nil")
	}
	if graph.Title != title {
		t.Errorf("title should be %s but %s", title, graph.Title)
	}
}

func TestGraph_AddTarget(t *testing.T) {
	var target = Target{
		RefID:      "A",
		Datasource: "Sample Source",
		Expr:       "sample request"}
	graph := NewGraph("")

	graph.AddTarget(&target)

	if len(graph.GraphPanel.Targets) != 1 {
		t.Errorf("should be 1 but %d", len(graph.GraphPanel.Targets))
	}
	if graph.GraphPanel.Targets[0].RefID != "A" {
		t.Errorf("should be equal A but %s", graph.GraphPanel.Targets[0].RefID)
	}
}

func TestGraph_SetTargetNew(t *testing.T) {
	var (
		target1 = Target{
			RefID:      "A",
			Datasource: "Sample Source 1",
			Expr:       "sample request 1"}
		target2 = Target{
			RefID:      "B",
			Datasource: "Sample Source 2",
			Expr:       "sample request 2"}
	)
	graph := NewGraph("")
	graph.AddTarget(&target1)

	graph.SetTarget(&target2)

	if len(graph.GraphPanel.Targets) != 2 {
		t.Errorf("should be 2 but %d", len(graph.GraphPanel.Targets))
	}
	if graph.GraphPanel.Targets[0].RefID != "A" {
		t.Errorf("should be equal A but %s", graph.GraphPanel.Targets[0].RefID)
	}
	if graph.GraphPanel.Targets[1].RefID != "B" {
		t.Errorf("should be equal B but %s", graph.GraphPanel.Targets[1].RefID)
	}
}

func TestGraph_SetTargetUpdate(t *testing.T) {
	var (
		target1 = Target{
			RefID:      "A",
			Datasource: "Sample Source 1",
			Expr:       "sample request 1"}
		target2 = Target{
			RefID:      "A",
			Datasource: "Sample Source 2",
			Expr:       "sample request 2"}
	)
	graph := NewGraph("")
	graph.AddTarget(&target1)

	graph.SetTarget(&target2)

	if len(graph.GraphPanel.Targets) != 1 {
		t.Errorf("should be 1 but %d", len(graph.GraphPanel.Targets))
	}
	if graph.GraphPanel.Targets[0].RefID != "A" {
		t.Errorf("should be equal A but %s", graph.GraphPanel.Targets[0].RefID)
	}
}

// Test on the panel sample with Elasticsearch datasource with Graylog query. Grafana 2.6.
func TastPanel_ElasticsearchSource_ParsedTargets(t *testing.T) {
	var rawPanel = []byte(`{
  "aliasColors": {},
  "bars": true,
  "datasource": "Example GrayLog",
  "editable": true,
  "error": false,
  "fill": 1,
  "grid": {
    "leftLogBase": 1,
    "leftMax": null,
    "leftMin": null,
    "rightLogBase": 1,
    "rightMax": null,
    "rightMin": null,
    "threshold1": null,
    "threshold1Color": "rgba(216, 200, 27, 0.27)",
    "threshold2": null,
    "threshold2Color": "rgba(234, 112, 112, 0.22)"
  },
  "id": 37,
  "isNew": true,
  "legend": {
    "avg": false,
    "current": false,
    "max": false,
    "min": false,
    "show": false,
    "total": false,
    "values": false
  },
  "lines": false,
  "linewidth": 2,
  "links": [
    {
      "params": "q=tag%3A%2Fid.*%2F+AND+level%3AERROR&rangetype=relative&relative=300#fields=message%2Csource",
      "title": "Example GrayLog Page",
      "type": "absolute",
      "url": "https://graylog/streams/xxx/messages"
    }
  ],
  "nullPointMode": "connected",
  "percentage": false,
  "pointradius": 5,
  "points": false,
  "renderer": "flot",
  "seriesOverrides": [],
  "span": 2,
  "stack": false,
  "steppedLine": false,
  "targets": [
    {
      "bucketAggs": [
        {
          "field": "timestamp",
          "id": "2",
          "settings": {
            "interval": "5m",
            "min_doc_count": 0
          },
          "type": "date_histogram"
        }
      ],
      "dsType": "elasticsearch",
      "metrics": [
        {
          "field": "select field",
          "id": "1",
          "type": "count"
        }
      ],
      "query": "tag:/.*.xxx.filtered/ AND tag:/id.*/ AND level:ERROR",
      "refId": "A",
      "target": "",
      "timeField": "timestamp"
    }
  ],
  "timeFrom": null,
  "timeShift": null,
  "title": "Example GrayLog Errors[5m]",
  "tooltip": {
    "shared": true,
    "value_type": "cumulative"
  },
  "transparent": true,
  "type": "graph",
  "x-axis": true,
  "y-axis": true,
  "y_formats": [
    "short",
    "short"
  ]
}`)

	var graph *Panel
	err := json.Unmarshal(rawPanel, graph)

	if err != nil {
		t.Error(err)
	}
	if len(graph.GraphPanel.Targets) != 1 {
		t.Errorf("should be 1 but %d", len(graph.GraphPanel.Targets))
	}
	if graph.GraphPanel.Targets[0].DsType == nil {
		t.Fatalf("should be \"elasticsearch\" but nil")
	}
	if *graph.GraphPanel.Targets[0].DsType != "elasticsearch" {
		t.Errorf("should be \"elasticsearch\" but %s", graph.GraphPanel.Targets[0].DsType)
	}

}
<|endoftext|>


<commit_before>package weed_server

import (
	"errors"
	"fmt"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/storage"
	"github.com/chrislusf/seaweedfs/weed/topology"
)

func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) {
	if e := r.ParseForm(); e != nil {
		glog.V(0).Infoln("form parse error:", e)
		writeJsonError(w, r, http.StatusBadRequest, e)
		return
	}
	vid, _, _, _, _ := parseURLPath(r.URL.Path)
	volumeId, ve := storage.NewVolumeId(vid)
	if ve != nil {
		glog.V(0).Infoln("NewVolumeId error:", ve)
		writeJsonError(w, r, http.StatusBadRequest, ve)
		return
	}
	needle, ne := storage.NewNeedle(r, vs.FixJpgOrientation)
	if ne != nil {
		writeJsonError(w, r, http.StatusBadRequest, ne)
		return
	}

	ret := operation.UploadResult{}
	size, errorStatus := topology.ReplicatedWrite(vs.GetMaster(),
		vs.store, volumeId, needle, r)
	httpStatus := http.StatusCreated
	if errorStatus != "" {
		httpStatus = http.StatusInternalServerError
		ret.Error = errorStatus
	}
	if needle.HasName() {
		ret.Name = string(needle.Name)
	}
	ret.Size = size
	setEtag(w, needle.Etag())
	writeJsonQuiet(w, r, httpStatus, ret)
}

func (vs *VolumeServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {
	n := new(storage.Needle)
	vid, fid, _, _, _ := parseURLPath(r.URL.Path)
	volumeId, _ := storage.NewVolumeId(vid)
	n.ParsePath(fid)

	glog.V(2).Infof("volume %s deleting %s", vid, n)

	cookie := n.Cookie

	_, ok := vs.store.ReadVolumeNeedle(volumeId, n)
	if ok != nil {
		m := make(map[string]uint32)
		m["size"] = 0
		writeJsonQuiet(w, r, http.StatusNotFound, m)
		return
	}

	if n.Cookie != cookie {
		glog.V(0).Infoln("delete", r.URL.Path, "with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent())
		writeJsonError(w, r, http.StatusBadRequest, errors.New("File Random Cookie does not match."))
		return
	}

	count := int64(n.Size)

	if n.IsChunkedManifest() {
		chunkManifest, e := operation.LoadChunkManifest(n.Data, n.IsGzipped())
		if e != nil {
			writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("Load chunks manifest error: %v", e))
			return
		}
		// make sure all chunks had deleted before delete manifest
		if e := chunkManifest.DeleteChunks(vs.GetMaster()); e != nil {
			writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("Delete chunks error: %v", e))
			return
		}
		count = chunkManifest.Size
	}

	n.LastModified = uint64(time.Now().Unix())
	if len(r.FormValue("ts")) > 0 {
		modifiedTime, err := strconv.ParseInt(r.FormValue("ts"), 10, 64)
		if err == nil {
			n.LastModified = uint64(modifiedTime)
		}
	}

	_, err := topology.ReplicatedDelete(vs.GetMaster(), vs.store, volumeId, n, r)

	if err == nil {
		m := make(map[string]int64)
		m["size"] = count
		writeJsonQuiet(w, r, http.StatusAccepted, m)
	} else {
		writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("Deletion Failed: %v", err))
	}

}

//Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas.
func (vs *VolumeServer) batchDeleteHandler(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	var ret []operation.DeleteResult
	now := uint64(time.Now().Unix())
	for _, fid := range r.Form["fid"] {
		vid, id_cookie, err := operation.ParseFileId(fid)
		if err != nil {
			ret = append(ret, operation.DeleteResult{
				Fid:    fid,
				Status: http.StatusBadRequest,
				Error:  err.Error()})
			continue
		}
		n := new(storage.Needle)
		volumeId, _ := storage.NewVolumeId(vid)
		n.ParsePath(id_cookie)
		glog.V(4).Infoln("batch deleting", n)
		cookie := n.Cookie
		if _, err := vs.store.ReadVolumeNeedle(volumeId, n); err != nil {
			ret = append(ret, operation.DeleteResult{
				Fid:    fid,
				Status: http.StatusNotFound,
				Error:  err.Error(),
			})
			continue
		}

		if n.IsChunkedManifest() {
			ret = append(ret, operation.DeleteResult{
				Fid:    fid,
				Status: http.StatusNotAcceptable,
				Error:  "ChunkManifest: not allowed in batch delete mode.",
			})
			continue
		}

		if n.Cookie != cookie {
			ret = append(ret, operation.DeleteResult{
				Fid:    fid,
				Status: http.StatusBadRequest,
				Error:  "File Random Cookie does not match.",
			})
			glog.V(0).Infoln("deleting", fid, "with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent())
			return
		}
		n.LastModified = now
		if size, err := vs.store.Delete(volumeId, n); err != nil {
			ret = append(ret, operation.DeleteResult{
				Fid:    fid,
				Status: http.StatusInternalServerError,
				Error:  err.Error()},
			)
		} else {
			ret = append(ret, operation.DeleteResult{
				Fid:    fid,
				Status: http.StatusAccepted,
				Size:   int(size)},
			)
		}
	}

	writeJsonQuiet(w, r, http.StatusAccepted, ret)
}

func setEtag(w http.ResponseWriter, etag string) {
	if etag != "" {
		if strings.HasPrefix(etag, "\"") {
			w.Header().Set("ETag", etag)
		} else {
			w.Header().Set("ETag", "\""+etag+"\"")
		}
	}
}
<commit_msg>adjust logging<commit_after>package weed_server

import (
	"errors"
	"fmt"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/storage"
	"github.com/chrislusf/seaweedfs/weed/topology"
)

func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) {
	if e := r.ParseForm(); e != nil {
		glog.V(0).Infoln("form parse error:", e)
		writeJsonError(w, r, http.StatusBadRequest, e)
		return
	}
	vid, _, _, _, _ := parseURLPath(r.URL.Path)
	volumeId, ve := storage.NewVolumeId(vid)
	if ve != nil {
		glog.V(0).Infoln("NewVolumeId error:", ve)
		writeJsonError(w, r, http.StatusBadRequest, ve)
		return
	}
	needle, ne := storage.NewNeedle(r, vs.FixJpgOrientation)
	if ne != nil {
		writeJsonError(w, r, http.StatusBadRequest, ne)
		return
	}

	ret := operation.UploadResult{}
	size, errorStatus := topology.ReplicatedWrite(vs.GetMaster(),
		vs.store, volumeId, needle, r)
	httpStatus := http.StatusCreated
	if errorStatus != "" {
		httpStatus = http.StatusInternalServerError
		ret.Error = errorStatus
	}
	if needle.HasName() {
		ret.Name = string(needle.Name)
	}
	ret.Size = size
	setEtag(w, needle.Etag())
	writeJsonQuiet(w, r, httpStatus, ret)
}

func (vs *VolumeServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {
	n := new(storage.Needle)
	vid, fid, _, _, _ := parseURLPath(r.URL.Path)
	volumeId, _ := storage.NewVolumeId(vid)
	n.ParsePath(fid)

	glog.V(2).Infof("volume %s deleting %s", vid, n)

	cookie := n.Cookie

	_, ok := vs.store.ReadVolumeNeedle(volumeId, n)
	if ok != nil {
		m := make(map[string]uint32)
		m["size"] = 0
		writeJsonQuiet(w, r, http.StatusNotFound, m)
		return
	}

	if n.Cookie != cookie {
		glog.V(0).Infoln("delete", r.URL.Path, "with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent())
		writeJsonError(w, r, http.StatusBadRequest, errors.New("File Random Cookie does not match."))
		return
	}

	count := int64(n.Size)

	if n.IsChunkedManifest() {
		chunkManifest, e := operation.LoadChunkManifest(n.Data, n.IsGzipped())
		if e != nil {
			writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("Load chunks manifest error: %v", e))
			return
		}
		// make sure all chunks had deleted before delete manifest
		if e := chunkManifest.DeleteChunks(vs.GetMaster()); e != nil {
			writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("Delete chunks error: %v", e))
			return
		}
		count = chunkManifest.Size
	}

	n.LastModified = uint64(time.Now().Unix())
	if len(r.FormValue("ts")) > 0 {
		modifiedTime, err := strconv.ParseInt(r.FormValue("ts"), 10, 64)
		if err == nil {
			n.LastModified = uint64(modifiedTime)
		}
	}

	_, err := topology.ReplicatedDelete(vs.GetMaster(), vs.store, volumeId, n, r)

	if err == nil {
		m := make(map[string]int64)
		m["size"] = count
		writeJsonQuiet(w, r, http.StatusAccepted, m)
	} else {
		writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("Deletion Failed: %v", err))
	}

}

//Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas.
func (vs *VolumeServer) batchDeleteHandler(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	var ret []operation.DeleteResult
	now := uint64(time.Now().Unix())
	for _, fid := range r.Form["fid"] {
		vid, id_cookie, err := operation.ParseFileId(fid)
		if err != nil {
			ret = append(ret, operation.DeleteResult{
				Fid:    fid,
				Status: http.StatusBadRequest,
				Error:  err.Error()})
			continue
		}
		n := new(storage.Needle)
		volumeId, _ := storage.NewVolumeId(vid)
		n.ParsePath(id_cookie)
		// glog.V(4).Infoln("batch deleting", n)
		cookie := n.Cookie
		if _, err := vs.store.ReadVolumeNeedle(volumeId, n); err != nil {
			ret = append(ret, operation.DeleteResult{
				Fid:    fid,
				Status: http.StatusNotFound,
				Error:  err.Error(),
			})
			continue
		}

		if n.IsChunkedManifest() {
			ret = append(ret, operation.DeleteResult{
				Fid:    fid,
				Status: http.StatusNotAcceptable,
				Error:  "ChunkManifest: not allowed in batch delete mode.",
			})
			continue
		}

		if n.Cookie != cookie {
			ret = append(ret, operation.DeleteResult{
				Fid:    fid,
				Status: http.StatusBadRequest,
				Error:  "File Random Cookie does not match.",
			})
			glog.V(0).Infoln("deleting", fid, "with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent())
			return
		}
		n.LastModified = now
		if size, err := vs.store.Delete(volumeId, n); err != nil {
			ret = append(ret, operation.DeleteResult{
				Fid:    fid,
				Status: http.StatusInternalServerError,
				Error:  err.Error()},
			)
		} else {
			ret = append(ret, operation.DeleteResult{
				Fid:    fid,
				Status: http.StatusAccepted,
				Size:   int(size)},
			)
		}
	}

	writeJsonQuiet(w, r, http.StatusAccepted, ret)
}

func setEtag(w http.ResponseWriter, etag string) {
	if etag != "" {
		if strings.HasPrefix(etag, "\"") {
			w.Header().Set("ETag", etag)
		} else {
			w.Header().Set("ETag", "\""+etag+"\"")
		}
	}
}
<|endoftext|>


<commit_before>package opentsdb

import (
	"bytes"
	"encoding/gob"
	"os"
	"testing"
	"time"

	"github.com/intelsdi-x/pulse/control/plugin"
	"github.com/intelsdi-x/pulse/core/ctypes"

	. "github.com/smartystreets/goconvey/convey"
)

func TestOpentsdbPublish(t *testing.T) {
	config := make(map[string]ctypes.ConfigValue)

	Convey("TestOpentsdb", t, func() {
		config["host"] = ctypes.ConfigValueStr{Value: os.Getenv("PULSE_OPENTSDB_HOST")}
		config["port"] = ctypes.ConfigValueInt{Value: 4242}

		ip := NewOpentsdbPublisher()
		So(ip, ShouldNotBeNil)

		policy := ip.GetConfigPolicy()
		So(policy, ShouldNotBeNil)

		Convey("Publish", func() {
			var buf bytes.Buffer
			metrics := []plugin.PluginMetricType{
				*plugin.NewPluginMetricType([]string{"/psutil/load/load15"}, time.Now(), "mac1", 23.1),
				*plugin.NewPluginMetricType([]string{"/psutil/vm/available"}, time.Now().Add(2*time.Second), "mac2", 23.2),
				*plugin.NewPluginMetricType([]string{"/psutil/load/load1"}, time.Now().Add(3*time.Second), "linux3", 23.3),
			}
			enc := gob.NewEncoder(&buf)
			enc.Encode(metrics)

			Convey("float", func() {
				err := ip.Publish(plugin.PulseGOBContentType, buf.Bytes(), config)
				So(err, ShouldBeNil)
			})

			Convey("int", func() {
				metrics = []plugin.PluginMetricType{
					*plugin.NewPluginMetricType([]string{"/psutil/vm/free"}, time.Now().Add(5*time.Second), "linux7", 23),
				}
				buf.Reset()
				enc = gob.NewEncoder(&buf)
				enc.Encode(metrics)

				err := ip.Publish(plugin.PulseGOBContentType, buf.Bytes(), config)
				So(err, ShouldBeNil)
			})
		})

	})
}
<commit_msg>Add integration tag and remove unit test checks<commit_after>//
// +build integration

package opentsdb

import (
	"bytes"
	"encoding/gob"
	"os"
	"testing"
	"time"

	"github.com/intelsdi-x/pulse/control/plugin"
	"github.com/intelsdi-x/pulse/core/ctypes"

	. "github.com/smartystreets/goconvey/convey"
)

func TestOpentsdbPublish(t *testing.T) {
	config := make(map[string]ctypes.ConfigValue)

	Convey("TestOpentsdb", t, func() {
		config["host"] = ctypes.ConfigValueStr{Value: os.Getenv("PULSE_OPENTSDB_HOST")}
		config["port"] = ctypes.ConfigValueInt{Value: 4242}

		ip := NewOpentsdbPublisher()
		policy := ip.GetConfigPolicy()

		Convey("Publish", func() {
			var buf bytes.Buffer
			metrics := []plugin.PluginMetricType{
				*plugin.NewPluginMetricType([]string{"/psutil/load/load15"}, time.Now(), "mac1", 23.1),
				*plugin.NewPluginMetricType([]string{"/psutil/vm/available"}, time.Now().Add(2*time.Second), "mac2", 23.2),
				*plugin.NewPluginMetricType([]string{"/psutil/load/load1"}, time.Now().Add(3*time.Second), "linux3", 23.3),
			}
			enc := gob.NewEncoder(&buf)
			enc.Encode(metrics)

			Convey("float", func() {
				err := ip.Publish(plugin.PulseGOBContentType, buf.Bytes(), config)
				So(err, ShouldBeNil)
			})

			Convey("int", func() {
				metrics = []plugin.PluginMetricType{
					*plugin.NewPluginMetricType([]string{"/psutil/vm/free"}, time.Now().Add(5*time.Second), "linux7", 23),
				}
				buf.Reset()
				enc = gob.NewEncoder(&buf)
				enc.Encode(metrics)

				err := ip.Publish(plugin.PulseGOBContentType, buf.Bytes(), config)
				So(err, ShouldBeNil)
			})
		})

	})
}
<|endoftext|>


<commit_before>// This package implements a provisioner for Packer that executes a
// saltstack highstate within the remote machine
package saltmasterless

import (
	"errors"
	"fmt"
	"github.com/mitchellh/packer/common"
	"github.com/mitchellh/packer/packer"
	"os"
	"path/filepath"
	"strings"
)

const DefaultTempConfigDir = "/tmp/salt"

type Config struct {
	common.PackerConfig `mapstructure:",squash"`

	// If true, run the salt-bootstrap script
	SkipBootstrap bool   `mapstructure:"skip_bootstrap"`
	BootstrapArgs string `mapstructure:"bootstrap_args"`

	// Local path to the salt state tree
	LocalStateTree string `mapstructure:"local_state_tree"`

	// Where files will be copied before moving to the /srv/salt directory
	TempConfigDir string `mapstructure:"temp_config_dir"`

	tpl *common.Template
}

type Provisioner struct {
	config Config
}

func (p *Provisioner) Prepare(raws ...interface{}) error {
	md, err := common.DecodeConfig(&p.config, raws...)
	if err != nil {
		return err
	}

	p.config.tpl, err = common.NewTemplate()
	if err != nil {
		return err
	}
	p.config.tpl.UserVars = p.config.PackerUserVars

	if p.config.TempConfigDir == "" {
		p.config.TempConfigDir = DefaultTempConfigDir
	}

	// Accumulate any errors
	errs := common.CheckUnusedConfig(md)

	templates := map[string]*string{
		"bootstrap_args":   &p.config.BootstrapArgs,
		"local_state_tree": &p.config.LocalStateTree,
		"temp_config_dir":  &p.config.TempConfigDir,
	}

	for n, ptr := range templates {
		var err error
		*ptr, err = p.config.tpl.Process(*ptr, nil)
		if err != nil {
			errs = packer.MultiErrorAppend(
				errs, fmt.Errorf("Error processing %s: %s", n, err))
		}
	}

	if p.config.LocalStateTree != "" {
		if _, err := os.Stat(p.config.LocalStateTree); err != nil {
			errs = packer.MultiErrorAppend(errs,
				errors.New("local_state_tree must exist and be accessible"))
		}
	}

	if errs != nil && len(errs.Errors) > 0 {
		return errs
	}

	return nil
}

func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error {
	var err error

	ui.Say("Provisioning with Salt...")
	if !p.config.SkipBootstrap {
		cmd := &packer.RemoteCmd{
			Command: fmt.Sprintf("wget -O - http://bootstrap.saltstack.org | sudo sh -s %s", p.config.BootstrapArgs),
		}
		ui.Message(fmt.Sprintf("Installing Salt with command %s", cmd))
		if err = cmd.StartWithUi(comm, ui); err != nil {
			return fmt.Errorf("Unable to install Salt: %d", err)
		}
	}

	ui.Message(fmt.Sprintf("Creating remote directory: %s", p.config.TempConfigDir))
	cmd := &packer.RemoteCmd{Command: fmt.Sprintf("mkdir -p %s", p.config.TempConfigDir)}
	if err = cmd.StartWithUi(comm, ui); err != nil || cmd.ExitStatus != 0 {
		if err == nil {
			err = fmt.Errorf("Bad exit status: %d", cmd.ExitStatus)
		}

		return fmt.Errorf("Error creating remote salt state directory: %s", err)
	}

	ui.Message(fmt.Sprintf("Uploading local state tree: %s", p.config.LocalStateTree))
	if err = UploadLocalDirectory(p.config.LocalStateTree, p.config.TempConfigDir, comm, ui); err != nil {
		return fmt.Errorf("Error uploading local state tree to remote: %s", err)
	}

	ui.Message(fmt.Sprintf("Moving %s to /srv/salt", p.config.TempConfigDir))
	cmd = &packer.RemoteCmd{Command: fmt.Sprintf("sudo mv %s /srv/salt", p.config.TempConfigDir)}
	if err = cmd.StartWithUi(comm, ui); err != nil || cmd.ExitStatus != 0 {
		if err == nil {
			err = fmt.Errorf("Bad exit status: %d", cmd.ExitStatus)
		}

		return fmt.Errorf("Unable to move %s to /srv/salt: %d", p.config.TempConfigDir, err)
	}

	ui.Message("Running highstate")
	cmd = &packer.RemoteCmd{Command: "sudo salt-call --local state.highstate -l info"}
	if err = cmd.StartWithUi(comm, ui); err != nil || cmd.ExitStatus != 0 {
		if err == nil {
			err = fmt.Errorf("Bad exit status: %d", cmd.ExitStatus)
		}

		return fmt.Errorf("Error executing highstate: %s", err)
	}

	return nil
}

func UploadLocalDirectory(localDir string, remoteDir string, comm packer.Communicator, ui packer.Ui) (err error) {
	visitPath := func(localPath string, f os.FileInfo, err error) (err2 error) {
		localPath = strings.Replace(localPath, "\\", "/", -1)
		localRelPath := strings.Replace(localPath, localDir, "", 1)
		remotePath := fmt.Sprintf("%s%s", remoteDir, localRelPath)
		if f.IsDir() && f.Name() == ".git" {
			return filepath.SkipDir
		}
		if f.IsDir() {
			// Make remote directory
			cmd := &packer.RemoteCmd{Command: fmt.Sprintf("mkdir -p %s", remotePath)}
			if err = cmd.StartWithUi(comm, ui); err != nil {
				return err
			}
		} else {
			// Upload file to existing directory
			file, err := os.Open(localPath)
			if err != nil {
				return fmt.Errorf("Error opening file: %s", err)
			}
			defer file.Close()

			ui.Message(fmt.Sprintf("Uploading file %s: %s", localPath, remotePath))
			if err = comm.Upload(remotePath, file); err != nil {
				return fmt.Errorf("Error uploading file: %s", err)
			}
		}
		return
	}

	err = filepath.Walk(localDir, visitPath)
	if err != nil {
		return fmt.Errorf("Error uploading local directory %s: %s", localDir, err)
	}

	return nil
}
<commit_msg>provisioner/salt-masterless: switch replace to that prior replace works<commit_after>// This package implements a provisioner for Packer that executes a
// saltstack highstate within the remote machine
package saltmasterless

import (
	"errors"
	"fmt"
	"github.com/mitchellh/packer/common"
	"github.com/mitchellh/packer/packer"
	"os"
	"path/filepath"
	"strings"
)

const DefaultTempConfigDir = "/tmp/salt"

type Config struct {
	common.PackerConfig `mapstructure:",squash"`

	// If true, run the salt-bootstrap script
	SkipBootstrap bool   `mapstructure:"skip_bootstrap"`
	BootstrapArgs string `mapstructure:"bootstrap_args"`

	// Local path to the salt state tree
	LocalStateTree string `mapstructure:"local_state_tree"`

	// Where files will be copied before moving to the /srv/salt directory
	TempConfigDir string `mapstructure:"temp_config_dir"`

	tpl *common.Template
}

type Provisioner struct {
	config Config
}

func (p *Provisioner) Prepare(raws ...interface{}) error {
	md, err := common.DecodeConfig(&p.config, raws...)
	if err != nil {
		return err
	}

	p.config.tpl, err = common.NewTemplate()
	if err != nil {
		return err
	}
	p.config.tpl.UserVars = p.config.PackerUserVars

	if p.config.TempConfigDir == "" {
		p.config.TempConfigDir = DefaultTempConfigDir
	}

	// Accumulate any errors
	errs := common.CheckUnusedConfig(md)

	templates := map[string]*string{
		"bootstrap_args":   &p.config.BootstrapArgs,
		"local_state_tree": &p.config.LocalStateTree,
		"temp_config_dir":  &p.config.TempConfigDir,
	}

	for n, ptr := range templates {
		var err error
		*ptr, err = p.config.tpl.Process(*ptr, nil)
		if err != nil {
			errs = packer.MultiErrorAppend(
				errs, fmt.Errorf("Error processing %s: %s", n, err))
		}
	}

	if p.config.LocalStateTree != "" {
		if _, err := os.Stat(p.config.LocalStateTree); err != nil {
			errs = packer.MultiErrorAppend(errs,
				errors.New("local_state_tree must exist and be accessible"))
		}
	}

	if errs != nil && len(errs.Errors) > 0 {
		return errs
	}

	return nil
}

func (p *Provisioner) Provision(ui packer.Ui, comm packer.Communicator) error {
	var err error

	ui.Say("Provisioning with Salt...")
	if !p.config.SkipBootstrap {
		cmd := &packer.RemoteCmd{
			Command: fmt.Sprintf("wget -O - http://bootstrap.saltstack.org | sudo sh -s %s", p.config.BootstrapArgs),
		}
		ui.Message(fmt.Sprintf("Installing Salt with command %s", cmd))
		if err = cmd.StartWithUi(comm, ui); err != nil {
			return fmt.Errorf("Unable to install Salt: %d", err)
		}
	}

	ui.Message(fmt.Sprintf("Creating remote directory: %s", p.config.TempConfigDir))
	cmd := &packer.RemoteCmd{Command: fmt.Sprintf("mkdir -p %s", p.config.TempConfigDir)}
	if err = cmd.StartWithUi(comm, ui); err != nil || cmd.ExitStatus != 0 {
		if err == nil {
			err = fmt.Errorf("Bad exit status: %d", cmd.ExitStatus)
		}

		return fmt.Errorf("Error creating remote salt state directory: %s", err)
	}

	ui.Message(fmt.Sprintf("Uploading local state tree: %s", p.config.LocalStateTree))
	if err = UploadLocalDirectory(p.config.LocalStateTree, p.config.TempConfigDir, comm, ui); err != nil {
		return fmt.Errorf("Error uploading local state tree to remote: %s", err)
	}

	ui.Message(fmt.Sprintf("Moving %s to /srv/salt", p.config.TempConfigDir))
	cmd = &packer.RemoteCmd{Command: fmt.Sprintf("sudo mv %s /srv/salt", p.config.TempConfigDir)}
	if err = cmd.StartWithUi(comm, ui); err != nil || cmd.ExitStatus != 0 {
		if err == nil {
			err = fmt.Errorf("Bad exit status: %d", cmd.ExitStatus)
		}

		return fmt.Errorf("Unable to move %s to /srv/salt: %d", p.config.TempConfigDir, err)
	}

	ui.Message("Running highstate")
	cmd = &packer.RemoteCmd{Command: "sudo salt-call --local state.highstate -l info"}
	if err = cmd.StartWithUi(comm, ui); err != nil || cmd.ExitStatus != 0 {
		if err == nil {
			err = fmt.Errorf("Bad exit status: %d", cmd.ExitStatus)
		}

		return fmt.Errorf("Error executing highstate: %s", err)
	}

	return nil
}

func UploadLocalDirectory(localDir string, remoteDir string, comm packer.Communicator, ui packer.Ui) (err error) {
	visitPath := func(localPath string, f os.FileInfo, err error) (err2 error) {
		localRelPath := strings.Replace(localPath, localDir, "", 1)
		localRelPath = strings.Replace(localRelPath, "\\", "/", -1)
		remotePath := fmt.Sprintf("%s%s", remoteDir, localRelPath)
		if f.IsDir() && f.Name() == ".git" {
			return filepath.SkipDir
		}
		if f.IsDir() {
			// Make remote directory
			cmd := &packer.RemoteCmd{Command: fmt.Sprintf("mkdir -p %s", remotePath)}
			if err = cmd.StartWithUi(comm, ui); err != nil {
				return err
			}
		} else {
			// Upload file to existing directory
			file, err := os.Open(localPath)
			if err != nil {
				return fmt.Errorf("Error opening file: %s", err)
			}
			defer file.Close()

			ui.Message(fmt.Sprintf("Uploading file %s: %s", localPath, remotePath))
			if err = comm.Upload(remotePath, file); err != nil {
				return fmt.Errorf("Error uploading file: %s", err)
			}
		}
		return
	}

	err = filepath.Walk(localDir, visitPath)
	if err != nil {
		return fmt.Errorf("Error uploading local directory %s: %s", localDir, err)
	}

	return nil
}
<|endoftext|>