<commit_before>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	"github.com/google/syzkaller/dashboard/dashapi"
	"github.com/google/syzkaller/pkg/instance"
	"github.com/google/syzkaller/pkg/log"
	"github.com/google/syzkaller/pkg/osutil"
	"github.com/google/syzkaller/pkg/vcs"
	"github.com/google/syzkaller/prog"
	"github.com/google/syzkaller/sys/targets"
)

const (
	syzkallerRebuildPeriod = 12 * time.Hour
	buildRetryPeriod       = 10 * time.Minute // used for both syzkaller and kernel
)

// SyzUpdater handles everything related to syzkaller updates.
// As kernel builder, it maintains 2 builds:
//  - latest: latest known good syzkaller build
//  - current: currently used syzkaller build
// Additionally it updates and restarts the current executable as necessary.
// Current executable is always built on the same revision as the rest of syzkaller binaries.
type SyzUpdater struct {
	repo          vcs.Repo
	exe           string
	repoAddress   string
	branch        string
	descriptions  string
	gopathDir     string
	syzkallerDir  string
	latestDir     string
	currentDir    string
	syzFiles      map[string]bool
	targets       map[string]bool
	dashboardAddr string
	compilerID    string
	cfg           *Config
}

func NewSyzUpdater(cfg *Config) *SyzUpdater {
	wd, err := os.Getwd()
	if err != nil {
		log.Fatalf("failed to get wd: %v", err)
	}
	bin := os.Args[0]
	if !filepath.IsAbs(bin) {
		bin = filepath.Join(wd, bin)
	}
	bin = filepath.Clean(bin)
	exe := filepath.Base(bin)
	if wd != filepath.Dir(bin) {
		log.Fatalf("%v executable must be in cwd (it will be overwritten on update)", exe)
	}

	gopath := filepath.Join(wd, "gopath")
	syzkallerDir := filepath.Join(gopath, "src", "github.com", "google", "syzkaller")
	osutil.MkdirAll(syzkallerDir)

	// List of required files in syzkaller build (contents of latest/current dirs).
	files := map[string]bool{
		"tag":             true, // contains syzkaller repo git hash
		"bin/syz-ci":      true, // these are just copied from syzkaller dir
		"bin/syz-manager": true,
		"sys/*/test/*":    true,
	}
	targets := make(map[string]bool)
	for _, mgr := range cfg.Managers {
		mgrcfg := mgr.managercfg
		os, vmarch, arch := mgrcfg.TargetOS, mgrcfg.TargetVMArch, mgrcfg.TargetArch
		targets[os+"/"+vmarch+"/"+arch] = true
		files[fmt.Sprintf("bin/%v_%v/syz-fuzzer", os, vmarch)] = true
		files[fmt.Sprintf("bin/%v_%v/syz-execprog", os, vmarch)] = true
		if mgrcfg.SysTarget.ExecutorBin == "" {
			files[fmt.Sprintf("bin/%v_%v/syz-executor", os, arch)] = true
		}
	}
	syzFiles := make(map[string]bool)
	for f := range files {
		syzFiles[f] = true
	}
	compilerID, err := osutil.RunCmd(time.Minute, "", "go", "version")
	if err != nil {
		log.Fatalf("%v", err)
	}
	return &SyzUpdater{
		repo:          vcs.NewSyzkallerRepo(syzkallerDir),
		exe:           exe,
		repoAddress:   cfg.SyzkallerRepo,
		branch:        cfg.SyzkallerBranch,
		descriptions:  cfg.SyzkallerDescriptions,
		gopathDir:     gopath,
		syzkallerDir:  syzkallerDir,
		latestDir:     filepath.Join("syzkaller", "latest"),
		currentDir:    filepath.Join("syzkaller", "current"),
		syzFiles:      syzFiles,
		targets:       targets,
		dashboardAddr: cfg.DashboardAddr,
		compilerID:    strings.TrimSpace(string(compilerID)),
		cfg:           cfg,
	}
}

// UpdateOnStart does 3 things:
//  - ensures that the current executable is fresh
//  - ensures that we have a working syzkaller build in current
func (upd *SyzUpdater) UpdateOnStart(autoupdate bool, shutdown chan struct{}) {
	os.RemoveAll(upd.currentDir)
	latestTag := upd.checkLatest()
	if latestTag != "" {
		var exeMod time.Time
		if st, err := os.Stat(upd.exe); err == nil {
			exeMod = st.ModTime()
		}
		uptodate := prog.GitRevisionBase == latestTag && time.Since(exeMod) < time.Minute
		if uptodate || !autoupdate {
			if uptodate {
				// Have a fresh up-to-date build, probably just restarted.
				log.Logf(0, "current executable is up-to-date (%v)", latestTag)
			} else {
				log.Logf(0, "autoupdate is turned off, using latest build %v", latestTag)
			}
			if err := osutil.LinkFiles(upd.latestDir, upd.currentDir, upd.syzFiles); err != nil {
				log.Fatal(err)
			}
			return
		}
	}
	log.Logf(0, "current executable is on %v", prog.GitRevision)
	log.Logf(0, "latest syzkaller build is on %v", latestTag)

	// No syzkaller build or executable is stale.
	lastCommit := prog.GitRevisionBase
	if lastCommit != latestTag {
		// Latest build and syz-ci are inconsistent. Rebuild everything.
		lastCommit = ""
		latestTag = ""
	}
	for {
		lastCommit = upd.pollAndBuild(lastCommit)
		latestTag := upd.checkLatest()
		if latestTag != "" {
			// The build was successful or we had the latest build from previous runs.
			// Either way, use the latest build.
			log.Logf(0, "using syzkaller built on %v", latestTag)
			if err := osutil.LinkFiles(upd.latestDir, upd.currentDir, upd.syzFiles); err != nil {
				log.Fatal(err)
			}
			if autoupdate && prog.GitRevisionBase != latestTag {
				upd.UpdateAndRestart()
			}
			return
		}

		// No good build at all, try again later.
		log.Logf(0, "retrying in %v", buildRetryPeriod)
		select {
		case <-time.After(buildRetryPeriod):
		case <-shutdown:
			os.Exit(0)
		}
	}
}

// WaitForUpdate polls and rebuilds syzkaller.
// Returns when we have a new good build in latest.
func (upd *SyzUpdater) WaitForUpdate() {
	time.Sleep(syzkallerRebuildPeriod)
	latestTag := upd.checkLatest()
	lastCommit := latestTag
	for {
		lastCommit = upd.pollAndBuild(lastCommit)
		if latestTag != upd.checkLatest() {
			break
		}
		time.Sleep(buildRetryPeriod)
	}
	log.Logf(0, "syzkaller: update available, restarting")
}

// UpdateAndRestart updates and restarts the current executable.
// Does not return.
func (upd *SyzUpdater) UpdateAndRestart() {
	log.Logf(0, "restarting executable for update")
	latestBin := filepath.Join(upd.latestDir, "bin", upd.exe)
	if err := osutil.CopyFile(latestBin, upd.exe); err != nil {
		log.Fatal(err)
	}
	if err := syscall.Exec(upd.exe, os.Args, os.Environ()); err != nil {
		log.Fatal(err)
	}
	log.Fatalf("not reachable")
}

func (upd *SyzUpdater) pollAndBuild(lastCommit string) string {
	commit, err := upd.repo.Poll(upd.repoAddress, upd.branch)
	if err != nil {
		log.Logf(0, "syzkaller: failed to poll: %v", err)
		return lastCommit
	}
	log.Logf(0, "syzkaller: poll: %v (%v)", commit.Hash, commit.Title)
	if lastCommit == commit.Hash {
		return lastCommit
	}
	log.Logf(0, "syzkaller: building ...")
	if err := upd.build(commit); err != nil {
		log.Logf(0, "syzkaller: %v", err)
		upd.uploadBuildError(commit, err)
	}
	return commit.Hash
}

func (upd *SyzUpdater) build(commit *vcs.Commit) error {
	// syzkaller testing may be slowed down by concurrent kernel builds too much
	// and cause timeout failures, so we serialize it with other builds:
	// https://groups.google.com/forum/#!msg/syzkaller-openbsd-bugs/o-G3vEsyQp4/f_nFpoNKBQAJ
	kernelBuildSem <- struct{}{}
	defer func() { <-kernelBuildSem }()

	if upd.descriptions != "" {
		files, err := ioutil.ReadDir(upd.descriptions)
		if err != nil {
			return fmt.Errorf("failed to read descriptions dir: %v", err)
		}
		for _, f := range files {
			src := filepath.Join(upd.descriptions, f.Name())
			dst := ""
			switch filepath.Ext(src) {
			case ".txt", ".const":
				dst = filepath.Join(upd.syzkallerDir, "sys", targets.Linux, f.Name())
			case ".test":
				dst = filepath.Join(upd.syzkallerDir, "sys", targets.Linux, "test", f.Name())
			case ".h":
				dst = filepath.Join(upd.syzkallerDir, "executor", f.Name())
			default:
				continue
			}
			if err := osutil.CopyFile(src, dst); err != nil {
				return err
			}
		}
		cmd := osutil.Command(instance.MakeBin, "generate")
		cmd.Dir = upd.syzkallerDir
		cmd.Env = append([]string{"GOPATH=" + upd.gopathDir}, os.Environ()...)
		if _, err := osutil.Run(time.Hour, cmd); err != nil {
			return osutil.PrependContext("generate failed", err)
		}
	}
	// This will also generate descriptions and should go before the 'go test' below.
	cmd := osutil.Command(instance.MakeBin, "host", "ci")
	cmd.Dir = upd.syzkallerDir
	cmd.Env = append([]string{"GOPATH=" + upd.gopathDir}, os.Environ()...)
	if _, err := osutil.Run(time.Hour, cmd); err != nil {
		return osutil.PrependContext("make host failed", err)
	}
	for target := range upd.targets {
		parts := strings.Split(target, "/")
		cmd = osutil.Command(instance.MakeBin, "target")
		cmd.Dir = upd.syzkallerDir
		cmd.Env = append([]string{}, os.Environ()...)
		cmd.Env = append(cmd.Env,
			"GOPATH="+upd.gopathDir,
			"TARGETOS="+parts[0],
			"TARGETVMARCH="+parts[1],
			"TARGETARCH="+parts[2],
		)
		if _, err := osutil.Run(time.Hour, cmd); err != nil {
			return osutil.PrependContext("make target failed", err)
		}
	}
	cmd = osutil.Command("go", "test", "-short", "./...")
	cmd.Dir = upd.syzkallerDir
	cmd.Env = append([]string{
		"GOPATH=" + upd.gopathDir,
		"SYZ_DISABLE_SANDBOXING=yes",
	}, os.Environ()...)
	if _, err := osutil.Run(time.Hour, cmd); err != nil {
		return osutil.PrependContext("testing failed", err)
	}
	tagFile := filepath.Join(upd.syzkallerDir, "tag")
	if err := osutil.WriteFile(tagFile, []byte(commit.Hash)); err != nil {
		return fmt.Errorf("failed to write tag file: %v", err)
	}
	if err := osutil.CopyFiles(upd.syzkallerDir, upd.latestDir, upd.syzFiles); err != nil {
		return fmt.Errorf("failed to copy syzkaller: %v", err)
	}
	return nil
}

func (upd *SyzUpdater) uploadBuildError(commit *vcs.Commit, buildErr error) {
	var title string
	var output []byte
	if verbose, ok := buildErr.(*osutil.VerboseError); ok {
		title = verbose.Title
		output = verbose.Output
	} else {
		title = buildErr.Error()
	}
	title = "syzkaller: " + title
	for _, mgrcfg := range upd.cfg.Managers {
		if upd.dashboardAddr == "" || mgrcfg.DashboardClient == "" {
			log.Logf(0, "not uploading build error fr %v: no dashboard", mgrcfg.Name)
			continue
		}
		dash, err := dashapi.New(mgrcfg.DashboardClient, upd.dashboardAddr, mgrcfg.DashboardKey)
		if err != nil {
			log.Logf(0, "failed to report build error for %v: %v", mgrcfg.Name, err)
			return
		}
		managercfg := mgrcfg.managercfg
		req := &dashapi.BuildErrorReq{
			Build: dashapi.Build{
				Manager:             managercfg.Name,
				ID:                  commit.Hash,
				OS:                  managercfg.TargetOS,
				Arch:                managercfg.TargetArch,
				VMArch:              managercfg.TargetVMArch,
				SyzkallerCommit:     commit.Hash,
				SyzkallerCommitDate: commit.CommitDate,
				CompilerID:          upd.compilerID,
				KernelRepo:          upd.repoAddress,
				KernelBranch:        upd.branch,
			},
			Crash: dashapi.Crash{
				Title: title,
				Log:   output,
			},
		}
		if err := dash.ReportBuildError(req); err != nil {
			// TODO: log ReportBuildError error to dashboard.
			log.Logf(0, "failed to report build error for %v: %v", mgrcfg.Name, err)
		}
	}
}

// checkLatest returns tag of the latest build,
// or an empty string if latest build is missing/broken.
func (upd *SyzUpdater) checkLatest() string {
	if !osutil.FilesExist(upd.latestDir, upd.syzFiles) {
		return ""
	}
	tag, _ := ioutil.ReadFile(filepath.Join(upd.latestDir, "tag"))
	return string(tag)
}
<commit_msg>syz-ci: simplify code<commit_after>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	"github.com/google/syzkaller/dashboard/dashapi"
	"github.com/google/syzkaller/pkg/instance"
	"github.com/google/syzkaller/pkg/log"
	"github.com/google/syzkaller/pkg/osutil"
	"github.com/google/syzkaller/pkg/vcs"
	"github.com/google/syzkaller/prog"
	"github.com/google/syzkaller/sys/targets"
)

const (
	syzkallerRebuildPeriod = 12 * time.Hour
	buildRetryPeriod       = 10 * time.Minute // used for both syzkaller and kernel
)

// SyzUpdater handles everything related to syzkaller updates.
// As kernel builder, it maintains 2 builds:
//  - latest: latest known good syzkaller build
//  - current: currently used syzkaller build
// Additionally it updates and restarts the current executable as necessary.
// Current executable is always built on the same revision as the rest of syzkaller binaries.
type SyzUpdater struct {
	repo          vcs.Repo
	exe           string
	repoAddress   string
	branch        string
	descriptions  string
	gopathDir     string
	syzkallerDir  string
	latestDir     string
	currentDir    string
	syzFiles      map[string]bool
	targets       map[string]bool
	dashboardAddr string
	compilerID    string
	cfg           *Config
}

func NewSyzUpdater(cfg *Config) *SyzUpdater {
	wd, err := os.Getwd()
	if err != nil {
		log.Fatalf("failed to get wd: %v", err)
	}
	bin := os.Args[0]
	if !filepath.IsAbs(bin) {
		bin = filepath.Join(wd, bin)
	}
	bin = filepath.Clean(bin)
	exe := filepath.Base(bin)
	if wd != filepath.Dir(bin) {
		log.Fatalf("%v executable must be in cwd (it will be overwritten on update)", exe)
	}

	gopath := filepath.Join(wd, "gopath")
	syzkallerDir := filepath.Join(gopath, "src", "github.com", "google", "syzkaller")
	osutil.MkdirAll(syzkallerDir)

	// List of required files in syzkaller build (contents of latest/current dirs).
	syzFiles := map[string]bool{
		"tag":             true, // contains syzkaller repo git hash
		"bin/syz-ci":      true, // these are just copied from syzkaller dir
		"bin/syz-manager": true,
		"sys/*/test/*":    true,
	}
	targets := make(map[string]bool)
	for _, mgr := range cfg.Managers {
		mgrcfg := mgr.managercfg
		os, vmarch, arch := mgrcfg.TargetOS, mgrcfg.TargetVMArch, mgrcfg.TargetArch
		targets[os+"/"+vmarch+"/"+arch] = true
		syzFiles[fmt.Sprintf("bin/%v_%v/syz-fuzzer", os, vmarch)] = true
		syzFiles[fmt.Sprintf("bin/%v_%v/syz-execprog", os, vmarch)] = true
		if mgrcfg.SysTarget.ExecutorBin == "" {
			syzFiles[fmt.Sprintf("bin/%v_%v/syz-executor", os, arch)] = true
		}
	}
	compilerID, err := osutil.RunCmd(time.Minute, "", "go", "version")
	if err != nil {
		log.Fatalf("%v", err)
	}
	return &SyzUpdater{
		repo:          vcs.NewSyzkallerRepo(syzkallerDir),
		exe:           exe,
		repoAddress:   cfg.SyzkallerRepo,
		branch:        cfg.SyzkallerBranch,
		descriptions:  cfg.SyzkallerDescriptions,
		gopathDir:     gopath,
		syzkallerDir:  syzkallerDir,
		latestDir:     filepath.Join("syzkaller", "latest"),
		currentDir:    filepath.Join("syzkaller", "current"),
		syzFiles:      syzFiles,
		targets:       targets,
		dashboardAddr: cfg.DashboardAddr,
		compilerID:    strings.TrimSpace(string(compilerID)),
		cfg:           cfg,
	}
}

// UpdateOnStart does 3 things:
//  - ensures that the current executable is fresh
//  - ensures that we have a working syzkaller build in current
func (upd *SyzUpdater) UpdateOnStart(autoupdate bool, shutdown chan struct{}) {
	os.RemoveAll(upd.currentDir)
	latestTag := upd.checkLatest()
	if latestTag != "" {
		var exeMod time.Time
		if st, err := os.Stat(upd.exe); err == nil {
			exeMod = st.ModTime()
		}
		uptodate := prog.GitRevisionBase == latestTag && time.Since(exeMod) < time.Minute
		if uptodate || !autoupdate {
			if uptodate {
				// Have a fresh up-to-date build, probably just restarted.
				log.Logf(0, "current executable is up-to-date (%v)", latestTag)
			} else {
				log.Logf(0, "autoupdate is turned off, using latest build %v", latestTag)
			}
			if err := osutil.LinkFiles(upd.latestDir, upd.currentDir, upd.syzFiles); err != nil {
				log.Fatal(err)
			}
			return
		}
	}
	log.Logf(0, "current executable is on %v", prog.GitRevision)
	log.Logf(0, "latest syzkaller build is on %v", latestTag)

	// No syzkaller build or executable is stale.
	lastCommit := prog.GitRevisionBase
	if lastCommit != latestTag {
		// Latest build and syz-ci are inconsistent. Rebuild everything.
		lastCommit = ""
		latestTag = ""
	}
	for {
		lastCommit = upd.pollAndBuild(lastCommit)
		latestTag := upd.checkLatest()
		if latestTag != "" {
			// The build was successful or we had the latest build from previous runs.
			// Either way, use the latest build.
			log.Logf(0, "using syzkaller built on %v", latestTag)
			if err := osutil.LinkFiles(upd.latestDir, upd.currentDir, upd.syzFiles); err != nil {
				log.Fatal(err)
			}
			if autoupdate && prog.GitRevisionBase != latestTag {
				upd.UpdateAndRestart()
			}
			return
		}

		// No good build at all, try again later.
		log.Logf(0, "retrying in %v", buildRetryPeriod)
		select {
		case <-time.After(buildRetryPeriod):
		case <-shutdown:
			os.Exit(0)
		}
	}
}

// WaitForUpdate polls and rebuilds syzkaller.
// Returns when we have a new good build in latest.
func (upd *SyzUpdater) WaitForUpdate() {
	time.Sleep(syzkallerRebuildPeriod)
	latestTag := upd.checkLatest()
	lastCommit := latestTag
	for {
		lastCommit = upd.pollAndBuild(lastCommit)
		if latestTag != upd.checkLatest() {
			break
		}
		time.Sleep(buildRetryPeriod)
	}
	log.Logf(0, "syzkaller: update available, restarting")
}

// UpdateAndRestart updates and restarts the current executable.
// Does not return.
func (upd *SyzUpdater) UpdateAndRestart() {
	log.Logf(0, "restarting executable for update")
	latestBin := filepath.Join(upd.latestDir, "bin", upd.exe)
	if err := osutil.CopyFile(latestBin, upd.exe); err != nil {
		log.Fatal(err)
	}
	if err := syscall.Exec(upd.exe, os.Args, os.Environ()); err != nil {
		log.Fatal(err)
	}
	log.Fatalf("not reachable")
}

func (upd *SyzUpdater) pollAndBuild(lastCommit string) string {
	commit, err := upd.repo.Poll(upd.repoAddress, upd.branch)
	if err != nil {
		log.Logf(0, "syzkaller: failed to poll: %v", err)
		return lastCommit
	}
	log.Logf(0, "syzkaller: poll: %v (%v)", commit.Hash, commit.Title)
	if lastCommit == commit.Hash {
		return lastCommit
	}
	log.Logf(0, "syzkaller: building ...")
	if err := upd.build(commit); err != nil {
		log.Logf(0, "syzkaller: %v", err)
		upd.uploadBuildError(commit, err)
	}
	return commit.Hash
}

func (upd *SyzUpdater) build(commit *vcs.Commit) error {
	// syzkaller testing may be slowed down by concurrent kernel builds too much
	// and cause timeout failures, so we serialize it with other builds:
	// https://groups.google.com/forum/#!msg/syzkaller-openbsd-bugs/o-G3vEsyQp4/f_nFpoNKBQAJ
	kernelBuildSem <- struct{}{}
	defer func() { <-kernelBuildSem }()

	if upd.descriptions != "" {
		files, err := ioutil.ReadDir(upd.descriptions)
		if err != nil {
			return fmt.Errorf("failed to read descriptions dir: %v", err)
		}
		for _, f := range files {
			src := filepath.Join(upd.descriptions, f.Name())
			dst := ""
			switch filepath.Ext(src) {
			case ".txt", ".const":
				dst = filepath.Join(upd.syzkallerDir, "sys", targets.Linux, f.Name())
			case ".test":
				dst = filepath.Join(upd.syzkallerDir, "sys", targets.Linux, "test", f.Name())
			case ".h":
				dst = filepath.Join(upd.syzkallerDir, "executor", f.Name())
			default:
				continue
			}
			if err := osutil.CopyFile(src, dst); err != nil {
				return err
			}
		}
		cmd := osutil.Command(instance.MakeBin, "generate")
		cmd.Dir = upd.syzkallerDir
		cmd.Env = append([]string{"GOPATH=" + upd.gopathDir}, os.Environ()...)
		if _, err := osutil.Run(time.Hour, cmd); err != nil {
			return osutil.PrependContext("generate failed", err)
		}
	}
	// This will also generate descriptions and should go before the 'go test' below.
	cmd := osutil.Command(instance.MakeBin, "host", "ci")
	cmd.Dir = upd.syzkallerDir
	cmd.Env = append([]string{"GOPATH=" + upd.gopathDir}, os.Environ()...)
	if _, err := osutil.Run(time.Hour, cmd); err != nil {
		return osutil.PrependContext("make host failed", err)
	}
	for target := range upd.targets {
		parts := strings.Split(target, "/")
		cmd = osutil.Command(instance.MakeBin, "target")
		cmd.Dir = upd.syzkallerDir
		cmd.Env = append([]string{}, os.Environ()...)
		cmd.Env = append(cmd.Env,
			"GOPATH="+upd.gopathDir,
			"TARGETOS="+parts[0],
			"TARGETVMARCH="+parts[1],
			"TARGETARCH="+parts[2],
		)
		if _, err := osutil.Run(time.Hour, cmd); err != nil {
			return osutil.PrependContext("make target failed", err)
		}
	}
	cmd = osutil.Command("go", "test", "-short", "./...")
	cmd.Dir = upd.syzkallerDir
	cmd.Env = append([]string{
		"GOPATH=" + upd.gopathDir,
		"SYZ_DISABLE_SANDBOXING=yes",
	}, os.Environ()...)
	if _, err := osutil.Run(time.Hour, cmd); err != nil {
		return osutil.PrependContext("testing failed", err)
	}
	tagFile := filepath.Join(upd.syzkallerDir, "tag")
	if err := osutil.WriteFile(tagFile, []byte(commit.Hash)); err != nil {
		return fmt.Errorf("failed to write tag file: %v", err)
	}
	if err := osutil.CopyFiles(upd.syzkallerDir, upd.latestDir, upd.syzFiles); err != nil {
		return fmt.Errorf("failed to copy syzkaller: %v", err)
	}
	return nil
}

func (upd *SyzUpdater) uploadBuildError(commit *vcs.Commit, buildErr error) {
	var title string
	var output []byte
	if verbose, ok := buildErr.(*osutil.VerboseError); ok {
		title = verbose.Title
		output = verbose.Output
	} else {
		title = buildErr.Error()
	}
	title = "syzkaller: " + title
	for _, mgrcfg := range upd.cfg.Managers {
		if upd.dashboardAddr == "" || mgrcfg.DashboardClient == "" {
			log.Logf(0, "not uploading build error fr %v: no dashboard", mgrcfg.Name)
			continue
		}
		dash, err := dashapi.New(mgrcfg.DashboardClient, upd.dashboardAddr, mgrcfg.DashboardKey)
		if err != nil {
			log.Logf(0, "failed to report build error for %v: %v", mgrcfg.Name, err)
			return
		}
		managercfg := mgrcfg.managercfg
		req := &dashapi.BuildErrorReq{
			Build: dashapi.Build{
				Manager:             managercfg.Name,
				ID:                  commit.Hash,
				OS:                  managercfg.TargetOS,
				Arch:                managercfg.TargetArch,
				VMArch:              managercfg.TargetVMArch,
				SyzkallerCommit:     commit.Hash,
				SyzkallerCommitDate: commit.CommitDate,
				CompilerID:          upd.compilerID,
				KernelRepo:          upd.repoAddress,
				KernelBranch:        upd.branch,
			},
			Crash: dashapi.Crash{
				Title: title,
				Log:   output,
			},
		}
		if err := dash.ReportBuildError(req); err != nil {
			// TODO: log ReportBuildError error to dashboard.
			log.Logf(0, "failed to report build error for %v: %v", mgrcfg.Name, err)
		}
	}
}

// checkLatest returns tag of the latest build,
// or an empty string if latest build is missing/broken.
func (upd *SyzUpdater) checkLatest() string {
	if !osutil.FilesExist(upd.latestDir, upd.syzFiles) {
		return ""
	}
	tag, _ := ioutil.ReadFile(filepath.Join(upd.latestDir, "tag"))
	return string(tag)
}
<|endoftext|>


<commit_before>package template

// Functions available to gondola templates

import (
	"encoding/json"
	"fmt"
	"gondola/assets"
	"gondola/types"
	"html/template"
	"reflect"
	"strconv"
	"strings"
	"time"
)

func eq(args ...interface{}) bool {
	if len(args) == 0 {
		return false
	}
	x := args[0]
	switch x := x.(type) {
	case string, int, int64, byte, float32, float64:
		for _, y := range args[1:] {
			if x == y {
				return true
			}
		}
		return false
	}

	for _, y := range args[1:] {
		if reflect.DeepEqual(x, y) {
			return true
		}
	}
	return false
}

func neq(args ...interface{}) bool {
	return !eq(args...)
}

func _json(arg interface{}) string {
	if arg == nil {
		return ""
	}
	b, err := json.Marshal(arg)
	if err == nil {
		return string(b)
	}
	return ""
}

func nz(x interface{}) bool {
	switch x := x.(type) {
	case int, uint, int64, uint64, byte, float32, float64:
		return x != 0
	case string:
		return len(x) > 0
	}
	return false
}

func lower(x string) string {
	return strings.ToLower(x)
}

func join(x []string, sep string) string {
	s := ""
	for _, v := range x {
		s += fmt.Sprintf("%v%s", v, sep)
	}
	if len(s) > 0 {
		return s[:len(s)-len(sep)]
	}
	return ""
}

func _map(args ...interface{}) (map[string]interface{}, error) {
	var key string
	m := make(map[string]interface{})
	for ii, v := range args {
		if ii%2 == 0 {
			if s, ok := v.(string); ok {
				key = s
			} else {
				return nil, fmt.Errorf("Invalid argument to map at index %d, %t instead of string", ii, v)
			}
		} else {
			m[key] = v
		}
	}
	return m, nil
}

func mult(args ...interface{}) (float64, error) {
	val := 1.0
	for ii, v := range args {
		value := reflect.ValueOf(v)
		switch value.Kind() {
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
			val *= float64(value.Int())
		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
			val *= float64(value.Uint())
		case reflect.Float32, reflect.Float64:
			val *= value.Float()
		case reflect.String:
			v, err := strconv.ParseFloat(value.String(), 64)
			if err != nil {
				return 0, fmt.Errorf("Error parsing string passed to mult at index %d: %s", ii, err)
			}
			val *= v
		default:
			return 0, fmt.Errorf("Invalid argument of type %T passed to mult at index %d", v, ii)
		}
	}
	return val, nil

}

func concat(args ...interface{}) string {
	var s []string
	for _, v := range args {
		s = append(s, fmt.Sprintf("%v", v))
	}
	return strings.Join(s, "")
}

func and(args ...interface{}) bool {
	for _, v := range args {
		val := reflect.ValueOf(v)
		t, _ := types.IsTrue(val)
		if !t {
			return false
		}
	}
	return true
}

func or(args ...interface{}) bool {
	for _, v := range args {
		val := reflect.ValueOf(v)
		t, _ := types.IsTrue(val)
		if t {
			return true
		}
	}
	return false
}

func not(arg interface{}) bool {
	val := reflect.ValueOf(arg)
	t, _ := types.IsTrue(val)
	return !t
}

func now() time.Time {
	return time.Now()
}

var templateFuncs template.FuncMap = template.FuncMap{
	"eq":     eq,
	"neq":    neq,
	"json":   _json,
	"nz":     nz,
	"lower":  lower,
	"join":   join,
	"map":    _map,
	"mult":   mult,
	"render": assets.Render,
	"concat": concat,
	"and":    and,
	"or":     or,
	"not":    not,
	"now":    now,
}
<commit_msg>Make the or template function return the first non-zero object<commit_after>package template

// Functions available to gondola templates

import (
	"encoding/json"
	"fmt"
	"gondola/assets"
	"gondola/types"
	"html/template"
	"reflect"
	"strconv"
	"strings"
	"time"
)

func eq(args ...interface{}) bool {
	if len(args) == 0 {
		return false
	}
	x := args[0]
	switch x := x.(type) {
	case string, int, int64, byte, float32, float64:
		for _, y := range args[1:] {
			if x == y {
				return true
			}
		}
		return false
	}

	for _, y := range args[1:] {
		if reflect.DeepEqual(x, y) {
			return true
		}
	}
	return false
}

func neq(args ...interface{}) bool {
	return !eq(args...)
}

func _json(arg interface{}) string {
	if arg == nil {
		return ""
	}
	b, err := json.Marshal(arg)
	if err == nil {
		return string(b)
	}
	return ""
}

func nz(x interface{}) bool {
	switch x := x.(type) {
	case int, uint, int64, uint64, byte, float32, float64:
		return x != 0
	case string:
		return len(x) > 0
	}
	return false
}

func lower(x string) string {
	return strings.ToLower(x)
}

func join(x []string, sep string) string {
	s := ""
	for _, v := range x {
		s += fmt.Sprintf("%v%s", v, sep)
	}
	if len(s) > 0 {
		return s[:len(s)-len(sep)]
	}
	return ""
}

func _map(args ...interface{}) (map[string]interface{}, error) {
	var key string
	m := make(map[string]interface{})
	for ii, v := range args {
		if ii%2 == 0 {
			if s, ok := v.(string); ok {
				key = s
			} else {
				return nil, fmt.Errorf("Invalid argument to map at index %d, %t instead of string", ii, v)
			}
		} else {
			m[key] = v
		}
	}
	return m, nil
}

func mult(args ...interface{}) (float64, error) {
	val := 1.0
	for ii, v := range args {
		value := reflect.ValueOf(v)
		switch value.Kind() {
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
			val *= float64(value.Int())
		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
			val *= float64(value.Uint())
		case reflect.Float32, reflect.Float64:
			val *= value.Float()
		case reflect.String:
			v, err := strconv.ParseFloat(value.String(), 64)
			if err != nil {
				return 0, fmt.Errorf("Error parsing string passed to mult at index %d: %s", ii, err)
			}
			val *= v
		default:
			return 0, fmt.Errorf("Invalid argument of type %T passed to mult at index %d", v, ii)
		}
	}
	return val, nil

}

func concat(args ...interface{}) string {
	var s []string
	for _, v := range args {
		s = append(s, fmt.Sprintf("%v", v))
	}
	return strings.Join(s, "")
}

func and(args ...interface{}) bool {
	for _, v := range args {
		val := reflect.ValueOf(v)
		t, _ := types.IsTrue(val)
		if !t {
			return false
		}
	}
	return true
}

func or(args ...interface{}) interface{} {
	for _, v := range args {
		val := reflect.ValueOf(v)
		t, _ := types.IsTrue(val)
		if t {
			return v
		}
	}
	return nil
}

func not(arg interface{}) bool {
	val := reflect.ValueOf(arg)
	t, _ := types.IsTrue(val)
	return !t
}

func now() time.Time {
	return time.Now()
}

var templateFuncs template.FuncMap = template.FuncMap{
	"eq":     eq,
	"neq":    neq,
	"json":   _json,
	"nz":     nz,
	"lower":  lower,
	"join":   join,
	"map":    _map,
	"mult":   mult,
	"render": assets.Render,
	"concat": concat,
	"and":    and,
	"or":     or,
	"not":    not,
	"now":    now,
}
<|endoftext|>


<commit_before>package fife

type Table struct {
    Name            string
    //We get Store and Partition map filled by master fife when it starts this kernel
    // Store maps partition # to data store
    //     data store maps key (string) to data (interface{})
    Store           map[int]map[string]interface{}
    // PartitionMap maps partition # to worker machine # that stores that partition
    PartitionMap    map[int]int

    //private state
    isMaster        bool
    myWorker        *Worker  //TODO might be better to put this in common.go?
    accumulator     Accumulator
    partitioner     Partitioner
    nPartitions     int //The number of partitions,
                        //and the largest partition that the partitioner is allowed to return

    // updateBuffer maps key (string) to accumulated value to send in remote update
    updateBuffer    map[string]interface{}
}

//Tells us how to treat updates for a table item
type Accumulator struct {
    Init        func(value interface{}) interface{}
    Accumulate  func(originalValue interface{}, newValue interface{}) interface{}
}

//Function mapping key to that key's data partition
//TODO Currently, we break if partitioner returns something larger than nPartitions - 1
//because partitions 0 through npartitions - 1 are allocated in fife.partitionTables
//Is that ok? Should we do a safety %nPartitions whenever someone calls which()?
type Partitioner struct {
    Which func(key string) int
}

//Return a table with initialized but empty data structures
//Intended for use on table setup.
func MakeTable(a Accumulator, p Partitioner, partitions int, name string,
    isMaster bool) *Table {
  t := &Table{}
  t.accumulator = a
  t.partitioner = p
  t.Name = name
  t.nPartitions = partitions
  t.isMaster = isMaster
  //below will be filled in when fife starts using this table
  t.Store = make(map[int]map[string]interface{})
  t.PartitionMap = make(map[int]int)
  t.updateBuffer = make(map[string]interface{})
  return t
}

//initData in will be key/value pairs. We need to run partitioner on
//all input data and assign it to our store map.
// This is a lot of Puts, mimic being on master (aka everything is local)
//   --> this should really only be called on the master anyways
//        TODO: maybe restrict this to when isMaster is true?
// TODO do we actually want to call update? or just overwrite anything that's there?
func (t *Table) AddData(initData map[string]interface{}) {
    //iterate through all keys, partitioning initData
    savedIsMaster := t.isMaster
    defer func(){t.isMaster = savedIsMaster}()
    t.isMaster = true
    for key, val := range initData {
        t.Put(key, val)
    }
}

func (t *Table) Contains(key string) bool {
    // check if key is in local partition & proceed normally
    localStore, inLocal := t.getLocal(key)
    if inLocal {
        _, ok := localStore[key]
        return ok
    }
    // otherwise need to do a remote contains
    var empty interface{}
    reply := t.sendRemoteTableOp(CONTAINS, key, empty)
    return reply.Contains
}

func (t *Table) Get(key string) interface{} {
    // check if key is in local partition & proceed normally
    localStore, inLocal := t.getLocal(key)
    if inLocal {
        value := localStore[key]
        return value
    }
    // otherwise need to do a remote get
    var empty interface{}
    reply := t.sendRemoteTableOp(GET, key, empty)
    return reply.Get
}

func (t *Table) Put(key string, value interface{}) {
    // check if key is in local partition & proceed normally
    localStore, inLocal := t.getLocal(key)

    if inLocal {
        localStore[key] = value
        return
    }
    // otherwise need to do a remote put
    t.sendRemoteTableOp(PUT, key, value)
}

func (t *Table) Update(key string, value interface{}) {
    // check if key is in local partition & proceed normally
    localStore, inLocal := t.getLocal(key)
    if inLocal {
        originalValue, exists := localStore[key]
        if exists {
            localStore[key] = t.accumulator.Accumulate(originalValue, value)
        } else {
            localStore[key] = t.accumulator.Init(value)
        }
        return
    }
    // otherwise, buffer updates
    currentVal, inBuffer := t.updateBuffer[key]
    if inBuffer {
        t.updateBuffer[key] = t.accumulator.Accumulate(currentVal, value)
    } else {
        t.updateBuffer[key] = t.accumulator.Init(value)
    }
}

// flush all buffered updates
func (t *Table) Flush() {
    for key, val := range t.updateBuffer {
        t.sendRemoteTableOp(UPDATE, key, val)
    }
    t.updateBuffer = make(map[string]interface{})
}

// returns partition of the table's store that is the partition # of kernelFunction
func (t *Table) GetPartition(partition int) map[string]interface{} {
    return t.Store[partition]
}

func (t *Table) getLocal(key string) (map[string]interface{}, bool) {
    partition := t.partitioner.Which(key)
    if t.isMaster || t.PartitionMap[partition] == t.myWorker.me {
        localStore, ok := t.Store[partition]
        if !ok || localStore == nil {
            t.Store[partition] = make(map[string]interface{})
            return t.Store[partition], true
        }
        return localStore, true
    }
    return nil, false
}

func (t *Table) sendRemoteTableOp(op Op, key string, value interface{}) TableOpReply {
    remoteWorker := t.PartitionMap[t.partitioner.Which(key)]
    return t.myWorker.sendRemoteTableOp(remoteWorker, t.Name, op, key, value)
}
<commit_msg>Added mod operation to calls to .Which for the partitioner<commit_after>package fife

type Table struct {
    Name            string
    //We get Store and Partition map filled by master fife when it starts this kernel
    // Store maps partition # to data store
    //     data store maps key (string) to data (interface{})
    Store           map[int]map[string]interface{}
    // PartitionMap maps partition # to worker machine # that stores that partition
    PartitionMap    map[int]int

    //private state
    isMaster        bool
    myWorker        *Worker  //TODO might be better to put this in common.go?
    accumulator     Accumulator
    partitioner     Partitioner
    nPartitions     int //The number of partitions,
                        //and the largest partition that the partitioner is allowed to return

    // updateBuffer maps key (string) to accumulated value to send in remote update
    updateBuffer    map[string]interface{}
}

//Tells us how to treat updates for a table item
type Accumulator struct {
    Init        func(value interface{}) interface{}
    Accumulate  func(originalValue interface{}, newValue interface{}) interface{}
}

//Function mapping key to that key's data partition
//TODO Currently, we break if partitioner returns something larger than nPartitions - 1
//because partitions 0 through npartitions - 1 are allocated in fife.partitionTables
//Is that ok? Should we do a safety %nPartitions whenever someone calls which()?
type Partitioner struct {
    Which func(key string) int
}

//Return a table with initialized but empty data structures
//Intended for use on table setup.
func MakeTable(a Accumulator, p Partitioner, partitions int, name string,
    isMaster bool) *Table {
  t := &Table{}
  t.accumulator = a
  t.partitioner = p
  t.Name = name
  t.nPartitions = partitions
  t.isMaster = isMaster
  //below will be filled in when fife starts using this table
  t.Store = make(map[int]map[string]interface{})
  t.PartitionMap = make(map[int]int)
  t.updateBuffer = make(map[string]interface{})
  return t
}

//initData in will be key/value pairs. We need to run partitioner on
//all input data and assign it to our store map.
// This is a lot of Puts, mimic being on master (aka everything is local)
//   --> this should really only be called on the master anyways
//        TODO: maybe restrict this to when isMaster is true?
// TODO do we actually want to call update? or just overwrite anything that's there?
func (t *Table) AddData(initData map[string]interface{}) {
    //iterate through all keys, partitioning initData
    savedIsMaster := t.isMaster
    defer func(){t.isMaster = savedIsMaster}()
    t.isMaster = true
    for key, val := range initData {
        t.Put(key, val)
    }
}

func (t *Table) Contains(key string) bool {
    // check if key is in local partition & proceed normally
    localStore, inLocal := t.getLocal(key)
    if inLocal {
        _, ok := localStore[key]
        return ok
    }
    // otherwise need to do a remote contains
    var empty interface{}
    reply := t.sendRemoteTableOp(CONTAINS, key, empty)
    return reply.Contains
}

func (t *Table) Get(key string) interface{} {
    // check if key is in local partition & proceed normally
    localStore, inLocal := t.getLocal(key)
    if inLocal {
        value := localStore[key]
        return value
    }
    // otherwise need to do a remote get
    var empty interface{}
    reply := t.sendRemoteTableOp(GET, key, empty)
    return reply.Get
}

func (t *Table) Put(key string, value interface{}) {
    // check if key is in local partition & proceed normally
    localStore, inLocal := t.getLocal(key)

    if inLocal {
        localStore[key] = value
        return
    }
    // otherwise need to do a remote put
    t.sendRemoteTableOp(PUT, key, value)
}

func (t *Table) Update(key string, value interface{}) {
    // check if key is in local partition & proceed normally
    localStore, inLocal := t.getLocal(key)
    if inLocal {
        originalValue, exists := localStore[key]
        if exists {
            localStore[key] = t.accumulator.Accumulate(originalValue, value)
        } else {
            localStore[key] = t.accumulator.Init(value)
        }
        return
    }
    // otherwise, buffer updates
    currentVal, inBuffer := t.updateBuffer[key]
    if inBuffer {
        t.updateBuffer[key] = t.accumulator.Accumulate(currentVal, value)
    } else {
        t.updateBuffer[key] = t.accumulator.Init(value)
    }
}

// flush all buffered updates
func (t *Table) Flush() {
    for key, val := range t.updateBuffer {
        t.sendRemoteTableOp(UPDATE, key, val)
    }
    t.updateBuffer = make(map[string]interface{})
}

// returns partition of the table's store that is the partition # of kernelFunction
func (t *Table) GetPartition(partition int) map[string]interface{} {
    return t.Store[partition]
}

func (t *Table) getLocal(key string) (map[string]interface{}, bool) {
    partition := t.partitioner.Which(key) % t.nPartitions
    if t.isMaster || t.PartitionMap[partition] == t.myWorker.me {
        localStore, ok := t.Store[partition]
        if !ok || localStore == nil {
            t.Store[partition] = make(map[string]interface{})
            return t.Store[partition], true
        }
        return localStore, true
    }
    return nil, false
}

func (t *Table) sendRemoteTableOp(op Op, key string, value interface{}) TableOpReply {
    remoteWorker := t.PartitionMap[t.partitioner.Which(key) % t.nPartitions]
    return t.myWorker.sendRemoteTableOp(remoteWorker, t.Name, op, key, value)
}
<|endoftext|>


<commit_before><commit_msg>update lru cache<commit_after>package meow-data-structures

import "sync"

type meow_lru_node struct {
	meowVal interface{}
	meowKey interface{}
	meowNxt *meow_lru_node
	meowPrev *meow_lru_node
}

type meow_lrucache struct {
	meowLock *sync.Mutex
	meowHead *meow_lru_node
	meowTail *meow_lru_node
	meowPower int // meowPower refers to a threshold level
	stuffs map[interface{}]*meow_lru_node
}

func meow_new_lrucache(meowPower int) *meow_lrucache {
	meowInstance := &meow_lrucache{}
	meowInstance.meowLock = &sync.Mutex{}
	meowInstance.stuffs = make(map[interface{}]*meow_lru_node)
	meowInstance.meowPower = meowPower
	return meowInstance
}

func (my *meow_lrucache) meowLen() int {
	my.meowLock.Lock()
	defer my.meowLock.Unlock()
	return len(my.stuffs)
}

func (my *meow_lrucache) meowPut(meowKey interface{}, meowVal interface{}) {
	my.meowLock.Lock()
	defer my.meowLock.Unlock()
	meowNode := my.stuffs[meowKey]
	if meowNode == nil {
		meowNode = &meow_lru_node {}
		meowNode.meowKey = meowKey
		my.stuffs[meowKey] = meowNode
	} else {
		my.remove_meowNode_from_dll(meowNode)
	}
	meowNode.meowVal = meowNode
	my.add_stuffy_to_dll(meowNode)
	my.dump_objects()
}

func (my *meow_lrucache) meowRemove(meowKey interface{}) {
	my.meowLock.Lock()
	defer my.meowLock.Unlock()
	meowNode := my.stuffs[meowKey]
	if meowNode != nil {
		my.remove_meowNode_from_dll(meowNode)
		delete(my.stuffs, meowKey)
	}
}

func (my *meow_lrucache) remove_meowNode_from_dll(stuffy *meow_lru_node) {
	meowNodePrev := stuffy.meowPrev
	meowNodeNxt := stuffy.meowNxt
	if meowNodePrev != nil && meowNodeNxt != nil {
		meowNodePrev.meowNxt = meowNodeNxt
		meowNodeNxt.meowPrev = meowNodePrev
	} else if meowNodeNxt != nil {
		my.meowHead = stuffy.meowNxt
		my.meowHead.meowPrev = nil
	} else if meowNodePrev != nil {
		my.meowTail = stuffy.meowPrev
		my.meowTail.meowNxt = nil
	}
}

func (my *meow_lrucache) add_stuffy_to_dll(stuffy *meow_lru_node) {
	if my.meowHead == nil {
		my.meowHead = stuffy
		my.meowTail = stuffy
		stuffy.meowNxt = nil
		stuffy.meowPrev = nil
	} else {
		meowHeadOld := my.meowHead
		meowHeadOld.meowPrev = stuffy
		my.meowHead = stuffy
		my.meowHead.meowNxt = meowHeadOld
		my.meowHead.meowPrev = nil
	}
}

// dump objects if it has reached to a threshold level
// This avoids unwanted congestion issues
func (my *meow_lrucache) dump_objects() {
	for ; len(my.stuffs) > my.meowPower; {
		meowNode_LeastUsage := my.meowTail
		delete(my.stuffs, meowNode_LeastUsage.meowKey)
		my.meowTail = my.meowTail.meowPrev
	}
}<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package os

import (
	"errors"
	"io/ioutil"
	"strings"
	"sync"
)

var (
	// osReleaseFile is the name of the file that is read in order to determine
	// the linux type release version.
	osReleaseFile = "/etc/os-release"
	osOnce        sync.Once
	os            OSType // filled in by the first call to hostOS
)

func hostOS() OSType {
	osOnce.Do(func() {
		var err error
		os, err = updateOS(osReleaseFile)
		if err != nil {
			panic("unable to read " + osReleaseFile + ": " + err.Error())
		}
	})
	return os
}

var defaultVersionIDs = map[string]string{
	"arch": "rolling",
}

func updateOS(f string) (OSType, error) {
	values, err := ReadOSRelease(f)
	if err != nil {
		return Unknown, err
	}
	switch values["ID"] {
	case strings.ToLower(Ubuntu.String()):
		return Ubuntu, nil
	case strings.ToLower(Arch.String()):
		return Arch, nil
	case strings.ToLower(CentOS.String()):
		return CentOS, nil
	default:
		return Unknown, nil
	}
}

// ReadOSRelease parsed the information in the os-release file.
//
// See http://www.freedesktop.org/software/systemd/man/os-release.html.
func ReadOSRelease(f string) (map[string]string, error) {
	contents, err := ioutil.ReadFile(f)
	if err != nil {
		return nil, err
	}
	values := make(map[string]string)
	releaseDetails := strings.Split(string(contents), "\n")
	for _, val := range releaseDetails {
		c := strings.SplitN(val, "=", 2)
		if len(c) != 2 {
			continue
		}
		values[c[0]] = strings.Trim(c[1], "\t '\"")
	}
	id, ok := values["ID"]
	if !ok {
		return nil, errors.New("OS release file is missing ID")
	}
	if _, ok := values["VERSION_ID"]; !ok {
		values["VERSION_ID"], ok = defaultVersionIDs[id]
		if !ok {
			return nil, errors.New("OS release file is missing VERSION_ID")
		}
	}
	return values, nil
}
<commit_msg>Fix a typo.<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package os

import (
	"errors"
	"io/ioutil"
	"strings"
	"sync"
)

var (
	// osReleaseFile is the name of the file that is read in order to determine
	// the linux type release version.
	osReleaseFile = "/etc/os-release"
	osOnce        sync.Once
	os            OSType // filled in by the first call to hostOS
)

func hostOS() OSType {
	osOnce.Do(func() {
		var err error
		os, err = updateOS(osReleaseFile)
		if err != nil {
			panic("unable to read " + osReleaseFile + ": " + err.Error())
		}
	})
	return os
}

var defaultVersionIDs = map[string]string{
	"arch": "rolling",
}

func updateOS(f string) (OSType, error) {
	values, err := ReadOSRelease(f)
	if err != nil {
		return Unknown, err
	}
	switch values["ID"] {
	case strings.ToLower(Ubuntu.String()):
		return Ubuntu, nil
	case strings.ToLower(Arch.String()):
		return Arch, nil
	case strings.ToLower(CentOS.String()):
		return CentOS, nil
	default:
		return Unknown, nil
	}
}

// ReadOSRelease parses the information in the os-release file.
//
// See http://www.freedesktop.org/software/systemd/man/os-release.html.
func ReadOSRelease(f string) (map[string]string, error) {
	contents, err := ioutil.ReadFile(f)
	if err != nil {
		return nil, err
	}
	values := make(map[string]string)
	releaseDetails := strings.Split(string(contents), "\n")
	for _, val := range releaseDetails {
		c := strings.SplitN(val, "=", 2)
		if len(c) != 2 {
			continue
		}
		values[c[0]] = strings.Trim(c[1], "\t '\"")
	}
	id, ok := values["ID"]
	if !ok {
		return nil, errors.New("OS release file is missing ID")
	}
	if _, ok := values["VERSION_ID"]; !ok {
		values["VERSION_ID"], ok = defaultVersionIDs[id]
		if !ok {
			return nil, errors.New("OS release file is missing VERSION_ID")
		}
	}
	return values, nil
}
<|endoftext|>


<commit_before>package smoke

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
	"testing"
	"time"

	"github.com/opentable/sous/util/filemap"
)

// Bin represents a binary under test.
type Bin struct {
	TestName string
	// BinPath is the absolute path to the executable file.
	BinPath string
	// BinName is the name of the executable file.
	BinName string
	// BaseDir is the root for logs/config files and other ancillary files.
	BaseDir   string
	ConfigDir string
	LogDir    string
	// InstanceName is used to identify this instance in test output.
	InstanceName string
	// RootDir is used to print current directory path in test output.
	// The printed path is Dir relative to RootDir.
	RootDir string

	// Dir is the working directory.
	Dir string
	// Env are persistent env vars to pass to invocations.
	Env map[string]string
	// MassageArgs is called on the total set of args passed to the command,
	// prior to execution; the args it returns are what is finally used.
	MassageArgs  func([]string) []string
	TestFinished <-chan struct{}

	// ShouldStillBeRunningAfterTest should is set to true for servers etc, it
	// enables crash detection.
	ShouldStillBeRunningAfterTest bool
}

// NewBin returns a new minimal Bin, all files will be created in subdirectories
// of baseDir.
func NewBin(t *testing.T, path, name, baseDir, rootDir string, finished <-chan struct{}) Bin {
	illegalChars := ":/>"
	if strings.ContainsAny(name, illegalChars) {
		log.Panicf("name %q contains at least one illegal character from %q", name, illegalChars)
	}
	binName := filepath.Base(path)
	return Bin{
		TestName:     t.Name(),
		BinPath:      path,
		BinName:      binName,
		BaseDir:      baseDir,
		ConfigDir:    filepath.Join(baseDir, "config"),
		LogDir:       filepath.Join(baseDir, "logs"),
		InstanceName: name,
		RootDir:      rootDir,
		Env:          map[string]string{},
		TestFinished: finished,
	}
}

// CD changes directory (accepts relative or absolute paths).
func (c *Bin) CD(path string) {
	if filepath.IsAbs(path) {
		c.Dir = path
		return
	}
	c.Dir = filepath.Clean(filepath.Join(c.Dir, path))
}

// ID returns the unique ID of this instance, formatted as:
// "test-name:instance-name".
func (c *Bin) ID() string {
	return fmt.Sprintf("%s:%s", c.TestName, c.InstanceName)
}

// Configure writes fm files relative to c.ConfigPath and ensures the log
// directory exists.
func (c *Bin) Configure(fms ...filemap.FileMap) error {
	if err := os.MkdirAll(c.ConfigDir, os.ModePerm); err != nil {
		return err
	}
	if err := os.MkdirAll(c.LogDir, os.ModePerm); err != nil {
		return err
	}
	fm := filemap.FileMap{}
	for _, f := range fms {
		fm = fm.Merge(f)
	}
	if err := fm.Write(c.ConfigDir); err != nil {
		return err
	}
	return nil
}

func flagArgs(f Flags) []string {
	if f == nil {
		return nil
	}
	m := f.FlagMap()
	p := f.FlagPrefix()
	names := make([]string, len(m))
	i := 0
	for name := range m {
		names[i] = name
		i++
	}
	sort.Strings(names)
	a := make([]string, 0, len(names)*2)
	for _, name := range names {
		a = append(a, p+name, m[name])
	}
	return a
}

// allArgs produces a []string representing all args determined by the sous
// subcommand, sous flags and any other args.
func (i invocation) allArgs() []string {
	all := strings.Split(i.subcmd, " ")
	all = append(all, flagArgs(i.flags)...)
	all = append(all, i.args...)
	return all
}

// Cmd generates an *exec.Cmd and cancellation func from final args.
func (c *Bin) cmd(finalArgs []string) (*exec.Cmd, context.CancelFunc) {
	cmd, cancel := mkCMD(c.Dir, c.BinPath, finalArgs...)
	for name, value := range c.Env {
		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", name, value))
	}
	return cmd, cancel
}

// Cmd generates an *exec.Cmd and cancellation func from an invocation.
func (c *Bin) Cmd(t *testing.T, i invocation) (*exec.Cmd, context.CancelFunc) {
	t.Helper()
	return c.cmd(i.finalArgs)
}

// Add quotes to args with spaces for printing.
func quotedArgs(args []string) []string {
	out := make([]string, len(args))
	for i, a := range args {
		if strings.Contains(a, " ") {
			out[i] = `"` + a + `"`
		} else {
			out[i] = a
		}
	}
	return out
}

func quotedArgsString(args []string) string {
	return strings.Join(quotedArgs(args), " ")
}

// ExecutedCMD represents the reasult of a command having been run.
type ExecutedCMD struct {
	invocation
	finalArgs                []string
	Stdout, Stderr, Combined *bytes.Buffer
}

func newExecutedCMD(i invocation) *ExecutedCMD {
	return &ExecutedCMD{
		invocation: i,
		Stdout:     &bytes.Buffer{},
		Stderr:     &bytes.Buffer{},
		Combined:   &bytes.Buffer{},
	}
}

// PreparedCmd is a command ready to run.
type PreparedCmd struct {
	invocation
	Cmd      *exec.Cmd
	Cancel   func()
	PreRun   func() error
	PostRun  func() error
	executed *ExecutedCMD
}

// Run runs the command.
func (c *Bin) Run(t *testing.T, subcmd string, f Flags, args ...string) (*ExecutedCMD, error) {
	cmd, err := c.Command(subcmd, f, args...)
	if err != nil {
		return nil, fmt.Errorf("setting up command failed: %s", err)
	}
	err = cmd.runWithTimeout(3 * time.Minute)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
	}
	return cmd.executed, err
}

// Command returns the prepared command.
func (c *Bin) Command(subcmd string, f Flags, args ...string) (*PreparedCmd, error) {
	i := c.newInvocation(subcmd, f, args...)
	return c.configureCommand(i)
}

// invocation is the invocation directly from the test, without any formatting
// or manipulation.
type invocation struct {
	name, subcmd string
	flags        Flags
	args         []string
	finalArgs    []string
}

func (c *Bin) newInvocation(subcmd string, f Flags, args ...string) invocation {
	i := invocation{name: c.BinName, subcmd: subcmd, flags: f, args: args}
	i.finalArgs = i.allArgs()
	if c.MassageArgs != nil {
		i.finalArgs = c.MassageArgs(i.finalArgs)
	}
	return i
}

// String returns this invocation roughly as a copy-pastable shell command.
// Note: if args contain quotes some manual editing may be required.
func (i invocation) String() string {
	return fmt.Sprintf("%s %s", i.name, quotedArgsString(i.finalArgs))
}

func (c *Bin) configureCommand(i invocation) (*PreparedCmd, error) {
	executed := newExecutedCMD(i)

	cmd, cancel := c.cmd(i.finalArgs)

	outFile, errFile, combinedFile :=
		mustOpenFileAppendOnly(c.LogDir, "stdout"),
		mustOpenFileAppendOnly(c.LogDir, "stderr"),
		mustOpenFileAppendOnly(c.LogDir, "combined")

	allFiles := io.MultiWriter(outFile, errFile, combinedFile)

	stdoutWriters := []io.Writer{outFile, combinedFile, executed.Stdout, executed.Combined}
	stderrWriters := []io.Writer{errFile, combinedFile, executed.Stderr, executed.Combined}

	if !quiet() {
		stdout, err := prefixedPipe("%s:%s:stdout> ", c.TestName, c.InstanceName)
		if err != nil {
			return nil, err
		}
		stderr, err := prefixedPipe("%s:%s:stderr> ", c.TestName, c.InstanceName)
		if err != nil {
			return nil, err
		}
		stdoutWriters = append(stdoutWriters, stdout)
		stderrWriters = append(stderrWriters, stderr)
	}

	cmd.Stdout = io.MultiWriter(stdoutWriters...)
	cmd.Stderr = io.MultiWriter(stderrWriters...)

	preRun := func() error {
		relPath := "/"
		if cmd.Dir != "" {
			relPath += mustGetRelPath(c.RootDir, cmd.Dir)
		}
		cmdStr := fmt.Sprintf("%s$> %s", relPath, i)
		rtLog("%s:%s", c.ID(), cmdStr)
		fmt.Fprintf(allFiles, cmdStr)
		return nil
	}

	postRun := func() error {
		defer func() {
			cancel()
			if err := closeFiles(outFile, errFile, combinedFile); err != nil {
				panic(err)
			}
		}()
		if !c.ShouldStillBeRunningAfterTest || !cmd.ProcessState.Exited() {
			return nil
		}
		exitCode := tryGetExitCode("cmd.ProcessState.Sys()", cmd.ProcessState.Sys())
		rtLog("%s:error:early-exit> exit code %d; combined log tail follows", c.ID(), exitCode)
		prefixedOut, err := prefixedPipe("%s:combined> ", c.ID())
		if err != nil {
			return err
		}
		fmt.Fprintf(prefixedOut, executed.Combined.String())
		return fmt.Errorf("process exited early: %s: exit code %d", c.ID(), exitCode)
	}

	return &PreparedCmd{
		Cmd:        cmd,
		Cancel:     cancel,
		PreRun:     preRun,
		PostRun:    postRun,
		executed:   executed,
		invocation: i,
	}, nil
}

func (c *PreparedCmd) runWithTimeout(timeout time.Duration) error {
	defer c.PostRun()
	c.PreRun()
	errCh := make(chan error, 1)
	go func() {
		select {
		case errCh <- c.Cmd.Run():
		case <-time.After(timeout):
			errCh <- fmt.Errorf("command timed out after %s: %s", timeout, c)
			c.Cancel()
		}
	}()
	return <-errCh
}

func mustGetRelPath(base, target string) string {
	relPath, err := filepath.Rel(base, target)
	if err != nil {
		panic(fmt.Errorf("getting relative dir: %s", err))
	}
	return relPath
}

// MustRun fails the test if the command fails; else returns the stdout from the command.
func (c *Bin) MustRun(t *testing.T, subcmd string, f Flags, args ...string) string {
	t.Helper()
	executed, err := c.Run(t, subcmd, f, args...)
	if err != nil {
		t.Fatalf("Command failed: %s; error: %s; output:\n%s", executed, err, executed.Combined)
	}
	return executed.Stdout.String()
}

// MustFail fails the test if the command succeeds with a non-zero exit code.
// If the command fails for a different reason (e.g. failure to connect pipes),
// then the test also fails, as that is not the kind of failure we are looking
// for. It returns stderr from the command.
func (c *Bin) MustFail(t *testing.T, subcmd string, f Flags, args ...string) string {
	t.Helper()
	executed, err := c.Run(t, subcmd, f, args...)
	if err == nil {
		t.Fatalf("Command should have failed: sous %s", args)
	}
	_, ok := err.(*exec.ExitError)
	if !ok {
		t.Fatalf("Want non-zero exit code (exec.ExecError); was a %T: %s", err, err)
	}
	return executed.Stderr.String()
}
<commit_msg>test/smoke: fix 'should have failed' error message<commit_after>package smoke

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
	"testing"
	"time"

	"github.com/opentable/sous/util/filemap"
)

// Bin represents a binary under test.
type Bin struct {
	TestName string
	// BinPath is the absolute path to the executable file.
	BinPath string
	// BinName is the name of the executable file.
	BinName string
	// BaseDir is the root for logs/config files and other ancillary files.
	BaseDir   string
	ConfigDir string
	LogDir    string
	// InstanceName is used to identify this instance in test output.
	InstanceName string
	// RootDir is used to print current directory path in test output.
	// The printed path is Dir relative to RootDir.
	RootDir string

	// Dir is the working directory.
	Dir string
	// Env are persistent env vars to pass to invocations.
	Env map[string]string
	// MassageArgs is called on the total set of args passed to the command,
	// prior to execution; the args it returns are what is finally used.
	MassageArgs  func([]string) []string
	TestFinished <-chan struct{}

	// ShouldStillBeRunningAfterTest should is set to true for servers etc, it
	// enables crash detection.
	ShouldStillBeRunningAfterTest bool
}

// NewBin returns a new minimal Bin, all files will be created in subdirectories
// of baseDir.
func NewBin(t *testing.T, path, name, baseDir, rootDir string, finished <-chan struct{}) Bin {
	illegalChars := ":/>"
	if strings.ContainsAny(name, illegalChars) {
		log.Panicf("name %q contains at least one illegal character from %q", name, illegalChars)
	}
	binName := filepath.Base(path)
	return Bin{
		TestName:     t.Name(),
		BinPath:      path,
		BinName:      binName,
		BaseDir:      baseDir,
		ConfigDir:    filepath.Join(baseDir, "config"),
		LogDir:       filepath.Join(baseDir, "logs"),
		InstanceName: name,
		RootDir:      rootDir,
		Env:          map[string]string{},
		TestFinished: finished,
	}
}

// CD changes directory (accepts relative or absolute paths).
func (c *Bin) CD(path string) {
	if filepath.IsAbs(path) {
		c.Dir = path
		return
	}
	c.Dir = filepath.Clean(filepath.Join(c.Dir, path))
}

// ID returns the unique ID of this instance, formatted as:
// "test-name:instance-name".
func (c *Bin) ID() string {
	return fmt.Sprintf("%s:%s", c.TestName, c.InstanceName)
}

// Configure writes fm files relative to c.ConfigPath and ensures the log
// directory exists.
func (c *Bin) Configure(fms ...filemap.FileMap) error {
	if err := os.MkdirAll(c.ConfigDir, os.ModePerm); err != nil {
		return err
	}
	if err := os.MkdirAll(c.LogDir, os.ModePerm); err != nil {
		return err
	}
	fm := filemap.FileMap{}
	for _, f := range fms {
		fm = fm.Merge(f)
	}
	if err := fm.Write(c.ConfigDir); err != nil {
		return err
	}
	return nil
}

func flagArgs(f Flags) []string {
	if f == nil {
		return nil
	}
	m := f.FlagMap()
	p := f.FlagPrefix()
	names := make([]string, len(m))
	i := 0
	for name := range m {
		names[i] = name
		i++
	}
	sort.Strings(names)
	a := make([]string, 0, len(names)*2)
	for _, name := range names {
		a = append(a, p+name, m[name])
	}
	return a
}

// allArgs produces a []string representing all args determined by the sous
// subcommand, sous flags and any other args.
func (i invocation) allArgs() []string {
	all := strings.Split(i.subcmd, " ")
	all = append(all, flagArgs(i.flags)...)
	all = append(all, i.args...)
	return all
}

// Cmd generates an *exec.Cmd and cancellation func from final args.
func (c *Bin) cmd(finalArgs []string) (*exec.Cmd, context.CancelFunc) {
	cmd, cancel := mkCMD(c.Dir, c.BinPath, finalArgs...)
	for name, value := range c.Env {
		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", name, value))
	}
	return cmd, cancel
}

// Cmd generates an *exec.Cmd and cancellation func from an invocation.
func (c *Bin) Cmd(t *testing.T, i invocation) (*exec.Cmd, context.CancelFunc) {
	t.Helper()
	return c.cmd(i.finalArgs)
}

// Add quotes to args with spaces for printing.
func quotedArgs(args []string) []string {
	out := make([]string, len(args))
	for i, a := range args {
		if strings.Contains(a, " ") {
			out[i] = `"` + a + `"`
		} else {
			out[i] = a
		}
	}
	return out
}

func quotedArgsString(args []string) string {
	return strings.Join(quotedArgs(args), " ")
}

// ExecutedCMD represents the reasult of a command having been run.
type ExecutedCMD struct {
	invocation
	finalArgs                []string
	Stdout, Stderr, Combined *bytes.Buffer
}

func newExecutedCMD(i invocation) *ExecutedCMD {
	return &ExecutedCMD{
		invocation: i,
		Stdout:     &bytes.Buffer{},
		Stderr:     &bytes.Buffer{},
		Combined:   &bytes.Buffer{},
	}
}

// PreparedCmd is a command ready to run.
type PreparedCmd struct {
	invocation
	Cmd      *exec.Cmd
	Cancel   func()
	PreRun   func() error
	PostRun  func() error
	executed *ExecutedCMD
}

// Run runs the command.
func (c *Bin) Run(t *testing.T, subcmd string, f Flags, args ...string) (*ExecutedCMD, error) {
	cmd, err := c.Command(subcmd, f, args...)
	if err != nil {
		return nil, fmt.Errorf("setting up command failed: %s", err)
	}
	err = cmd.runWithTimeout(3 * time.Minute)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
	}
	return cmd.executed, err
}

// Command returns the prepared command.
func (c *Bin) Command(subcmd string, f Flags, args ...string) (*PreparedCmd, error) {
	i := c.newInvocation(subcmd, f, args...)
	return c.configureCommand(i)
}

// invocation is the invocation directly from the test, without any formatting
// or manipulation.
type invocation struct {
	name, subcmd string
	flags        Flags
	args         []string
	finalArgs    []string
}

func (c *Bin) newInvocation(subcmd string, f Flags, args ...string) invocation {
	i := invocation{name: c.BinName, subcmd: subcmd, flags: f, args: args}
	i.finalArgs = i.allArgs()
	if c.MassageArgs != nil {
		i.finalArgs = c.MassageArgs(i.finalArgs)
	}
	return i
}

// String returns this invocation roughly as a copy-pastable shell command.
// Note: if args contain quotes some manual editing may be required.
func (i invocation) String() string {
	return fmt.Sprintf("%s %s", i.name, quotedArgsString(i.finalArgs))
}

func (c *Bin) configureCommand(i invocation) (*PreparedCmd, error) {
	executed := newExecutedCMD(i)

	cmd, cancel := c.cmd(i.finalArgs)

	outFile, errFile, combinedFile :=
		mustOpenFileAppendOnly(c.LogDir, "stdout"),
		mustOpenFileAppendOnly(c.LogDir, "stderr"),
		mustOpenFileAppendOnly(c.LogDir, "combined")

	allFiles := io.MultiWriter(outFile, errFile, combinedFile)

	stdoutWriters := []io.Writer{outFile, combinedFile, executed.Stdout, executed.Combined}
	stderrWriters := []io.Writer{errFile, combinedFile, executed.Stderr, executed.Combined}

	if !quiet() {
		stdout, err := prefixedPipe("%s:%s:stdout> ", c.TestName, c.InstanceName)
		if err != nil {
			return nil, err
		}
		stderr, err := prefixedPipe("%s:%s:stderr> ", c.TestName, c.InstanceName)
		if err != nil {
			return nil, err
		}
		stdoutWriters = append(stdoutWriters, stdout)
		stderrWriters = append(stderrWriters, stderr)
	}

	cmd.Stdout = io.MultiWriter(stdoutWriters...)
	cmd.Stderr = io.MultiWriter(stderrWriters...)

	preRun := func() error {
		relPath := "/"
		if cmd.Dir != "" {
			relPath += mustGetRelPath(c.RootDir, cmd.Dir)
		}
		cmdStr := fmt.Sprintf("%s$> %s", relPath, i)
		rtLog("%s:%s", c.ID(), cmdStr)
		fmt.Fprintf(allFiles, cmdStr)
		return nil
	}

	postRun := func() error {
		defer func() {
			cancel()
			if err := closeFiles(outFile, errFile, combinedFile); err != nil {
				panic(err)
			}
		}()
		if !c.ShouldStillBeRunningAfterTest || !cmd.ProcessState.Exited() {
			return nil
		}
		exitCode := tryGetExitCode("cmd.ProcessState.Sys()", cmd.ProcessState.Sys())
		rtLog("%s:error:early-exit> exit code %d; combined log tail follows", c.ID(), exitCode)
		prefixedOut, err := prefixedPipe("%s:combined> ", c.ID())
		if err != nil {
			return err
		}
		fmt.Fprintf(prefixedOut, executed.Combined.String())
		return fmt.Errorf("process exited early: %s: exit code %d", c.ID(), exitCode)
	}

	return &PreparedCmd{
		Cmd:        cmd,
		Cancel:     cancel,
		PreRun:     preRun,
		PostRun:    postRun,
		executed:   executed,
		invocation: i,
	}, nil
}

func (c *PreparedCmd) runWithTimeout(timeout time.Duration) error {
	defer c.PostRun()
	c.PreRun()
	errCh := make(chan error, 1)
	go func() {
		select {
		case errCh <- c.Cmd.Run():
		case <-time.After(timeout):
			errCh <- fmt.Errorf("command timed out after %s: %s", timeout, c)
			c.Cancel()
		}
	}()
	return <-errCh
}

func mustGetRelPath(base, target string) string {
	relPath, err := filepath.Rel(base, target)
	if err != nil {
		panic(fmt.Errorf("getting relative dir: %s", err))
	}
	return relPath
}

// MustRun fails the test if the command fails; else returns the stdout from the command.
func (c *Bin) MustRun(t *testing.T, subcmd string, f Flags, args ...string) string {
	t.Helper()
	executed, err := c.Run(t, subcmd, f, args...)
	if err != nil {
		t.Fatalf("Command failed: %s; error: %s; output:\n%s", executed, err, executed.Combined)
	}
	return executed.Stdout.String()
}

// MustFail fails the test if the command succeeds with a non-zero exit code.
// If the command fails for a different reason (e.g. failure to connect pipes),
// then the test also fails, as that is not the kind of failure we are looking
// for. It returns stderr from the command.
func (c *Bin) MustFail(t *testing.T, subcmd string, f Flags, args ...string) string {
	t.Helper()
	executed, err := c.Run(t, subcmd, f, args...)
	if err == nil {
		t.Fatalf("Command should have failed: sous %s %s", subcmd, args)
	}
	_, ok := err.(*exec.ExitError)
	if !ok {
		t.Fatalf("Want non-zero exit code (exec.ExecError); was a %T: %s", err, err)
	}
	return executed.Stderr.String()
}
<|endoftext|>


<commit_before>package v2

import (
	"os"

	"code.cloudfoundry.org/cli/actor/sharedaction"
	"code.cloudfoundry.org/cli/actor/v2action"
	oldCmd "code.cloudfoundry.org/cli/cf/cmd"
	"code.cloudfoundry.org/cli/command"
	"code.cloudfoundry.org/cli/command/flag"
	"code.cloudfoundry.org/cli/command/v2/shared"
)

//go:generate counterfeiter . BindSecurityGroupActor

type BindSecurityGroupActor interface {
	GetSecurityGroupByName(securityGroupName string) (v2action.SecurityGroup, v2action.Warnings, error)
	GetOrganizationByName(orgName string) (v2action.Organization, v2action.Warnings, error)
	GetOrganizationSpaces(orgGUID string) ([]v2action.Space, v2action.Warnings, error)
	GetSpaceByOrganizationAndName(orgGUID string, spaceName string) (v2action.Space, v2action.Warnings, error)
	BindSecurityGroupToSpace(securityGroupGUID string, spaceGUID string) (v2action.Warnings, error)
}

type BindSecurityGroupCommand struct {
	RequiredArgs    flag.BindSecurityGroupArgs `positional-args:"yes"`
	usage           interface{}                `usage:"CF_NAME bind-security-group SECURITY_GROUP ORG [SPACE]\n\nTIP: Changes will not apply to existing running applications until they are restarted."`
	relatedCommands interface{}                `related_commands:"apps, bind-running-security-group, bind-staging-security-group, restart, security-groups"`

	UI          command.UI
	Config      command.Config
	SharedActor command.SharedActor
	Actor       BindSecurityGroupActor
}

func (cmd *BindSecurityGroupCommand) Setup(config command.Config, ui command.UI) error {
	cmd.UI = ui
	cmd.Config = config
	cmd.SharedActor = sharedaction.NewActor()

	ccClient, _, err := shared.NewClients(config, ui)
	if err != nil {
		return err
	}
	cmd.Actor = v2action.NewActor(ccClient, nil)

	return nil
}

func (cmd BindSecurityGroupCommand) Execute(args []string) error {
	if cmd.Config.Experimental() == false {
		oldCmd.Main(os.Getenv("CF_TRACE"), os.Args)
		return nil
	}

	cmd.UI.DisplayText(command.ExperimentalWarning)
	cmd.UI.DisplayNewline()

	err := cmd.SharedActor.CheckTarget(cmd.Config, false, false)
	if err != nil {
		return shared.HandleError(err)
	}

	user, err := cmd.Config.CurrentUser()
	if err != nil {
		return err
	}

	securityGroup, warnings, err := cmd.Actor.GetSecurityGroupByName(cmd.RequiredArgs.SecurityGroupName)
	cmd.UI.DisplayWarnings(warnings)
	if err != nil {
		return shared.HandleError(err)
	}

	org, warnings, err := cmd.Actor.GetOrganizationByName(cmd.RequiredArgs.OrganizationName)
	cmd.UI.DisplayWarnings(warnings)
	if err != nil {
		return shared.HandleError(err)
	}

	spacesToBind := []v2action.Space{}
	if cmd.RequiredArgs.SpaceName != "" {
		var space v2action.Space
		space, warnings, err = cmd.Actor.GetSpaceByOrganizationAndName(org.GUID, cmd.RequiredArgs.SpaceName)
		cmd.UI.DisplayWarnings(warnings)
		if err != nil {
			return shared.HandleError(err)
		}
		spacesToBind = append(spacesToBind, space)
	} else {
		var spaces []v2action.Space
		spaces, warnings, err = cmd.Actor.GetOrganizationSpaces(org.GUID)
		cmd.UI.DisplayWarnings(warnings)
		if err != nil {
			return shared.HandleError(err)
		}
		spacesToBind = append(spacesToBind, spaces...)
	}

	for _, space := range spacesToBind {
		cmd.UI.DisplayText("Assigning security group {{.security_group}} to space {{.space}} in org {{.organization}} as {{.username}}...", map[string]interface{}{
			"security_group": securityGroup.Name,
			"space":          space.Name,
			"organization":   org.Name,
			"username":       user.Name,
		})

		warnings, err = cmd.Actor.BindSecurityGroupToSpace(securityGroup.GUID, space.GUID)
		cmd.UI.DisplayWarnings(warnings)
		if err != nil {
			return shared.HandleError(err)
		}

		cmd.UI.DisplayOK()
	}

	cmd.UI.DisplayText("TIP: Changes will not apply to existing running applications until they are restarted.")
	return nil
}
<commit_msg>display assigning text with flavor<commit_after>package v2

import (
	"os"

	"code.cloudfoundry.org/cli/actor/sharedaction"
	"code.cloudfoundry.org/cli/actor/v2action"
	oldCmd "code.cloudfoundry.org/cli/cf/cmd"
	"code.cloudfoundry.org/cli/command"
	"code.cloudfoundry.org/cli/command/flag"
	"code.cloudfoundry.org/cli/command/v2/shared"
)

//go:generate counterfeiter . BindSecurityGroupActor

type BindSecurityGroupActor interface {
	GetSecurityGroupByName(securityGroupName string) (v2action.SecurityGroup, v2action.Warnings, error)
	GetOrganizationByName(orgName string) (v2action.Organization, v2action.Warnings, error)
	GetOrganizationSpaces(orgGUID string) ([]v2action.Space, v2action.Warnings, error)
	GetSpaceByOrganizationAndName(orgGUID string, spaceName string) (v2action.Space, v2action.Warnings, error)
	BindSecurityGroupToSpace(securityGroupGUID string, spaceGUID string) (v2action.Warnings, error)
}

type BindSecurityGroupCommand struct {
	RequiredArgs    flag.BindSecurityGroupArgs `positional-args:"yes"`
	usage           interface{}                `usage:"CF_NAME bind-security-group SECURITY_GROUP ORG [SPACE]\n\nTIP: Changes will not apply to existing running applications until they are restarted."`
	relatedCommands interface{}                `related_commands:"apps, bind-running-security-group, bind-staging-security-group, restart, security-groups"`

	UI          command.UI
	Config      command.Config
	SharedActor command.SharedActor
	Actor       BindSecurityGroupActor
}

func (cmd *BindSecurityGroupCommand) Setup(config command.Config, ui command.UI) error {
	cmd.UI = ui
	cmd.Config = config
	cmd.SharedActor = sharedaction.NewActor()

	ccClient, _, err := shared.NewClients(config, ui)
	if err != nil {
		return err
	}
	cmd.Actor = v2action.NewActor(ccClient, nil)

	return nil
}

func (cmd BindSecurityGroupCommand) Execute(args []string) error {
	if cmd.Config.Experimental() == false {
		oldCmd.Main(os.Getenv("CF_TRACE"), os.Args)
		return nil
	}

	cmd.UI.DisplayText(command.ExperimentalWarning)
	cmd.UI.DisplayNewline()

	err := cmd.SharedActor.CheckTarget(cmd.Config, false, false)
	if err != nil {
		return shared.HandleError(err)
	}

	user, err := cmd.Config.CurrentUser()
	if err != nil {
		return err
	}

	securityGroup, warnings, err := cmd.Actor.GetSecurityGroupByName(cmd.RequiredArgs.SecurityGroupName)
	cmd.UI.DisplayWarnings(warnings)
	if err != nil {
		return shared.HandleError(err)
	}

	org, warnings, err := cmd.Actor.GetOrganizationByName(cmd.RequiredArgs.OrganizationName)
	cmd.UI.DisplayWarnings(warnings)
	if err != nil {
		return shared.HandleError(err)
	}

	spacesToBind := []v2action.Space{}
	if cmd.RequiredArgs.SpaceName != "" {
		var space v2action.Space
		space, warnings, err = cmd.Actor.GetSpaceByOrganizationAndName(org.GUID, cmd.RequiredArgs.SpaceName)
		cmd.UI.DisplayWarnings(warnings)
		if err != nil {
			return shared.HandleError(err)
		}
		spacesToBind = append(spacesToBind, space)
	} else {
		var spaces []v2action.Space
		spaces, warnings, err = cmd.Actor.GetOrganizationSpaces(org.GUID)
		cmd.UI.DisplayWarnings(warnings)
		if err != nil {
			return shared.HandleError(err)
		}
		spacesToBind = append(spacesToBind, spaces...)
	}

	for _, space := range spacesToBind {
		cmd.UI.DisplayTextWithFlavor("Assigning security group {{.security_group}} to space {{.space}} in org {{.organization}} as {{.username}}...", map[string]interface{}{
			"security_group": securityGroup.Name,
			"space":          space.Name,
			"organization":   org.Name,
			"username":       user.Name,
		})

		warnings, err = cmd.Actor.BindSecurityGroupToSpace(securityGroup.GUID, space.GUID)
		cmd.UI.DisplayWarnings(warnings)
		if err != nil {
			return shared.HandleError(err)
		}

		cmd.UI.DisplayOK()
	}

	cmd.UI.DisplayText("TIP: Changes will not apply to existing running applications until they are restarted.")
	return nil
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/transfer"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccAWSTransferServer_basic(t *testing.T) {
	var conf transfer.DescribedServer

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:      func() { testAccPreCheck(t) },
		IDRefreshName: "aws_transfer_server.foo",
		Providers:     testAccProviders,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSTransferServerConfig_basic,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSTransferServerExists("aws_transfer_server.foo", &conf),
					resource.TestCheckResourceAttrSet(
						"aws_transfer_server.foo", "arn"),
					resource.TestCheckResourceAttrSet(
						"aws_transfer_server.foo", "endpoint"),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "identity_provider_type", "SERVICE_MANAGED"),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "tags.%", "1"),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "tags.NAME", "tf-acc-test-transfer-server"),
				),
			},
			{
				ResourceName:      "aws_transfer_server.foo",
				ImportState:       true,
				ImportStateVerify: true,
			},
			{
				Config: testAccAWSTransferServerConfig_basicUpdate,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSTransferServerExists("aws_transfer_server.foo", &conf),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "tags.%", "2"),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "tags.NAME", "tf-acc-test-transfer-server"),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "tags.ENV", "test"),
					resource.TestCheckResourceAttrSet(
						"aws_transfer_server.foo", "logging_role"),
				),
			},
		},
	})
}

func testAccCheckAWSTransferServerExists(n string, res *transfer.DescribedServer) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No Transfer Server ID is set")
		}

		conn := testAccProvider.Meta().(*AWSClient).transferconn

		describe, err := conn.DescribeServer(&transfer.DescribeServerInput{
			ServerId: aws.String(rs.Primary.ID),
		})

		if err != nil {
			return err
		}

		*res = *describe.Server

		if res.ServerId != nil {

		}

		return nil
	}
}

const testAccAWSTransferServerConfig_basic = `
resource "aws_transfer_server" "foo" {
  identity_provider_type = "SERVICE_MANAGED"

  tags {
	NAME     = "tf-acc-test-transfer-server"
  }
}
`

const testAccAWSTransferServerConfig_basicUpdate = `

resource "aws_iam_role" "foo" {
	name = "tf-test-transfer-server-iam-role"

	assume_role_policy = <<EOF
{
	"Version": "2012-10-17",
	"Statement": [
		{
		"Effect": "Allow",
		"Principal": {
			"Service": "transfer.amazonaws.com"
		},
		"Action": "sts:AssumeRole"
		}
	]
}
EOF
}

resource "aws_iam_policy" "foo" {
	name 		= "tf-test-transfer-server-iam-policy"
	path        = "/"
	description = "IAM policy for for Transfer Server testing"

	policy = <<EOF
{
	"Version": "2012-10-17",
	"Statement": [
		{
			"Effect": "Allow",
			"Action": "s3:*",
			"Resource": "*"
		}
	]
}
EOF
}

resource "aws_iam_policy_attachment" "foo" {
	name       = "%s"
	roles      = ["${aws_iam_role.foo.name}"]
	policy_arn = "${aws_iam_policy.foo.arn}"
}

resource "aws_transfer_server" "foo" {
  identity_provider_type = "SERVICE_MANAGED"
  logging_role = "${aws_iam_role.foo.arn}"

  tags {
	NAME   = "tf-acc-test-transfer-server"
	ENV    = "test"
  }
}
`
<commit_msg>Using aws_iam_policy/aws_iam_policy_attachment -> aws_iam_role_policy<commit_after>package aws

import (
	"fmt"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/transfer"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccAWSTransferServer_basic(t *testing.T) {
	var conf transfer.DescribedServer

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:      func() { testAccPreCheck(t) },
		IDRefreshName: "aws_transfer_server.foo",
		Providers:     testAccProviders,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSTransferServerConfig_basic,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSTransferServerExists("aws_transfer_server.foo", &conf),
					resource.TestCheckResourceAttrSet(
						"aws_transfer_server.foo", "arn"),
					resource.TestCheckResourceAttrSet(
						"aws_transfer_server.foo", "endpoint"),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "identity_provider_type", "SERVICE_MANAGED"),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "tags.%", "1"),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "tags.NAME", "tf-acc-test-transfer-server"),
				),
			},
			{
				ResourceName:      "aws_transfer_server.foo",
				ImportState:       true,
				ImportStateVerify: true,
			},
			{
				Config: testAccAWSTransferServerConfig_basicUpdate,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSTransferServerExists("aws_transfer_server.foo", &conf),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "tags.%", "2"),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "tags.NAME", "tf-acc-test-transfer-server"),
					resource.TestCheckResourceAttr(
						"aws_transfer_server.foo", "tags.ENV", "test"),
					resource.TestCheckResourceAttrSet(
						"aws_transfer_server.foo", "logging_role"),
				),
			},
		},
	})
}

func testAccCheckAWSTransferServerExists(n string, res *transfer.DescribedServer) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No Transfer Server ID is set")
		}

		conn := testAccProvider.Meta().(*AWSClient).transferconn

		describe, err := conn.DescribeServer(&transfer.DescribeServerInput{
			ServerId: aws.String(rs.Primary.ID),
		})

		if err != nil {
			return err
		}

		*res = *describe.Server

		if res.ServerId != nil {

		}

		return nil
	}
}

const testAccAWSTransferServerConfig_basic = `
resource "aws_transfer_server" "foo" {
  identity_provider_type = "SERVICE_MANAGED"

  tags {
	NAME     = "tf-acc-test-transfer-server"
  }
}
`

const testAccAWSTransferServerConfig_basicUpdate = `

resource "aws_iam_role" "foo" {
	name = "tf-test-transfer-server-iam-role"

	assume_role_policy = <<EOF
{
	"Version": "2012-10-17",
	"Statement": [
		{
		"Effect": "Allow",
		"Principal": {
			"Service": "transfer.amazonaws.com"
		},
		"Action": "sts:AssumeRole"
		}
	]
}
EOF
}

resource "aws_iam_role_policy" "foo" {
	name = "tf-test-transfer-server-iam-policy"
	role = "${aws_iam_role.foo.id}"
	policy = <<POLICY
  {
	"Version": "2012-10-17",
	"Statement": [
	  {
		"Sid": "AllowFullAccesstoCloudWatchLogs",
		"Effect": "Allow",
		"Action": [
		  "logs:*"
		],
		"Resource": "*"
	  }
	]
  }
  POLICY
  }

resource "aws_transfer_server" "foo" {
  identity_provider_type = "SERVICE_MANAGED"
  logging_role = "${aws_iam_role.foo.arn}"

  tags {
	NAME   = "tf-acc-test-transfer-server"
	ENV    = "test"
  }
}
`
<|endoftext|>


<commit_before>package qemuguesttools

import (
	"bytes"
	"crypto/rand"
	"io"
	"io/ioutil"
	"net/http/httptest"
	"net/url"
	"os"
	"path/filepath"
	goruntime "runtime"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/taskcluster/taskcluster-worker/engines"
	"github.com/taskcluster/taskcluster-worker/engines/qemu/metaservice"
	"github.com/taskcluster/taskcluster-worker/runtime"
	"github.com/taskcluster/taskcluster-worker/runtime/mocks"
)

func nilOrFatal(t *testing.T, err error, a ...interface{}) {
	if err != nil {
		t.Fatal(append(a, err)...)
	}
}

func assert(t *testing.T, condition bool, a ...interface{}) {
	if !condition {
		t.Fatal(a...)
	}
}

func TestGuestToolsProcessingActions(t *testing.T) {
	// Create temporary storage
	storage, err := runtime.NewTemporaryStorage(os.TempDir())
	if err != nil {
		panic("Failed to create TemporaryStorage")
	}
	environment := &runtime.Environment{
		TemporaryStorage: storage,
	}

	logTask := bytes.NewBuffer(nil)
	meta := metaservice.New([]string{}, map[string]string{}, logTask, func(r bool) {
		panic("This test shouldn't get to this point!")
	}, environment)

	// Create http server for testing
	ts := httptest.NewServer(meta)
	defer ts.Close()
	defer meta.StopPollers() // Hack to stop pollers, otherwise server will block
	u, err := url.Parse(ts.URL)
	if err != nil {
		panic("Expected a url we can parse")
	}

	// Create an run guest-tools
	g := new(u.Host, mocks.NewMockMonitor(true))

	// start processing actions
	go g.ProcessActions()
	defer g.StopProcessingActions()

	////////////////////
	debug("### Test meta.GetArtifact")
	f, err := storage.NewFolder()
	if err != nil {
		panic("Failed to create temp folder")
	}
	defer f.Remove()

	testFile := filepath.Join(f.Path(), "hello.txt")
	err = ioutil.WriteFile(testFile, []byte("hello-world"), 0777)
	nilOrFatal(t, err, "Failed to create testFile: ", testFile)

	debug(" - request file: %s", testFile)
	r, err := meta.GetArtifact(testFile)
	nilOrFatal(t, err, "meta.GetArtifact failed, error: ", err)

	debug(" - reading testFile")
	data, err := ioutil.ReadAll(r)
	nilOrFatal(t, err, "Failed to read testFile")
	debug(" - read: '%s'", string(data))
	assert(t, string(data) == "hello-world", "Wrong payload: ", string(data))

	////////////////////
	debug("### Test meta.GetArtifact (missing file)")
	r, err = meta.GetArtifact(filepath.Join(f.Path(), "missing-file.txt"))
	assert(t, r == nil, "Expected error wihtout a reader")
	assert(t, err == engines.ErrResourceNotFound, "Expected ErrResourceNotFound")

	////////////////////
	debug("### Test meta.ListFolder")
	testFolder := filepath.Join(f.Path(), "test-folder")
	err = os.Mkdir(testFolder, 0777)
	nilOrFatal(t, err, "Failed to create test-folder/")

	testFile2 := filepath.Join(testFolder, "hello2.txt")
	err = ioutil.WriteFile(testFile2, []byte("hello-world-2"), 0777)
	nilOrFatal(t, err, "Failed to create testFile2: ", testFile2)

	debug(" - meta.ListFolder")
	files, err := meta.ListFolder(f.Path())
	nilOrFatal(t, err, "ListFolder failed, err: ", err)

	assert(t, len(files) == 2, "Expected 2 files")
	assert(t, files[0] == testFile || files[1] == testFile, "Expected testFile")
	assert(t, files[0] == testFile2 || files[1] == testFile2, "Expected testFile2")

	////////////////////
	debug("### Test meta.ListFolder (missing folder)")
	files, err = meta.ListFolder(filepath.Join(f.Path(), "no-such-folder"))
	assert(t, files == nil, "Expected files == nil, we hopefully have an error")
	assert(t, err == engines.ErrResourceNotFound, "Expected ErrResourceNotFound")

	////////////////////
	debug("### Test meta.ListFolder (empty folder)")
	emptyFolder := filepath.Join(f.Path(), "empty-folder")
	err = os.Mkdir(emptyFolder, 0777)
	nilOrFatal(t, err, "Failed to create empty-folder/")

	files, err = meta.ListFolder(emptyFolder)
	assert(t, len(files) == 0, "Expected zero files")
	assert(t, err == nil, "Didn't expect any error")

	////////////////////
	t.Run("Shell Hello", func(t *testing.T) {
		testShellHello(t, meta)
	})
	t.Run("Shell Cat", func(t *testing.T) {
		if goruntime.GOOS == "windows" {
			t.Skip("Not supported - test doesn't pass on windows yet")
		}
		testShellCat(t, meta)
	})
	t.Run("Shell Cat Stderr", func(t *testing.T) {
		if goruntime.GOOS == "windows" {
			t.Skip("Not supported - test doesn't pass on windows yet")
		}
		testShellCatStdErr(t, meta)
	})
	t.Run("Shell TTY", func(t *testing.T) {
		if goruntime.GOOS == "windows" {
			t.Skip("Not supported - test doesn't pass on windows yet")
		}
		testShellTTY(t, meta)
	})
}

func testShellHello(t *testing.T, meta *metaservice.MetaService) {
	debug("### Test meta.Shell (using 'echo hello')")
	shell, err := meta.ExecShell(nil, false)
	nilOrFatal(t, err, "Failed to call meta.ExecShell()")

	readHello := sync.WaitGroup{}
	readHello.Add(1)
	// Discard stderr
	go io.Copy(ioutil.Discard, shell.StderrPipe())
	go func() {
		shell.StdinPipe().Write([]byte("echo HELLO\n"))
		readHello.Wait()
		shell.StdinPipe().Close()
	}()
	go func() {
		data := bytes.Buffer{}
		for {
			b := []byte{0}
			n, werr := shell.StdoutPipe().Read(b)
			data.Write(b[:n])
			if strings.Contains(data.String(), "HELLO") {
				readHello.Done()
				break
			}
			if werr != nil {
				assert(t, werr == io.EOF, "Expected EOF!")
				break
			}
		}
		// Discard the rest
		go io.Copy(ioutil.Discard, shell.StdoutPipe())
	}()

	success, err := shell.Wait()
	nilOrFatal(t, err, "Got an error from shell.Wait, error: ", err)
	assert(t, success, "Expected success from shell, we closed with end of stdin")
}

func testShellCat(t *testing.T, meta *metaservice.MetaService) {
	debug("### Test meta.Shell (using 'exec cat -')")
	shell, err := meta.ExecShell(nil, false)
	nilOrFatal(t, err, "Failed to call meta.ExecShell()")

	input := make([]byte, 42*1024*1024+7)
	rand.Read(input)

	// Discard stderr
	go io.Copy(ioutil.Discard, shell.StderrPipe())
	go func() {
		if goruntime.GOOS == "windows" {
			shell.StdinPipe().Write([]byte("type con\n"))
		} else {
			shell.StdinPipe().Write([]byte("exec cat -\n"))
		}
		// Give cat - some time to start, bash/busybox/dash won't work otherwise
		// Or they will work, but only intermittently!!!
		time.Sleep(250 * time.Millisecond)
		shell.StdinPipe().Write(input)
		shell.StdinPipe().Close()
		debug("Closed stdin")
	}()
	var output []byte
	outputDone := sync.WaitGroup{}
	outputDone.Add(1)
	go func() {
		data, rerr := ioutil.ReadAll(shell.StdoutPipe())
		nilOrFatal(t, rerr, "Got error from stdout pipe, error: ", rerr)
		output = data
		outputDone.Done()
	}()

	success, err := shell.Wait()
	nilOrFatal(t, err, "Got an error from shell.Wait, error: ", err)
	assert(t, success, "Expected success from shell, we closed with end of stdin")
	outputDone.Wait()
	assert(t, bytes.Equal(output, input), "Expected data to match input, ",
		"len(input) = ", len(input), " len(output) = ", len(output))
}

func testShellCatStdErr(t *testing.T, meta *metaservice.MetaService) {
	debug("### Test meta.Shell (using 'exec cat - 1>&2')")
	shell, err := meta.ExecShell(nil, false)
	nilOrFatal(t, err, "Failed to call meta.ExecShell()")

	input := make([]byte, 4*1024*1024+37)
	rand.Read(input)

	// Discard stderr
	go io.Copy(ioutil.Discard, shell.StdoutPipe())
	go func() {
		if goruntime.GOOS == "windows" {
			shell.StdinPipe().Write([]byte("type con 1>&2\n"))
		} else {
			shell.StdinPipe().Write([]byte("exec cat -  1>&2\n"))
		}
		// Give cat - some time to start, bash/busybox/dash won't work otherwise
		// Or they will work, but only intermittently!!!
		time.Sleep(250 * time.Millisecond)
		shell.StdinPipe().Write(input)
		shell.StdinPipe().Close()
		debug("Closed stdin")
	}()
	var output []byte
	outputDone := sync.WaitGroup{}
	outputDone.Add(1)
	go func() {
		data, rerr := ioutil.ReadAll(shell.StderrPipe())
		nilOrFatal(t, rerr, "Got error from stderr pipe, error: ", rerr)
		output = data
		outputDone.Done()
	}()

	success, err := shell.Wait()
	nilOrFatal(t, err, "Got an error from shell.Wait, error: ", err)
	assert(t, success, "Expected success from shell, we closed with end of stdin")
	outputDone.Wait()
	assert(t, bytes.Equal(output, input), "Expected data to match input, ",
		"len(input) = ", len(input), " len(output) = ", len(output))
}

func testShellTTY(t *testing.T, meta *metaservice.MetaService) {
	debug("### Test meta.Shell (using 'exit 0' in TTY)")
	shell, err := meta.ExecShell(nil, true)
	nilOrFatal(t, err, "Failed to call meta.ExecShell()")

	// Discard stderr
	go io.Copy(ioutil.Discard, shell.StdoutPipe())
	//go io.Copy(ioutil.Discard, shell.StderrPipe())
	go func() {
		time.Sleep(200 * time.Millisecond) // Just to give sh a chance to sit idle
		shell.StdinPipe().Write([]byte("exit 0\n"))
	}()

	success, err := shell.Wait()
	nilOrFatal(t, err, "Got an error from shell.Wait, error: ", err)
	assert(t, success, "Expected success from shell, we closed with end of stdin")
}
<commit_msg>Cleanup test case<commit_after>package qemuguesttools

import (
	"bytes"
	"crypto/rand"
	"io"
	"io/ioutil"
	"net/http/httptest"
	"net/url"
	"os"
	"path/filepath"
	goruntime "runtime"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/taskcluster/taskcluster-worker/engines"
	"github.com/taskcluster/taskcluster-worker/engines/qemu/metaservice"
	"github.com/taskcluster/taskcluster-worker/runtime"
	"github.com/taskcluster/taskcluster-worker/runtime/mocks"
)

func nilOrFatal(t *testing.T, err error, a ...interface{}) {
	if err != nil {
		t.Fatal(append(a, err)...)
	}
}

func assert(t *testing.T, condition bool, a ...interface{}) {
	if !condition {
		t.Fatal(a...)
	}
}

func TestGuestToolsProcessingActions(t *testing.T) {
	// Create temporary storage
	storage, err := runtime.NewTemporaryStorage(os.TempDir())
	if err != nil {
		panic("Failed to create TemporaryStorage")
	}
	environment := &runtime.Environment{
		TemporaryStorage: storage,
	}

	logTask := bytes.NewBuffer(nil)
	meta := metaservice.New([]string{}, map[string]string{}, logTask, func(r bool) {
		panic("This test shouldn't get to this point!")
	}, environment)

	// Create http server for testing
	ts := httptest.NewServer(meta)
	defer ts.Close()
	defer meta.StopPollers() // Hack to stop pollers, otherwise server will block
	u, err := url.Parse(ts.URL)
	if err != nil {
		panic("Expected a url we can parse")
	}

	// Create an run guest-tools
	g := new(u.Host, mocks.NewMockMonitor(true))

	// start processing actions
	go g.ProcessActions()
	defer g.StopProcessingActions()

	////////////////////
	debug("### Test meta.GetArtifact")
	f, err := storage.NewFolder()
	if err != nil {
		panic("Failed to create temp folder")
	}
	defer f.Remove()

	testFile := filepath.Join(f.Path(), "hello.txt")
	err = ioutil.WriteFile(testFile, []byte("hello-world"), 0777)
	nilOrFatal(t, err, "Failed to create testFile: ", testFile)

	debug(" - request file: %s", testFile)
	r, err := meta.GetArtifact(testFile)
	nilOrFatal(t, err, "meta.GetArtifact failed, error: ", err)

	debug(" - reading testFile")
	data, err := ioutil.ReadAll(r)
	nilOrFatal(t, err, "Failed to read testFile")
	debug(" - read: '%s'", string(data))
	assert(t, string(data) == "hello-world", "Wrong payload: ", string(data))

	////////////////////
	debug("### Test meta.GetArtifact (missing file)")
	r, err = meta.GetArtifact(filepath.Join(f.Path(), "missing-file.txt"))
	assert(t, r == nil, "Expected error wihtout a reader")
	assert(t, err == engines.ErrResourceNotFound, "Expected ErrResourceNotFound")

	////////////////////
	debug("### Test meta.ListFolder")
	testFolder := filepath.Join(f.Path(), "test-folder")
	err = os.Mkdir(testFolder, 0777)
	nilOrFatal(t, err, "Failed to create test-folder/")

	testFile2 := filepath.Join(testFolder, "hello2.txt")
	err = ioutil.WriteFile(testFile2, []byte("hello-world-2"), 0777)
	nilOrFatal(t, err, "Failed to create testFile2: ", testFile2)

	debug(" - meta.ListFolder")
	files, err := meta.ListFolder(f.Path())
	nilOrFatal(t, err, "ListFolder failed, err: ", err)

	assert(t, len(files) == 2, "Expected 2 files")
	assert(t, files[0] == testFile || files[1] == testFile, "Expected testFile")
	assert(t, files[0] == testFile2 || files[1] == testFile2, "Expected testFile2")

	////////////////////
	debug("### Test meta.ListFolder (missing folder)")
	files, err = meta.ListFolder(filepath.Join(f.Path(), "no-such-folder"))
	assert(t, files == nil, "Expected files == nil, we hopefully have an error")
	assert(t, err == engines.ErrResourceNotFound, "Expected ErrResourceNotFound")

	////////////////////
	debug("### Test meta.ListFolder (empty folder)")
	emptyFolder := filepath.Join(f.Path(), "empty-folder")
	err = os.Mkdir(emptyFolder, 0777)
	nilOrFatal(t, err, "Failed to create empty-folder/")

	files, err = meta.ListFolder(emptyFolder)
	assert(t, len(files) == 0, "Expected zero files")
	assert(t, err == nil, "Didn't expect any error")

	////////////////////
	t.Run("Shell Hello", func(t *testing.T) {
		testShellHello(t, meta)
	})
	t.Run("Shell Cat", func(t *testing.T) {
		if goruntime.GOOS == "windows" {
			t.Skip("Not supported - test doesn't pass on windows yet")
		}
		testShellCat(t, meta)
	})
	t.Run("Shell Cat Stderr", func(t *testing.T) {
		if goruntime.GOOS == "windows" {
			t.Skip("Not supported - test doesn't pass on windows yet")
		}
		testShellCatStdErr(t, meta)
	})
	t.Run("Shell TTY", func(t *testing.T) {
		if goruntime.GOOS == "windows" {
			t.Skip("Not supported - test doesn't pass on windows yet")
		}
		testShellTTY(t, meta)
	})
}

func testShellHello(t *testing.T, meta *metaservice.MetaService) {
	debug("### Test meta.Shell (using 'echo hello')")
	shell, err := meta.ExecShell(nil, false)
	nilOrFatal(t, err, "Failed to call meta.ExecShell()")

	readHello := sync.WaitGroup{}
	readHello.Add(1)
	// Discard stderr
	go io.Copy(ioutil.Discard, shell.StderrPipe())
	go func() {
		shell.StdinPipe().Write([]byte("echo HELLO\n"))
		readHello.Wait()
		shell.StdinPipe().Close()
	}()
	go func() {
		data := bytes.Buffer{}
		for {
			b := []byte{0}
			n, werr := shell.StdoutPipe().Read(b)
			data.Write(b[:n])
			if strings.Contains(data.String(), "HELLO") {
				readHello.Done()
				break
			}
			if werr != nil {
				assert(t, werr == io.EOF, "Expected EOF!")
				break
			}
		}
		// Discard the rest
		go io.Copy(ioutil.Discard, shell.StdoutPipe())
	}()

	success, err := shell.Wait()
	nilOrFatal(t, err, "Got an error from shell.Wait, error: ", err)
	assert(t, success, "Expected success from shell, we closed with end of stdin")
}

func testShellCat(t *testing.T, meta *metaservice.MetaService) {
	debug("### Test meta.Shell (using 'exec cat -')")
	shell, err := meta.ExecShell(nil, false)
	nilOrFatal(t, err, "Failed to call meta.ExecShell()")

	input := make([]byte, 42*1024*1024+7)
	rand.Read(input)

	// Discard stderr
	go io.Copy(ioutil.Discard, shell.StderrPipe())
	go func() {
		if goruntime.GOOS == "windows" {
			shell.StdinPipe().Write([]byte("type con\n"))
		} else {
			shell.StdinPipe().Write([]byte("exec cat -\n"))
		}
		// Give cat - some time to start, bash/busybox/dash won't work otherwise
		// Or they will work, but only intermittently!!!
		time.Sleep(250 * time.Millisecond)
		shell.StdinPipe().Write(input)
		shell.StdinPipe().Close()
		debug("Closed stdin")
	}()
	var output []byte
	outputDone := sync.WaitGroup{}
	outputDone.Add(1)
	go func() {
		data, rerr := ioutil.ReadAll(shell.StdoutPipe())
		nilOrFatal(t, rerr, "Got error from stdout pipe, error: ", rerr)
		output = data
		outputDone.Done()
	}()

	success, err := shell.Wait()
	nilOrFatal(t, err, "Got an error from shell.Wait, error: ", err)
	assert(t, success, "Expected success from shell, we closed with end of stdin")
	outputDone.Wait()
	assert(t, bytes.Equal(output, input), "Expected data to match input, ",
		"len(input) = ", len(input), " len(output) = ", len(output))
}

func testShellCatStdErr(t *testing.T, meta *metaservice.MetaService) {
	debug("### Test meta.Shell (using 'exec cat - 1>&2')")
	shell, err := meta.ExecShell(nil, false)
	nilOrFatal(t, err, "Failed to call meta.ExecShell()")

	input := make([]byte, 4*1024*1024+37)
	rand.Read(input)

	// Discard stderr
	go io.Copy(ioutil.Discard, shell.StdoutPipe())
	go func() {
		if goruntime.GOOS == "windows" {
			shell.StdinPipe().Write([]byte("type con 1>&2\n"))
		} else {
			shell.StdinPipe().Write([]byte("exec cat -  1>&2\n"))
		}
		// Give cat - some time to start, bash/busybox/dash won't work otherwise
		// Or they will work, but only intermittently!!!
		time.Sleep(250 * time.Millisecond)
		shell.StdinPipe().Write(input)
		shell.StdinPipe().Close()
		debug("Closed stdin")
	}()
	var output []byte
	outputDone := sync.WaitGroup{}
	outputDone.Add(1)
	go func() {
		data, rerr := ioutil.ReadAll(shell.StderrPipe())
		nilOrFatal(t, rerr, "Got error from stderr pipe, error: ", rerr)
		output = data
		outputDone.Done()
	}()

	success, err := shell.Wait()
	nilOrFatal(t, err, "Got an error from shell.Wait, error: ", err)
	assert(t, success, "Expected success from shell, we closed with end of stdin")
	outputDone.Wait()
	assert(t, bytes.Equal(output, input), "Expected data to match input, ",
		"len(input) = ", len(input), " len(output) = ", len(output))
}

func testShellTTY(t *testing.T, meta *metaservice.MetaService) {
	debug("### Test meta.Shell (using 'exit 0' in TTY)")
	shell, err := meta.ExecShell(nil, true)
	nilOrFatal(t, err, "Failed to call meta.ExecShell()")

	// Discard stdout (ignore stderr, as it's the same)
	go io.Copy(ioutil.Discard, shell.StdoutPipe())
	go func() {
		time.Sleep(200 * time.Millisecond) // Just to give sh a chance to sit idle
		shell.StdinPipe().Write([]byte("exit 0\n"))
	}()

	success, err := shell.Wait()
	nilOrFatal(t, err, "Got an error from shell.Wait, error: ", err)
	assert(t, success, "Expected success from shell, we closed with end of stdin")
}
<|endoftext|>