<commit_before>// Copyright 2015 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//+build ignore

// Release is a tool for building the NDK tarballs hosted on dl.google.com.
//
// The Go toolchain only needs the gcc compiler and headers, which are ~10MB.
// The entire NDK is ~400MB. Building smaller toolchain binaries reduces the
// run time of gomobile init significantly.
package main

import (
	"archive/tar"
	"bufio"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
)

const ndkVersion = "ndk-r10d"

type version struct {
	os   string
	arch string
}

var hosts = []version{
	// TODO: windows
	{"darwin", "x86"},
	{"darwin", "x86_64"},
	{"linux", "x86"},
	{"linux", "x86_64"},
}

var tmpdir string

func main() {
	var err error
	tmpdir, err = ioutil.TempDir("", "gomobile-release-")
	if err != nil {
		log.Fatal(err)
	}
	defer os.RemoveAll(tmpdir)

	for _, host := range hosts {
		if err := mkpkg(host); err != nil {
			log.Fatal(err)
		}
	}
}

func mkpkg(host version) (err error) {
	ndkName := "android-" + ndkVersion + "-" + host.os + "-" + host.arch + "."
	if host.os == "windows" {
		ndkName += "exe"
	} else {
		ndkName += "bin"
	}
	url := "http://dl.google.com/android/ndk/" + ndkName
	log.Printf("%s\n", url)
	binPath := tmpdir + "/" + ndkName
	if err := fetch(binPath, url); err != nil {
		log.Fatal(err)
	}

	src := tmpdir + "/" + host.os + "-" + host.arch + "-src"
	dst := tmpdir + "/" + host.os + "-" + host.arch + "-dst"
	if err := os.Mkdir(src, 0755); err != nil {
		return err
	}
	if err := inflate(src, binPath); err != nil {
		return err
	}

	// The NDK is unpacked into tmpdir/linux-x86_64-src/android-ndk-r10d.
	// Move the files we want into tmpdir/linux-x86_64-dst/android-ndk-r10d.
	// We preserve the same file layout to make the full NDK interchangable
	// with the cut down file.
	usr := "android-" + ndkVersion + "/platforms/android-15/arch-arm/usr"
	gcc := "android-" + ndkVersion + "/toolchains/arm-linux-androideabi-4.8/prebuilt/" + host.os + "-" + host.arch
	if err := os.MkdirAll(dst+"/"+usr, 0755); err != nil {
		return err
	}
	if err := os.MkdirAll(dst+"/"+gcc, 0755); err != nil {
		return err
	}
	if err := move(dst+"/"+usr, src+"/"+usr, "include", "lib"); err != nil {
		return err
	}
	if err := move(dst+"/"+gcc, src+"/"+gcc, "bin", "lib", "libexec", "COPYING", "COPYING.LIB"); err != nil {
		return err
	}

	// Build the tarball.
	f, err := os.Create("gomobile-ndk-r10d-" + host.os + "-" + host.arch + ".tar.gz")
	if err != nil {
		return err
	}
	bw := bufio.NewWriter(f)
	zw, err := gzip.NewWriterLevel(bw, gzip.BestCompression)
	if err != nil {
		return err
	}
	tw := tar.NewWriter(zw)
	defer func() {
		err2 := f.Close()
		if err == nil {
			err = err2
		}
	}()
	defer func() {
		err2 := bw.Flush()
		if err == nil {
			err = err2
		}
	}()
	defer func() {
		err2 := zw.Close()
		if err == nil {
			err = err2
		}
	}()
	defer func() {
		err2 := tw.Close()
		if err == nil {
			err = err2
		}
	}()

	readme := "Stripped down copy of:\n\n\t" + url + "\n\nGenerated by golang.org/x/mobile/cmd/gomobile/release.go."
	err = tw.WriteHeader(&tar.Header{
		Name: "README",
		Mode: 0644,
		Size: int64(len(readme)),
	})
	if err != nil {
		return err
	}
	_, err = tw.Write([]byte(readme))
	if err != nil {
		return err
	}

	return filepath.Walk(dst, func(path string, fi os.FileInfo, err error) error {
		defer func() {
			if err != nil {
				err = fmt.Errorf("%s: %v", path, err)
			}
		}()
		if err != nil {
			return err
		}
		if path == dst {
			return nil
		}
		name := path[len(dst)+1:]
		if fi.IsDir() {
			return nil
		}
		if fi.Mode()&os.ModeSymlink != 0 {
			dst, err := os.Readlink(path)
			if err != nil {
				log.Printf("bad symlink: %s", name)
				return nil
			}
			//log.Printf("linking %s to %s", name, dst)
			return tw.WriteHeader(&tar.Header{
				Name:     name,
				Linkname: dst,
				Typeflag: tar.TypeSymlink,
			})
		}
		//log.Printf("writing %s (%d)", name, fi.Size())
		err = tw.WriteHeader(&tar.Header{
			Name: name,
			Mode: int64(fi.Mode()),
			Size: fi.Size(),
		})
		if err != nil {
			return err
		}
		f, err := os.Open(path)
		if err != nil {
			return err
		}
		_, err = io.Copy(tw, f)
		f.Close()
		return err
	})
}

func fetch(dst, url string) error {
	f, err := os.OpenFile(dst, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0755)
	if err != nil {
		return err
	}
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	_, err = io.Copy(f, resp.Body)
	err2 := resp.Body.Close()
	err3 := f.Close()
	if err != nil {
		return err
	}
	if err2 != nil {
		return err2
	}
	return err3
}

func inflate(dst, path string) error {
	p7zip := "7z"
	if runtime.GOOS == "darwin" {
		p7zip = "/Applications/Keka.app/Contents/Resources/keka7z"
	}
	cmd := exec.Command(p7zip, "x", path)
	cmd.Dir = dst
	out, err := cmd.CombinedOutput()
	if err != nil {
		os.Stderr.Write(out)
		return err
	}
	return nil
}

func move(dst, src string, names ...string) error {
	for _, name := range names {
		if err := os.Rename(src+"/"+name, dst+"/"+name); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>cmd/gomobile/release: add windows android ndk dist.<commit_after>// Copyright 2015 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//+build ignore

// Release is a tool for building the NDK tarballs hosted on dl.google.com.
//
// The Go toolchain only needs the gcc compiler and headers, which are ~10MB.
// The entire NDK is ~400MB. Building smaller toolchain binaries reduces the
// run time of gomobile init significantly.
package main

import (
	"archive/tar"
	"bufio"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
)

const ndkVersion = "ndk-r10d"

type version struct {
	os   string
	arch string
}

var hosts = []version{
	{"darwin", "x86"},
	{"darwin", "x86_64"},
	{"linux", "x86"},
	{"linux", "x86_64"},
	{"windows", "x86"},
	{"windows", "x86_64"},
}

var tmpdir string

func main() {
	var err error
	tmpdir, err = ioutil.TempDir("", "gomobile-release-")
	if err != nil {
		log.Fatal(err)
	}
	defer os.RemoveAll(tmpdir)

	for _, host := range hosts {
		if err := mkpkg(host); err != nil {
			log.Fatal(err)
		}
	}
}

func mkpkg(host version) (err error) {
	ndkName := "android-" + ndkVersion + "-" + host.os + "-" + host.arch + "."
	if host.os == "windows" {
		ndkName += "exe"
	} else {
		ndkName += "bin"
	}
	url := "http://dl.google.com/android/ndk/" + ndkName
	log.Printf("%s\n", url)
	binPath := tmpdir + "/" + ndkName
	if err := fetch(binPath, url); err != nil {
		log.Fatal(err)
	}

	src := tmpdir + "/" + host.os + "-" + host.arch + "-src"
	dst := tmpdir + "/" + host.os + "-" + host.arch + "-dst"
	if err := os.Mkdir(src, 0755); err != nil {
		return err
	}
	if err := inflate(src, binPath); err != nil {
		return err
	}

	// The NDK is unpacked into tmpdir/linux-x86_64-src/android-ndk-r10d.
	// Move the files we want into tmpdir/linux-x86_64-dst/android-ndk-r10d.
	// We preserve the same file layout to make the full NDK interchangable
	// with the cut down file.
	usr := "android-" + ndkVersion + "/platforms/android-15/arch-arm/usr"
	gcc := "android-" + ndkVersion + "/toolchains/arm-linux-androideabi-4.8/prebuilt/"
	if host.os == "windows" && host.arch == "x86" {
		gcc += "windows"
	} else {
		gcc += host.os + "-" + host.arch
	}

	if err := os.MkdirAll(dst+"/"+usr, 0755); err != nil {
		return err
	}
	if err := os.MkdirAll(dst+"/"+gcc, 0755); err != nil {
		return err
	}
	if err := move(dst+"/"+usr, src+"/"+usr, "include", "lib"); err != nil {
		return err
	}
	if err := move(dst+"/"+gcc, src+"/"+gcc, "bin", "lib", "libexec", "COPYING", "COPYING.LIB"); err != nil {
		return err
	}

	// Build the tarball.
	f, err := os.Create("gomobile-ndk-r10d-" + host.os + "-" + host.arch + ".tar.gz")
	if err != nil {
		return err
	}
	bw := bufio.NewWriter(f)
	zw, err := gzip.NewWriterLevel(bw, gzip.BestCompression)
	if err != nil {
		return err
	}
	tw := tar.NewWriter(zw)
	defer func() {
		err2 := f.Close()
		if err == nil {
			err = err2
		}
	}()
	defer func() {
		err2 := bw.Flush()
		if err == nil {
			err = err2
		}
	}()
	defer func() {
		err2 := zw.Close()
		if err == nil {
			err = err2
		}
	}()
	defer func() {
		err2 := tw.Close()
		if err == nil {
			err = err2
		}
	}()

	readme := "Stripped down copy of:\n\n\t" + url + "\n\nGenerated by golang.org/x/mobile/cmd/gomobile/release.go."
	err = tw.WriteHeader(&tar.Header{
		Name: "README",
		Mode: 0644,
		Size: int64(len(readme)),
	})
	if err != nil {
		return err
	}
	_, err = tw.Write([]byte(readme))
	if err != nil {
		return err
	}

	return filepath.Walk(dst, func(path string, fi os.FileInfo, err error) error {
		defer func() {
			if err != nil {
				err = fmt.Errorf("%s: %v", path, err)
			}
		}()
		if err != nil {
			return err
		}
		if path == dst {
			return nil
		}
		name := path[len(dst)+1:]
		if fi.IsDir() {
			return nil
		}
		if fi.Mode()&os.ModeSymlink != 0 {
			dst, err := os.Readlink(path)
			if err != nil {
				log.Printf("bad symlink: %s", name)
				return nil
			}
			//log.Printf("linking %s to %s", name, dst)
			return tw.WriteHeader(&tar.Header{
				Name:     name,
				Linkname: dst,
				Typeflag: tar.TypeSymlink,
			})
		}
		//log.Printf("writing %s (%d)", name, fi.Size())
		err = tw.WriteHeader(&tar.Header{
			Name: name,
			Mode: int64(fi.Mode()),
			Size: fi.Size(),
		})
		if err != nil {
			return err
		}
		f, err := os.Open(path)
		if err != nil {
			return err
		}
		_, err = io.Copy(tw, f)
		f.Close()
		return err
	})
}

func fetch(dst, url string) error {
	f, err := os.OpenFile(dst, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0755)
	if err != nil {
		return err
	}
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	_, err = io.Copy(f, resp.Body)
	err2 := resp.Body.Close()
	err3 := f.Close()
	if err != nil {
		return err
	}
	if err2 != nil {
		return err2
	}
	return err3
}

func inflate(dst, path string) error {
	p7zip := "7z"
	if runtime.GOOS == "darwin" {
		p7zip = "/Applications/Keka.app/Contents/Resources/keka7z"
	}
	cmd := exec.Command(p7zip, "x", path)
	cmd.Dir = dst
	out, err := cmd.CombinedOutput()
	if err != nil {
		os.Stderr.Write(out)
		return err
	}
	return nil
}

func move(dst, src string, names ...string) error {
	for _, name := range names {
		if err := os.Rename(src+"/"+name, dst+"/"+name); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"text/template"
	"time"

	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/testing"
	. "launchpad.net/juju-core/testing/checkers"
)

type PluginSuite struct {
	oldPath string
	home    *testing.FakeHome
}

var _ = Suite(&PluginSuite{})

func (suite *PluginSuite) SetUpTest(c *C) {
	suite.oldPath = os.Getenv("PATH")
	suite.home = testing.MakeSampleHome(c)
	os.Setenv("PATH", "/bin:"+testing.HomePath())
}

func (suite *PluginSuite) TearDownTest(c *C) {
	suite.home.Restore()
	os.Setenv("PATH", suite.oldPath)
}

func (*PluginSuite) TestFindPlugins(c *C) {
	plugins := findPlugins()
	c.Assert(plugins, DeepEquals, []string{})
}

func (suite *PluginSuite) TestFindPluginsOrder(c *C) {
	suite.makePlugin("foo", 0744)
	suite.makePlugin("bar", 0654)
	suite.makePlugin("baz", 0645)
	plugins := findPlugins()
	c.Assert(plugins, DeepEquals, []string{"juju-bar", "juju-baz", "juju-foo"})
}

func (suite *PluginSuite) TestFindPluginsIgnoreNotExec(c *C) {
	suite.makePlugin("foo", 0644)
	suite.makePlugin("bar", 0666)
	plugins := findPlugins()
	c.Assert(plugins, DeepEquals, []string{})
}

func (suite *PluginSuite) TestRunPluginExising(c *C) {
	suite.makePlugin("foo", 0755)
	ctx := testing.Context(c)
	err := RunPlugin(ctx, "foo", []string{"some params"})
	c.Assert(err, IsNil)
	c.Assert(testing.Stdout(ctx), Equals, "foo erewhemos some params\n")
	c.Assert(testing.Stderr(ctx), Equals, "")
}

func (suite *PluginSuite) TestRunPluginExisingJujuEnv(c *C) {
	suite.makePlugin("foo", 0755)
	os.Setenv("JUJU_ENV", "omg")
	ctx := testing.Context(c)
	err := RunPlugin(ctx, "foo", []string{"some params"})
	c.Assert(err, IsNil)
	c.Assert(testing.Stdout(ctx), Equals, "foo omg some params\n")
	c.Assert(testing.Stderr(ctx), Equals, "")
}

func (suite *PluginSuite) TestRunPluginExisingDashE(c *C) {
	suite.makePlugin("foo", 0755)
	ctx := testing.Context(c)
	err := RunPlugin(ctx, "foo", []string{"-e plugins-rock some params"})
	c.Assert(err, IsNil)
	c.Assert(testing.Stdout(ctx), Equals, "foo plugins-rock some params\n")
	c.Assert(testing.Stderr(ctx), Equals, "")
}

func (suite *PluginSuite) TestRunPluginWithFailing(c *C) {
	suite.makeFailingPlugin("foo", 2)
	ctx := testing.Context(c)
	err := RunPlugin(ctx, "foo", []string{"some params"})
	c.Assert(err, ErrorMatches, "exit status 2")
	c.Assert(testing.Stdout(ctx), Equals, "failing\n")
	c.Assert(testing.Stderr(ctx), Equals, "")
}

func (suite *PluginSuite) TestGatherDescriptionsInParallel(c *C) {
	suite.makeFullPlugin(PluginParams{Name: "foo", Sleep: 100 * time.Millisecond})
	suite.makeFullPlugin(PluginParams{Name: "bar", Sleep: 150 * time.Millisecond})
	suite.makeFullPlugin(PluginParams{Name: "baz", Sleep: 300 * time.Millisecond})
	suite.makeFullPlugin(PluginParams{Name: "error", ExitStatus: 1, Sleep: 100 * time.Millisecond})
	suite.makeFullPlugin(PluginParams{Name: "slow", Sleep: 200 * time.Millisecond})

	start := time.Now()
	results := GetPluginDescriptions()
	elapsed := time.Since(start)

	// 300 for baz above + 50ms wiggle room
	expectedDuration := 350 * time.Millisecond

	c.Assert(results, HasLen, 5)
	c.Check(elapsed, DurationLessThan, expectedDuration)
	c.Assert(results[0].name, Equals, "bar")
	c.Assert(results[0].description, Equals, "bar description")
	c.Assert(results[1].name, Equals, "baz")
	c.Assert(results[1].description, Equals, "baz description")
	c.Assert(results[2].name, Equals, "error")
	c.Assert(results[2].description, Equals, "error occurred running 'juju-error --description'")
	c.Assert(results[3].name, Equals, "foo")
	c.Assert(results[3].description, Equals, "foo description")
	c.Assert(results[4].name, Equals, "slow")
	c.Assert(results[4].description, Equals, "slow description")
}

func (suite *PluginSuite) TestHelpPluginsWithNoPlugins(c *C) {
	output := badrun(c, 0, "help", "plugins")
	c.Assert(output, HasPrefix, PluginTopicText)
	c.Assert(output, HasSuffix, "\n\nNo plugins found.\n")
}

func (suite *PluginSuite) TestHelpPluginsWithPlugins(c *C) {
	suite.makeFullPlugin(PluginParams{Name: "foo"})
	suite.makeFullPlugin(PluginParams{Name: "bar"})
	output := badrun(c, 0, "help", "plugins")
	c.Assert(output, HasPrefix, PluginTopicText)
	expectedPlugins := `

bar  bar description
foo  foo description
`
	c.Assert(output, HasSuffix, expectedPlugins)
}

func (suite *PluginSuite) TestHelpPluginName(c *C) {
	suite.makeFullPlugin(PluginParams{Name: "foo"})
	output := badrun(c, 0, "help", "foo")
	expectedHelp := `foo longer help

something useful
`
	c.Assert(output, Matches, expectedHelp)
}

func (suite *PluginSuite) TestHelpPluginNameNotAPlugin(c *C) {
	output := badrun(c, 0, "help", "foo")
	expectedHelp := "error: unknown command or topic for foo\n"
	c.Assert(output, Matches, expectedHelp)
}

func (suite *PluginSuite) makePlugin(name string, perm os.FileMode) {
	content := fmt.Sprintf("#!/bin/bash\necho %s $JUJU_ENV $*", name)
	filename := testing.HomePath(JujuPluginPrefix + name)
	ioutil.WriteFile(filename, []byte(content), perm)
}

func (suite *PluginSuite) makeFailingPlugin(name string, exitStatus int) {
	content := fmt.Sprintf("#!/bin/bash\necho failing\nexit %d", exitStatus)
	filename := testing.HomePath(JujuPluginPrefix + name)
	ioutil.WriteFile(filename, []byte(content), 0755)
}

type PluginParams struct {
	Name       string
	ExitStatus int
	Sleep      time.Duration
}

const pluginTemplate = `#!/bin/bash

if [ "$1" = "--description" ]; then
  sleep {{.Sleep.Seconds}}
  echo "{{.Name}} description"
  exit {{.ExitStatus}}
fi

if [ "$1" = "--help" ]; then
  echo "{{.Name}} longer help"
  echo ""
  echo "something useful"
  exit {{.ExitStatus}}
fi

echo {{.Name}} $*
exit {{.ExitStatus}}
`

func (suite *PluginSuite) makeFullPlugin(params PluginParams) {
	// Create a new template and parse the plugin into it.
	t := template.Must(template.New("plugin").Parse(pluginTemplate))
	content := &bytes.Buffer{}
	t.Execute(content, params)
	filename := testing.HomePath("juju-" + params.Name)
	ioutil.WriteFile(filename, content.Bytes(), 0755)
}
<commit_msg>change TestGatherDescriptionsInParallel<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"text/template"
	"time"

	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/testing"
	. "launchpad.net/juju-core/testing/checkers"
)

type PluginSuite struct {
	oldPath string
	home    *testing.FakeHome
}

var _ = Suite(&PluginSuite{})

func (suite *PluginSuite) SetUpTest(c *C) {
	suite.oldPath = os.Getenv("PATH")
	suite.home = testing.MakeSampleHome(c)
	os.Setenv("PATH", "/bin:"+testing.HomePath())
}

func (suite *PluginSuite) TearDownTest(c *C) {
	suite.home.Restore()
	os.Setenv("PATH", suite.oldPath)
}

func (*PluginSuite) TestFindPlugins(c *C) {
	plugins := findPlugins()
	c.Assert(plugins, DeepEquals, []string{})
}

func (suite *PluginSuite) TestFindPluginsOrder(c *C) {
	suite.makePlugin("foo", 0744)
	suite.makePlugin("bar", 0654)
	suite.makePlugin("baz", 0645)
	plugins := findPlugins()
	c.Assert(plugins, DeepEquals, []string{"juju-bar", "juju-baz", "juju-foo"})
}

func (suite *PluginSuite) TestFindPluginsIgnoreNotExec(c *C) {
	suite.makePlugin("foo", 0644)
	suite.makePlugin("bar", 0666)
	plugins := findPlugins()
	c.Assert(plugins, DeepEquals, []string{})
}

func (suite *PluginSuite) TestRunPluginExising(c *C) {
	suite.makePlugin("foo", 0755)
	ctx := testing.Context(c)
	err := RunPlugin(ctx, "foo", []string{"some params"})
	c.Assert(err, IsNil)
	c.Assert(testing.Stdout(ctx), Equals, "foo erewhemos some params\n")
	c.Assert(testing.Stderr(ctx), Equals, "")
}

func (suite *PluginSuite) TestRunPluginExisingJujuEnv(c *C) {
	suite.makePlugin("foo", 0755)
	os.Setenv("JUJU_ENV", "omg")
	ctx := testing.Context(c)
	err := RunPlugin(ctx, "foo", []string{"some params"})
	c.Assert(err, IsNil)
	c.Assert(testing.Stdout(ctx), Equals, "foo omg some params\n")
	c.Assert(testing.Stderr(ctx), Equals, "")
}

func (suite *PluginSuite) TestRunPluginExisingDashE(c *C) {
	suite.makePlugin("foo", 0755)
	ctx := testing.Context(c)
	err := RunPlugin(ctx, "foo", []string{"-e plugins-rock some params"})
	c.Assert(err, IsNil)
	c.Assert(testing.Stdout(ctx), Equals, "foo plugins-rock some params\n")
	c.Assert(testing.Stderr(ctx), Equals, "")
}

func (suite *PluginSuite) TestRunPluginWithFailing(c *C) {
	suite.makeFailingPlugin("foo", 2)
	ctx := testing.Context(c)
	err := RunPlugin(ctx, "foo", []string{"some params"})
	c.Assert(err, ErrorMatches, "exit status 2")
	c.Assert(testing.Stdout(ctx), Equals, "failing\n")
	c.Assert(testing.Stderr(ctx), Equals, "")
}

func (suite *PluginSuite) TestGatherDescriptionsInParallel(c *C) {
	// Each plugin depends on another one being started before they will complete.
	// Thus if we don't start them in parallel, we would deadlock
	suite.makeFullPlugin(PluginParams{Name: "foo", Creates: "foo", DependsOn: "bar"})
	suite.makeFullPlugin(PluginParams{Name: "bar", Creates: "bar", DependsOn: "baz"})
	suite.makeFullPlugin(PluginParams{Name: "baz", Creates: "baz", DependsOn: "error"})
	suite.makeFullPlugin(PluginParams{Name: "error", ExitStatus: 1, Creates: "error", DependsOn: "foo"})

	// If the code was wrong, GetPluginDescriptions would deadlock,
	// so timeout after a short while
	resultChan := make(chan []PluginDescription)
	go func() {
		resultChan <- GetPluginDescriptions()
	}()
	waitTime := 10 * time.Second
	var results []PluginDescription
	select {
	case results = <-resultChan:
		break
	case <-time.After(waitTime):
		c.Fatalf("Took too more than %fs to complete.", waitTime.Seconds())
	}

	c.Assert(results, HasLen, 4)
	c.Assert(results[0].name, Equals, "bar")
	c.Assert(results[0].description, Equals, "bar description")
	c.Assert(results[1].name, Equals, "baz")
	c.Assert(results[1].description, Equals, "baz description")
	c.Assert(results[2].name, Equals, "error")
	c.Assert(results[2].description, Equals, "error occurred running 'juju-error --description'")
	c.Assert(results[3].name, Equals, "foo")
	c.Assert(results[3].description, Equals, "foo description")
}

func (suite *PluginSuite) TestHelpPluginsWithNoPlugins(c *C) {
	output := badrun(c, 0, "help", "plugins")
	c.Assert(output, HasPrefix, PluginTopicText)
	c.Assert(output, HasSuffix, "\n\nNo plugins found.\n")
}

func (suite *PluginSuite) TestHelpPluginsWithPlugins(c *C) {
	suite.makeFullPlugin(PluginParams{Name: "foo"})
	suite.makeFullPlugin(PluginParams{Name: "bar"})
	output := badrun(c, 0, "help", "plugins")
	c.Assert(output, HasPrefix, PluginTopicText)
	expectedPlugins := `

bar  bar description
foo  foo description
`
	c.Assert(output, HasSuffix, expectedPlugins)
}

func (suite *PluginSuite) TestHelpPluginName(c *C) {
	suite.makeFullPlugin(PluginParams{Name: "foo"})
	output := badrun(c, 0, "help", "foo")
	expectedHelp := `foo longer help

something useful
`
	c.Assert(output, Matches, expectedHelp)
}

func (suite *PluginSuite) TestHelpPluginNameNotAPlugin(c *C) {
	output := badrun(c, 0, "help", "foo")
	expectedHelp := "error: unknown command or topic for foo\n"
	c.Assert(output, Matches, expectedHelp)
}

func (suite *PluginSuite) makePlugin(name string, perm os.FileMode) {
	content := fmt.Sprintf("#!/bin/bash\necho %s $JUJU_ENV $*", name)
	filename := testing.HomePath(JujuPluginPrefix + name)
	ioutil.WriteFile(filename, []byte(content), perm)
}

func (suite *PluginSuite) makeFailingPlugin(name string, exitStatus int) {
	content := fmt.Sprintf("#!/bin/bash\necho failing\nexit %d", exitStatus)
	filename := testing.HomePath(JujuPluginPrefix + name)
	ioutil.WriteFile(filename, []byte(content), 0755)
}

type PluginParams struct {
	Name       string
	ExitStatus int
	Creates    string
	DependsOn  string
}

const pluginTemplate = `#!/bin/bash

if [ "$1" = "--description" ]; then
  if [ -n "{{.Creates}}" ]; then
    touch "{{.Creates}}"
  fi
  if [ -n "{{.DependsOn}}" ]; then
    # Sleep 10ms while waiting to allow other stuff to do work
    while [ ! -e "{{.DependsOn}}" ]; do sleep 0.010; done
  fi
  echo "{{.Name}} description"
  exit {{.ExitStatus}}
else
  echo "No --description" >2
fi

if [ "$1" = "--help" ]; then
  echo "{{.Name}} longer help"
  echo ""
  echo "something useful"
  exit {{.ExitStatus}}
fi

echo {{.Name}} $*
exit {{.ExitStatus}}
`

func (suite *PluginSuite) makeFullPlugin(params PluginParams) {
	// Create a new template and parse the plugin into it.
	t := template.Must(template.New("plugin").Parse(pluginTemplate))
	content := &bytes.Buffer{}
	filename := testing.HomePath("juju-" + params.Name)
	// Create the files in the temp dirs, so we don't pollute the working space
	if params.Creates != "" {
		params.Creates = testing.HomePath(params.Creates)
	}
	if params.DependsOn != "" {
		params.DependsOn = testing.HomePath(params.DependsOn)
	}
	t.Execute(content, params)
	ioutil.WriteFile(filename, content.Bytes(), 0755)
}
<|endoftext|>


<commit_before>//go:build plan9 || solaris
// +build plan9 solaris

// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"io"
	"os"
	"syscall"
)

func access(name string) error {
	_, err := os.Lstat(name)
	return err
}

// readDirFn applies the fn() function on each entries at dirPath, doesn't recurse into
// the directory itself, if the dirPath doesn't exist this function doesn't return
// an error.
func readDirFn(dirPath string, filter func(name string, typ os.FileMode) error) error {
	d, err := Open(dirPath)
	if err != nil {
		if osErrToFileErr(err) == errFileNotFound {
			return nil
		}
		return osErrToFileErr(err)
	}
	defer d.Close()

	maxEntries := 1000
	for {
		// Read up to max number of entries.
		fis, err := d.Readdir(maxEntries)
		if err != nil {
			if err == io.EOF {
				break
			}
			err = osErrToFileErr(err)
			if err == errFileNotFound {
				return nil
			}
			return err
		}
		for _, fi := range fis {
			if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
				fi, err = Stat(pathJoin(dirPath, fi.Name()))
				if err != nil {
					// It got deleted in the meantime, not found
					// or returns too many symlinks ignore this
					// file/directory.
					if osIsNotExist(err) || isSysErrPathNotFound(err) ||
						isSysErrTooManySymlinks(err) {
						continue
					}
					return err
				}

				// Ignore symlinked directories.
				if fi.IsDir() {
					continue
				}
			}
			if err = filter(fi.Name(), fi.Mode()); err == errDoneForNow {
				// filtering requested to return by caller.
				return nil
			}
		}
	}
	return nil
}

// Return entries at the directory dirPath.
func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err error) {
	d, err := Open(dirPath)
	if err != nil {
		return nil, osErrToFileErr(err)
	}
	defer d.Close()

	maxEntries := 1000
	if opts.count > 0 && opts.count < maxEntries {
		maxEntries = count
	}

	done := false
	remaining := opts.count

	for !done {
		// Read up to max number of entries.
		fis, err := d.Readdir(maxEntries)
		if err != nil {
			if err == io.EOF {
				break
			}
			return nil, osErrToFileErr(err)
		}
		if opts.count > -1 {
			if remaining <= len(fis) {
				fis = fis[:remaining]
				done = true
			}
		}
		for _, fi := range fis {
			if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
				fi, err = Stat(pathJoin(dirPath, fi.Name()))
				if err != nil {
					// It got deleted in the meantime, not found
					// or returns too many symlinks ignore this
					// file/directory.
					if osIsNotExist(err) || isSysErrPathNotFound(err) ||
						isSysErrTooManySymlinks(err) {
						continue
					}
					return nil, err
				}

				// Ignore symlinked directories.
				if !opts.followDirSymlink && fi.IsDir() {
					continue
				}
			}

			if fi.IsDir() {
				// Append SlashSeparator instead of "\" so that sorting is achieved as expected.
				entries = append(entries, fi.Name()+SlashSeparator)
			} else if fi.Mode().IsRegular() {
				entries = append(entries, fi.Name())
			}
			if opts.count > 0 {
				remaining--
			}
		}
	}
	return entries, nil
}

func globalSync() {
	// no-op not sure about plan9/solaris support for syscall support
	syscall.Sync()
}
<commit_msg>fix: build on illumos (Solaris) (#13097)<commit_after>//go:build plan9 || solaris
// +build plan9 solaris

// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"io"
	"os"
	"syscall"
)

func access(name string) error {
	_, err := os.Lstat(name)
	return err
}

// readDirFn applies the fn() function on each entries at dirPath, doesn't recurse into
// the directory itself, if the dirPath doesn't exist this function doesn't return
// an error.
func readDirFn(dirPath string, filter func(name string, typ os.FileMode) error) error {
	d, err := Open(dirPath)
	if err != nil {
		if osErrToFileErr(err) == errFileNotFound {
			return nil
		}
		return osErrToFileErr(err)
	}
	defer d.Close()

	maxEntries := 1000
	for {
		// Read up to max number of entries.
		fis, err := d.Readdir(maxEntries)
		if err != nil {
			if err == io.EOF {
				break
			}
			err = osErrToFileErr(err)
			if err == errFileNotFound {
				return nil
			}
			return err
		}
		for _, fi := range fis {
			if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
				fi, err = Stat(pathJoin(dirPath, fi.Name()))
				if err != nil {
					// It got deleted in the meantime, not found
					// or returns too many symlinks ignore this
					// file/directory.
					if osIsNotExist(err) || isSysErrPathNotFound(err) ||
						isSysErrTooManySymlinks(err) {
						continue
					}
					return err
				}

				// Ignore symlinked directories.
				if fi.IsDir() {
					continue
				}
			}
			if err = filter(fi.Name(), fi.Mode()); err == errDoneForNow {
				// filtering requested to return by caller.
				return nil
			}
		}
	}
	return nil
}

// Return entries at the directory dirPath.
func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err error) {
	d, err := Open(dirPath)
	if err != nil {
		return nil, osErrToFileErr(err)
	}
	defer d.Close()

	maxEntries := 1000
	if opts.count > 0 && opts.count < maxEntries {
		maxEntries = opts.count
	}

	done := false
	remaining := opts.count

	for !done {
		// Read up to max number of entries.
		fis, err := d.Readdir(maxEntries)
		if err != nil {
			if err == io.EOF {
				break
			}
			return nil, osErrToFileErr(err)
		}
		if opts.count > -1 {
			if remaining <= len(fis) {
				fis = fis[:remaining]
				done = true
			}
		}
		for _, fi := range fis {
			if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
				fi, err = Stat(pathJoin(dirPath, fi.Name()))
				if err != nil {
					// It got deleted in the meantime, not found
					// or returns too many symlinks ignore this
					// file/directory.
					if osIsNotExist(err) || isSysErrPathNotFound(err) ||
						isSysErrTooManySymlinks(err) {
						continue
					}
					return nil, err
				}

				// Ignore symlinked directories.
				if !opts.followDirSymlink && fi.IsDir() {
					continue
				}
			}

			if fi.IsDir() {
				// Append SlashSeparator instead of "\" so that sorting is achieved as expected.
				entries = append(entries, fi.Name()+SlashSeparator)
			} else if fi.Mode().IsRegular() {
				entries = append(entries, fi.Name())
			}
			if opts.count > 0 {
				remaining--
			}
		}
	}
	return entries, nil
}

func globalSync() {
	// no-op not sure about plan9/solaris support for syscall support
	syscall.Sync()
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"encoding/json"
	"fmt"
	"path/filepath"

	"github.com/restic/restic/internal/backend"
	"github.com/restic/restic/internal/restic"
	"github.com/restic/restic/internal/walker"

	"github.com/minio/sha256-simd"
	"github.com/spf13/cobra"
)

var cmdStats = &cobra.Command{
	Use:   "stats [flags] [snapshot ID] [...]",
	Short: "Scan the repository and show basic statistics",
	Long: `
The "stats" command walks one or multiple snapshots in a repository
and accumulates statistics about the data stored therein. It reports
on the number of unique files and their sizes, according to one of
the counting modes as given by the --mode flag.

It operates on all snapshots matching the selection criteria or all
snapshots if nothing is specified. The special snapshot ID "latest"
is also supported. Some modes make more sense over
just a single snapshot, while others are useful across all snapshots,
depending on what you are trying to calculate.

The modes are:

* restore-size: (default) Counts the size of the restored files.
* files-by-contents: Counts total size of files, where a file is
   considered unique if it has unique contents.
* raw-data: Counts the size of blobs in the repository, regardless of
  how many files reference them.
* blobs-per-file: A combination of files-by-contents and raw-data.

Refer to the online manual for more details about each mode.

EXIT STATUS
===========

Exit status is 0 if the command was successful, and non-zero if there was any error.
`,
	DisableAutoGenTag: true,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runStats(globalOptions, args)
	},
}

// StatsOptions collects all options for the stats command.
type StatsOptions struct {
	// the mode of counting to perform (see consts for available modes)
	countMode string

	// filter snapshots by, if given by user
	Hosts []string
	Tags  restic.TagLists
	Paths []string
}

var statsOptions StatsOptions

func init() {
	cmdRoot.AddCommand(cmdStats)
	f := cmdStats.Flags()
	f.StringVar(&statsOptions.countMode, "mode", countModeRestoreSize, "counting mode: restore-size (default), files-by-contents, blobs-per-file or raw-data")
	f.StringArrayVarP(&statsOptions.Hosts, "host", "H", nil, "only consider snapshots with the given `host` (can be specified multiple times)")
	f.Var(&statsOptions.Tags, "tag", "only consider snapshots which include this `taglist` in the format `tag[,tag,...]` (can be specified multiple times)")
	f.StringArrayVar(&statsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` (can be specified multiple times)")
}

func runStats(gopts GlobalOptions, args []string) error {
	err := verifyStatsInput(gopts, args)
	if err != nil {
		return err
	}

	ctx, cancel := context.WithCancel(gopts.ctx)
	defer cancel()

	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	if !gopts.NoLock {
		lock, err := lockRepo(ctx, repo)
		defer unlockRepo(lock)
		if err != nil {
			return err
		}
	}

	snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile)
	if err != nil {
		return err
	}

	if err = repo.LoadIndex(ctx); err != nil {
		return err
	}

	if !gopts.JSON {
		Printf("scanning...\n")
	}

	// create a container for the stats (and other needed state)
	stats := &statsContainer{
		uniqueFiles:    make(map[fileID]struct{}),
		fileBlobs:      make(map[string]restic.IDSet),
		blobs:          restic.NewBlobSet(),
		snapshotsCount: 0,
	}

	for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, statsOptions.Hosts, statsOptions.Tags, statsOptions.Paths, args) {
		err = statsWalkSnapshot(ctx, sn, repo, stats)
		if err != nil {
			return fmt.Errorf("error walking snapshot: %v", err)
		}
	}

	if err != nil {
		return err
	}

	if statsOptions.countMode == countModeRawData {
		// the blob handles have been collected, but not yet counted
		for blobHandle := range stats.blobs {
			pbs := repo.Index().Lookup(blobHandle)
			if len(pbs) == 0 {
				return fmt.Errorf("blob %v not found", blobHandle)
			}
			stats.TotalSize += uint64(pbs[0].Length)
			stats.TotalBlobCount++
		}
	}

	if gopts.JSON {
		err = json.NewEncoder(globalOptions.stdout).Encode(stats)
		if err != nil {
			return fmt.Errorf("encoding output: %v", err)
		}
		return nil
	}

	Printf("Stats in %s mode:\n", statsOptions.countMode)
	Printf("Snapshots processed:   %d\n", stats.snapshotsCount)

	if stats.TotalBlobCount > 0 {
		Printf("   Total Blob Count:   %d\n", stats.TotalBlobCount)
	}
	if stats.TotalFileCount > 0 {
		Printf("   Total File Count:   %d\n", stats.TotalFileCount)
	}
	Printf("         Total Size:   %-5s\n", formatBytes(stats.TotalSize))

	return nil
}

func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo restic.Repository, stats *statsContainer) error {
	if snapshot.Tree == nil {
		return fmt.Errorf("snapshot %s has nil tree", snapshot.ID().Str())
	}

	stats.snapshotsCount++

	if statsOptions.countMode == countModeRawData {
		// count just the sizes of unique blobs; we don't need to walk the tree
		// ourselves in this case, since a nifty function does it for us
		return restic.FindUsedBlobs(ctx, repo, restic.IDs{*snapshot.Tree}, stats.blobs, nil)
	}

	uniqueInodes := make(map[uint64]struct{})
	err := walker.Walk(ctx, repo, *snapshot.Tree, restic.NewIDSet(), statsWalkTree(repo, stats, uniqueInodes))
	if err != nil {
		return fmt.Errorf("walking tree %s: %v", *snapshot.Tree, err)
	}

	return nil
}

func statsWalkTree(repo restic.Repository, stats *statsContainer, uniqueInodes map[uint64]struct{}) walker.WalkFunc {
	return func(parentTreeID restic.ID, npath string, node *restic.Node, nodeErr error) (bool, error) {
		if nodeErr != nil {
			return true, nodeErr
		}
		if node == nil {
			return true, nil
		}

		if statsOptions.countMode == countModeUniqueFilesByContents || statsOptions.countMode == countModeBlobsPerFile {
			// only count this file if we haven't visited it before
			fid := makeFileIDByContents(node)
			if _, ok := stats.uniqueFiles[fid]; !ok {
				// mark the file as visited
				stats.uniqueFiles[fid] = struct{}{}

				if statsOptions.countMode == countModeUniqueFilesByContents {
					// simply count the size of each unique file (unique by contents only)
					stats.TotalSize += node.Size
					stats.TotalFileCount++
				}
				if statsOptions.countMode == countModeBlobsPerFile {
					// count the size of each unique blob reference, which is
					// by unique file (unique by contents and file path)
					for _, blobID := range node.Content {
						// ensure we have this file (by path) in our map; in this
						// mode, a file is unique by both contents and path
						nodePath := filepath.Join(npath, node.Name)
						if _, ok := stats.fileBlobs[nodePath]; !ok {
							stats.fileBlobs[nodePath] = restic.NewIDSet()
							stats.TotalFileCount++
						}
						if _, ok := stats.fileBlobs[nodePath][blobID]; !ok {
							// is always a data blob since we're accessing it via a file's Content array
							blobSize, found := repo.LookupBlobSize(blobID, restic.DataBlob)
							if !found {
								return true, fmt.Errorf("blob %s not found for tree %s", blobID, parentTreeID)
							}

							// count the blob's size, then add this blob by this
							// file (path) so we don't double-count it
							stats.TotalSize += uint64(blobSize)
							stats.fileBlobs[nodePath].Insert(blobID)
							// this mode also counts total unique blob _references_ per file
							stats.TotalBlobCount++
						}
					}
				}
			}
		}

		if statsOptions.countMode == countModeRestoreSize {
			// as this is a file in the snapshot, we can simply count its
			// size without worrying about uniqueness, since duplicate files
			// will still be restored
			stats.TotalFileCount++

			// if inodes are present, only count each inode once
			// (hard links do not increase restore size)
			if _, ok := uniqueInodes[node.Inode]; !ok || node.Inode == 0 {
				uniqueInodes[node.Inode] = struct{}{}
				stats.TotalSize += node.Size
			}

			return false, nil
		}

		return true, nil
	}
}

// makeFileIDByContents returns a hash of the blob IDs of the
// node's Content in sequence.
func makeFileIDByContents(node *restic.Node) fileID {
	var bb []byte
	for _, c := range node.Content {
		bb = append(bb, []byte(c[:])...)
	}
	return sha256.Sum256(bb)
}

func verifyStatsInput(gopts GlobalOptions, args []string) error {
	// require a recognized counting mode
	switch statsOptions.countMode {
	case countModeRestoreSize:
	case countModeUniqueFilesByContents:
	case countModeBlobsPerFile:
	case countModeRawData:
	default:
		return fmt.Errorf("unknown counting mode: %s (use the -h flag to get a list of supported modes)", statsOptions.countMode)
	}

	return nil
}

// statsContainer holds information during a walk of a repository
// to collect information about it, as well as state needed
// for a successful and efficient walk.
type statsContainer struct {
	TotalSize      uint64 `json:"total_size"`
	TotalFileCount uint64 `json:"total_file_count"`
	TotalBlobCount uint64 `json:"total_blob_count,omitempty"`

	// uniqueFiles marks visited files according to their
	// contents (hashed sequence of content blob IDs)
	uniqueFiles map[fileID]struct{}

	// fileBlobs maps a file name (path) to the set of
	// blobs that have been seen as a part of the file
	fileBlobs map[string]restic.IDSet

	// blobs is used to count individual unique blobs,
	// independent of references to files
	blobs restic.BlobSet

	// holds count of all considered snapshots
	snapshotsCount int
}

// fileID is a 256-bit hash that distinguishes unique files.
type fileID [32]byte

const (
	countModeRestoreSize           = "restore-size"
	countModeUniqueFilesByContents = "files-by-contents"
	countModeBlobsPerFile          = "blobs-per-file"
	countModeRawData               = "raw-data"
)
<commit_msg>stats: Add snapshots count to json output<commit_after>package main

import (
	"context"
	"encoding/json"
	"fmt"
	"path/filepath"

	"github.com/restic/restic/internal/backend"
	"github.com/restic/restic/internal/restic"
	"github.com/restic/restic/internal/walker"

	"github.com/minio/sha256-simd"
	"github.com/spf13/cobra"
)

var cmdStats = &cobra.Command{
	Use:   "stats [flags] [snapshot ID] [...]",
	Short: "Scan the repository and show basic statistics",
	Long: `
The "stats" command walks one or multiple snapshots in a repository
and accumulates statistics about the data stored therein. It reports
on the number of unique files and their sizes, according to one of
the counting modes as given by the --mode flag.

It operates on all snapshots matching the selection criteria or all
snapshots if nothing is specified. The special snapshot ID "latest"
is also supported. Some modes make more sense over
just a single snapshot, while others are useful across all snapshots,
depending on what you are trying to calculate.

The modes are:

* restore-size: (default) Counts the size of the restored files.
* files-by-contents: Counts total size of files, where a file is
   considered unique if it has unique contents.
* raw-data: Counts the size of blobs in the repository, regardless of
  how many files reference them.
* blobs-per-file: A combination of files-by-contents and raw-data.

Refer to the online manual for more details about each mode.

EXIT STATUS
===========

Exit status is 0 if the command was successful, and non-zero if there was any error.
`,
	DisableAutoGenTag: true,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runStats(globalOptions, args)
	},
}

// StatsOptions collects all options for the stats command.
type StatsOptions struct {
	// the mode of counting to perform (see consts for available modes)
	countMode string

	// filter snapshots by, if given by user
	Hosts []string
	Tags  restic.TagLists
	Paths []string
}

var statsOptions StatsOptions

func init() {
	cmdRoot.AddCommand(cmdStats)
	f := cmdStats.Flags()
	f.StringVar(&statsOptions.countMode, "mode", countModeRestoreSize, "counting mode: restore-size (default), files-by-contents, blobs-per-file or raw-data")
	f.StringArrayVarP(&statsOptions.Hosts, "host", "H", nil, "only consider snapshots with the given `host` (can be specified multiple times)")
	f.Var(&statsOptions.Tags, "tag", "only consider snapshots which include this `taglist` in the format `tag[,tag,...]` (can be specified multiple times)")
	f.StringArrayVar(&statsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` (can be specified multiple times)")
}

func runStats(gopts GlobalOptions, args []string) error {
	err := verifyStatsInput(gopts, args)
	if err != nil {
		return err
	}

	ctx, cancel := context.WithCancel(gopts.ctx)
	defer cancel()

	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	if !gopts.NoLock {
		lock, err := lockRepo(ctx, repo)
		defer unlockRepo(lock)
		if err != nil {
			return err
		}
	}

	snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile)
	if err != nil {
		return err
	}

	if err = repo.LoadIndex(ctx); err != nil {
		return err
	}

	if !gopts.JSON {
		Printf("scanning...\n")
	}

	// create a container for the stats (and other needed state)
	stats := &statsContainer{
		uniqueFiles:    make(map[fileID]struct{}),
		fileBlobs:      make(map[string]restic.IDSet),
		blobs:          restic.NewBlobSet(),
		SnapshotsCount: 0,
	}

	for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, statsOptions.Hosts, statsOptions.Tags, statsOptions.Paths, args) {
		err = statsWalkSnapshot(ctx, sn, repo, stats)
		if err != nil {
			return fmt.Errorf("error walking snapshot: %v", err)
		}
	}

	if err != nil {
		return err
	}

	if statsOptions.countMode == countModeRawData {
		// the blob handles have been collected, but not yet counted
		for blobHandle := range stats.blobs {
			pbs := repo.Index().Lookup(blobHandle)
			if len(pbs) == 0 {
				return fmt.Errorf("blob %v not found", blobHandle)
			}
			stats.TotalSize += uint64(pbs[0].Length)
			stats.TotalBlobCount++
		}
	}

	if gopts.JSON {
		err = json.NewEncoder(globalOptions.stdout).Encode(stats)
		if err != nil {
			return fmt.Errorf("encoding output: %v", err)
		}
		return nil
	}

	Printf("Stats in %s mode:\n", statsOptions.countMode)
	Printf("Snapshots processed:   %d\n", stats.SnapshotsCount)

	if stats.TotalBlobCount > 0 {
		Printf("   Total Blob Count:   %d\n", stats.TotalBlobCount)
	}
	if stats.TotalFileCount > 0 {
		Printf("   Total File Count:   %d\n", stats.TotalFileCount)
	}
	Printf("         Total Size:   %-5s\n", formatBytes(stats.TotalSize))

	return nil
}

func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo restic.Repository, stats *statsContainer) error {
	if snapshot.Tree == nil {
		return fmt.Errorf("snapshot %s has nil tree", snapshot.ID().Str())
	}

	stats.SnapshotsCount++

	if statsOptions.countMode == countModeRawData {
		// count just the sizes of unique blobs; we don't need to walk the tree
		// ourselves in this case, since a nifty function does it for us
		return restic.FindUsedBlobs(ctx, repo, restic.IDs{*snapshot.Tree}, stats.blobs, nil)
	}

	uniqueInodes := make(map[uint64]struct{})
	err := walker.Walk(ctx, repo, *snapshot.Tree, restic.NewIDSet(), statsWalkTree(repo, stats, uniqueInodes))
	if err != nil {
		return fmt.Errorf("walking tree %s: %v", *snapshot.Tree, err)
	}

	return nil
}

func statsWalkTree(repo restic.Repository, stats *statsContainer, uniqueInodes map[uint64]struct{}) walker.WalkFunc {
	return func(parentTreeID restic.ID, npath string, node *restic.Node, nodeErr error) (bool, error) {
		if nodeErr != nil {
			return true, nodeErr
		}
		if node == nil {
			return true, nil
		}

		if statsOptions.countMode == countModeUniqueFilesByContents || statsOptions.countMode == countModeBlobsPerFile {
			// only count this file if we haven't visited it before
			fid := makeFileIDByContents(node)
			if _, ok := stats.uniqueFiles[fid]; !ok {
				// mark the file as visited
				stats.uniqueFiles[fid] = struct{}{}

				if statsOptions.countMode == countModeUniqueFilesByContents {
					// simply count the size of each unique file (unique by contents only)
					stats.TotalSize += node.Size
					stats.TotalFileCount++
				}
				if statsOptions.countMode == countModeBlobsPerFile {
					// count the size of each unique blob reference, which is
					// by unique file (unique by contents and file path)
					for _, blobID := range node.Content {
						// ensure we have this file (by path) in our map; in this
						// mode, a file is unique by both contents and path
						nodePath := filepath.Join(npath, node.Name)
						if _, ok := stats.fileBlobs[nodePath]; !ok {
							stats.fileBlobs[nodePath] = restic.NewIDSet()
							stats.TotalFileCount++
						}
						if _, ok := stats.fileBlobs[nodePath][blobID]; !ok {
							// is always a data blob since we're accessing it via a file's Content array
							blobSize, found := repo.LookupBlobSize(blobID, restic.DataBlob)
							if !found {
								return true, fmt.Errorf("blob %s not found for tree %s", blobID, parentTreeID)
							}

							// count the blob's size, then add this blob by this
							// file (path) so we don't double-count it
							stats.TotalSize += uint64(blobSize)
							stats.fileBlobs[nodePath].Insert(blobID)
							// this mode also counts total unique blob _references_ per file
							stats.TotalBlobCount++
						}
					}
				}
			}
		}

		if statsOptions.countMode == countModeRestoreSize {
			// as this is a file in the snapshot, we can simply count its
			// size without worrying about uniqueness, since duplicate files
			// will still be restored
			stats.TotalFileCount++

			// if inodes are present, only count each inode once
			// (hard links do not increase restore size)
			if _, ok := uniqueInodes[node.Inode]; !ok || node.Inode == 0 {
				uniqueInodes[node.Inode] = struct{}{}
				stats.TotalSize += node.Size
			}

			return false, nil
		}

		return true, nil
	}
}

// makeFileIDByContents returns a hash of the blob IDs of the
// node's Content in sequence.
func makeFileIDByContents(node *restic.Node) fileID {
	var bb []byte
	for _, c := range node.Content {
		bb = append(bb, []byte(c[:])...)
	}
	return sha256.Sum256(bb)
}

func verifyStatsInput(gopts GlobalOptions, args []string) error {
	// require a recognized counting mode
	switch statsOptions.countMode {
	case countModeRestoreSize:
	case countModeUniqueFilesByContents:
	case countModeBlobsPerFile:
	case countModeRawData:
	default:
		return fmt.Errorf("unknown counting mode: %s (use the -h flag to get a list of supported modes)", statsOptions.countMode)
	}

	return nil
}

// statsContainer holds information during a walk of a repository
// to collect information about it, as well as state needed
// for a successful and efficient walk.
type statsContainer struct {
	TotalSize      uint64 `json:"total_size"`
	TotalFileCount uint64 `json:"total_file_count"`
	TotalBlobCount uint64 `json:"total_blob_count,omitempty"`
	// holds count of all considered snapshots
	SnapshotsCount int `json:"snapshots_count"`

	// uniqueFiles marks visited files according to their
	// contents (hashed sequence of content blob IDs)
	uniqueFiles map[fileID]struct{}

	// fileBlobs maps a file name (path) to the set of
	// blobs that have been seen as a part of the file
	fileBlobs map[string]restic.IDSet

	// blobs is used to count individual unique blobs,
	// independent of references to files
	blobs restic.BlobSet
}

// fileID is a 256-bit hash that distinguishes unique files.
type fileID [32]byte

const (
	countModeRestoreSize           = "restore-size"
	countModeUniqueFilesByContents = "files-by-contents"
	countModeBlobsPerFile          = "blobs-per-file"
	countModeRawData               = "raw-data"
)
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/opencontainers/specs"
)

func loadSpecConfig() (spec *specs.LinuxSpec, rspec *specs.LinuxRuntimeSpec, err error) {
	cPath := "config.json"
	cf, err := os.Open(cPath)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil, fmt.Errorf("config.json not found")
		}
	}
	defer cf.Close()

	rPath := "runtime.json"
	rf, err := os.Open(rPath)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil, fmt.Errorf("runtime.json not found")
		}
	}
	defer rf.Close()

	if err = json.NewDecoder(cf).Decode(&spec); err != nil {
		return
	}
	if err = json.NewDecoder(rf).Decode(&rspec); err != nil {
		return
	}
	return spec, rspec, nil
}

func validateProcess(spec *specs.LinuxSpec, rspec *specs.LinuxRuntimeSpec) error {
	uid := os.Getuid()
	if uint32(uid) != spec.Process.User.UID {
		return fmt.Errorf("UID expected: %v, actual: %v", spec.Process.User.UID, uid)
	}
	gid := os.Getgid()
	if uint32(gid) != spec.Process.User.GID {
		return fmt.Errorf("GID expected: %v, actual: %v", spec.Process.User.GID, gid)
	}

	if spec.Process.Cwd != "" {
		cwd, err := os.Getwd()
		if err != nil {
			return err
		}
		if cwd != spec.Process.Cwd {
			return fmt.Errorf("Cwd expected: %v, actual: %v", spec.Process.Cwd, cwd)
		}
	}

	cmdlineBytes, err := ioutil.ReadFile("/proc/1/cmdline")
	if err != nil {
		return err
	}

	args := strings.Split(string(bytes.Trim(cmdlineBytes, "\x00")), " ")
	if len(args) != len(spec.Process.Args) {
		return fmt.Errorf("Processs arguments expected: %v, actual: %v")
	}
	for i, a := range args {
		if a != spec.Process.Args[i] {
			return fmt.Errorf("Processs arguments expected: %v, actual: %v", a, spec.Process.Args[i])
		}
	}

	return nil
}

func main() {
	spec, rspec, err := loadSpecConfig()
	if err != nil {
		logrus.Fatalf("Failed to load configuration: %q", err)
	}
	if err := validateProcess(spec, rspec); err != nil {
		logrus.Fatalf("Validation failed: %q", err)
	}
}
<commit_msg>Add validation for groups<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/opencontainers/specs"
)

func loadSpecConfig() (spec *specs.LinuxSpec, rspec *specs.LinuxRuntimeSpec, err error) {
	cPath := "config.json"
	cf, err := os.Open(cPath)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil, fmt.Errorf("config.json not found")
		}
	}
	defer cf.Close()

	rPath := "runtime.json"
	rf, err := os.Open(rPath)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil, fmt.Errorf("runtime.json not found")
		}
	}
	defer rf.Close()

	if err = json.NewDecoder(cf).Decode(&spec); err != nil {
		return
	}
	if err = json.NewDecoder(rf).Decode(&rspec); err != nil {
		return
	}
	return spec, rspec, nil
}

func validateProcess(spec *specs.LinuxSpec, rspec *specs.LinuxRuntimeSpec) error {
	uid := os.Getuid()
	if uint32(uid) != spec.Process.User.UID {
		return fmt.Errorf("UID expected: %v, actual: %v", spec.Process.User.UID, uid)
	}
	gid := os.Getgid()
	if uint32(gid) != spec.Process.User.GID {
		return fmt.Errorf("GID expected: %v, actual: %v", spec.Process.User.GID, gid)
	}

	groups, err := os.Getgroups()
	if err != nil {
		return err
	}

	if len(groups) != len(spec.Process.User.AdditionalGids) {
		return fmt.Errorf("Groups expected: %v, actual: %v", spec.Process.User.AdditionalGids, groups)
	}

	groupsMap := make(map[int]bool)
	for _, g := range spec.Process.User.AdditionalGids {
		groupsMap[int(g)] = true
	}

	for _, g := range groups {
		if !groupsMap[g] {
			return fmt.Errorf("Groups expected: %v, actual: %v", spec.Process.User.AdditionalGids, groups)
		}
	}

	if spec.Process.Cwd != "" {
		cwd, err := os.Getwd()
		if err != nil {
			return err
		}
		if cwd != spec.Process.Cwd {
			return fmt.Errorf("Cwd expected: %v, actual: %v", spec.Process.Cwd, cwd)
		}
	}

	cmdlineBytes, err := ioutil.ReadFile("/proc/1/cmdline")
	if err != nil {
		return err
	}

	args := strings.Split(string(bytes.Trim(cmdlineBytes, "\x00")), " ")
	if len(args) != len(spec.Process.Args) {
		return fmt.Errorf("Processs arguments expected: %v, actual: %v")
	}
	for i, a := range args {
		if a != spec.Process.Args[i] {
			return fmt.Errorf("Processs arguments expected: %v, actual: %v", a, spec.Process.Args[i])
		}
	}

	return nil
}

func main() {
	spec, rspec, err := loadSpecConfig()
	if err != nil {
		logrus.Fatalf("Failed to load configuration: %q", err)
	}
	if err := validateProcess(spec, rspec); err != nil {
		logrus.Fatalf("Validation failed: %q", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/cloudfoundry-incubator/cf-lager"
	. "github.com/pivotal-cf-experimental/switchboard"
	"github.com/pivotal-golang/lager"
)

var (
	pidfile = flag.String("pidfile", "", "The location for the pidfile")
	port    = flag.Uint("port", 3306, "Port to listen on")

	backendIPsFlag       = flag.String("backendIPs", "", "Comma-separated list of backend IP addresses")
	backendPortsFlag     = flag.String("backendPorts", "3306", "Comma-separated list of backend ports")
	healthcheckPortsFlag = flag.String("healthcheckPorts", "9200", "Comma-separated list of healthcheck ports")
	healthcheckTimeout   = flag.Duration("healthcheckTimeout", 5*time.Second, "Timeout for healthcheck")

	backendIPs                     []string
	backendPorts, healthcheckPorts []uint
	logger                         lager.Logger
)

func main() {
	flag.Parse()

	logger = cf_lager.New("switchboard")
	logger.Info("Logging for the switchbord")

	fmt.Println("printing")

	listener, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", *port))
	if err != nil {
		logger.Fatal("Error listening on port.", err, lager.Data{"port": *port})
	}
	defer listener.Close()

	err = ioutil.WriteFile(*pidfile, []byte(strconv.Itoa(os.Getpid())), 0644)
	if err != nil {
		logger.Fatal("Cannot write pid to file", err, lager.Data{"pidfile": *pidfile})
	}

	backendIPs = strings.Split(*backendIPsFlag, ",")

	backendPorts, err = stringsToUints(strings.Split(*backendPortsFlag, ","))
	if err != nil {
		log.Fatal(fmt.Sprintf("Error parsing backendPorts: %v", err))
	}

	healthcheckPorts, err = stringsToUints(strings.Split(*healthcheckPortsFlag, ","))
	if err != nil {
		log.Fatal(fmt.Sprintf("Error parsing healthcheckPorts: %v", err))
	}

	replicatePorts()

	logger.Info(fmt.Sprintf("Proxy started on port %d\n", *port))
	logger.Info(fmt.Sprintf("Backend ipAddress: %s\n", backendIPs[0]))
	logger.Info(fmt.Sprintf("Backend port: %d\n", backendPorts[0]))
	logger.Info(fmt.Sprintf("Healthcheck port: %d\n", healthcheckPorts[0]))

	backends := NewCluster(
		backendIPs,
		backendPorts,
		healthcheckPorts,
		*healthcheckTimeout,
		logger,
	)

	switchboard := New(listener, backends, logger)
	switchboard.Run()
}

func stringsToUints(s []string) ([]uint, error) {
	dest_slice := make([]uint, len(s))
	for i, val := range s {
		intVal, err := strconv.ParseUint(val, 10, 64)
		if err != nil {
			return nil, err
		}
		dest_slice[i] = uint(intVal)
	}
	return dest_slice, nil
}

func replicatePorts() {
	if len(backendPorts) != len(backendIPs) {
		port := backendPorts[0]
		backendPorts = make([]uint, len(backendIPs))
		for i, _ := range backendIPs {
			backendPorts[i] = port
		}
	}
	if len(healthcheckPorts) != len(backendIPs) {
		port := healthcheckPorts[0]
		healthcheckPorts = make([]uint, len(backendIPs))
		for i, _ := range backendIPs {
			healthcheckPorts[i] = port
		}
	}
}
<commit_msg>Import switchboard into main.go without .<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/cloudfoundry-incubator/cf-lager"
	"github.com/pivotal-cf-experimental/switchboard"
	"github.com/pivotal-golang/lager"
)

var (
	pidfile = flag.String("pidfile", "", "The location for the pidfile")
	port    = flag.Uint("port", 3306, "Port to listen on")

	backendIPsFlag       = flag.String("backendIPs", "", "Comma-separated list of backend IP addresses")
	backendPortsFlag     = flag.String("backendPorts", "3306", "Comma-separated list of backend ports")
	healthcheckPortsFlag = flag.String("healthcheckPorts", "9200", "Comma-separated list of healthcheck ports")
	healthcheckTimeout   = flag.Duration("healthcheckTimeout", 5*time.Second, "Timeout for healthcheck")

	backendIPs                     []string
	backendPorts, healthcheckPorts []uint
	logger                         lager.Logger
)

func main() {
	flag.Parse()

	logger = cf_lager.New("switchboard")
	logger.Info("Logging for the switchbord")

	listener, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%d", *port))
	if err != nil {
		logger.Fatal("Error listening on port.", err, lager.Data{"port": *port})
	}
	defer listener.Close()

	err = ioutil.WriteFile(*pidfile, []byte(strconv.Itoa(os.Getpid())), 0644)
	if err != nil {
		logger.Fatal("Cannot write pid to file", err, lager.Data{"pidfile": *pidfile})
	}

	backendIPs = strings.Split(*backendIPsFlag, ",")

	backendPorts, err = stringsToUints(strings.Split(*backendPortsFlag, ","))
	if err != nil {
		log.Fatal(fmt.Sprintf("Error parsing backendPorts: %v", err))
	}

	healthcheckPorts, err = stringsToUints(strings.Split(*healthcheckPortsFlag, ","))
	if err != nil {
		log.Fatal(fmt.Sprintf("Error parsing healthcheckPorts: %v", err))
	}

	replicatePorts()

	logger.Info(fmt.Sprintf("Proxy started on port %d\n", *port))
	logger.Info(fmt.Sprintf("Backend ipAddress: %s\n", backendIPs[0]))
	logger.Info(fmt.Sprintf("Backend port: %d\n", backendPorts[0]))
	logger.Info(fmt.Sprintf("Healthcheck port: %d\n", healthcheckPorts[0]))

	backends := switchboard.NewCluster(
		backendIPs,
		backendPorts,
		healthcheckPorts,
		*healthcheckTimeout,
		logger,
	)

	switchboard := switchboard.New(listener, backends, logger)
	switchboard.Run()
}

func stringsToUints(s []string) ([]uint, error) {
	dest_slice := make([]uint, len(s))
	for i, val := range s {
		intVal, err := strconv.ParseUint(val, 10, 64)
		if err != nil {
			return nil, err
		}
		dest_slice[i] = uint(intVal)
	}
	return dest_slice, nil
}

func replicatePorts() {
	if len(backendPorts) != len(backendIPs) {
		port := backendPorts[0]
		backendPorts = make([]uint, len(backendIPs))
		for i, _ := range backendIPs {
			backendPorts[i] = port
		}
	}
	if len(healthcheckPorts) != len(backendIPs) {
		port := healthcheckPorts[0]
		healthcheckPorts = make([]uint, len(backendIPs))
		for i, _ := range backendIPs {
			healthcheckPorts[i] = port
		}
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"os"
	"os/exec"

	"github.com/pkg/errors"

	isatty "github.com/mattn/go-isatty"

	"github.com/havoc-io/mutagen/pkg/process"
)

// HandleTerminalCompatibility automatically restarts the current process inside
// a terminal compatibility emulator if necessary. It currently only handles the
// case of mintty consoles on Windows requiring a relaunch of the current
// command inside winpty.
func HandleTerminalCompatibility() {
	// If we're not running inside a mintty-based terminal, then there's nothing
	// that we need to do.
	if !isatty.IsCygwinTerminal(os.Stdout.Fd()) {
		return
	}

	// Since we're running inside a mintty-based terminal, we need to relaunch
	// using winpty, so first attempt to locate it.
	winpty, err := exec.LookPath("winpty")
	if err != nil {
		Fatal(errors.New("running inside mintty terminal and unable to locate winpty"))
	}

	// Compute the path to the current executable.
	executable, err := os.Executable()
	if err != nil {
		Fatal(errors.New("unable to locate path to current executable"))
	}

	// Build the argument list for winpty.
	arguments := make([]string, 0, len(os.Args))
	arguments = append(arguments, executable)
	arguments = append(arguments, os.Args[1:]...)

	// Create the command that we'll run.
	command := exec.Command(winpty, arguments...)

	// Set up its input/output streams.
	command.Stdin = os.Stdin
	command.Stdout = os.Stdout
	command.Stderr = os.Stderr

	// Run the command and terminate with its exit code.
	if err := command.Run(); err != nil {
		if exitCode, exitCodeErr := process.ExitCodeForError(err); exitCodeErr == nil {
			os.Exit(exitCode)
		}
		Fatal(errors.Wrap(err, "unable to restart process"))
	} else {
		os.Exit(0)
	}
}
<commit_msg>Clarified errors for HandleTerminalCompatibility.<commit_after>package cmd

import (
	"os"
	"os/exec"

	"github.com/pkg/errors"

	isatty "github.com/mattn/go-isatty"

	"github.com/havoc-io/mutagen/pkg/process"
)

// HandleTerminalCompatibility automatically restarts the current process inside
// a terminal compatibility emulator if necessary. It currently only handles the
// case of mintty consoles on Windows requiring a relaunch of the current
// command inside winpty.
func HandleTerminalCompatibility() {
	// If we're not running inside a mintty-based terminal, then there's nothing
	// that we need to do.
	if !isatty.IsCygwinTerminal(os.Stdout.Fd()) {
		return
	}

	// Since we're running inside a mintty-based terminal, we need to relaunch
	// using winpty, so first attempt to locate it.
	winpty, err := exec.LookPath("winpty")
	if err != nil {
		Fatal(errors.New("running inside mintty terminal and unable to locate winpty"))
	}

	// Compute the path to the current executable.
	executable, err := os.Executable()
	if err != nil {
		Fatal(errors.Wrap(err, "running inside mintty terminal and unable to locate current executable"))
	}

	// Build the argument list for winpty.
	arguments := make([]string, 0, len(os.Args))
	arguments = append(arguments, executable)
	arguments = append(arguments, os.Args[1:]...)

	// Create the command that we'll run.
	command := exec.Command(winpty, arguments...)

	// Set up its input/output streams.
	command.Stdin = os.Stdin
	command.Stdout = os.Stdout
	command.Stderr = os.Stderr

	// Run the command and terminate with its exit code.
	if err := command.Run(); err != nil {
		if exitCode, exitCodeErr := process.ExitCodeForError(err); exitCodeErr == nil {
			os.Exit(exitCode)
		}
		Fatal(errors.Wrap(err, "running inside mintty terminal and unable to restart process"))
	} else {
		os.Exit(0)
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"github.com/qiniu/qshell/v2/cmd_test/test"
	"testing"
)

func TestDelete(t *testing.T) {
	deleteKey := "qshell_delete.json"
	_, errs := test.RunCmdWithError("copy", test.Bucket, test.Key, test.Bucket, "-k", deleteKey, "-w")
	if len(errs) > 0 {
		t.Fail()
	}

	_, errs = test.RunCmdWithError("delete", test.Bucket, deleteKey)
	if len(errs) > 0 {
		t.Fail()
	}
}

func TestBatchDelete(t *testing.T) {
	batchConfig := ""
	for _, key := range test.Keys {
		batchConfig += "copy_" + key + "\n"
	}

	path, err := test.CreateFileWithContent("batch_delete.txt", batchConfig)
	if err != nil {
		t.Fatal("create cdn config file error:", err)
	}

	_, errs := test.RunCmdWithError("batchdelete", test.Bucket, "-i", path, "-y")
	if len(errs) > 0 {
		t.Fail()
	}
}

func TestDeleteAfter(t *testing.T) {
	deleteKey := "qshell_delete_after.json"
	_, errs := test.RunCmdWithError("copy", test.Bucket, test.Key, test.Bucket, "-k", deleteKey, "-w")
	if len(errs) > 0 {
		t.Fail()
	}

	_, errs = test.RunCmdWithError("expire", test.Bucket, deleteKey, "1")
	if len(errs) > 0 {
		t.Fail()
	}
}

func TestBatchDeleteAfter(t *testing.T) {
	// copy
	batchConfig := ""
	for _, key := range test.Keys {
		batchConfig += key + "\t" + "delete_after_" + key + "\n"
	}

	path, err := test.CreateFileWithContent("batch_delete_after_copy.txt", batchConfig)
	if err != nil {
		t.Fatal("create cdn config file error:", err)
	}

	_, errs := test.RunCmdWithError("batchcopy", test.Bucket, test.Bucket, "-i", path, "-w", "-y")
	if len(errs) > 0 {
		t.Fail()
	}

	// delete
	batchConfig = ""
	for _, key := range test.Keys {
		batchConfig += "delete_after_" + key + "\t" + "1" + "\n"
	}

	path, err = test.CreateFileWithContent("batch_delete_after.txt", batchConfig)
	if err != nil {
		t.Fatal("create batch expire after config file error:", err)
	}

	_, errs = test.RunCmdWithError("batchexpire", test.Bucket, "-i", path, "-y")
	if len(errs) > 0 {
		t.Fail()
	}
}
<commit_msg>delete & batchdelete add test case<commit_after>package cmd

import (
	"github.com/qiniu/qshell/v2/cmd_test/test"
	"path/filepath"
	"strings"
	"testing"
)

func TestDelete(t *testing.T) {
	deleteKey := "qshell_delete.json"
	_, errs := test.RunCmdWithError("copy", test.Bucket, test.Key, test.Bucket, "-k", deleteKey, "-w")
	if len(errs) > 0 {
		t.Fail()
	}

	_, errs = test.RunCmdWithError("delete", test.Bucket, deleteKey)
	if len(errs) > 0 {
		t.Fail()
	}
}

func TestDeleteNoExistBucket(t *testing.T) {
	_, errs := test.RunCmdWithError("delete", test.BucketNotExist, test.Key)
	if !strings.Contains(errs, "no such bucket") {
		t.Fail()
	}
}

func TestDeleteNoExistKey(t *testing.T) {
	_, errs := test.RunCmdWithError("delete", test.Bucket, test.KeyNotExist)
	if !strings.Contains(errs, "no such file or directory") {
		t.Fail()
	}
}

func TestDeleteNoBucket(t *testing.T) {
	_, errs := test.RunCmdWithError("delete")
	if !strings.Contains(errs, "Bucket can't empty") {
		t.Fail()
	}
}

func TestDeleteNoKey(t *testing.T) {
	_, errs := test.RunCmdWithError("delete", test.Bucket)
	if !strings.Contains(errs, "Key can't empty") {
		t.Fail()
	}
}

func TestDeleteDocument(t *testing.T) {
	test.TestDocument("delete", t)
}

func TestBatchDelete(t *testing.T) {
	TestBatchCopy(t)

	batchConfig := ""
	keys := test.Keys
	keys = append(keys, "hello10.json")
	for _, key := range keys {
		batchConfig += "copy_" + key + "\n"
	}

	resultDir, err := test.ResultPath()
	if err != nil {
		t.Fatal("get result dir error:", err)
	}

	successLogPath := filepath.Join(resultDir, "batch_delete_success.txt")
	failLogPath :=  filepath.Join(resultDir, "batch_delete_fail.txt")

	path, err := test.CreateFileWithContent("batch_delete.txt", batchConfig)
	if err != nil {
		t.Fatal("create cdn config file error:", err)
	}

	test.RunCmdWithError("batchdelete", test.Bucket,
		"-i", path,
		"--success-list", successLogPath,
		"--failure-list", failLogPath,
		"--worker", "4",
		"-y")
	defer func() {
		test.RemoveFile(successLogPath)
		test.RemoveFile(failLogPath)
	}()

	if !test.IsFileHasContent(successLogPath) {
		t.Fatal("batch result: success log to file error: file empty")
	}

	if !test.IsFileHasContent(failLogPath) {
		t.Fatal("batch result: fail log  to file error: file empty")
	}
}

func TestBatchDeleteDocument(t *testing.T) {
	test.TestDocument("batchdelete", t)
}

func TestDeleteAfter(t *testing.T) {
	deleteKey := "qshell_delete_after.json"
	_, errs := test.RunCmdWithError("copy", test.Bucket, test.Key, test.Bucket, "-k", deleteKey, "-w")
	if len(errs) > 0 {
		t.Fail()
	}

	_, errs = test.RunCmdWithError("expire", test.Bucket, deleteKey, "1")
	if len(errs) > 0 {
		t.Fail()
	}
}

func TestBatchDeleteAfter(t *testing.T) {
	// copy
	batchConfig := ""
	for _, key := range test.Keys {
		batchConfig += key + "\t" + "delete_after_" + key + "\n"
	}

	path, err := test.CreateFileWithContent("batch_delete_after_copy.txt", batchConfig)
	if err != nil {
		t.Fatal("create cdn config file error:", err)
	}

	_, errs := test.RunCmdWithError("batchcopy", test.Bucket, test.Bucket, "-i", path, "-w", "-y")
	if len(errs) > 0 {
		t.Fail()
	}

	// delete
	batchConfig = ""
	for _, key := range test.Keys {
		batchConfig += "delete_after_" + key + "\t" + "1" + "\n"
	}

	path, err = test.CreateFileWithContent("batch_delete_after.txt", batchConfig)
	if err != nil {
		t.Fatal("create batch expire after config file error:", err)
	}

	_, errs = test.RunCmdWithError("batchexpire", test.Bucket, "-i", path, "-y")
	if len(errs) > 0 {
		t.Fail()
	}
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"exec"
	"flag"
	"fmt"
	"go/build"
	"go/token"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
)

func usage() {
	fmt.Fprint(os.Stderr, "usage: goinstall importpath...\n")
	fmt.Fprintf(os.Stderr, "\tgoinstall -a\n")
	flag.PrintDefaults()
	os.Exit(2)
}

const logfile = "goinstall.log"

var (
	fset          = token.NewFileSet()
	argv0         = os.Args[0]
	errors        = false
	parents       = make(map[string]string)
	visit         = make(map[string]status)
	installedPkgs = make(map[string]map[string]bool)
	schemeRe      = regexp.MustCompile(`^[a-z]+://`)

	allpkg            = flag.Bool("a", false, "install all previously installed packages")
	reportToDashboard = flag.Bool("dashboard", true, "report public packages at "+dashboardURL)
	update            = flag.Bool("u", false, "update already-downloaded packages")
	doInstall         = flag.Bool("install", true, "build and install")
	clean             = flag.Bool("clean", false, "clean the package directory before installing")
	nuke              = flag.Bool("nuke", false, "clean the package directory and target before installing")
	useMake           = flag.Bool("make", true, "use make to build and install")
	verbose           = flag.Bool("v", false, "verbose")
)

type status int // status for visited map
const (
	unvisited status = iota
	visiting
	done
)

func logf(format string, args ...interface{}) {
	format = "%s: " + format
	args = append([]interface{}{argv0}, args...)
	fmt.Fprintf(os.Stderr, format, args...)
}

func printf(format string, args ...interface{}) {
	if *verbose {
		logf(format, args...)
	}
}

func errorf(format string, args ...interface{}) {
	errors = true
	logf(format, args...)
}

func terrorf(tree *build.Tree, format string, args ...interface{}) {
	if tree != nil && tree.Goroot && os.Getenv("GOPATH") == "" {
		format = strings.TrimRight(format, "\n") + " ($GOPATH not set)\n"
	}
	errorf(format, args...)
}

func main() {
	flag.Usage = usage
	flag.Parse()
	if runtime.GOROOT() == "" {
		fmt.Fprintf(os.Stderr, "%s: no $GOROOT\n", argv0)
		os.Exit(1)
	}
	readPackageList()

	// special case - "unsafe" is already installed
	visit["unsafe"] = done

	args := flag.Args()
	if *allpkg {
		if len(args) != 0 {
			usage() // -a and package list both provided
		}
		// install all packages that were ever installed
		n := 0
		for _, pkgs := range installedPkgs {
			for pkg := range pkgs {
				args = append(args, pkg)
				n++
			}
		}
		if n == 0 {
			logf("no installed packages\n")
			os.Exit(1)
		}
	}
	if len(args) == 0 {
		usage()
	}
	for _, path := range args {
		if s := schemeRe.FindString(path); s != "" {
			errorf("%q used in import path, try %q\n", s, path[len(s):])
			continue
		}

		install(path, "")
	}
	if errors {
		os.Exit(1)
	}
}

// printDeps prints the dependency path that leads to pkg.
func printDeps(pkg string) {
	if pkg == "" {
		return
	}
	if visit[pkg] != done {
		printDeps(parents[pkg])
	}
	fmt.Fprintf(os.Stderr, "\t%s ->\n", pkg)
}

// readPackageList reads the list of installed packages from the
// goinstall.log files in GOROOT and the GOPATHs and initalizes
// the installedPkgs variable.
func readPackageList() {
	for _, t := range build.Path {
		installedPkgs[t.Path] = make(map[string]bool)
		name := filepath.Join(t.Path, logfile)
		pkglistdata, err := ioutil.ReadFile(name)
		if err != nil {
			printf("%s\n", err)
			continue
		}
		pkglist := strings.Fields(string(pkglistdata))
		for _, pkg := range pkglist {
			installedPkgs[t.Path][pkg] = true
		}
	}
}

// logPackage logs the named package as installed in the goinstall.log file
// in the given tree if the package is not already in that file.
func logPackage(pkg string, tree *build.Tree) (logged bool) {
	if installedPkgs[tree.Path][pkg] {
		return false
	}
	name := filepath.Join(tree.Path, logfile)
	fout, err := os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		terrorf(tree, "package log: %s\n", err)
		return false
	}
	fmt.Fprintf(fout, "%s\n", pkg)
	fout.Close()
	return true
}

// install installs the package named by path, which is needed by parent.
func install(pkg, parent string) {
	// Make sure we're not already trying to install pkg.
	switch visit[pkg] {
	case done:
		return
	case visiting:
		fmt.Fprintf(os.Stderr, "%s: package dependency cycle\n", argv0)
		printDeps(parent)
		fmt.Fprintf(os.Stderr, "\t%s\n", pkg)
		os.Exit(2)
	}
	parents[pkg] = parent
	visit[pkg] = visiting
	defer func() {
		visit[pkg] = done
	}()

	// Don't allow trailing '/'
	if _, f := filepath.Split(pkg); f == "" {
		errorf("%s should not have trailing '/'\n", pkg)
		return
	}

	// Check whether package is local or remote.
	// If remote, download or update it.
	tree, pkg, err := build.FindTree(pkg)
	// Don't build the standard library.
	if err == nil && tree.Goroot && isStandardPath(pkg) {
		if parent == "" {
			errorf("%s: can not goinstall the standard library\n", pkg)
		} else {
			printf("%s: skipping standard library\n", pkg)
		}
		return
	}
	// Download remote packages if not found or forced with -u flag.
	remote, public := isRemote(pkg), false
	if remote {
		if err == build.ErrNotFound || (err == nil && *update) {
			// Download remote package.
			printf("%s: download\n", pkg)
			public, err = download(pkg, tree.SrcDir())
		} else {
			// Test if this is a public repository
			// (for reporting to dashboard).
			m, _ := findPublicRepo(pkg)
			public = m != nil
		}
	}
	if err != nil {
		terrorf(tree, "%s: %v\n", pkg, err)
		return
	}
	dir := filepath.Join(tree.SrcDir(), pkg)

	// Install prerequisites.
	dirInfo, err := build.ScanDir(dir, parent == "")
	if err != nil {
		terrorf(tree, "%s: %v\n", pkg, err)
		return
	}
	if len(dirInfo.GoFiles)+len(dirInfo.CgoFiles) == 0 {
		terrorf(tree, "%s: package has no files\n", pkg)
		return
	}
	for _, p := range dirInfo.Imports {
		if p != "C" {
			install(p, pkg)
		}
	}
	if errors {
		return
	}

	// Install this package.
	if *useMake {
		err := domake(dir, pkg, tree, dirInfo.IsCommand())
		if err != nil {
			terrorf(tree, "%s: install: %v\n", pkg, err)
			return
		}
	} else {
		script, err := build.Build(tree, pkg, dirInfo)
		if err != nil {
			terrorf(tree, "%s: install: %v\n", pkg, err)
			return
		}
		if *nuke {
			printf("%s: nuke\n", pkg)
			script.Nuke()
		} else if *clean {
			printf("%s: clean\n", pkg)
			script.Clean()
		}
		if *doInstall {
			if script.Stale() {
				printf("%s: install\n", pkg)
				if err := script.Run(); err != nil {
					terrorf(tree, "%s: install: %v\n", pkg, err)
					return
				}
			} else {
				printf("%s: up-to-date\n", pkg)
			}
		}
	}

	if remote {
		// mark package as installed in goinstall.log
		logged := logPackage(pkg, tree)

		// report installation to the dashboard if this is the first
		// install from a public repository.
		if logged && public {
			maybeReportToDashboard(pkg)
		}
	}
}

// Is this a standard package path?  strings container/vector etc.
// Assume that if the first element has a dot, it's a domain name
// and is not the standard package path.
func isStandardPath(s string) bool {
	dot := strings.Index(s, ".")
	slash := strings.Index(s, "/")
	return dot < 0 || 0 < slash && slash < dot
}

// run runs the command cmd in directory dir with standard input stdin.
// If the command fails, run prints the command and output on standard error
// in addition to returning a non-nil os.Error.
func run(dir string, stdin []byte, cmd ...string) os.Error {
	return genRun(dir, stdin, cmd, false)
}

// quietRun is like run but prints nothing on failure unless -v is used.
func quietRun(dir string, stdin []byte, cmd ...string) os.Error {
	return genRun(dir, stdin, cmd, true)
}

// genRun implements run and quietRun.
func genRun(dir string, stdin []byte, arg []string, quiet bool) os.Error {
	cmd := exec.Command(arg[0], arg[1:]...)
	cmd.Stdin = bytes.NewBuffer(stdin)
	cmd.Dir = dir
	printf("%s: %s %s\n", dir, cmd.Path, strings.Join(arg[1:], " "))
	out, err := cmd.CombinedOutput()
	if err != nil {
		if !quiet || *verbose {
			if dir != "" {
				dir = "cd " + dir + "; "
			}
			fmt.Fprintf(os.Stderr, "%s: === %s%s\n", cmd.Path, dir, strings.Join(cmd.Args, " "))
			os.Stderr.Write(out)
			fmt.Fprintf(os.Stderr, "--- %s\n", err)
		}
		return os.NewError("running " + arg[0] + ": " + err.String())
	}
	return nil
}
<commit_msg>goinstall: better usage message<commit_after>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"exec"
	"flag"
	"fmt"
	"go/build"
	"go/token"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
)

func usage() {
	fmt.Fprintln(os.Stderr, "usage: goinstall [flags] importpath...")
	fmt.Fprintln(os.Stderr, "       goinstall [flags] -a")
	flag.PrintDefaults()
	os.Exit(2)
}

const logfile = "goinstall.log"

var (
	fset          = token.NewFileSet()
	argv0         = os.Args[0]
	errors        = false
	parents       = make(map[string]string)
	visit         = make(map[string]status)
	installedPkgs = make(map[string]map[string]bool)
	schemeRe      = regexp.MustCompile(`^[a-z]+://`)

	allpkg            = flag.Bool("a", false, "install all previously installed packages")
	reportToDashboard = flag.Bool("dashboard", true, "report public packages at "+dashboardURL)
	update            = flag.Bool("u", false, "update already-downloaded packages")
	doInstall         = flag.Bool("install", true, "build and install")
	clean             = flag.Bool("clean", false, "clean the package directory before installing")
	nuke              = flag.Bool("nuke", false, "clean the package directory and target before installing")
	useMake           = flag.Bool("make", true, "use make to build and install")
	verbose           = flag.Bool("v", false, "verbose")
)

type status int // status for visited map
const (
	unvisited status = iota
	visiting
	done
)

func logf(format string, args ...interface{}) {
	format = "%s: " + format
	args = append([]interface{}{argv0}, args...)
	fmt.Fprintf(os.Stderr, format, args...)
}

func printf(format string, args ...interface{}) {
	if *verbose {
		logf(format, args...)
	}
}

func errorf(format string, args ...interface{}) {
	errors = true
	logf(format, args...)
}

func terrorf(tree *build.Tree, format string, args ...interface{}) {
	if tree != nil && tree.Goroot && os.Getenv("GOPATH") == "" {
		format = strings.TrimRight(format, "\n") + " ($GOPATH not set)\n"
	}
	errorf(format, args...)
}

func main() {
	flag.Usage = usage
	flag.Parse()
	if runtime.GOROOT() == "" {
		fmt.Fprintf(os.Stderr, "%s: no $GOROOT\n", argv0)
		os.Exit(1)
	}
	readPackageList()

	// special case - "unsafe" is already installed
	visit["unsafe"] = done

	args := flag.Args()
	if *allpkg {
		if len(args) != 0 {
			usage() // -a and package list both provided
		}
		// install all packages that were ever installed
		n := 0
		for _, pkgs := range installedPkgs {
			for pkg := range pkgs {
				args = append(args, pkg)
				n++
			}
		}
		if n == 0 {
			logf("no installed packages\n")
			os.Exit(1)
		}
	}
	if len(args) == 0 {
		usage()
	}
	for _, path := range args {
		if s := schemeRe.FindString(path); s != "" {
			errorf("%q used in import path, try %q\n", s, path[len(s):])
			continue
		}

		install(path, "")
	}
	if errors {
		os.Exit(1)
	}
}

// printDeps prints the dependency path that leads to pkg.
func printDeps(pkg string) {
	if pkg == "" {
		return
	}
	if visit[pkg] != done {
		printDeps(parents[pkg])
	}
	fmt.Fprintf(os.Stderr, "\t%s ->\n", pkg)
}

// readPackageList reads the list of installed packages from the
// goinstall.log files in GOROOT and the GOPATHs and initalizes
// the installedPkgs variable.
func readPackageList() {
	for _, t := range build.Path {
		installedPkgs[t.Path] = make(map[string]bool)
		name := filepath.Join(t.Path, logfile)
		pkglistdata, err := ioutil.ReadFile(name)
		if err != nil {
			printf("%s\n", err)
			continue
		}
		pkglist := strings.Fields(string(pkglistdata))
		for _, pkg := range pkglist {
			installedPkgs[t.Path][pkg] = true
		}
	}
}

// logPackage logs the named package as installed in the goinstall.log file
// in the given tree if the package is not already in that file.
func logPackage(pkg string, tree *build.Tree) (logged bool) {
	if installedPkgs[tree.Path][pkg] {
		return false
	}
	name := filepath.Join(tree.Path, logfile)
	fout, err := os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		terrorf(tree, "package log: %s\n", err)
		return false
	}
	fmt.Fprintf(fout, "%s\n", pkg)
	fout.Close()
	return true
}

// install installs the package named by path, which is needed by parent.
func install(pkg, parent string) {
	// Make sure we're not already trying to install pkg.
	switch visit[pkg] {
	case done:
		return
	case visiting:
		fmt.Fprintf(os.Stderr, "%s: package dependency cycle\n", argv0)
		printDeps(parent)
		fmt.Fprintf(os.Stderr, "\t%s\n", pkg)
		os.Exit(2)
	}
	parents[pkg] = parent
	visit[pkg] = visiting
	defer func() {
		visit[pkg] = done
	}()

	// Don't allow trailing '/'
	if _, f := filepath.Split(pkg); f == "" {
		errorf("%s should not have trailing '/'\n", pkg)
		return
	}

	// Check whether package is local or remote.
	// If remote, download or update it.
	tree, pkg, err := build.FindTree(pkg)
	// Don't build the standard library.
	if err == nil && tree.Goroot && isStandardPath(pkg) {
		if parent == "" {
			errorf("%s: can not goinstall the standard library\n", pkg)
		} else {
			printf("%s: skipping standard library\n", pkg)
		}
		return
	}
	// Download remote packages if not found or forced with -u flag.
	remote, public := isRemote(pkg), false
	if remote {
		if err == build.ErrNotFound || (err == nil && *update) {
			// Download remote package.
			printf("%s: download\n", pkg)
			public, err = download(pkg, tree.SrcDir())
		} else {
			// Test if this is a public repository
			// (for reporting to dashboard).
			m, _ := findPublicRepo(pkg)
			public = m != nil
		}
	}
	if err != nil {
		terrorf(tree, "%s: %v\n", pkg, err)
		return
	}
	dir := filepath.Join(tree.SrcDir(), pkg)

	// Install prerequisites.
	dirInfo, err := build.ScanDir(dir, parent == "")
	if err != nil {
		terrorf(tree, "%s: %v\n", pkg, err)
		return
	}
	if len(dirInfo.GoFiles)+len(dirInfo.CgoFiles) == 0 {
		terrorf(tree, "%s: package has no files\n", pkg)
		return
	}
	for _, p := range dirInfo.Imports {
		if p != "C" {
			install(p, pkg)
		}
	}
	if errors {
		return
	}

	// Install this package.
	if *useMake {
		err := domake(dir, pkg, tree, dirInfo.IsCommand())
		if err != nil {
			terrorf(tree, "%s: install: %v\n", pkg, err)
			return
		}
	} else {
		script, err := build.Build(tree, pkg, dirInfo)
		if err != nil {
			terrorf(tree, "%s: install: %v\n", pkg, err)
			return
		}
		if *nuke {
			printf("%s: nuke\n", pkg)
			script.Nuke()
		} else if *clean {
			printf("%s: clean\n", pkg)
			script.Clean()
		}
		if *doInstall {
			if script.Stale() {
				printf("%s: install\n", pkg)
				if err := script.Run(); err != nil {
					terrorf(tree, "%s: install: %v\n", pkg, err)
					return
				}
			} else {
				printf("%s: up-to-date\n", pkg)
			}
		}
	}

	if remote {
		// mark package as installed in goinstall.log
		logged := logPackage(pkg, tree)

		// report installation to the dashboard if this is the first
		// install from a public repository.
		if logged && public {
			maybeReportToDashboard(pkg)
		}
	}
}

// Is this a standard package path?  strings container/vector etc.
// Assume that if the first element has a dot, it's a domain name
// and is not the standard package path.
func isStandardPath(s string) bool {
	dot := strings.Index(s, ".")
	slash := strings.Index(s, "/")
	return dot < 0 || 0 < slash && slash < dot
}

// run runs the command cmd in directory dir with standard input stdin.
// If the command fails, run prints the command and output on standard error
// in addition to returning a non-nil os.Error.
func run(dir string, stdin []byte, cmd ...string) os.Error {
	return genRun(dir, stdin, cmd, false)
}

// quietRun is like run but prints nothing on failure unless -v is used.
func quietRun(dir string, stdin []byte, cmd ...string) os.Error {
	return genRun(dir, stdin, cmd, true)
}

// genRun implements run and quietRun.
func genRun(dir string, stdin []byte, arg []string, quiet bool) os.Error {
	cmd := exec.Command(arg[0], arg[1:]...)
	cmd.Stdin = bytes.NewBuffer(stdin)
	cmd.Dir = dir
	printf("%s: %s %s\n", dir, cmd.Path, strings.Join(arg[1:], " "))
	out, err := cmd.CombinedOutput()
	if err != nil {
		if !quiet || *verbose {
			if dir != "" {
				dir = "cd " + dir + "; "
			}
			fmt.Fprintf(os.Stderr, "%s: === %s%s\n", cmd.Path, dir, strings.Join(cmd.Args, " "))
			os.Stderr.Write(out)
			fmt.Fprintf(os.Stderr, "--- %s\n", err)
		}
		return os.NewError("running " + arg[0] + ": " + err.String())
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Run "make install" to build package.

package main

import (
	"bytes"
	"os"
	"template"
)

// domake builds the package in dir.
// If local is false, the package was copied from an external system.
// For non-local packages or packages without Makefiles,
// domake generates a standard Makefile and passes it
// to make on standard input.
func domake(dir, pkg string, local bool) (err os.Error) {
	needMakefile := true
	if local {
		_, err := os.Stat(dir + "/Makefile")
		if err == nil {
			needMakefile = false
		}
	}
	cmd := []string{"gomake"}
	var makefile []byte
	if needMakefile {
		if makefile, err = makeMakefile(dir, pkg); err != nil {
			return err
		}
		cmd = append(cmd, "-f-")
	}
	if *clean {
		cmd = append(cmd, "clean")
	}
	cmd = append(cmd, "install")
	return run(dir, makefile, cmd...)
}

// makeMakefile computes the standard Makefile for the directory dir
// installing as package pkg.  It includes all *.go files in the directory
// except those in package main and those ending in _test.go.
func makeMakefile(dir, pkg string) ([]byte, os.Error) {
	dirInfo, err := scanDir(dir, false)
	if err != nil {
		return nil, err
	}

	if len(dirInfo.cgoFiles) == 0 && len(dirInfo.cFiles) > 0 {
		// When using cgo, .c files are compiled with gcc.  Without cgo,
		// they may be intended for 6c.  Just error out for now.
		return nil, os.ErrorString("C files found in non-cgo package")
	}

	cgoFiles := dirInfo.cgoFiles
	isCgo := make(map[string]bool, len(cgoFiles))
	for _, file := range cgoFiles {
		isCgo[file] = true
	}

	oFiles := make([]string, 0, len(dirInfo.cFiles))
	for _, file := range dirInfo.cFiles {
		oFiles = append(oFiles, file[:len(file)-2]+".o")
	}

	goFiles := make([]string, 0, len(dirInfo.goFiles))
	for _, file := range dirInfo.goFiles {
		if !isCgo[file] {
			goFiles = append(goFiles, file)
		}
	}

	var buf bytes.Buffer
	md := makedata{pkg, goFiles, cgoFiles, oFiles}
	if err := makefileTemplate.Execute(&md, &buf); err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

// makedata is the data type for the makefileTemplate.
type makedata struct {
	pkg      string   // package import path
	goFiles  []string // list of non-cgo .go files
	cgoFiles []string // list of cgo .go files
	oFiles   []string // list of ofiles for cgo
}

var makefileTemplate = template.MustParse(`
include $(GOROOT)/src/Make.inc

TARG={pkg}

{.section goFiles}
GOFILES=\
{.repeated section goFiles}
	{@}\
{.end}

{.end}
{.section cgoFiles}
CGOFILES=\
{.repeated section cgoFiles}
	{@}\
{.end}

{.end}
{.section oFiles}
CGO_OFILES=\
{.repeated section oFiles}
	{@}\
{.end}

{.end}
include $(GOROOT)/src/Make.pkg
`,
	nil)
<commit_msg>goinstall: Fix template to use exported fields<commit_after>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Run "make install" to build package.

package main

import (
	"bytes"
	"os"
	"template"
)

// domake builds the package in dir.
// If local is false, the package was copied from an external system.
// For non-local packages or packages without Makefiles,
// domake generates a standard Makefile and passes it
// to make on standard input.
func domake(dir, pkg string, local bool) (err os.Error) {
	needMakefile := true
	if local {
		_, err := os.Stat(dir + "/Makefile")
		if err == nil {
			needMakefile = false
		}
	}
	cmd := []string{"gomake"}
	var makefile []byte
	if needMakefile {
		if makefile, err = makeMakefile(dir, pkg); err != nil {
			return err
		}
		cmd = append(cmd, "-f-")
	}
	if *clean {
		cmd = append(cmd, "clean")
	}
	cmd = append(cmd, "install")
	return run(dir, makefile, cmd...)
}

// makeMakefile computes the standard Makefile for the directory dir
// installing as package pkg.  It includes all *.go files in the directory
// except those in package main and those ending in _test.go.
func makeMakefile(dir, pkg string) ([]byte, os.Error) {
	dirInfo, err := scanDir(dir, false)
	if err != nil {
		return nil, err
	}

	if len(dirInfo.cgoFiles) == 0 && len(dirInfo.cFiles) > 0 {
		// When using cgo, .c files are compiled with gcc.  Without cgo,
		// they may be intended for 6c.  Just error out for now.
		return nil, os.ErrorString("C files found in non-cgo package")
	}

	cgoFiles := dirInfo.cgoFiles
	isCgo := make(map[string]bool, len(cgoFiles))
	for _, file := range cgoFiles {
		isCgo[file] = true
	}

	oFiles := make([]string, 0, len(dirInfo.cFiles))
	for _, file := range dirInfo.cFiles {
		oFiles = append(oFiles, file[:len(file)-2]+".o")
	}

	goFiles := make([]string, 0, len(dirInfo.goFiles))
	for _, file := range dirInfo.goFiles {
		if !isCgo[file] {
			goFiles = append(goFiles, file)
		}
	}

	var buf bytes.Buffer
	md := makedata{pkg, goFiles, cgoFiles, oFiles}
	if err := makefileTemplate.Execute(&md, &buf); err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

// makedata is the data type for the makefileTemplate.
type makedata struct {
	Pkg      string   // package import path
	GoFiles  []string // list of non-cgo .go files
	CgoFiles []string // list of cgo .go files
	OFiles   []string // list of ofiles for cgo
}

var makefileTemplate = template.MustParse(`
include $(GOROOT)/src/Make.inc

TARG={Pkg}

{.section GoFiles}
GOFILES=\
{.repeated section GoFiles}
	{@}\
{.end}

{.end}
{.section CgoFiles}
CGOFILES=\
{.repeated section CgoFiles}
	{@}\
{.end}

{.end}
{.section OFiles}
CGO_OFILES=\
{.repeated section OFiles}
	{@}\
{.end}

{.end}
include $(GOROOT)/src/Make.pkg
`,
	nil)
<|endoftext|>