<commit_before>package artifactory

import (
	"fmt"
	"net"
	"net/url"
	"os"
	"path"
	"regexp"
	"sort"

	"github.com/fsouza/go-dockerclient"
	"github.com/rafecolton/go-dockerclient-sort"
)

var client *DockerClient

// DockerClient wraps docker.Client, adding a few handy functions
type DockerClient docker.Client

// NewDockerClient returns the dockerclient used by the artifactory package
func NewDockerClient() (*DockerClient, error) {
	if client != nil {
		return client, nil
	}

	endpoint, err := getEndpoint()
	if err != nil {
		return nil, err
	}
	certPath := os.Getenv("DOCKER_CERT_PATH")
	tlsVerify := os.Getenv("DOCKER_TLS_VERIFY") != ""

	var dclient *docker.Client
	if endpoint.Scheme == "https" {
		cert := path.Join(certPath, "cert.pem")
		key := path.Join(certPath, "key.pem")
		ca := ""
		if tlsVerify {
			ca = path.Join(certPath, "ca.pem")
		}

		dclient, err = docker.NewTLSClient(endpoint.String(), cert, key, ca)
		if err != nil {
			return nil, err
		}
	} else {
		dclient, err = docker.NewClient(endpoint.String())
		if err != nil {
			return nil, err
		}
	}
	client = (*DockerClient)(dclient)

	return client, nil
}

func getEndpoint() (*url.URL, error) {
	endpoint := os.Getenv("DOCKER_HOST")
	if endpoint == "" {
		endpoint = "unix:///var/run/docker.sock"
	}

	u, err := url.Parse(endpoint)
	if err != nil {
		return nil, fmt.Errorf("couldn't parse endpoint %s as URL", endpoint)
	}
	if u.Scheme == "tcp" {
		_, port, err := net.SplitHostPort(u.Host)
		if err != nil {
			return nil, fmt.Errorf("error parsing %s for port", u.Host)
		}

		// Only reliable way to determine if we should be using HTTPS appears to be via port
		if os.Getenv("DOCKER_HOST_SCHEME") != "" {
			u.Scheme = os.Getenv("DOCKER_HOST_SCHEME")
		} else if port == "2376" {
			u.Scheme = "https"
		} else {
			u.Scheme = "http"
		}
	}
	return u, nil
}

// LatestImageIDByName uses the provided docker client to get the id
// most-recently-created image with a name matching `name`
func (client *DockerClient) LatestImageIDByName(name string) (string, error) {
	images, err := (*docker.Client)(client).ListImages(false)
	if err != nil {
		return "", err
	}
	sort.Sort(dockersort.ByCreatedDescending(images))
	for _, image := range images {
		for _, tag := range image.RepoTags {
			matched, err := regexp.MatchString("^"+name+"$", tag)
			if err != nil {
				return "", nil
			}
			if matched {
				return image.ID, nil
			}
		}
	}

	return "", fmt.Errorf("unable to find image named %s", name)
}
<commit_msg>Adding a function that returns the underlying client for easy access<commit_after>package artifactory

import (
	"fmt"
	"net"
	"net/url"
	"os"
	"path"
	"regexp"
	"sort"

	"github.com/fsouza/go-dockerclient"
	"github.com/rafecolton/go-dockerclient-sort"
)

var client *DockerClient

// DockerClient wraps docker.Client, adding a few handy functions
type DockerClient docker.Client

// NewDockerClient returns the dockerclient used by the artifactory package
func NewDockerClient() (*DockerClient, error) {
	if client != nil {
		return client, nil
	}

	endpoint, err := getEndpoint()
	if err != nil {
		return nil, err
	}
	certPath := os.Getenv("DOCKER_CERT_PATH")
	tlsVerify := os.Getenv("DOCKER_TLS_VERIFY") != ""

	var dclient *docker.Client
	if endpoint.Scheme == "https" {
		cert := path.Join(certPath, "cert.pem")
		key := path.Join(certPath, "key.pem")
		ca := ""
		if tlsVerify {
			ca = path.Join(certPath, "ca.pem")
		}

		dclient, err = docker.NewTLSClient(endpoint.String(), cert, key, ca)
		if err != nil {
			return nil, err
		}
	} else {
		dclient, err = docker.NewClient(endpoint.String())
		if err != nil {
			return nil, err
		}
	}
	client = (*DockerClient)(dclient)

	return client, nil
}

func getEndpoint() (*url.URL, error) {
	endpoint := os.Getenv("DOCKER_HOST")
	if endpoint == "" {
		endpoint = "unix:///var/run/docker.sock"
	}

	u, err := url.Parse(endpoint)
	if err != nil {
		return nil, fmt.Errorf("couldn't parse endpoint %s as URL", endpoint)
	}
	if u.Scheme == "tcp" {
		_, port, err := net.SplitHostPort(u.Host)
		if err != nil {
			return nil, fmt.Errorf("error parsing %s for port", u.Host)
		}

		// Only reliable way to determine if we should be using HTTPS appears to be via port
		if os.Getenv("DOCKER_HOST_SCHEME") != "" {
			u.Scheme = os.Getenv("DOCKER_HOST_SCHEME")
		} else if port == "2376" {
			u.Scheme = "https"
		} else {
			u.Scheme = "http"
		}
	}
	return u, nil
}

// LatestImageIDByName uses the provided docker client to get the id
// most-recently-created image with a name matching `name`
func (client *DockerClient) LatestImageIDByName(name string) (string, error) {
	images, err := (*docker.Client)(client).ListImages(false)
	if err != nil {
		return "", err
	}
	sort.Sort(dockersort.ByCreatedDescending(images))
	for _, image := range images {
		for _, tag := range image.RepoTags {
			matched, err := regexp.MatchString("^"+name+"$", tag)
			if err != nil {
				return "", nil
			}
			if matched {
				return image.ID, nil
			}
		}
	}

	return "", fmt.Errorf("unable to find image named %s", name)
}

// Client returns the underlying *docker.Client for calling all of its functions
func (client *DockerClient) Client() *docker.Client {
	return (*docker.Client)(client)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"runtime"
	"strconv"
	"strings"
	"sync"
)

import docker "github.com/fsouza/go-dockerclient"

func ok(err error) {
	if err != nil {
		pc, file, line, ok := runtime.Caller(1)

		fn := runtime.FuncForPC(pc)
		var name string
		if fn != nil {
			name = fn.Name()
		} else {
			name = file
		}
		if ok && false {
			log.Fatalf("ERROR [%s:%d] %s\n", name, line, err)
		}
		panic(err)
	}
}

var c *docker.Client

func init() {

	// connect docker
	// c, err := docker.NewClientFromEnv()
	var err error
	c, err = docker.NewClient("http://127.0.0.1:8080")
	ok(err)
}

// // //////////
//   docker  //
// ////////////

func getAll(all bool) []string {
	opts := docker.ListContainersOptions{All: all}
	containers, err := c.ListContainers(opts)
	ok(err)
	ids := []string{}
	for _, con := range containers {
		ids = append(ids, con.ID)
	}
	return ids
}

func rmAll() {
	ids := getAll(true)
	for _, id := range ids {
		rm(id)
	}
}

func rm(id string) {

	opts := docker.RemoveContainerOptions{
		Force: true,
		ID:    id,
	}
	err := c.RemoveContainer(opts)
	ok(err)
}

func runB(b int, baseName, image, cmd string) {
	for i := 0; i < b; i++ {
		name := fmt.Sprintf("%s-%d", baseName, i)
		run(name, image, cmd)
	}
}

func runN(n int, baseName, image, cmd string) {

	wg := sync.WaitGroup{}
	wg.Add(n)
	for i := 0; i < n; i++ {
		go func(i int) {
			name := fmt.Sprintf("%s-%d", baseName, i)
			run(name, image, cmd)
			wg.Done()
		}(i)
	}
	wg.Wait()
}

func cnt() int {
	info, err := c.Info()
	ok(err)
	m := info.Map()
	v, err := strconv.Atoi(m["Containers"])
	ok(err)
	return v
}

func pull(name string) {
	// get or create an image
	i, err := c.InspectImage(name)
	switch err {
	case docker.ErrNoSuchImage:
		// pull stress image
		err = c.PullImage(docker.PullImageOptions{
			Repository: "alpine",
			Tag:        "latest",
		}, docker.AuthConfiguration{})
		ok(err)
		i, err = c.InspectImage("alpine")
		ok(err)
	default:
		ok(err)
	}
	fmt.Printf("using image = %v\n", i.ID)

}

// run returns container.ID
func create(name, image, cmd string) string {
	cmds := strings.Split(cmd, " ")
	config := &docker.Config{Cmd: cmds, Image: image, NetworkDisabled: true}
	cc := docker.CreateContainerOptions{Name: name, Config: config}
	cont, err := c.CreateContainer(cc)
	ok(err)
	return cont.ID
}

func start(id string) {
	hc := &docker.HostConfig{}
	err := c.StartContainer(id, hc)
	ok(err)

}

func run(name, image, cmd string) string {
	id := create(name, image, cmd)
	start(id)
	return id

}

// // ////////
//   main  //
// //////////

func main() {

	err := c.Ping()
	ok(err)
	pull("alpine")

	rmAll()

	// run("alpine-1", "alpine", "sleep 1000")
	runB(10000, "co1oxx", "alpine", "sleep 32600000")
	// runN(10000, "c", "alpine", "sleep 1000")
	// runNxB(1, 1, "c2", "alpine", "sleep 36000000")

	fmt.Printf("cnt = %+v\n", cnt())

}
<commit_msg>events<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"
)

import docker "github.com/fsouza/go-dockerclient"

func ok(err error) {
	if err != nil {
		pc, file, line, ok := runtime.Caller(1)

		fn := runtime.FuncForPC(pc)
		var name string
		if fn != nil {
			name = fn.Name()
		} else {
			name = file
		}
		if ok && false {
			log.Fatalf("ERROR [%s:%d] %s\n", name, line, err)
		}
		panic(err)
	}
}

var (
	c *docker.Client
)

func init() {

	// connect docker
	// c, err := docker.NewClientFromEnv()
	var err error
	c, err = docker.NewClient("http://127.0.0.1:8080")
	ok(err)

	//  check connection
	err = c.Ping()
	ok(err)

}

// // //////////
//   docker  //
// ////////////

func getAll(all bool) []string {
	opts := docker.ListContainersOptions{All: all}
	containers, err := c.ListContainers(opts)
	ok(err)
	ids := []string{}
	for _, con := range containers {
		ids = append(ids, con.ID)
	}
	return ids
}

func rmAll() {
	ids := getAll(true)
	for _, id := range ids {
		rm(id)
	}
}

func rm(id string) {

	opts := docker.RemoveContainerOptions{
		Force: true,
		ID:    id,
	}
	err := c.RemoveContainer(opts)
	ok(err)
}

func runB(b int, baseName, image, cmd string) {
	for i := 0; i < b; i++ {
		name := fmt.Sprintf("%s-%d", baseName, i)
		run(name, image, cmd)
	}
}

func runN(n int, baseName, image, cmd string) {

	wg := sync.WaitGroup{}
	wg.Add(n)
	for i := 0; i < n; i++ {
		go func(i int) {
			name := fmt.Sprintf("%s-%d", baseName, i)
			run(name, image, cmd)
			wg.Done()
		}(i)
	}
	wg.Wait()
}

func cnt() int {
	info, err := c.Info()
	ok(err)
	m := info.Map()
	v, err := strconv.Atoi(m["Containers"])
	ok(err)
	return v
}

func pull(name string) {
	// get or create an image
	i, err := c.InspectImage(name)
	switch err {
	case docker.ErrNoSuchImage:
		// pull stress image
		err = c.PullImage(docker.PullImageOptions{
			Repository: "alpine",
			Tag:        "latest",
		}, docker.AuthConfiguration{})
		ok(err)
		i, err = c.InspectImage("alpine")
		ok(err)
	default:
		ok(err)
	}
	log.Printf("using image %q = %v\n", name, i.ID)

}

// run returns container.ID
func create(name, image, cmd string) string {
	cmds := strings.Split(cmd, " ")
	config := &docker.Config{Cmd: cmds, Image: image, NetworkDisabled: true}
	cc := docker.CreateContainerOptions{Name: name, Config: config}
	cont, err := c.CreateContainer(cc)
	ok(err)
	return cont.ID
}

func start(id string) {
	hc := &docker.HostConfig{}
	err := c.StartContainer(id, hc)
	ok(err)

}

func run(name, image, cmd string) string {
	id := create(name, image, cmd)
	start(id)
	return id

}

// start an goroutine and print all events
func events() {
	listener := make(chan *docker.APIEvents)
	err := c.AddEventListener(listener)
	ok(err)
	go func() {
		for {
			select {
			case e := <-listener:
				log.Printf("e = %+v\n", e)
			case <-time.After(1 * time.Second):
				log.Println("no events observed")

			}
		}
	}()
}

// // ////////
//   main  //
// //////////

func main() {
	// flags
	rmAllFlag := flag.Bool("rmall", false, "kurwa")
	flag.Parse()
	events()

	if *rmAllFlag {
		log.Println("rmall")
		rmAll()
	}

	rmAll()

	pull("alpine")

	run("alpine-1", "alpine", "sleep 864000")
	runN(100, "c", "alpine", "sleep 864000")
	runB(2000, "co1oxx", "alpine", "sleep 864000")
	runNxB(100, 100, "c2", "alpine", "sleep 864000")

	// fmt.Printf("cnt = %+v\n", cnt())

	time.Sleep(5 * time.Second)

}
<|endoftext|>


<commit_before>// +build ignore

// Run tests for all the remotes
//
// Run with go run test_all.go
package main

import (
	"flag"
	"log"
	"os"
	"os/exec"
	"regexp"
	"runtime"
	"strings"
	"time"

	"github.com/ncw/rclone/fs"
	_ "github.com/ncw/rclone/fs/all" // import all fs
	"github.com/ncw/rclone/fstest"
)

var (
	remotes = []string{
		"TestSwift:",
		"TestS3:",
		"TestDrive:",
		"TestGoogleCloudStorage:",
		"TestDropbox:",
		"TestAmazonCloudDrive:",
		"TestOneDrive:",
		"TestHubic:",
		"TestB2:",
		"TestYandex:",
	}
	binary = "fs.test"
	// Flags
	maxTries = flag.Int("maxtries", 5, "Number of times to try each test")
	runTests = flag.String("remotes", "", "Comma separated list of remotes to test, eg 'TestSwift:,TestS3'")
	verbose  = flag.Bool("verbose", false, "Run the tests with -v")
	clean    = flag.Bool("clean", false, "Instead of testing, clean all left over test directories")
	runOnly  = flag.String("run-only", "", "Run only those tests matching the regexp supplied")
)

// test holds info about a running test
type test struct {
	remote      string
	subdir      bool
	cmdLine     []string
	cmdString   string
	try         int
	err         error
	output      []byte
	failedTests []string
	runFlag     string
}

// newTest creates a new test
func newTest(remote string, subdir bool) *test {
	t := &test{
		remote:  remote,
		subdir:  subdir,
		cmdLine: []string{"./" + binary, "-remote", remote},
		try:     1,
	}
	if *verbose {
		t.cmdLine = append(t.cmdLine, "-test.v")
	}
	if *runOnly != "" {
		t.cmdLine = append(t.cmdLine, "-test.run", *runOnly)
	}
	if subdir {
		t.cmdLine = append(t.cmdLine, "-subdir")
	}
	t.cmdString = strings.Join(t.cmdLine, " ")
	return t
}

var failRe = regexp.MustCompile(`(?m)^--- FAIL: (Test\w*) \(`)

// findFailures looks for all the tests which failed
func (t *test) findFailures() {
	t.failedTests = nil
	for _, matches := range failRe.FindAllSubmatch(t.output, -1) {
		t.failedTests = append(t.failedTests, string(matches[1]))
	}
	if len(t.failedTests) != 0 {
		t.runFlag = "^(" + strings.Join(t.failedTests, "|") + ")$"
	} else {
		t.runFlag = ""
	}
	if t.passed() && len(t.failedTests) != 0 {
		log.Printf("%q - Expecting no errors but got: %v", t.cmdString, t.failedTests)
	} else if !t.passed() && len(t.failedTests) == 0 {
		log.Printf("%q - Expecting errors but got none: %v", t.cmdString, t.failedTests)
	}
}

// trial runs a single test
func (t *test) trial() {
	cmdLine := t.cmdLine[:]
	if t.runFlag != "" {
		cmdLine = append(cmdLine, "-test.run", t.runFlag)
	}
	cmdString := strings.Join(cmdLine, " ")
	log.Printf("%q - Starting (try %d/%d)", cmdString, t.try, *maxTries)
	cmd := exec.Command(cmdLine[0], cmdLine[1:]...)
	start := time.Now()
	t.output, t.err = cmd.CombinedOutput()
	duration := time.Since(start)
	t.findFailures()
	if t.passed() {
		log.Printf("%q - Finished OK in %v (try %d/%d)", cmdString, duration, t.try, *maxTries)
	} else {
		log.Printf("%q - Finished ERROR in %v (try %d/%d): %v: Failed %v", cmdString, duration, t.try, *maxTries, t.err, t.failedTests)
	}
}

// cleanFs runs a single clean fs for left over directories
func (t *test) cleanFs() error {
	f, err := fs.NewFs(t.remote)
	if err != nil {
		return err
	}
	dirs, err := fs.NewLister().SetLevel(1).Start(f, "").GetDirs()
	for _, dir := range dirs {
		if fstest.MatchTestRemote.MatchString(dir.Name) {
			log.Printf("Purging %s%s", t.remote, dir.Name)
			dir, err := fs.NewFs(t.remote + dir.Name)
			if err != nil {
				return err
			}
			err = fs.Purge(dir)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

// clean runs a single clean on a fs for left over directories
func (t *test) clean() {
	log.Printf("%q - Starting clean (try %d/%d)", t.remote, t.try, *maxTries)
	start := time.Now()
	t.err = t.cleanFs()
	if t.err != nil {
		log.Printf("%q - Failed to purge %v", t.remote, t.err)
	}
	duration := time.Since(start)
	if t.passed() {
		log.Printf("%q - Finished OK in %v (try %d/%d)", t.cmdString, duration, t.try, *maxTries)
	} else {
		log.Printf("%q - Finished ERROR in %v (try %d/%d): %v", t.cmdString, duration, t.try, *maxTries, t.err)
	}
}

// passed returns true if the test passed
func (t *test) passed() bool {
	return t.err == nil
}

// run runs all the trials for this test
func (t *test) run(result chan<- *test) {
	for t.try = 1; t.try <= *maxTries; t.try++ {
		if *clean {
			if !t.subdir {
				t.clean()
			}
		} else {
			t.trial()
		}
		if t.passed() {
			break
		}
	}
	if !t.passed() {
		log.Println("------------------------------------------------------------")
		log.Println(string(t.output))
		log.Println("------------------------------------------------------------")
	}
	result <- t
}

// makeTestBinary makes the binary we will run
func makeTestBinary() {
	if runtime.GOOS == "windows" {
		binary += ".exe"
	}
	log.Printf("Making test binary %q", binary)
	err := exec.Command("go", "test", "-c", "-o", binary).Run()
	if err != nil {
		log.Fatalf("Failed to make test binary: %v", err)
	}
	if _, err := os.Stat(binary); err != nil {
		log.Fatalf("Couldn't find test binary %q", binary)
	}
}

// removeTestBinary removes the binary made in makeTestBinary
func removeTestBinary() {
	err := os.Remove(binary) // Delete the binary when finished
	if err != nil {
		log.Printf("Error removing test binary %q: %v", binary, err)
	}
}

func main() {
	flag.Parse()
	if *runTests != "" {
		remotes = strings.Split(*runTests, ",")
	}
	log.Printf("Testing remotes: %s", strings.Join(remotes, ", "))

	start := time.Now()
	if *clean {
		fs.LoadConfig()
	} else {
		makeTestBinary()
		defer removeTestBinary()
	}

	// start the tests
	results := make(chan *test, 8)
	awaiting := 0
	for _, remote := range remotes {
		awaiting += 2
		go newTest(remote, false).run(results)
		go newTest(remote, true).run(results)
	}

	// Wait for the tests to finish
	var failed []*test
	for ; awaiting > 0; awaiting-- {
		t := <-results
		if !t.passed() {
			failed = append(failed, t)
		}
	}
	duration := time.Since(start)

	// Summarise results
	if len(failed) == 0 {
		log.Printf("PASS: All tests finished OK in %v", duration)
	} else {
		log.Printf("FAIL: %d tests failed in %v", len(failed), duration)
		for _, t := range failed {
			log.Printf("  * %s", t.cmdString)
		}
		os.Exit(1)
	}
}
<commit_msg>Dump out unexpected state in integration test<commit_after>// +build ignore

// Run tests for all the remotes
//
// Run with go run test_all.go
package main

import (
	"flag"
	"log"
	"os"
	"os/exec"
	"regexp"
	"runtime"
	"strings"
	"time"

	"github.com/ncw/rclone/fs"
	_ "github.com/ncw/rclone/fs/all" // import all fs
	"github.com/ncw/rclone/fstest"
)

var (
	remotes = []string{
		"TestSwift:",
		"TestS3:",
		"TestDrive:",
		"TestGoogleCloudStorage:",
		"TestDropbox:",
		"TestAmazonCloudDrive:",
		"TestOneDrive:",
		"TestHubic:",
		"TestB2:",
		"TestYandex:",
	}
	binary = "fs.test"
	// Flags
	maxTries = flag.Int("maxtries", 5, "Number of times to try each test")
	runTests = flag.String("remotes", "", "Comma separated list of remotes to test, eg 'TestSwift:,TestS3'")
	verbose  = flag.Bool("verbose", false, "Run the tests with -v")
	clean    = flag.Bool("clean", false, "Instead of testing, clean all left over test directories")
	runOnly  = flag.String("run-only", "", "Run only those tests matching the regexp supplied")
)

// test holds info about a running test
type test struct {
	remote      string
	subdir      bool
	cmdLine     []string
	cmdString   string
	try         int
	err         error
	output      []byte
	failedTests []string
	runFlag     string
}

// newTest creates a new test
func newTest(remote string, subdir bool) *test {
	t := &test{
		remote:  remote,
		subdir:  subdir,
		cmdLine: []string{"./" + binary, "-remote", remote},
		try:     1,
	}
	if *verbose {
		t.cmdLine = append(t.cmdLine, "-test.v")
	}
	if *runOnly != "" {
		t.cmdLine = append(t.cmdLine, "-test.run", *runOnly)
	}
	if subdir {
		t.cmdLine = append(t.cmdLine, "-subdir")
	}
	t.cmdString = strings.Join(t.cmdLine, " ")
	return t
}

// dumpOutput prints the error output
func (t *test) dumpOutput() {
	log.Println("------------------------------------------------------------")
	log.Printf("---- %q ----", t.cmdString)
	log.Println(string(t.output))
	log.Println("------------------------------------------------------------")
}

var failRe = regexp.MustCompile(`(?m)^--- FAIL: (Test\w*) \(`)

// findFailures looks for all the tests which failed
func (t *test) findFailures() {
	oldFailedTests := t.failedTests
	t.failedTests = nil
	for _, matches := range failRe.FindAllSubmatch(t.output, -1) {
		t.failedTests = append(t.failedTests, string(matches[1]))
	}
	if len(t.failedTests) != 0 {
		t.runFlag = "^(" + strings.Join(t.failedTests, "|") + ")$"
	} else {
		t.runFlag = ""
	}
	if t.passed() && len(t.failedTests) != 0 {
		log.Printf("%q - Expecting no errors but got: %v", t.cmdString, t.failedTests)
		t.dumpOutput()
	} else if !t.passed() && len(t.failedTests) == 0 {
		log.Printf("%q - Expecting errors but got none: %v", t.cmdString, t.failedTests)
		t.dumpOutput()
		t.failedTests = oldFailedTests
	}
}

// trial runs a single test
func (t *test) trial() {
	cmdLine := t.cmdLine[:]
	if t.runFlag != "" {
		cmdLine = append(cmdLine, "-test.run", t.runFlag)
	}
	cmdString := strings.Join(cmdLine, " ")
	log.Printf("%q - Starting (try %d/%d)", cmdString, t.try, *maxTries)
	cmd := exec.Command(cmdLine[0], cmdLine[1:]...)
	start := time.Now()
	t.output, t.err = cmd.CombinedOutput()
	duration := time.Since(start)
	t.findFailures()
	if t.passed() {
		log.Printf("%q - Finished OK in %v (try %d/%d)", cmdString, duration, t.try, *maxTries)
	} else {
		log.Printf("%q - Finished ERROR in %v (try %d/%d): %v: Failed %v", cmdString, duration, t.try, *maxTries, t.err, t.failedTests)
	}
}

// cleanFs runs a single clean fs for left over directories
func (t *test) cleanFs() error {
	f, err := fs.NewFs(t.remote)
	if err != nil {
		return err
	}
	dirs, err := fs.NewLister().SetLevel(1).Start(f, "").GetDirs()
	for _, dir := range dirs {
		if fstest.MatchTestRemote.MatchString(dir.Name) {
			log.Printf("Purging %s%s", t.remote, dir.Name)
			dir, err := fs.NewFs(t.remote + dir.Name)
			if err != nil {
				return err
			}
			err = fs.Purge(dir)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

// clean runs a single clean on a fs for left over directories
func (t *test) clean() {
	log.Printf("%q - Starting clean (try %d/%d)", t.remote, t.try, *maxTries)
	start := time.Now()
	t.err = t.cleanFs()
	if t.err != nil {
		log.Printf("%q - Failed to purge %v", t.remote, t.err)
	}
	duration := time.Since(start)
	if t.passed() {
		log.Printf("%q - Finished OK in %v (try %d/%d)", t.cmdString, duration, t.try, *maxTries)
	} else {
		log.Printf("%q - Finished ERROR in %v (try %d/%d): %v", t.cmdString, duration, t.try, *maxTries, t.err)
	}
}

// passed returns true if the test passed
func (t *test) passed() bool {
	return t.err == nil
}

// run runs all the trials for this test
func (t *test) run(result chan<- *test) {
	for t.try = 1; t.try <= *maxTries; t.try++ {
		if *clean {
			if !t.subdir {
				t.clean()
			}
		} else {
			t.trial()
		}
		if t.passed() {
			break
		}
	}
	if !t.passed() {
		t.dumpOutput()
	}
	result <- t
}

// makeTestBinary makes the binary we will run
func makeTestBinary() {
	if runtime.GOOS == "windows" {
		binary += ".exe"
	}
	log.Printf("Making test binary %q", binary)
	err := exec.Command("go", "test", "-c", "-o", binary).Run()
	if err != nil {
		log.Fatalf("Failed to make test binary: %v", err)
	}
	if _, err := os.Stat(binary); err != nil {
		log.Fatalf("Couldn't find test binary %q", binary)
	}
}

// removeTestBinary removes the binary made in makeTestBinary
func removeTestBinary() {
	err := os.Remove(binary) // Delete the binary when finished
	if err != nil {
		log.Printf("Error removing test binary %q: %v", binary, err)
	}
}

func main() {
	flag.Parse()
	if *runTests != "" {
		remotes = strings.Split(*runTests, ",")
	}
	log.Printf("Testing remotes: %s", strings.Join(remotes, ", "))

	start := time.Now()
	if *clean {
		fs.LoadConfig()
	} else {
		makeTestBinary()
		defer removeTestBinary()
	}

	// start the tests
	results := make(chan *test, 8)
	awaiting := 0
	for _, remote := range remotes {
		awaiting += 2
		go newTest(remote, false).run(results)
		go newTest(remote, true).run(results)
	}

	// Wait for the tests to finish
	var failed []*test
	for ; awaiting > 0; awaiting-- {
		t := <-results
		if !t.passed() {
			failed = append(failed, t)
		}
	}
	duration := time.Since(start)

	// Summarise results
	if len(failed) == 0 {
		log.Printf("PASS: All tests finished OK in %v", duration)
	} else {
		log.Printf("FAIL: %d tests failed in %v", len(failed), duration)
		for _, t := range failed {
			log.Printf("  * %s", t.cmdString)
		}
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package es

import (
	"encoding/json"
	"fmt"
	"github.com/tiaguinho/esmsync/mongo"
	"gopkg.in/mgo.v2/bson"
	"io/ioutil"
	"reflect"
	"strings"
)

//Mapping struct
type Node struct {
	MongoField string `json:"mongo"`
	Type       string `json:"type"`
	EsField    string `json:"es"`
}

func getNodesFile() []Node {
	var nodes []Node

	content, err := ioutil.ReadFile("./config/mapping.json")
	if err == nil {
		json.Unmarshal(content, &nodes)
	}

	return nodes
}

//map a struct to the model defined in mapping.json
func Mapping(oplog interface{}) (object Elasticsearch) {
	nodes := getNodesFile()

	var data map[string]interface{}
	switch reflect.ValueOf(oplog).Field(0).FieldByName("Op").String() {
	case "i":
		s := oplog.(mongo.OplogInsert)
		data = s.O
		object = Elasticsearch{
			Id:        s.O["_id"].(bson.ObjectId).Hex(),
			Operation: "i",
		}
	case "u":
		s := oplog.(mongo.OplogUpdate)
		data = s.O
		object = Elasticsearch{
			Id:        s.O2["_id"].Hex(),
			Operation: "u",
		}
	case "d":
		s := oplog.(mongo.OplogDelete)
		object = Elasticsearch{
			Id:        s.O["_id"].Hex(),
			Operation: "d",
		}
	}

	if len(data) != 0 {
		object.Data = make(map[string]interface{}, len(nodes))

		for _, node := range nodes {
			rs := getValue(node.MongoField, node.Type, data)
			if rs != nil {
				object.Data[node.EsField] = rs
			}
		}
	}
	fmt.Println(object.Data)
	fmt.Println(".........................................................")

	return object
}

//return a value of the field
func getValue(key, data_type string, data map[string]interface{}) (resp interface{}) {
	if data[key] == nil {
		fields := strings.Split(key, ">")

		r := extractValue(fields, data)
		if r != nil {
			resp = r
		}
	} else {
		resp = data[key]
	}

	if resp != nil {
		if data_type != reflect.TypeOf(resp).Kind().String() {
			temps := make([]interface{}, 1)
			temps[0] = resp

			resp = temps
		}
	}

	return
}

func extractValue(fields []string, data interface{}) (result interface{}) {
	var index int
	if len(fields) > 1 {
		index = 1
	}

	if data != nil {
		if reflect.TypeOf(data).Kind() == reflect.Map {
			result = extractValue(fields[index:], data.(map[string]interface{})[fields[0]])
		} else if reflect.TypeOf(data).Kind() == reflect.Slice {
			length := reflect.ValueOf(data).Len()

			results := make([]interface{}, length)
			for i := 0; i < length; i++ {
				results[i] = extractValue(fields[index:], reflect.ValueOf(data).Index(i).Interface())
			}

			result = results
		} else if data != nil {
			result = data
		}
	}

	return
}
<commit_msg>remove debug<commit_after>package es

import (
	"encoding/json"
	"github.com/tiaguinho/esmsync/mongo"
	"gopkg.in/mgo.v2/bson"
	"io/ioutil"
	"reflect"
	"strings"
)

//Mapping struct
type Node struct {
	MongoField string `json:"mongo"`
	Type       string `json:"type"`
	EsField    string `json:"es"`
}

func getNodesFile() []Node {
	var nodes []Node

	content, err := ioutil.ReadFile("./config/mapping.json")
	if err == nil {
		json.Unmarshal(content, &nodes)
	}

	return nodes
}

//map a struct to the model defined in mapping.json
func Mapping(oplog interface{}) (object Elasticsearch) {
	nodes := getNodesFile()

	var data map[string]interface{}
	switch reflect.ValueOf(oplog).Field(0).FieldByName("Op").String() {
	case "i":
		s := oplog.(mongo.OplogInsert)
		data = s.O
		object = Elasticsearch{
			Id:        s.O["_id"].(bson.ObjectId).Hex(),
			Operation: "i",
		}
	case "u":
		s := oplog.(mongo.OplogUpdate)
		data = s.O
		object = Elasticsearch{
			Id:        s.O2["_id"].Hex(),
			Operation: "u",
		}
	case "d":
		s := oplog.(mongo.OplogDelete)
		object = Elasticsearch{
			Id:        s.O["_id"].Hex(),
			Operation: "d",
		}
	}

	if len(data) != 0 {
		object.Data = make(map[string]interface{}, len(nodes))

		for _, node := range nodes {
			rs := getValue(node.MongoField, node.Type, data)
			if rs != nil {
				object.Data[node.EsField] = rs
			}
		}
	}

	return object
}

//return a value of the field
func getValue(key, data_type string, data map[string]interface{}) (resp interface{}) {
	if data[key] == nil {
		fields := strings.Split(key, ">")

		r := extractValue(fields, data)
		if r != nil {
			resp = r
		}
	} else {
		resp = data[key]
	}

	if resp != nil {
		if data_type != reflect.TypeOf(resp).Kind().String() {
			temps := make([]interface{}, 1)
			temps[0] = resp

			resp = temps
		}
	}

	return
}

func extractValue(fields []string, data interface{}) (result interface{}) {
	var index int
	if len(fields) > 1 {
		index = 1
	}

	if data != nil {
		if reflect.TypeOf(data).Kind() == reflect.Map {
			result = extractValue(fields[index:], data.(map[string]interface{})[fields[0]])
		} else if reflect.TypeOf(data).Kind() == reflect.Slice {
			length := reflect.ValueOf(data).Len()

			results := make([]interface{}, length)
			for i := 0; i < length; i++ {
				results[i] = extractValue(fields[index:], reflect.ValueOf(data).Index(i).Interface())
			}

			result = results
		} else if data != nil {
			result = data
		}
	}

	return
}
<|endoftext|>


<commit_before>/*
 *  Copyright (c) 2012, Muhammed Uluyol <uluyol0@gmail.com>
 *  All rights reserved.
 *
 *  Redistribution and use in source and binary forms, with or without
 *  modification, are permitted provided that the following conditions are met:
 *
 *   - Redistributions of source code must retain the above copyright notice,
 *     this list of conditions and the following disclaimer.
 *
 *   - Redistributions in binary form must reproduce the above copyright notice,
 *     this list of conditions and the following disclaimer in the documentation
 *     and/or other materials provided with the distribution.
 *
 *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 *  POSSIBILITY OF SUCH DAMAGE.
 */

package main

import (
	"bytes"
	"fmt"
	"github.com/mattn/go-gtk/glib"
	"github.com/mattn/go-gtk/gtk"
	"io/ioutil"
	"os"
	"strconv"
)

const (
	TIMEOUT            = 3000
	NEWLINE            = byte(10)
	STATUS_CHARGING    = "Charging"
	STATUS_DISCHARGING = "Discharging"
)

var (
	status_icon *gtk.GtkStatusIcon
	full        int64
)

func main() {
	gtk.Init(&os.Args)
	glib.SetApplicationName("zzcleanbattery")

	buf, err := ioutil.ReadFile("/sys/class/power_supply/BAT0/energy_full")
	if err != nil {
		panic(err)
	}
	str := string(bytes.Split(buf, []byte{NEWLINE})[0])
	full, err = strconv.ParseInt(str, 10, 64)
	if err != nil {
		panic(err)
	}

	status_icon = gtk.StatusIcon()
	status_icon.SetTitle("zzcleanbattery")

	update_icon()

	glib.TimeoutAdd(TIMEOUT, update_icon)
	gtk.Main()
}

func update_icon() bool {

	var (
		hours   int64
		minutes int64
		seconds int64
		pfull   int64
		rate    int64
		now     int64
		status  string
		text    string
	)

	buf, err := ioutil.ReadFile("/sys/class/power_supply/BAT0/energy_now")
	if err != nil {
		panic(err)
	}
	str := string(bytes.Split(buf, []byte{NEWLINE})[0])
	now, err = strconv.ParseInt(str, 10, 64)
	if err != nil {
		panic(err)
	}

	buf, err = ioutil.ReadFile("/sys/class/power_supply/BAT0/power_now")
	if err != nil {
		panic(err)
	}
	str = string(bytes.Split(buf, []byte{NEWLINE})[0])
	rate, err = strconv.ParseInt(str, 10, 64)
	if err != nil {
		panic(err)
	}

	buf, err = ioutil.ReadFile("/sys/class/power_supply/BAT0/status")
	if err != nil {
		panic(err)
	}
	status = string(bytes.Split(buf, []byte{NEWLINE})[0])

	pfull = now * 100 / full

	if rate > 0 {
		switch status {
		case STATUS_CHARGING:
			seconds = 3600 * (full - now) / rate
		case STATUS_DISCHARGING:
			seconds = 3600 * now / rate
		default:
			seconds = 0
		}
	} else {
		seconds = 0
	}
	hours = seconds / 3600
	seconds -= hours * 3600
	minutes = seconds / 60
	seconds -= minutes * 60
	if seconds == 0 {
		text = fmt.Sprintf("%s, %d%%", status, pfull)
	} else {
		text = fmt.Sprintf("%s, %d%%, %d:%02d remaining",
			status,
			pfull,
			hours,
			minutes)
	}

	status_icon.SetTooltipText(text)
	status_icon.SetFromIconName(get_icon_name(status, pfull))
	return true
}

func get_icon_name(status string, pfull int64) string {
	if status == STATUS_DISCHARGING {
		switch {
		case pfull < 10:
			return "battery_empty"
		case pfull < 20:
			return "battery_caution"
		case pfull < 40:
			return "battery_low"
		case pfull < 60:
			return "battery_two_thirds"
		case pfull < 75:
			return "battery_third_fouth"
		default:
			return "battery_full"
		}
	} else if status == STATUS_CHARGING {
		return "battery_charged"
	}
	return "battery_plugged"
}
<commit_msg>Update style, follow Go conventions<commit_after>/*

  Copyright (c) 2012-2014, Muhammed Uluyol <uluyol0@gmail.com>
  All rights reserved.

  Redistribution and use in source and binary forms, with or without
  modification, are permitted provided that the following conditions are met:

   - Redistributions of source code must retain the above copyright notice,
     this list of conditions and the following disclaimer.

   - Redistributions in binary form must reproduce the above copyright notice,
     this list of conditions and the following disclaimer in the documentation
     and/or other materials provided with the distribution.

  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  POSSIBILITY OF SUCH DAMAGE.

*/

package main

import (
	"bytes"
	"fmt"
	"github.com/mattn/go-gtk/glib"
	"github.com/mattn/go-gtk/gtk"
	"io/ioutil"
	"os"
	"strconv"
)

const (
	timeout            = 3000
	newline            = byte(10)
	statusCharging    = "Charging"
	statusDischarging = "Discharging"
)

var (
	statusIcon *gtk.GtkStatusIcon
	full        int64
)

func main() {
	gtk.Init(&os.Args)
	glib.SetApplicationName("zzcleanbattery")

	buf, err := ioutil.ReadFile("/sys/class/power_supply/BAT0/energy_full")
	if err != nil {
		panic(err)
	}
	str := string(bytes.Split(buf, []byte{newline})[0])
	full, err = strconv.ParseInt(str, 10, 64)
	if err != nil {
		panic(err)
	}

	statusIcon = gtk.StatusIcon()
	statusIcon.SetTitle("zzcleanbattery")

	updateIcon()

	glib.TimeoutAdd(timeout, updateIcon)
	gtk.Main()
}

func updateIcon() bool {

	var (
		hours   int64
		minutes int64
		seconds int64
		pfull   int64
		rate    int64
		now     int64
		status  string
		text    string
	)

	buf, err := ioutil.ReadFile("/sys/class/power_supply/BAT0/energy_now")
	if err != nil {
		panic(err)
	}
	str := string(bytes.Split(buf, []byte{newline})[0])
	now, err = strconv.ParseInt(str, 10, 64)
	if err != nil {
		panic(err)
	}

	buf, err = ioutil.ReadFile("/sys/class/power_supply/BAT0/power_now")
	if err != nil {
		panic(err)
	}
	str = string(bytes.Split(buf, []byte{newline})[0])
	rate, err = strconv.ParseInt(str, 10, 64)
	if err != nil {
		panic(err)
	}

	buf, err = ioutil.ReadFile("/sys/class/power_supply/BAT0/status")
	if err != nil {
		panic(err)
	}
	status = string(bytes.Split(buf, []byte{newline})[0])

	pfull = now * 100 / full

	if rate > 0 {
		switch status {
		case statusCharging:
			seconds = 3600 * (full - now) / rate
		case statusDischarging:
			seconds = 3600 * now / rate
		default:
			seconds = 0
		}
	} else {
		seconds = 0
	}
	hours = seconds / 3600
	seconds -= hours * 3600
	minutes = seconds / 60
	seconds -= minutes * 60
	if seconds == 0 {
		text = fmt.Sprintf("%s, %d%%", status, pfull)
	} else {
		text = fmt.Sprintf("%s, %d%%, %d:%02d remaining",
			status,
			pfull,
			hours,
			minutes)
	}

	statusIcon.SetTooltipText(text)
	statusIcon.SetFromIconName(getIconName(status, pfull))
	return true
}

func getIconName(status string, pfull int64) string {
	if status == statusDischarging {
		switch {
		case pfull < 10:
			return "battery_empty"
		case pfull < 20:
			return "battery_caution"
		case pfull < 40:
			return "battery_low"
		case pfull < 60:
			return "battery_two_thirds"
		case pfull < 75:
			return "battery_third_fouth"
		default:
			return "battery_full"
		}
	} else if status == statusCharging {
		return "battery_charged"
	}
	return "battery_plugged"
}
<|endoftext|>


<commit_before>// Copyright 2019 NetApp, Inc. All Rights Reserved.

package cmd

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"

	"github.com/spf13/cobra"
	k8s "k8s.io/api/core/v1"

	"github.com/netapp/trident/cli/api"
	"github.com/netapp/trident/config"
)

const (
	FormatJSON = "json"
	FormatName = "name"
	FormatWide = "wide"
	FormatYAML = "yaml"

	ModeDirect  = "direct"
	ModeTunnel  = "tunnel"
	ModeInstall = "install"

	CLIKubernetes = "kubectl"
	CLIOpenshift  = "oc"

	PodServer = "127.0.0.1:8000"

	ExitCodeSuccess = 0
	ExitCodeFailure = 1

	TridentLegacyLabelKey   = "app"
	TridentLegacyLabelValue = "trident.netapp.io"
	TridentLegacyLabel      = TridentLegacyLabelKey + "=" + TridentLegacyLabelValue

	TridentCSILabelKey   = "app"
	TridentCSILabelValue = "controller.csi.trident.netapp.io"
	TridentCSILabel      = TridentCSILabelKey + "=" + TridentCSILabelValue

	TridentNodeLabelKey   = "app"
	TridentNodeLabelValue = "node.csi.trident.netapp.io"
	TridentNodeLabel      = TridentNodeLabelKey + "=" + TridentNodeLabelValue

	TridentInstallerLabelKey   = "app"
	TridentInstallerLabelValue = "trident-installer.netapp.io"
	TridentInstallerLabel      = TridentInstallerLabelKey + "=" + TridentInstallerLabelValue

	TridentMigratorLabelKey   = "app"
	TridentMigratorLabelValue = "trident-migrator.netapp.io"
	TridentMigratorLabel      = TridentMigratorLabelKey + "=" + TridentMigratorLabelValue
)

var (
	OperatingMode       string
	KubernetesCLI       string
	TridentPodName      string
	TridentPodNamespace string
	ExitCode            int

	Debug        bool
	Server       string
	OutputFormat string
)

var RootCmd = &cobra.Command{
	SilenceUsage: true,
	Use:          "tridentctl",
	Short:        "A CLI tool for NetApp Trident",
	Long:         `A CLI tool for managing the NetApp Trident external storage provisioner for Kubernetes`,
}

func init() {
	RootCmd.PersistentFlags().BoolVarP(&Debug, "debug", "d", false, "Debug output")
	RootCmd.PersistentFlags().StringVarP(&Server, "server", "s", "", "Address/port of Trident REST interface")
	RootCmd.PersistentFlags().StringVarP(&OutputFormat, "output", "o", "", "Output format. One of json|yaml|name|wide|ps (default)")
	RootCmd.PersistentFlags().StringVarP(&TridentPodNamespace, "namespace", "n", "", "Namespace of Trident deployment")
}

func discoverOperatingMode(_ *cobra.Command) error {

	defer func() {
		if !Debug {
			return
		}

		switch OperatingMode {
		case ModeDirect:
			fmt.Printf("Operating mode = %s, Server = %s\n", OperatingMode, Server)
		case ModeTunnel:
			fmt.Printf("Operating mode = %s, Trident pod = %s, Namespace = %s, CLI = %s\n",
				OperatingMode, TridentPodName, TridentPodNamespace, KubernetesCLI)
		}
	}()

	var err error

	envServer := os.Getenv("TRIDENT_SERVER")

	if Server != "" {

		// Server specified on command line takes precedence
		OperatingMode = ModeDirect
		return nil
	} else if envServer != "" {

		// Consider environment variable next
		Server = envServer
		OperatingMode = ModeDirect
		return nil
	}

	// To work with pods, we need to discover which CLI to invoke
	err = discoverKubernetesCLI()
	if err != nil {
		return err
	}

	// Server not specified, so try tunneling to a pod
	if TridentPodNamespace == "" {
		if TridentPodNamespace, err = getCurrentNamespace(); err != nil {
			return err
		}
	}

	// Find the CSI Trident pod
	if TridentPodName, err = getTridentPod(TridentPodNamespace, TridentCSILabel); err != nil {
		// Fall back to non-CSI Trident pod
		if TridentPodName, err = getTridentPod(TridentPodNamespace, TridentLegacyLabel); err != nil {
			return err
		}
	}

	OperatingMode = ModeTunnel
	Server = PodServer
	return nil
}

func discoverKubernetesCLI() error {

	// Try the OpenShift CLI first
	_, err := exec.Command(CLIOpenshift, "version").Output()
	if GetExitCodeFromError(err) == ExitCodeSuccess {
		KubernetesCLI = CLIOpenshift
		return nil
	}

	// Fall back to the K8S CLI
	_, err = exec.Command(CLIKubernetes, "version").Output()
	if GetExitCodeFromError(err) == ExitCodeSuccess {
		KubernetesCLI = CLIKubernetes
		return nil
	}

	if ee, ok := err.(*exec.ExitError); ok {
		return fmt.Errorf("found the Kubernetes CLI, but it exited with error: %s",
			strings.TrimRight(string(ee.Stderr), "\n"))
	}

	return fmt.Errorf("could not find the Kubernetes CLI: %v", err)
}

// getCurrentNamespace returns the default namespace from service account info
func getCurrentNamespace() (string, error) {

	// Get current namespace from service account info
	cmd := exec.Command(KubernetesCLI, "get", "serviceaccount", "default", "-o=json")
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return "", err
	}
	if err := cmd.Start(); err != nil {
		return "", err
	}

	var serviceAccount k8s.ServiceAccount
	if err := json.NewDecoder(stdout).Decode(&serviceAccount); err != nil {
		return "", err
	}
	if err := cmd.Wait(); err != nil {
		return "", err
	}

	//fmt.Printf("%+v\n", serviceAccount)

	// Get Trident pod name & namespace
	namespace := serviceAccount.ObjectMeta.Namespace

	return namespace, nil
}

// getTridentPod returns the name of the Trident pod in the specified namespace
func getTridentPod(namespace, appLabel string) (string, error) {

	// Get 'trident' pod info
	cmd := exec.Command(
		KubernetesCLI,
		"get", "pod",
		"-n", namespace,
		"-l", appLabel,
		"-o=json",
		"--field-selector=status.phase=Running",
	)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return "", err
	}
	if err = cmd.Start(); err != nil {
		return "", err
	}

	var tridentPod k8s.PodList
	if err = json.NewDecoder(stdout).Decode(&tridentPod); err != nil {
		return "", err
	}
	if err = cmd.Wait(); err != nil {
		return "", err
	}

	if len(tridentPod.Items) != 1 {
		return "", fmt.Errorf("could not find a Trident pod in the %s namespace. "+
			"You may need to use the -n option to specify the correct namespace", namespace)
	}

	// Get Trident pod name & namespace
	name := tridentPod.Items[0].ObjectMeta.Name

	return name, nil
}

// listTridentSidecars returns a list of sidecar container names inside the trident controller pod
func listTridentSidecars(podName, podNameSpace string) ([]string, error) {
	// Get 'trident' pod info
	var sidecarNames []string
	cmd := exec.Command(
		KubernetesCLI,
		"get", "pod",
		podName,
		"-n", podNameSpace,
		"-o=json",
	)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return sidecarNames, err
	}
	if err = cmd.Start(); err != nil {
		return sidecarNames, err
	}

	var tridentPod k8s.Pod
	if err = json.NewDecoder(stdout).Decode(&tridentPod); err != nil {
		return sidecarNames, err
	}
	if err = cmd.Wait(); err != nil {
		return sidecarNames, err
	}

	for _, sidecar := range tridentPod.Spec.Containers {
		// Ignore Trident's main container
		if sidecar.Name != config.ContainerTrident {
			sidecarNames = append(sidecarNames, sidecar.Name)
		}
	}

	return sidecarNames, nil
}

func getTridentNode(nodeName, namespace string) (string, error) {
	selector := fmt.Sprintf("--field-selector=spec.nodeName=%s", nodeName)
	cmd := exec.Command(
		KubernetesCLI,
		"get", "pod",
		"-n", namespace,
		"-l", TridentNodeLabel,
		"-o=json",
		selector,
	)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return "", err
	}
	if err = cmd.Start(); err != nil {
		return "", err
	}

	var tridentPods k8s.PodList
	if err = json.NewDecoder(stdout).Decode(&tridentPods); err != nil {
		return "", err
	}
	if err = cmd.Wait(); err != nil {
		return "", err
	}

	if len(tridentPods.Items) != 1 {
		return "", fmt.Errorf("could not find a Trident node pod in the %s namespace on node %s. "+
			"You may need to use the -n option to specify the correct namespace", namespace, nodeName)
	}

	// Get Trident node pod name
	name := tridentPods.Items[0].ObjectMeta.Name

	return name, nil
}

// listTridentNodes returns a list of names of the Trident node pods in the specified namespace
func listTridentNodes(namespace string) (map[string]string, error) {
	// Get trident node pods info
	tridentNodes := make(map[string]string)
	cmd := exec.Command(
		KubernetesCLI,
		"get", "pod",
		"-n", namespace,
		"-l", TridentNodeLabel,
		"-o=json",
		"--field-selector=status.phase=Running",
	)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return tridentNodes, err
	}
	if err = cmd.Start(); err != nil {
		return tridentNodes, err
	}

	var tridentPods k8s.PodList
	if err = json.NewDecoder(stdout).Decode(&tridentPods); err != nil {
		return tridentNodes, err
	}
	if err = cmd.Wait(); err != nil {
		return tridentNodes, err
	}

	if len(tridentPods.Items) < 1 {
		return tridentNodes, fmt.Errorf("could not find any Trident node pods in the %s namespace. "+
			"You may need to use the -n option to specify the correct namespace", namespace)
	}

	// Get Trident node and pod name
	for _, pod := range tridentPods.Items {
		tridentNodes[pod.Spec.NodeName] = pod.Name
	}

	return tridentNodes, nil
}

func BaseURL() string {

	url := fmt.Sprintf("http://%s%s", Server, config.BaseURL)

	if Debug {
		fmt.Printf("Trident URL: %s\n", url)
	}

	return url
}

func TunnelCommand(commandArgs []string) {

	// Build tunnel command to exec command in container
	execCommand := []string{"exec", TridentPodName, "-n", TridentPodNamespace, "-c", config.ContainerTrident, "--"}

	// Build CLI command
	cliCommand := []string{"tridentctl"}
	if Debug {
		cliCommand = append(cliCommand, "--debug")
	}
	if OutputFormat != "" {
		cliCommand = append(cliCommand, []string{"--output", OutputFormat}...)
	}
	cliCommand = append(cliCommand, commandArgs...)

	// Combine tunnel and CLI commands
	execCommand = append(execCommand, cliCommand...)

	if Debug {
		fmt.Printf("Invoking tunneled command: %s %v\n", KubernetesCLI, strings.Join(execCommand, " "))
	}

	// Invoke tridentctl inside the Trident pod
	out, err := exec.Command(KubernetesCLI, execCommand...).CombinedOutput()

	SetExitCodeFromError(err)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s", string(out))
	} else {
		fmt.Print(string(out))
	}
}

func TunnelCommandRaw(commandArgs []string) ([]byte, error) {

	// Build tunnel command to exec command in container
	execCommand := []string{"exec", TridentPodName, "-n", TridentPodNamespace, "-c", config.ContainerTrident, "--"}

	// Build CLI command
	cliCommand := []string{"tridentctl"}
	cliCommand = append(cliCommand, commandArgs...)

	// Combine tunnel and CLI commands
	execCommand = append(execCommand, cliCommand...)

	if Debug {
		fmt.Printf("Invoking tunneled command: %s %v\n", KubernetesCLI, strings.Join(execCommand, " "))
	}

	// Invoke tridentctl inside the Trident pod
	output, err := exec.Command(KubernetesCLI, execCommand...).CombinedOutput()

	SetExitCodeFromError(err)
	return output, err
}

func GetErrorFromHTTPResponse(response *http.Response, responseBody []byte) error {

	var errorResponse api.ErrorResponse
	if err := json.Unmarshal(responseBody, &errorResponse); err == nil {
		return fmt.Errorf("%s (%s)", errorResponse.Error, response.Status)
	}
	return errors.New(response.Status)
}

func SetExitCodeFromError(err error) {
	ExitCode = GetExitCodeFromError(err)
}

func GetExitCodeFromError(err error) int {
	if err == nil {
		return ExitCodeSuccess
	} else {

		// Default to 1 in case we can't determine a process exit code
		code := ExitCodeFailure

		if exitError, ok := err.(*exec.ExitError); ok {
			ws := exitError.Sys().(syscall.WaitStatus)
			code = ws.ExitStatus()
		}

		return code
	}
}

func getUserConfirmation(s string) (bool, error) {

	reader := bufio.NewReader(os.Stdin)

	for {
		fmt.Printf("%s [y/n]: ", s)

		input, err := reader.ReadString('\n')
		if err != nil {
			return false, err
		}

		input = strings.ToLower(strings.TrimSpace(input))

		if input == "y" || input == "yes" {
			return true, nil
		} else if input == "n" || input == "no" {
			return false, nil
		}
	}
}

func homeDir() string {
	if h := os.Getenv("HOME"); h != "" {
		return h
	}
	return os.Getenv("USERPROFILE")
}

func kubeConfigPath() string {

	// If KUBECONFIG contains multiple paths, return the first one.
	if paths := os.Getenv("KUBECONFIG"); paths != "" {
		for _, path := range strings.Split(paths, ":") {
			if len(path) > 0 {
				return path
			}
		}
	}

	if home := homeDir(); home != "" {
		return filepath.Join(home, ".kube", "config")
	}

	return ""
}
<commit_msg>Fix no Auth Provider found error (#256)<commit_after>// Copyright 2019 NetApp, Inc. All Rights Reserved.

package cmd

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"

	"github.com/spf13/cobra"
	k8s "k8s.io/api/core/v1"

	"github.com/netapp/trident/cli/api"
	"github.com/netapp/trident/config"

	// Load all auth plugins
	_ "k8s.io/client-go/plugin/pkg/client/auth"
)

const (
	FormatJSON = "json"
	FormatName = "name"
	FormatWide = "wide"
	FormatYAML = "yaml"

	ModeDirect  = "direct"
	ModeTunnel  = "tunnel"
	ModeInstall = "install"

	CLIKubernetes = "kubectl"
	CLIOpenshift  = "oc"

	PodServer = "127.0.0.1:8000"

	ExitCodeSuccess = 0
	ExitCodeFailure = 1

	TridentLegacyLabelKey   = "app"
	TridentLegacyLabelValue = "trident.netapp.io"
	TridentLegacyLabel      = TridentLegacyLabelKey + "=" + TridentLegacyLabelValue

	TridentCSILabelKey   = "app"
	TridentCSILabelValue = "controller.csi.trident.netapp.io"
	TridentCSILabel      = TridentCSILabelKey + "=" + TridentCSILabelValue

	TridentNodeLabelKey   = "app"
	TridentNodeLabelValue = "node.csi.trident.netapp.io"
	TridentNodeLabel      = TridentNodeLabelKey + "=" + TridentNodeLabelValue

	TridentInstallerLabelKey   = "app"
	TridentInstallerLabelValue = "trident-installer.netapp.io"
	TridentInstallerLabel      = TridentInstallerLabelKey + "=" + TridentInstallerLabelValue

	TridentMigratorLabelKey   = "app"
	TridentMigratorLabelValue = "trident-migrator.netapp.io"
	TridentMigratorLabel      = TridentMigratorLabelKey + "=" + TridentMigratorLabelValue
)

var (
	OperatingMode       string
	KubernetesCLI       string
	TridentPodName      string
	TridentPodNamespace string
	ExitCode            int

	Debug        bool
	Server       string
	OutputFormat string
)

var RootCmd = &cobra.Command{
	SilenceUsage: true,
	Use:          "tridentctl",
	Short:        "A CLI tool for NetApp Trident",
	Long:         `A CLI tool for managing the NetApp Trident external storage provisioner for Kubernetes`,
}

func init() {
	RootCmd.PersistentFlags().BoolVarP(&Debug, "debug", "d", false, "Debug output")
	RootCmd.PersistentFlags().StringVarP(&Server, "server", "s", "", "Address/port of Trident REST interface")
	RootCmd.PersistentFlags().StringVarP(&OutputFormat, "output", "o", "", "Output format. One of json|yaml|name|wide|ps (default)")
	RootCmd.PersistentFlags().StringVarP(&TridentPodNamespace, "namespace", "n", "", "Namespace of Trident deployment")
}

func discoverOperatingMode(_ *cobra.Command) error {

	defer func() {
		if !Debug {
			return
		}

		switch OperatingMode {
		case ModeDirect:
			fmt.Printf("Operating mode = %s, Server = %s\n", OperatingMode, Server)
		case ModeTunnel:
			fmt.Printf("Operating mode = %s, Trident pod = %s, Namespace = %s, CLI = %s\n",
				OperatingMode, TridentPodName, TridentPodNamespace, KubernetesCLI)
		}
	}()

	var err error

	envServer := os.Getenv("TRIDENT_SERVER")

	if Server != "" {

		// Server specified on command line takes precedence
		OperatingMode = ModeDirect
		return nil
	} else if envServer != "" {

		// Consider environment variable next
		Server = envServer
		OperatingMode = ModeDirect
		return nil
	}

	// To work with pods, we need to discover which CLI to invoke
	err = discoverKubernetesCLI()
	if err != nil {
		return err
	}

	// Server not specified, so try tunneling to a pod
	if TridentPodNamespace == "" {
		if TridentPodNamespace, err = getCurrentNamespace(); err != nil {
			return err
		}
	}

	// Find the CSI Trident pod
	if TridentPodName, err = getTridentPod(TridentPodNamespace, TridentCSILabel); err != nil {
		// Fall back to non-CSI Trident pod
		if TridentPodName, err = getTridentPod(TridentPodNamespace, TridentLegacyLabel); err != nil {
			return err
		}
	}

	OperatingMode = ModeTunnel
	Server = PodServer
	return nil
}

func discoverKubernetesCLI() error {

	// Try the OpenShift CLI first
	_, err := exec.Command(CLIOpenshift, "version").Output()
	if GetExitCodeFromError(err) == ExitCodeSuccess {
		KubernetesCLI = CLIOpenshift
		return nil
	}

	// Fall back to the K8S CLI
	_, err = exec.Command(CLIKubernetes, "version").Output()
	if GetExitCodeFromError(err) == ExitCodeSuccess {
		KubernetesCLI = CLIKubernetes
		return nil
	}

	if ee, ok := err.(*exec.ExitError); ok {
		return fmt.Errorf("found the Kubernetes CLI, but it exited with error: %s",
			strings.TrimRight(string(ee.Stderr), "\n"))
	}

	return fmt.Errorf("could not find the Kubernetes CLI: %v", err)
}

// getCurrentNamespace returns the default namespace from service account info
func getCurrentNamespace() (string, error) {

	// Get current namespace from service account info
	cmd := exec.Command(KubernetesCLI, "get", "serviceaccount", "default", "-o=json")
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return "", err
	}
	if err := cmd.Start(); err != nil {
		return "", err
	}

	var serviceAccount k8s.ServiceAccount
	if err := json.NewDecoder(stdout).Decode(&serviceAccount); err != nil {
		return "", err
	}
	if err := cmd.Wait(); err != nil {
		return "", err
	}

	//fmt.Printf("%+v\n", serviceAccount)

	// Get Trident pod name & namespace
	namespace := serviceAccount.ObjectMeta.Namespace

	return namespace, nil
}

// getTridentPod returns the name of the Trident pod in the specified namespace
func getTridentPod(namespace, appLabel string) (string, error) {

	// Get 'trident' pod info
	cmd := exec.Command(
		KubernetesCLI,
		"get", "pod",
		"-n", namespace,
		"-l", appLabel,
		"-o=json",
		"--field-selector=status.phase=Running",
	)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return "", err
	}
	if err = cmd.Start(); err != nil {
		return "", err
	}

	var tridentPod k8s.PodList
	if err = json.NewDecoder(stdout).Decode(&tridentPod); err != nil {
		return "", err
	}
	if err = cmd.Wait(); err != nil {
		return "", err
	}

	if len(tridentPod.Items) != 1 {
		return "", fmt.Errorf("could not find a Trident pod in the %s namespace. "+
			"You may need to use the -n option to specify the correct namespace", namespace)
	}

	// Get Trident pod name & namespace
	name := tridentPod.Items[0].ObjectMeta.Name

	return name, nil
}

// listTridentSidecars returns a list of sidecar container names inside the trident controller pod
func listTridentSidecars(podName, podNameSpace string) ([]string, error) {
	// Get 'trident' pod info
	var sidecarNames []string
	cmd := exec.Command(
		KubernetesCLI,
		"get", "pod",
		podName,
		"-n", podNameSpace,
		"-o=json",
	)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return sidecarNames, err
	}
	if err = cmd.Start(); err != nil {
		return sidecarNames, err
	}

	var tridentPod k8s.Pod
	if err = json.NewDecoder(stdout).Decode(&tridentPod); err != nil {
		return sidecarNames, err
	}
	if err = cmd.Wait(); err != nil {
		return sidecarNames, err
	}

	for _, sidecar := range tridentPod.Spec.Containers {
		// Ignore Trident's main container
		if sidecar.Name != config.ContainerTrident {
			sidecarNames = append(sidecarNames, sidecar.Name)
		}
	}

	return sidecarNames, nil
}

func getTridentNode(nodeName, namespace string) (string, error) {
	selector := fmt.Sprintf("--field-selector=spec.nodeName=%s", nodeName)
	cmd := exec.Command(
		KubernetesCLI,
		"get", "pod",
		"-n", namespace,
		"-l", TridentNodeLabel,
		"-o=json",
		selector,
	)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return "", err
	}
	if err = cmd.Start(); err != nil {
		return "", err
	}

	var tridentPods k8s.PodList
	if err = json.NewDecoder(stdout).Decode(&tridentPods); err != nil {
		return "", err
	}
	if err = cmd.Wait(); err != nil {
		return "", err
	}

	if len(tridentPods.Items) != 1 {
		return "", fmt.Errorf("could not find a Trident node pod in the %s namespace on node %s. "+
			"You may need to use the -n option to specify the correct namespace", namespace, nodeName)
	}

	// Get Trident node pod name
	name := tridentPods.Items[0].ObjectMeta.Name

	return name, nil
}

// listTridentNodes returns a list of names of the Trident node pods in the specified namespace
func listTridentNodes(namespace string) (map[string]string, error) {
	// Get trident node pods info
	tridentNodes := make(map[string]string)
	cmd := exec.Command(
		KubernetesCLI,
		"get", "pod",
		"-n", namespace,
		"-l", TridentNodeLabel,
		"-o=json",
		"--field-selector=status.phase=Running",
	)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return tridentNodes, err
	}
	if err = cmd.Start(); err != nil {
		return tridentNodes, err
	}

	var tridentPods k8s.PodList
	if err = json.NewDecoder(stdout).Decode(&tridentPods); err != nil {
		return tridentNodes, err
	}
	if err = cmd.Wait(); err != nil {
		return tridentNodes, err
	}

	if len(tridentPods.Items) < 1 {
		return tridentNodes, fmt.Errorf("could not find any Trident node pods in the %s namespace. "+
			"You may need to use the -n option to specify the correct namespace", namespace)
	}

	// Get Trident node and pod name
	for _, pod := range tridentPods.Items {
		tridentNodes[pod.Spec.NodeName] = pod.Name
	}

	return tridentNodes, nil
}

func BaseURL() string {

	url := fmt.Sprintf("http://%s%s", Server, config.BaseURL)

	if Debug {
		fmt.Printf("Trident URL: %s\n", url)
	}

	return url
}

func TunnelCommand(commandArgs []string) {

	// Build tunnel command to exec command in container
	execCommand := []string{"exec", TridentPodName, "-n", TridentPodNamespace, "-c", config.ContainerTrident, "--"}

	// Build CLI command
	cliCommand := []string{"tridentctl"}
	if Debug {
		cliCommand = append(cliCommand, "--debug")
	}
	if OutputFormat != "" {
		cliCommand = append(cliCommand, []string{"--output", OutputFormat}...)
	}
	cliCommand = append(cliCommand, commandArgs...)

	// Combine tunnel and CLI commands
	execCommand = append(execCommand, cliCommand...)

	if Debug {
		fmt.Printf("Invoking tunneled command: %s %v\n", KubernetesCLI, strings.Join(execCommand, " "))
	}

	// Invoke tridentctl inside the Trident pod
	out, err := exec.Command(KubernetesCLI, execCommand...).CombinedOutput()

	SetExitCodeFromError(err)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s", string(out))
	} else {
		fmt.Print(string(out))
	}
}

func TunnelCommandRaw(commandArgs []string) ([]byte, error) {

	// Build tunnel command to exec command in container
	execCommand := []string{"exec", TridentPodName, "-n", TridentPodNamespace, "-c", config.ContainerTrident, "--"}

	// Build CLI command
	cliCommand := []string{"tridentctl"}
	cliCommand = append(cliCommand, commandArgs...)

	// Combine tunnel and CLI commands
	execCommand = append(execCommand, cliCommand...)

	if Debug {
		fmt.Printf("Invoking tunneled command: %s %v\n", KubernetesCLI, strings.Join(execCommand, " "))
	}

	// Invoke tridentctl inside the Trident pod
	output, err := exec.Command(KubernetesCLI, execCommand...).CombinedOutput()

	SetExitCodeFromError(err)
	return output, err
}

func GetErrorFromHTTPResponse(response *http.Response, responseBody []byte) error {

	var errorResponse api.ErrorResponse
	if err := json.Unmarshal(responseBody, &errorResponse); err == nil {
		return fmt.Errorf("%s (%s)", errorResponse.Error, response.Status)
	}
	return errors.New(response.Status)
}

func SetExitCodeFromError(err error) {
	ExitCode = GetExitCodeFromError(err)
}

func GetExitCodeFromError(err error) int {
	if err == nil {
		return ExitCodeSuccess
	} else {

		// Default to 1 in case we can't determine a process exit code
		code := ExitCodeFailure

		if exitError, ok := err.(*exec.ExitError); ok {
			ws := exitError.Sys().(syscall.WaitStatus)
			code = ws.ExitStatus()
		}

		return code
	}
}

func getUserConfirmation(s string) (bool, error) {

	reader := bufio.NewReader(os.Stdin)

	for {
		fmt.Printf("%s [y/n]: ", s)

		input, err := reader.ReadString('\n')
		if err != nil {
			return false, err
		}

		input = strings.ToLower(strings.TrimSpace(input))

		if input == "y" || input == "yes" {
			return true, nil
		} else if input == "n" || input == "no" {
			return false, nil
		}
	}
}

func homeDir() string {
	if h := os.Getenv("HOME"); h != "" {
		return h
	}
	return os.Getenv("USERPROFILE")
}

func kubeConfigPath() string {

	// If KUBECONFIG contains multiple paths, return the first one.
	if paths := os.Getenv("KUBECONFIG"); paths != "" {
		for _, path := range strings.Split(paths, ":") {
			if len(path) > 0 {
				return path
			}
		}
	}

	if home := homeDir(); home != "" {
		return filepath.Join(home, ".kube", "config")
	}

	return ""
}
<|endoftext|>


<commit_before>// +build gofuzz
package filewritecache

import (
	"bytes"
	"encoding/binary"

	"github.com/nyaxt/otaru/blobstore"
	"github.com/nyaxt/otaru/filewritecache"
	"github.com/nyaxt/otaru/flags"
	"github.com/nyaxt/otaru/logger"
	tu "github.com/nyaxt/otaru/testutils"
	"github.com/nyaxt/otaru/util"
)

var mylog = logger.Registry().Category("otaru-fuzz-filewritecache")

func init() { tu.EnsureLogger() }

const (
	InvalidInput     = -1
	NeutralInput     = 0
	InterestingInput = 1
)

type ReadAtAdaptor struct {
	bh blobstore.BlobHandle
}

func (a ReadAtAdaptor) ReadAt(p []byte, offset int64) (int, error) {
	n := len(p)

	currLen := a.bh.Size()
	if offset+int64(len(p)) > currLen {
		zoff := util.Int64Max(currLen-offset, 0)
		logger.Debugf(mylog, "offset: %d, len(p): %d, currLen: %d, zoff: %d", offset, len(p), currLen, zoff)
		z := p[zoff:]
		for i, _ := range z {
			z[i] = 0
		}
		p = p[:zoff]
	}
	if len(p) == 0 {
		return n, nil
	}

	if err := a.bh.PRead(p, offset); err != nil {
		return 0, err
	}
	return n, nil
}

type cmdpack struct {
	IsWrite uint8
	Offset  uint32
	OpLen   uint32
}

func Fuzz(data []byte) int {
	filewritecache.MaxPatches = 4
	filewritecache.MaxPatchContentLen = 16
	const AbsoluteMaxLen uint32 = 32

	bs := blobstore.NewMemBlobStore()
	bh, err := bs.Open("hoge", flags.O_RDWRCREATE)
	if err != nil {
		panic(err)
	}

	wc := filewritecache.New()

	currLen := uint32(0)

	cmdReader := bytes.NewBuffer(data)
	cmdp := cmdpack{}
	wbuf := make([]byte, AbsoluteMaxLen)
	rbuf := make([]byte, AbsoluteMaxLen)
	mirror := make([]byte, AbsoluteMaxLen)
	for n := byte(0); true; n++ {
		if err := binary.Read(cmdReader, binary.BigEndian, &cmdp); err != nil {
			if n < 4 {
				return InvalidInput
			} else {
				return NeutralInput
			}
		}
		logger.Infof(mylog, "Cmd %d", n)

		isWrite := (cmdp.IsWrite & 1) == 1
		if isWrite {
			offset := cmdp.Offset % AbsoluteMaxLen
			opLen := cmdp.OpLen % (AbsoluteMaxLen - offset)

			w := wbuf[:opLen]
			for i, _ := range w {
				w[i] = n
			}
			if err := wc.PWrite(w[:opLen], int64(offset)); err != nil {
				panic(err)
			}
			if wc.NeedsSync() {
				if err := wc.Sync(bh); err != nil {
					panic(err)
				}
			}
			copy(mirror[offset:offset+opLen], w)

			if currLen < offset+opLen {
				currLen = offset + opLen
			}
		} else {
			if currLen == 0 {
				return InvalidInput
			}
			offset := cmdp.Offset % currLen
			maxLen := currLen - offset
			if maxLen == 0 {
				return InvalidInput
			}
			opLen := cmdp.OpLen % maxLen

			r := rbuf[:opLen]
			adaptor := ReadAtAdaptor{bh}
			if _, err := wc.ReadAtThrough(r, int64(offset), adaptor); err != nil {
				panic(err)
			}
			r2 := mirror[offset : offset+opLen]
			if !bytes.Equal(r, r2) {
				logger.Warningf(mylog, "mismatch!!! | wc     := %+v", r)
				logger.Warningf(mylog, "mismatch!!! | mirror := %+v", r2)
				return InterestingInput
			}
		}
	}
	{
		r := rbuf[:currLen]
		adaptor := ReadAtAdaptor{bh}
		if _, err := wc.ReadAtThrough(r, 0, adaptor); err != nil {
			panic(err)
		}
		r2 := mirror[:currLen]
		if !bytes.Equal(r, r2) {
			logger.Warningf(mylog, "mismatch!!! | wc     := %+v", r)
			logger.Warningf(mylog, "mismatch!!! | mirror := %+v", r2)
			return InterestingInput
		}
	}
	return NeutralInput
}
<commit_msg>go-fuzz/filewritecache: Add more debug outputs<commit_after>// +build gofuzz
package filewritecache

import (
	"bytes"
	"encoding/binary"
	"errors"

	"github.com/nyaxt/otaru/blobstore"
	"github.com/nyaxt/otaru/filewritecache"
	"github.com/nyaxt/otaru/flags"
	"github.com/nyaxt/otaru/logger"
	tu "github.com/nyaxt/otaru/testutils"
	"github.com/nyaxt/otaru/util"
)

var mylog = logger.Registry().Category("otaru-fuzz-filewritecache")

func init() { tu.EnsureLogger() }

const (
	InvalidInput     = -1
	NeutralInput     = 0
	InterestingInput = 1
)

type ReadAtAdaptor struct {
	bh blobstore.BlobHandle
}

func (a ReadAtAdaptor) ReadAt(p []byte, offset int64) (int, error) {
	n := len(p)

	currLen := a.bh.Size()
	if offset+int64(len(p)) > currLen {
		zoff := util.Int64Max(currLen-offset, 0)
		logger.Debugf(mylog, "offset: %d, len(p): %d, currLen: %d, zoff: %d", offset, len(p), currLen, zoff)
		z := p[zoff:]
		for i, _ := range z {
			z[i] = 0
		}
		p = p[:zoff]
	}
	if len(p) == 0 {
		return n, nil
	}

	if err := a.bh.PRead(p, offset); err != nil {
		return 0, err
	}
	return n, nil
}

type cmdpack struct {
	IsWrite uint8
	Offset  uint32
	OpLen   uint32
}

func Fuzz(data []byte) int {
	filewritecache.MaxPatches = 4
	filewritecache.MaxPatchContentLen = 16
	const AbsoluteMaxLen uint32 = 32

	bs := blobstore.NewMemBlobStore()
	bh, err := bs.Open("hoge", flags.O_RDWRCREATE)
	if err != nil {
		panic(err)
	}

	wc := filewritecache.New()

	currLen := uint32(0)

	cmdReader := bytes.NewBuffer(data)
	cmdp := cmdpack{}
	wbuf := make([]byte, AbsoluteMaxLen)
	rbuf := make([]byte, AbsoluteMaxLen)
	mirror := make([]byte, AbsoluteMaxLen)
	for n := byte(0); true; n++ {
		if err := binary.Read(cmdReader, binary.BigEndian, &cmdp); err != nil {
			if n < 4 {
				return InvalidInput
			} else {
				return NeutralInput
			}
		}
		logger.Infof(mylog, "Cmd %d %+v", n, cmdp)

		isWrite := (cmdp.IsWrite & 1) == 1
		if isWrite {
			offset := cmdp.Offset % AbsoluteMaxLen
			opLen := cmdp.OpLen % (AbsoluteMaxLen - offset)

			w := wbuf[:opLen]
			for i, _ := range w {
				w[i] = n
			}
			logger.Debugf(mylog, "PWrite offset %d opLen %d currLen %d", offset, opLen, currLen)
			if err := wc.PWrite(w[:opLen], int64(offset)); err != nil {
				panic(err)
			}
			if wc.NeedsSync() {
				logger.Debugf(mylog, "NeedsSync!")
				if err := wc.Sync(bh); err != nil {
					panic(err)
				}
			}
			copy(mirror[offset:offset+opLen], w)

			if currLen < offset+opLen {
				currLen = offset + opLen
			}
		} else {
			if currLen == 0 {
				return InvalidInput
			}
			offset := cmdp.Offset % currLen
			maxLen := currLen - offset
			if maxLen == 0 {
				return InvalidInput
			}
			opLen := cmdp.OpLen % maxLen

			r := rbuf[:opLen]
			adaptor := ReadAtAdaptor{bh}
			logger.Debugf(mylog, "ReadAtThrough offset %d opLen %d currLen %d", offset, opLen, currLen)
			if _, err := wc.ReadAtThrough(r, int64(offset), adaptor); err != nil {
				panic(err)
			}
			r2 := mirror[offset : offset+opLen]
			if !bytes.Equal(r, r2) {
				logger.Warningf(mylog, "mismatch!!! | wc     := %+v", r)
				logger.Warningf(mylog, "mismatch!!! | mirror := %+v", r2)
				panic(errors.New("mismatch"))
			}
		}
	}
	{
		r := rbuf[:currLen]
		adaptor := ReadAtAdaptor{bh}
		if _, err := wc.ReadAtThrough(r, 0, adaptor); err != nil {
			panic(err)
		}
		r2 := mirror[:currLen]
		if !bytes.Equal(r, r2) {
			logger.Warningf(mylog, "mismatch!!! | wc     := %+v", r)
			logger.Warningf(mylog, "mismatch!!! | mirror := %+v", r2)
			panic(errors.New("mismatch"))
		}
	}
	return NeutralInput
}
<|endoftext|>


<commit_before>package kloud

import (
	"encoding/json"
	"errors"
	"fmt"
	"koding/db/models"
	"koding/kites/kloud/contexthelper/request"
	"koding/kites/kloud/contexthelper/session"
	"koding/kites/kloud/provider/generic"
	"koding/kites/kloud/terraformer"
	"strconv"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"

	"golang.org/x/net/context"

	"github.com/koding/kite"
)

func (k *Kloud) Apply(r *kite.Request) (interface{}, error) {
	if r.Args == nil {
		return nil, NewError(ErrNoArguments)
	}

	var args *TerraformKloudRequest
	if err := r.Args.One().Unmarshal(&args); err != nil {
		return nil, err
	}

	if args.TerraformContext == "" {
		return nil, NewError(ErrTerraformContextIsMissing)
	}

	if len(args.PublicKeys) == 0 {
		return nil, errors.New("publicKeys are not passed")
	}

	if len(args.MachineIds) == 0 {
		return nil, errors.New("machine ids are not passed")
	}

	if len(args.MachineIds) != len(args.PublicKeys) {
		return nil, errors.New("machineIds and publicKeys do not match")
	}

	ctx := request.NewContext(context.Background(), r)
	ctx = k.ContextCreator(ctx)

	sess, ok := session.FromContext(ctx)
	if !ok {
		return nil, errors.New("session context is not passed")
	}

	machines, err := fetchMachines(ctx, args.MachineIds...)
	if err != nil {
		return nil, err
	}

	creds, err := fetchCredentials(r.Username, sess.DB, args.PublicKeys)
	if err != nil {
		return nil, err
	}

	tfKite, err := terraformer.Connect(sess.Kite)
	if err != nil {
		return nil, err
	}
	defer tfKite.Close()

	args.TerraformContext = appendVariables(args.TerraformContext, creds)
	state, err := tfKite.Apply(args.TerraformContext)
	if err != nil {
		return nil, err
	}

	region, err := regionFromHCL(args.TerraformContext)
	if err != nil {
		return nil, err
	}

	output, err := machinesFromState(state)
	if err != nil {
		return nil, err
	}
	output.AppendRegion(region)

	if err := updateMachines(ctx, output, machines); err != nil {
		return nil, err
	}

	d, err := json.MarshalIndent(output, "", " ")
	if err != nil {
		return nil, err
	}

	fmt.Printf(string(d))

	return nil, errors.New("not implemented yet")
}

func fetchMachines(ctx context.Context, ids ...string) ([]*generic.Machine, error) {
	sess, ok := session.FromContext(ctx)
	if !ok {
		return nil, errors.New("session context is not passed")
	}

	mongodbIds := make([]bson.ObjectId, len(ids))
	for i, id := range ids {
		mongodbIds[i] = bson.ObjectIdHex(id)
	}

	machines := make([]*generic.Machine, 0)
	if err := sess.DB.Run("jMachines", func(c *mgo.Collection) error {
		return c.Find(bson.M{"_id": bson.M{"$in": mongodbIds}}).All(&machines)
	}); err != nil {
		return nil, err
	}

	allowedIds := make([]bson.ObjectId, len(machines))

	for i, machine := range machines {
		for _, perm := range machine.Users {
			// we only going to fetch users that are allowed
			if perm.Sudo && perm.Owner {
				allowedIds[i] = perm.Id
			} else {
				return nil, fmt.Errorf("machine '%s' is not valid. Aborting apply", machine.Id.Hex())
			}
		}
	}

	var allowedUsers []*models.User
	if err := sess.DB.Run("jUsers", func(c *mgo.Collection) error {
		return c.Find(bson.M{"_id": bson.M{"$in": allowedIds}}).All(&allowedUsers)
	}); err != nil {
		return nil, fmt.Errorf("username lookup error: %v", err)
	}

	req, ok := request.FromContext(ctx)
	if !ok {
		return nil, errors.New("request context is not passed")
	}

	// validate users
	for _, u := range allowedUsers {
		if u.Name != req.Username {
			return nil, fmt.Errorf("machine is only allowed for user: %s. But have: %s", req.Username, u.Name)
		}
	}

	return machines, nil
}

func updateMachines(ctx context.Context, data *Machines, jMachines []*generic.Machine) error {
	sess, ok := session.FromContext(ctx)
	if !ok {
		return errors.New("session context is not passed")
	}

	for _, machine := range jMachines {
		terraformMachine, err := data.Label(machine.Label)
		if err != nil {
			return fmt.Errorf("machine label '%s' doesn't exist in terraform output", machine.Label)
		}

		storageSize, err := strconv.Atoi(terraformMachine.Attributes["root_block_device.0.volume_size"])
		if err != nil {
			return err
		}

		if err := sess.DB.Run("jMachines", func(c *mgo.Collection) error {
			return c.UpdateId(
				machine.Id,
				bson.M{"$set": bson.M{
					"provider":          terraformMachine.Provider,
					"meta.region":       terraformMachine.Region,
					"ipAddress":         terraformMachine.Attributes["public_ip"],
					"meta.instanceId":   terraformMachine.Attributes["id"],
					"meta.instanceType": terraformMachine.Attributes["instance_type"],
					"meta.source_ami":   terraformMachine.Attributes["ami"],
					"meta.storage_size": storageSize,
				}},
			)
		}); err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>kloud/apply: change name to tf<commit_after>package kloud

import (
	"encoding/json"
	"errors"
	"fmt"
	"koding/db/models"
	"koding/kites/kloud/contexthelper/request"
	"koding/kites/kloud/contexthelper/session"
	"koding/kites/kloud/provider/generic"
	"koding/kites/kloud/terraformer"
	"strconv"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"

	"golang.org/x/net/context"

	"github.com/koding/kite"
)

func (k *Kloud) Apply(r *kite.Request) (interface{}, error) {
	if r.Args == nil {
		return nil, NewError(ErrNoArguments)
	}

	var args *TerraformKloudRequest
	if err := r.Args.One().Unmarshal(&args); err != nil {
		return nil, err
	}

	if args.TerraformContext == "" {
		return nil, NewError(ErrTerraformContextIsMissing)
	}

	if len(args.PublicKeys) == 0 {
		return nil, errors.New("publicKeys are not passed")
	}

	if len(args.MachineIds) == 0 {
		return nil, errors.New("machine ids are not passed")
	}

	if len(args.MachineIds) != len(args.PublicKeys) {
		return nil, errors.New("machineIds and publicKeys do not match")
	}

	ctx := request.NewContext(context.Background(), r)
	ctx = k.ContextCreator(ctx)

	sess, ok := session.FromContext(ctx)
	if !ok {
		return nil, errors.New("session context is not passed")
	}

	machines, err := fetchMachines(ctx, args.MachineIds...)
	if err != nil {
		return nil, err
	}

	creds, err := fetchCredentials(r.Username, sess.DB, args.PublicKeys)
	if err != nil {
		return nil, err
	}

	tfKite, err := terraformer.Connect(sess.Kite)
	if err != nil {
		return nil, err
	}
	defer tfKite.Close()

	args.TerraformContext = appendVariables(args.TerraformContext, creds)
	state, err := tfKite.Apply(args.TerraformContext)
	if err != nil {
		return nil, err
	}

	region, err := regionFromHCL(args.TerraformContext)
	if err != nil {
		return nil, err
	}

	output, err := machinesFromState(state)
	if err != nil {
		return nil, err
	}
	output.AppendRegion(region)

	if err := updateMachines(ctx, output, machines); err != nil {
		return nil, err
	}

	d, err := json.MarshalIndent(output, "", " ")
	if err != nil {
		return nil, err
	}

	fmt.Printf(string(d))

	return nil, errors.New("not implemented yet")
}

func fetchMachines(ctx context.Context, ids ...string) ([]*generic.Machine, error) {
	sess, ok := session.FromContext(ctx)
	if !ok {
		return nil, errors.New("session context is not passed")
	}

	mongodbIds := make([]bson.ObjectId, len(ids))
	for i, id := range ids {
		mongodbIds[i] = bson.ObjectIdHex(id)
	}

	machines := make([]*generic.Machine, 0)
	if err := sess.DB.Run("jMachines", func(c *mgo.Collection) error {
		return c.Find(bson.M{"_id": bson.M{"$in": mongodbIds}}).All(&machines)
	}); err != nil {
		return nil, err
	}

	allowedIds := make([]bson.ObjectId, len(machines))

	for i, machine := range machines {
		for _, perm := range machine.Users {
			// we only going to fetch users that are allowed
			if perm.Sudo && perm.Owner {
				allowedIds[i] = perm.Id
			} else {
				return nil, fmt.Errorf("machine '%s' is not valid. Aborting apply", machine.Id.Hex())
			}
		}
	}

	var allowedUsers []*models.User
	if err := sess.DB.Run("jUsers", func(c *mgo.Collection) error {
		return c.Find(bson.M{"_id": bson.M{"$in": allowedIds}}).All(&allowedUsers)
	}); err != nil {
		return nil, fmt.Errorf("username lookup error: %v", err)
	}

	req, ok := request.FromContext(ctx)
	if !ok {
		return nil, errors.New("request context is not passed")
	}

	// validate users
	for _, u := range allowedUsers {
		if u.Name != req.Username {
			return nil, fmt.Errorf("machine is only allowed for user: %s. But have: %s", req.Username, u.Name)
		}
	}

	return machines, nil
}

func updateMachines(ctx context.Context, data *Machines, jMachines []*generic.Machine) error {
	sess, ok := session.FromContext(ctx)
	if !ok {
		return errors.New("session context is not passed")
	}

	for _, machine := range jMachines {
		tf, err := data.Label(machine.Label)
		if err != nil {
			return fmt.Errorf("machine label '%s' doesn't exist in terraform output", machine.Label)
		}

		size, err := strconv.Atoi(tf.Attributes["root_block_device.0.volume_size"])
		if err != nil {
			return err
		}

		if err := sess.DB.Run("jMachines", func(c *mgo.Collection) error {
			return c.UpdateId(
				machine.Id,
				bson.M{"$set": bson.M{
					"provider":          tf.Provider,
					"meta.region":       tf.Region,
					"ipAddress":         tf.Attributes["public_ip"],
					"meta.instanceId":   tf.Attributes["id"],
					"meta.instanceType": tf.Attributes["instance_type"],
					"meta.source_ami":   tf.Attributes["ami"],
					"meta.storage_size": size,
				}},
			)
		}); err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// config package contains reused config variables.
package config

import (
	"fmt"
	"net/url"
	"path"
	"runtime"
	"strconv"

	"koding/kites/config"
	"koding/kites/config/configstore"
)

const (
	// Name is the user facing name for this binary. Internally we call it
	// klientctl to avoid confusion.
	Name = "kd"

	// KlientName is the user facing name for klient.
	KlientName = "KD Daemon"

	// SSHDefaultKeyDir is the default directory that stores users ssh key pairs.
	SSHDefaultKeyDir = ".ssh"

	// SSHDefaultKeyName is the default name of the ssh key pair.
	SSHDefaultKeyName = "kd-ssh-key"

	// used in combination with os-specific log paths under _linux and _darwin.
	kdLogName     = "kd.log"
	klientLogName = "klient.log"
)

var environments = map[string]string{
	"production":  "managed",
	"development": "devmanaged",
}

func kd2klient(kdEnv string) string {
	if klientEnv, ok := environments[kdEnv]; ok {
		return klientEnv
	}

	return "devmanaged"
}

var (
	// Version is the current version of klientctl. This number is used
	// by CheckUpdate to determine if current version is behind or equal to latest
	// version on S3 bucket.
	//
	// Version is overwritten during deploy via linker flag.
	Version = "0"

	// Environment is the target channel of klientctl. This value is used
	// to register with Kontrol and to install klient.
	//
	// Environment is overwritten during deploy via linker flag.
	Environment = "development"

	// KiteVersion is the version identifier used to connect to Kontrol.
	KiteVersion = "0.0." + Version

	// Used to send basic error metrics.
	//
	// Injected on build.
	SegmentKey = ""
)

var Konfig = configstore.Read(&config.Environments{
	Env:       Environment,
	KlientEnv: kd2klient(Environment),
})

func dirURL(s, env string) string {
	u, err := url.Parse(s)
	if err != nil {
		panic(err)
	}

	if env == "" {
		u.Path = path.Dir(u.Path)
	} else {
		u.Path = env
	}

	return u.String()
}

func VersionNum() int {
	version, err := strconv.ParseUint(Version, 10, 32)
	if err != nil {
		return 0
	}

	return int(version)
}

func S3Klient(version int, env string) string {
	s3dir := dirURL(Konfig.Endpoints.KlientLatest.Public.String(), kd2klient(env))

	// TODO(rjeczalik): klient uses a URL without $GOOS_$GOARCH suffix for
	// auto-updates. Remove the special case when a redirect is deployed
	// to the suffixed file.
	if runtime.GOOS == "linux" {
		return fmt.Sprintf("%[1]s/%[2]d/klient-0.1.%[2]d.gz", s3dir, version)
	}

	return fmt.Sprintf("%[1]s/%[2]d/klient-0.1.%[2]d.%[3]s_%[4]s.gz",
		s3dir, version, runtime.GOOS, runtime.GOARCH)
}

func S3Klientctl(version int, env string) string {
	return fmt.Sprintf("%s/kd-0.1.%d.%s_%s.gz", dirURL(Konfig.Endpoints.KDLatest.Public.String(), env),
		version, runtime.GOOS, runtime.GOARCH,
	)
}
<commit_msg>kd/config: add Environments<commit_after>// config package contains reused config variables.
package config

import (
	"fmt"
	"net/url"
	"path"
	"runtime"
	"strconv"

	"koding/kites/config"
	"koding/kites/config/configstore"
)

const (
	// Name is the user facing name for this binary. Internally we call it
	// klientctl to avoid confusion.
	Name = "kd"

	// KlientName is the user facing name for klient.
	KlientName = "KD Daemon"

	// SSHDefaultKeyDir is the default directory that stores users ssh key pairs.
	SSHDefaultKeyDir = ".ssh"

	// SSHDefaultKeyName is the default name of the ssh key pair.
	SSHDefaultKeyName = "kd-ssh-key"

	// used in combination with os-specific log paths under _linux and _darwin.
	kdLogName     = "kd.log"
	klientLogName = "klient.log"
)

var environments = map[string]string{
	"production":  "managed",
	"development": "devmanaged",
}

func kd2klient(kdEnv string) string {
	if klientEnv, ok := environments[kdEnv]; ok {
		return klientEnv
	}

	return "devmanaged"
}

var (
	// Version is the current version of klientctl. This number is used
	// by CheckUpdate to determine if current version is behind or equal to latest
	// version on S3 bucket.
	//
	// Version is overwritten during deploy via linker flag.
	Version = "0"

	// Environment is the target channel of klientctl. This value is used
	// to register with Kontrol and to install klient.
	//
	// Environment is overwritten during deploy via linker flag.
	Environment = "development"

	// KiteVersion is the version identifier used to connect to Kontrol.
	KiteVersion = "0.0." + Version

	// Used to send basic error metrics.
	//
	// Injected on build.
	SegmentKey = ""
)

var Environments = &config.Environments{
	Env:       Environment,
	KlientEnv: kd2klient(Environment),
}

var Konfig = configstore.Read(Environments)

func dirURL(s, env string) string {
	u, err := url.Parse(s)
	if err != nil {
		panic(err)
	}

	if env == "" {
		u.Path = path.Dir(u.Path)
	} else {
		u.Path = env
	}

	return u.String()
}

func VersionNum() int {
	version, err := strconv.ParseUint(Version, 10, 32)
	if err != nil {
		return 0
	}

	return int(version)
}

func S3Klient(version int, env string) string {
	s3dir := dirURL(Konfig.Endpoints.KlientLatest.Public.String(), kd2klient(env))

	// TODO(rjeczalik): klient uses a URL without $GOOS_$GOARCH suffix for
	// auto-updates. Remove the special case when a redirect is deployed
	// to the suffixed file.
	if runtime.GOOS == "linux" {
		return fmt.Sprintf("%[1]s/%[2]d/klient-0.1.%[2]d.gz", s3dir, version)
	}

	return fmt.Sprintf("%[1]s/%[2]d/klient-0.1.%[2]d.%[3]s_%[4]s.gz",
		s3dir, version, runtime.GOOS, runtime.GOARCH)
}

func S3Klientctl(version int, env string) string {
	return fmt.Sprintf("%s/kd-0.1.%d.%s_%s.gz", dirURL(Konfig.Endpoints.KDLatest.Public.String(), env),
		version, runtime.GOOS, runtime.GOARCH,
	)
}
<|endoftext|>