<commit_before>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// Package tailer provides a class that is responsible for tailing a log file
// and extracting new log lines to be passed into the virtual machines.
package tailer

// mtail gets notified on modifications (i.e. appends) to log files that are
// being watched, in order to read the new lines. Log files can also be
// rotated, so mtail is also notified of creates in the log file directory.

import (
	"expvar"
	"io"
	"os"
	"path"
	"path/filepath"
	"sync"
	"syscall"
	"unicode/utf8"

	"github.com/golang/glog"

	"github.com/google/mtail/watcher"

	"github.com/jaqx0r/afero"
)

var (
	logCount     = expvar.NewInt("log_count")
	logErrors    = expvar.NewMap("log_errors_total")
	logRotations = expvar.NewMap("log_rotations_total")
)

// Tailer receives notification of changes from a Watcher and extracts new log
// lines from files. It also handles new log file creation events and log
// rotations.
type Tailer struct {
	w watcher.Watcher

	watched     map[string]struct{}   // Names of logs being watched.
	watchedLock sync.RWMutex          // protects `watched'
	lines       chan<- string         // Logfile lines being emitted.
	files       map[string]afero.File // File handles for each pathname.
	filesLock   sync.Mutex            // protects `files'
	partials    map[string]string     // Accumulator for the currently read line for each pathname.

	fs afero.Fs // mockable filesystem interface
}

// Options configures a Tailer
type Options struct {
	Lines chan<- string
	W     watcher.Watcher // not required, will use a watcher.LogWatcher if it is zero.
	FS    afero.Fs        // not required, will use afero.OsFs{} if it is zero.
}

// New returns a new Tailer, configured with the supplied Options
func New(o Options) *Tailer {
	fs := o.FS
	if fs == nil {
		fs = &afero.OsFs{}
	}
	w := o.W
	if w == nil {
		var err error
		w, err = watcher.NewLogWatcher()
		if err != nil {
			glog.Fatalf("Couldn't create a watcher for tailer: %s", err)
		}
	}
	t := &Tailer{
		w:        w,
		watched:  make(map[string]struct{}),
		lines:    o.Lines,
		files:    make(map[string]afero.File),
		partials: make(map[string]string),
		fs:       fs,
	}
	go t.run()
	return t
}

func (t *Tailer) addWatched(path string) {
	t.watchedLock.Lock()
	defer t.watchedLock.Unlock()
	t.watched[path] = struct{}{}
}

func (t *Tailer) isWatching(path string) bool {
	t.watchedLock.RLock()
	defer t.watchedLock.RUnlock()
	_, ok := t.watched[path]
	return ok
}

// Tail registers a file to be tailed.
func (t *Tailer) Tail(pathname string) {
	fullpath, err := filepath.Abs(pathname)
	if err != nil {
		glog.Infof("Failed to find absolute path for %q: %s\n", pathname, err)
		return
	}
	if !t.isWatching(fullpath) {
		t.addWatched(fullpath)
		logCount.Add(1)
		t.openLogFile(fullpath, false)
	}
}

// handleLogUpdate reads all available bytes from an already opened file
// identified by pathname, and sends them to be processed on the lines channel.
func (t *Tailer) handleLogUpdate(pathname string) {
	for {
		b := make([]byte, 4096)
		f, ok := t.files[pathname]
		if !ok {
			glog.Infof("No file found for %q", pathname)
			return
		}
		n, err := f.Read(b)
		if err != nil {
			if err == io.EOF && n == 0 {
				// end of file for now, return
				return
			}
			glog.Infof("Failed to read updates from %q: %s", pathname, err)
			return // TODO(jaq): handle this path better
		}
		for i, width := 0, 0; i < len(b) && i < n; i += width {
			var rune rune
			rune, width = utf8.DecodeRune(b[i:])
			switch {
			case rune != '\n':
				t.partials[pathname] += string(rune)
			default:
				// send off line for processing
				t.lines <- t.partials[pathname]
				// reset accumulator
				t.partials[pathname] = ""
			}
		}
	}
}

// inode returns the inode number of a file, or 0 if the file has no underlying Sys implementation.
func inode(f os.FileInfo) uint64 {
	s := f.Sys()
	if s == nil {
		return 0
	}
	switch s := s.(type) {
	case *syscall.Stat_t:
		return s.Ino
	default:
		return 0
	}
}

// handleLogCreate handles both new and rotated log files.
func (t *Tailer) handleLogCreate(pathname string) {
	if !t.isWatching(pathname) {
		glog.V(1).Info("Not watching path %q, ignoring.\n", pathname)
		return
	}

	t.filesLock.Lock()
	fd, ok := t.files[pathname]
	t.filesLock.Unlock()
	if ok {
		s1, err := fd.Stat()
		if err != nil {
			glog.Infof("Stat failed on %q: %s", t.files[pathname].Name(), err)
			return
		}
		s2, err := t.fs.Stat(pathname)
		if err != nil {
			glog.Infof("Stat failed on %q: %s", pathname, err)
			return
		}
		if inode(s1) != inode(s2) {
			logRotations.Add(pathname, 1)
			// flush the old log, pathname is still an index into t.files with the old inode.
			t.handleLogUpdate(pathname)
			fd.Close()
			err := t.w.Remove(pathname)
			if err != nil {
				glog.Info("Failed removing watches on", pathname)
			}
			// Always seek to start on log rotation.
			glog.Infof("Seek to start on %s", pathname)
			t.openLogFile(pathname, true)
		} else {
			glog.Infof("Path %s already being watched, and inode not changed.",
				pathname)
		}
	} else {
		// Freshly opened log file, never seen before.
		t.openLogFile(pathname, true)
	}
}

// openLogFile opens a new log file at pathname, and optionally seeks to the
// start or end of the file. Rotated logs should start at the start, but logs
// opened for the first time start at the end.
func (t *Tailer) openLogFile(pathname string, seekStart bool) {
	if !t.isWatching(pathname) {
		glog.Infof("Not watching %q, ignoring.", pathname)
		return
	}

	d := path.Dir(pathname)
	if !t.isWatching(d) {
		err := t.w.Add(d)
		if err != nil {
			glog.Infof("Adding a create watch failed on %q: %s", d, err)
		}
		t.addWatched(d)
	}

	fd, err := t.fs.Open(pathname)
	if err != nil {
		// Doesn't exist yet. We're watching the directory, so we'll pick it up
		// again on create; return successfully.
		if os.IsNotExist(err) {
			return
		}
		glog.Infof("Failed to open %q for reading: %s", pathname, err)
		logErrors.Add(pathname, 1)
		return
	}
	t.filesLock.Lock()
	t.files[pathname] = fd

	if seekStart {
		t.files[pathname].Seek(0, os.SEEK_SET)
	} else {
		t.files[pathname].Seek(0, os.SEEK_END)
	}

	t.filesLock.Unlock()

	err = t.w.Add(pathname)
	if err != nil {
		glog.Infof("Adding a change watch failed on %q: %s", pathname, err)
	}

	glog.Infof("Tailing %s", pathname)

	// In case the new log has been written to already, attempt to read the first lines.
	t.handleLogUpdate(pathname)
}

// start is the main event loop for the Tailer.
// It receives notification of log file changes from the watcher channel, and
// handles them.
func (t *Tailer) run() {
	for e := range t.w.Events() {
		switch e := e.(type) {
		case watcher.UpdateEvent:
			t.handleLogUpdate(e.Pathname)
		case watcher.CreateEvent:
			t.handleLogCreate(e.Pathname)
		default:
			glog.Infof("Unexpected event %q", e)
		}
	}
	glog.Infof("Shutting down tailer.")
	close(t.lines)
}

// Close signals termination to the watcher.
func (t *Tailer) Close() {
	t.w.Close()
}
<commit_msg>Update comments for tailer options.<commit_after>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// Package tailer provides a class that is responsible for tailing a log file
// and extracting new log lines to be passed into the virtual machines.
package tailer

// mtail gets notified on modifications (i.e. appends) to log files that are
// being watched, in order to read the new lines. Log files can also be
// rotated, so mtail is also notified of creates in the log file directory.

import (
	"expvar"
	"io"
	"os"
	"path"
	"path/filepath"
	"sync"
	"syscall"
	"unicode/utf8"

	"github.com/golang/glog"

	"github.com/google/mtail/watcher"

	"github.com/jaqx0r/afero"
)

var (
	logCount     = expvar.NewInt("log_count")
	logErrors    = expvar.NewMap("log_errors_total")
	logRotations = expvar.NewMap("log_rotations_total")
)

// Tailer receives notification of changes from a Watcher and extracts new log
// lines from files. It also handles new log file creation events and log
// rotations.
type Tailer struct {
	w watcher.Watcher

	watched     map[string]struct{}   // Names of logs being watched.
	watchedLock sync.RWMutex          // protects `watched'
	lines       chan<- string         // Logfile lines being emitted.
	files       map[string]afero.File // File handles for each pathname.
	filesLock   sync.Mutex            // protects `files'
	partials    map[string]string     // Accumulator for the currently read line for each pathname.

	fs afero.Fs // mockable filesystem interface
}

// Options configures a Tailer
type Options struct {
	Lines chan<- string
	W     watcher.Watcher // Not required, will use watcher.LogWatcher if it is zero.
	FS    afero.Fs        // Not required, will use afero.OsFs if it is zero.
}

// New returns a new Tailer, configured with the supplied Options
func New(o Options) *Tailer {
	fs := o.FS
	if fs == nil {
		fs = &afero.OsFs{}
	}
	w := o.W
	if w == nil {
		var err error
		w, err = watcher.NewLogWatcher()
		if err != nil {
			glog.Fatalf("Couldn't create a watcher for tailer: %s", err)
		}
	}
	t := &Tailer{
		w:        w,
		watched:  make(map[string]struct{}),
		lines:    o.Lines,
		files:    make(map[string]afero.File),
		partials: make(map[string]string),
		fs:       fs,
	}
	go t.run()
	return t
}

func (t *Tailer) addWatched(path string) {
	t.watchedLock.Lock()
	defer t.watchedLock.Unlock()
	t.watched[path] = struct{}{}
}

func (t *Tailer) isWatching(path string) bool {
	t.watchedLock.RLock()
	defer t.watchedLock.RUnlock()
	_, ok := t.watched[path]
	return ok
}

// Tail registers a file to be tailed.
func (t *Tailer) Tail(pathname string) {
	fullpath, err := filepath.Abs(pathname)
	if err != nil {
		glog.Infof("Failed to find absolute path for %q: %s\n", pathname, err)
		return
	}
	if !t.isWatching(fullpath) {
		t.addWatched(fullpath)
		logCount.Add(1)
		t.openLogFile(fullpath, false)
	}
}

// handleLogUpdate reads all available bytes from an already opened file
// identified by pathname, and sends them to be processed on the lines channel.
func (t *Tailer) handleLogUpdate(pathname string) {
	for {
		b := make([]byte, 4096)
		f, ok := t.files[pathname]
		if !ok {
			glog.Infof("No file found for %q", pathname)
			return
		}
		n, err := f.Read(b)
		if err != nil {
			if err == io.EOF && n == 0 {
				// end of file for now, return
				return
			}
			glog.Infof("Failed to read updates from %q: %s", pathname, err)
			return // TODO(jaq): handle this path better
		}
		for i, width := 0, 0; i < len(b) && i < n; i += width {
			var rune rune
			rune, width = utf8.DecodeRune(b[i:])
			switch {
			case rune != '\n':
				t.partials[pathname] += string(rune)
			default:
				// send off line for processing
				t.lines <- t.partials[pathname]
				// reset accumulator
				t.partials[pathname] = ""
			}
		}
	}
}

// inode returns the inode number of a file, or 0 if the file has no underlying Sys implementation.
func inode(f os.FileInfo) uint64 {
	s := f.Sys()
	if s == nil {
		return 0
	}
	switch s := s.(type) {
	case *syscall.Stat_t:
		return s.Ino
	default:
		return 0
	}
}

// handleLogCreate handles both new and rotated log files.
func (t *Tailer) handleLogCreate(pathname string) {
	if !t.isWatching(pathname) {
		glog.V(1).Info("Not watching path %q, ignoring.\n", pathname)
		return
	}

	t.filesLock.Lock()
	fd, ok := t.files[pathname]
	t.filesLock.Unlock()
	if ok {
		s1, err := fd.Stat()
		if err != nil {
			glog.Infof("Stat failed on %q: %s", t.files[pathname].Name(), err)
			return
		}
		s2, err := t.fs.Stat(pathname)
		if err != nil {
			glog.Infof("Stat failed on %q: %s", pathname, err)
			return
		}
		if inode(s1) != inode(s2) {
			logRotations.Add(pathname, 1)
			// flush the old log, pathname is still an index into t.files with the old inode.
			t.handleLogUpdate(pathname)
			fd.Close()
			err := t.w.Remove(pathname)
			if err != nil {
				glog.Info("Failed removing watches on", pathname)
			}
			// Always seek to start on log rotation.
			glog.Infof("Seek to start on %s", pathname)
			t.openLogFile(pathname, true)
		} else {
			glog.Infof("Path %s already being watched, and inode not changed.",
				pathname)
		}
	} else {
		// Freshly opened log file, never seen before.
		t.openLogFile(pathname, true)
	}
}

// openLogFile opens a new log file at pathname, and optionally seeks to the
// start or end of the file. Rotated logs should start at the start, but logs
// opened for the first time start at the end.
func (t *Tailer) openLogFile(pathname string, seekStart bool) {
	if !t.isWatching(pathname) {
		glog.Infof("Not watching %q, ignoring.", pathname)
		return
	}

	d := path.Dir(pathname)
	if !t.isWatching(d) {
		err := t.w.Add(d)
		if err != nil {
			glog.Infof("Adding a create watch failed on %q: %s", d, err)
		}
		t.addWatched(d)
	}

	fd, err := t.fs.Open(pathname)
	if err != nil {
		// Doesn't exist yet. We're watching the directory, so we'll pick it up
		// again on create; return successfully.
		if os.IsNotExist(err) {
			return
		}
		glog.Infof("Failed to open %q for reading: %s", pathname, err)
		logErrors.Add(pathname, 1)
		return
	}
	t.filesLock.Lock()
	t.files[pathname] = fd

	if seekStart {
		t.files[pathname].Seek(0, os.SEEK_SET)
	} else {
		t.files[pathname].Seek(0, os.SEEK_END)
	}

	t.filesLock.Unlock()

	err = t.w.Add(pathname)
	if err != nil {
		glog.Infof("Adding a change watch failed on %q: %s", pathname, err)
	}

	glog.Infof("Tailing %s", pathname)

	// In case the new log has been written to already, attempt to read the first lines.
	t.handleLogUpdate(pathname)
}

// start is the main event loop for the Tailer.
// It receives notification of log file changes from the watcher channel, and
// handles them.
func (t *Tailer) run() {
	for e := range t.w.Events() {
		switch e := e.(type) {
		case watcher.UpdateEvent:
			t.handleLogUpdate(e.Pathname)
		case watcher.CreateEvent:
			t.handleLogCreate(e.Pathname)
		default:
			glog.Infof("Unexpected event %q", e)
		}
	}
	glog.Infof("Shutting down tailer.")
	close(t.lines)
}

// Close signals termination to the watcher.
func (t *Tailer) Close() {
	t.w.Close()
}
<|endoftext|>


<commit_before>// +build acceptance

package v2

import (
	"testing"

	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/acceptance/tools"
	os "github.com/rackspace/gophercloud/openstack/compute/v2/servers"
	"github.com/rackspace/gophercloud/pagination"
	"github.com/rackspace/gophercloud/rackspace/compute/v2/servers"
	th "github.com/rackspace/gophercloud/testhelper"
)

func createServer(t *testing.T, client *gophercloud.ServiceClient) *os.Server {
	options, err := optionsFromEnv()
	th.AssertNoErr(t, err)

	name := tools.RandomString("Gophercloud-", 8)
	t.Logf("Creating server [%s].", name)
	s, err := servers.Create(client, &os.CreateOpts{
		Name:      name,
		ImageRef:  options.imageID,
		FlavorRef: options.flavorID,
	}).Extract()
	th.AssertNoErr(t, err)
	t.Logf("Server created successfully.")
	return s
}

func deleteServer(t *testing.T, client *gophercloud.ServiceClient, server *os.Server) {
	t.Logf("Deleting server [%s].", server.ID)
	err := servers.Delete(client, server.ID)
	th.AssertNoErr(t, err)
	t.Logf("Server deleted successfully.")
}

func logServer(t *testing.T, server *os.Server, index int) {
	if index == -1 {
		t.Logf("             id=[%s]", server.ID)
	} else {
		t.Logf("[%02d]             id=[%s]", index, server.ID)
	}
	t.Logf("           name=[%s]", server.Name)
	t.Logf("      tenant ID=[%s]", server.TenantID)
	t.Logf("        user ID=[%s]", server.UserID)
	t.Logf("        updated=[%s]", server.Updated)
	t.Logf("        created=[%s]", server.Created)
	t.Logf("        host ID=[%s]", server.HostID)
	t.Logf("    access IPv4=[%s]", server.AccessIPv4)
	t.Logf("    access IPv6=[%s]", server.AccessIPv6)
	t.Logf("          image=[%v]", server.Image)
	t.Logf("         flavor=[%v]", server.Flavor)
	t.Logf("      addresses=[%v]", server.Addresses)
	t.Logf("       metadata=[%v]", server.Metadata)
	t.Logf("          links=[%v]", server.Links)
	t.Logf("        keyname=[%s]", server.KeyName)
	t.Logf(" admin password=[%s]", server.AdminPass)
	t.Logf("         status=[%s]", server.Status)
	t.Logf("       progress=[%d]", server.Progress)
}

func TestCreateServer(t *testing.T) {
	t.Parallel()

	client, err := newClient()
	th.AssertNoErr(t, err)

	s := createServer(t, client)
	defer deleteServer(t, client, s)

	t.Logf("Waiting for server to become active ...")
	err = servers.WaitForStatus(client, s.ID, "ACTIVE", 300)
	th.AssertNoErr(t, err)

	t.Logf("Server launched.")
	logServer(t, s, -1)

	t.Logf("Getting additional server details.")
	details, err := servers.Get(client, s.ID).Extract()
	logServer(t, details, -1)
}

func TestListServers(t *testing.T) {
	t.Parallel()

	client, err := newClient()
	th.AssertNoErr(t, err)

	count := 0
	err = servers.List(client, nil).EachPage(func(page pagination.Page) (bool, error) {
		count++
		t.Logf("-- Page %02d --", count)

		s, err := servers.ExtractServers(page)
		th.AssertNoErr(t, err)
		for index, server := range s {
			logServer(t, &server, index)
		}

		return true, nil
	})
	th.AssertNoErr(t, err)
}

func TestChangeAdminPassword(t *testing.T) {
	t.Parallel()

	client, err := newClient()
	th.AssertNoErr(t, err)

	s := createServer(t, client)
	defer deleteServer(t, client, s)
	original := s.AdminPass

	err = servers.WaitForStatus(client, s.ID, "ACTIVE", 300)
	th.AssertNoErr(t, err)

	t.Logf("Changing server password.")
	err = servers.ChangeAdminPassword(client, s.ID, tools.MakeNewPassword(original)).Extract()
	th.AssertNoErr(t, err)

	err = servers.WaitForStatus(client, s.ID, "ACTIVE", 300)
	th.AssertNoErr(t, err)
	t.Logf("Password changed successfully.")
}

func TestReboot(t *testing.T) {
	t.Parallel()

	client, err := newClient()
	th.AssertNoErr(t, err)

	s := createServer(t, client)
	defer deleteServer(t, client, s)

	err = servers.WaitForStatus(client, s.ID, "ACTIVE", 300)
	th.AssertNoErr(t, err)

	t.Logf("Rebooting server.")
	err = servers.Reboot(client, s.ID, os.HardReboot).Extract()
	th.AssertNoErr(t, err)

	err = servers.WaitForStatus(client, s.ID, "ACTIVE", 300)
	th.AssertNoErr(t, err)
	t.Logf("Server successfully rebooted.")
}

func TestRebuild(t *testing.T) {
	t.Parallel()

	options, err := optionsFromEnv()
	th.AssertNoErr(t, err)

	client, err := newClient()
	th.AssertNoErr(t, err)

	s := createServer(t, client)
	defer deleteServer(t, client, s)

	err = servers.WaitForStatus(client, s.ID, "ACTIVE", 300)
	th.AssertNoErr(t, err)

	t.Logf("Rebuilding server.")
	opts := os.RebuildOpts{
		Name:      tools.RandomString("RenamedGopher", 16),
		AdminPass: tools.MakeNewPassword(s.AdminPass),
		ImageID:   options.imageID,
	}
	after, err := servers.Rebuild(client, s.ID, opts).Extract()
	th.AssertNoErr(t, err)

	err = servers.WaitForStatus(client, after.ID, "ACTIVE", 300)
	th.AssertNoErr(t, err)

	t.Logf("Server successfully rebuilt.")
	logServer(t, after, -1)
}
<commit_msg>Make the compute acceptance tests sequential.<commit_after>// +build acceptance

package v2

import (
	"testing"

	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/acceptance/tools"
	os "github.com/rackspace/gophercloud/openstack/compute/v2/servers"
	"github.com/rackspace/gophercloud/pagination"
	"github.com/rackspace/gophercloud/rackspace/compute/v2/servers"
	th "github.com/rackspace/gophercloud/testhelper"
)

func createServer(t *testing.T, client *gophercloud.ServiceClient) *os.Server {
	options, err := optionsFromEnv()
	th.AssertNoErr(t, err)

	name := tools.RandomString("Gophercloud-", 8)
	t.Logf("Creating server [%s].", name)
	s, err := servers.Create(client, &os.CreateOpts{
		Name:      name,
		ImageRef:  options.imageID,
		FlavorRef: options.flavorID,
	}).Extract()
	th.AssertNoErr(t, err)
	t.Logf("Creating server.")

	err = servers.WaitForStatus(client, s.ID, "ACTIVE", 300)
	th.AssertNoErr(t, err)
	t.Logf("Server created successfully.")

	return s
}

func logServer(t *testing.T, server *os.Server, index int) {
	if index == -1 {
		t.Logf("             id=[%s]", server.ID)
	} else {
		t.Logf("[%02d]             id=[%s]", index, server.ID)
	}
	t.Logf("           name=[%s]", server.Name)
	t.Logf("      tenant ID=[%s]", server.TenantID)
	t.Logf("        user ID=[%s]", server.UserID)
	t.Logf("        updated=[%s]", server.Updated)
	t.Logf("        created=[%s]", server.Created)
	t.Logf("        host ID=[%s]", server.HostID)
	t.Logf("    access IPv4=[%s]", server.AccessIPv4)
	t.Logf("    access IPv6=[%s]", server.AccessIPv6)
	t.Logf("          image=[%v]", server.Image)
	t.Logf("         flavor=[%v]", server.Flavor)
	t.Logf("      addresses=[%v]", server.Addresses)
	t.Logf("       metadata=[%v]", server.Metadata)
	t.Logf("          links=[%v]", server.Links)
	t.Logf("        keyname=[%s]", server.KeyName)
	t.Logf(" admin password=[%s]", server.AdminPass)
	t.Logf("         status=[%s]", server.Status)
	t.Logf("       progress=[%d]", server.Progress)
}

func getServer(t *testing.T, client *gophercloud.ServiceClient, server *os.Server) {
	t.Logf("> servers.Get")

	details, err := servers.Get(client, server.ID).Extract()
	th.AssertNoErr(t, err)
	logServer(t, details, -1)
}

func listServers(t *testing.T, client *gophercloud.ServiceClient) {
	t.Logf("> servers.List")

	count := 0
	err := servers.List(client, nil).EachPage(func(page pagination.Page) (bool, error) {
		count++
		t.Logf("--- Page %02d ---", count)

		s, err := servers.ExtractServers(page)
		th.AssertNoErr(t, err)
		for index, server := range s {
			logServer(t, &server, index)
		}

		return true, nil
	})
	th.AssertNoErr(t, err)
}

func changeAdminPassword(t *testing.T, client *gophercloud.ServiceClient, server *os.Server) {
	t.Logf("> servers.ChangeAdminPassword")

	original := server.AdminPass

	t.Logf("Changing server password.")
	err := servers.ChangeAdminPassword(client, server.ID, tools.MakeNewPassword(original)).Extract()
	th.AssertNoErr(t, err)

	err = servers.WaitForStatus(client, server.ID, "ACTIVE", 300)
	th.AssertNoErr(t, err)
	t.Logf("Password changed successfully.")
}

func rebootServer(t *testing.T, client *gophercloud.ServiceClient, server *os.Server) {
	t.Logf("> servers.Reboot")

	err := servers.Reboot(client, server.ID, os.HardReboot).Extract()
	th.AssertNoErr(t, err)

	err = servers.WaitForStatus(client, server.ID, "ACTIVE", 300)
	th.AssertNoErr(t, err)

	t.Logf("Server successfully rebooted.")
}

func rebuildServer(t *testing.T, client *gophercloud.ServiceClient, server *os.Server) {
	t.Logf("> servers.Rebuild")

	options, err := optionsFromEnv()
	th.AssertNoErr(t, err)

	opts := os.RebuildOpts{
		Name:      tools.RandomString("RenamedGopher", 16),
		AdminPass: tools.MakeNewPassword(server.AdminPass),
		ImageID:   options.imageID,
	}
	after, err := servers.Rebuild(client, server.ID, opts).Extract()
	th.AssertNoErr(t, err)
	th.CheckEquals(t, after.ID, server.ID)

	err = servers.WaitForStatus(client, after.ID, "ACTIVE", 300)
	th.AssertNoErr(t, err)

	t.Logf("Server successfully rebuilt.")
	logServer(t, after, -1)
}

func deleteServer(t *testing.T, client *gophercloud.ServiceClient, server *os.Server) {
	t.Logf("> servers.Delete")

	err := servers.Delete(client, server.ID)
	th.AssertNoErr(t, err)

	t.Logf("Server deleted successfully.")
}

func TestServerOperations(t *testing.T) {
	client, err := newClient()
	th.AssertNoErr(t, err)

	server := createServer(t, client)
	defer deleteServer(t, client, server)

	getServer(t, client, server)
	listServers(t, client)
	changeAdminPassword(t, client, server)
	rebootServer(t, client, server)
	rebuildServer(t, client, server)
}
<|endoftext|>


<commit_before>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A command that scans all of the blobs in a GCS bucket, verifying their
// scores against their content.
//
// Output is space-separated lines of the form:
//
//     <score> <child score>*
//
// where <score> is a verified score, and the list is a list of scores of
// children for directories (and empty for files).

package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/crypto"
	"github.com/jacobsa/comeback/repr"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
)

var cmdScanBlobs = &Command{
	Name: "scan_blobs",
}

var fOutputFile = cmdScanBlobs.Flags.String(
	"output_file", "", "Path to scan_blobs output file. Will be truncated.")

func init() {
	cmdScanBlobs.Run = runScanBlobs // Break flag-related dependency loop.
}

type scoreAndContents struct {
	score    blob.Score
	contents []byte
}

type verifiedScore struct {
	score    blob.Score
	children []blob.Score
}

// List all scores in the GCS bucket into the channel. Do not close the
// channel.
func listBlobs(
	ctx context.Context,
	bucket gcs.Bucket,
	scores chan<- blob.Score) (err error) {
	req := &gcs.ListObjectsRequest{
		Prefix: blobKeyPrefix,
	}

	// List until we run out.
	for {
		// Fetch the next batch.
		var listing *gcs.Listing
		listing, err = bucket.ListObjects(ctx, req)
		if err != nil {
			err = fmt.Errorf("ListObjects: %v", err)
			return
		}

		// Transform to scores.
		var batch []blob.Score
		for _, o := range listing.Objects {
			var score blob.Score
			score, err = blob.ParseHexScore(strings.TrimPrefix(o.Name, blobKeyPrefix))
			if err != nil {
				err = fmt.Errorf("Parsing object name \"%s\": %v", o.Name, err)
				return
			}

			batch = append(batch, score)
		}

		// Feed out each score.
		for _, score := range batch {
			select {
			case scores <- score:

				// Cancelled?
			case <-ctx.Done():
				err = ctx.Err()
				return
			}
		}

		// Are we done?
		if listing.ContinuationToken == "" {
			break
		}

		req.ContinuationToken = listing.ContinuationToken
	}

	return
}

func readAndClose(rc io.ReadCloser) (d []byte, err error) {
	// Read.
	d, err = ioutil.ReadAll(rc)
	if err != nil {
		rc.Close()
		err = fmt.Errorf("ReadAll: %v", err)
		return
	}

	// Close.
	err = rc.Close()
	if err != nil {
		err = fmt.Errorf("Close: %v", err)
		return
	}

	return
}

func readBlob(
	ctx context.Context,
	bucket gcs.Bucket,
	score blob.Score) (blob scoreAndContents, err error) {
	// Fill in the score.
	blob.score = score

	// Obtain a reader.
	req := &gcs.ReadObjectRequest{
		Name: blobKeyPrefix + score.Hex(),
	}

	rc, err := bucket.NewReader(ctx, req)
	if err != nil {
		err = fmt.Errorf("NewReader: %v", err)
		return
	}

	// Consume it.
	blob.contents, err = readAndClose(rc)
	if err != nil {
		err = fmt.Errorf("readAndClose: %v", err)
		return
	}

	return
}

func readBlobWithRetryLoop(
	ctx context.Context,
	bucket gcs.Bucket,
	score blob.Score) (blob scoreAndContents, err error) {
	const maxRetries = 5
	for n := 0; n < maxRetries; n++ {
		blob, err = readBlob(ctx, bucket, score)
		if err == nil {
			return
		}

		time.Sleep(500 * time.Millisecond)
	}

	return
}

// Read the contents of blobs specified on the incoming channel. Do not close
// the outgoing channel.
func readBlobs(
	ctx context.Context,
	bucket gcs.Bucket,
	scores <-chan blob.Score,
	blobs chan<- scoreAndContents) (err error) {
	for score := range scores {
		var blob scoreAndContents

		// Read the contents for this score.
		blob, err = readBlobWithRetryLoop(ctx, bucket, score)
		if err != nil {
			err = fmt.Errorf("readBlob: %v", err)
			return
		}

		select {
		case blobs <- blob:
		case <-ctx.Done():
			err = ctx.Err()
			return
		}
	}

	return
}

func scoresEqual(a, b blob.Score) bool {
	for i := 0; i < blob.ScoreLength; i++ {
		if a[i] != b[i] {
			return false
		}
	}

	return true
}

// Parse the blob ciphertext as a directory if appropriate, returning a list of
// children. If not a directory, return the empty list.
func unmarshalBlob(
	crypter crypto.Crypter,
	b []byte) (children []blob.Score, err error) {
	// Attempt to decrypt.
	b, err = crypter.Decrypt(b)
	if err != nil {
		err = fmt.Errorf("Decrypt: %v", err)
		return
	}

	// If this is a file, simply make sure it is in a legal format.
	if !repr.IsDir(b) {
		_, err = repr.UnmarshalFile(b)
		if err != nil {
			err = fmt.Errorf("UnmarshalFile: %v", err)
			return
		}

		return
	}

	// Otherwise, parse it as a directory.
	entries, err := repr.UnmarshalDir(b)
	if err != nil {
		err = fmt.Errorf("UnmarshalDir: %v", err)
		return
	}

	// Pull out the child scores.
	for _, entry := range entries {
		for _, score := range entry.Scores {
			children = append(children, score)
		}
	}

	return
}

// Verify the contents of the incoming blobs. Do not close the outgoing
// channel.
func verifyScores(
	ctx context.Context,
	crypter crypto.Crypter,
	blobs <-chan scoreAndContents,
	results chan<- verifiedScore) (err error) {
	for b := range blobs {
		result := verifiedScore{
			score: b.score,
		}

		// Make sure the score matches.
		computed := blob.ComputeScore(b.contents)
		if !scoresEqual(b.score, computed) {
			err = fmt.Errorf(
				"Score mismatch: %v vs. %v",
				b.score.Hex(),
				computed.Hex())

			return
		}

		// Parse the blob.
		result.children, err = unmarshalBlob(crypter, b.contents)
		if err != nil {
			err = fmt.Errorf("unmarshalBlob: %v", err)
			return
		}

		// Write out the result.
		select {
		case results <- result:
		case <-ctx.Done():
			err = ctx.Err()
			return
		}
	}

	return
}

// Write results to stdout.
func writeResults(
	ctx context.Context,
	output io.Writer,
	results <-chan verifiedScore) (err error) {
	// Log also to stdout.
	output = io.MultiWriter(output, os.Stdout)

	// Process each result.
	for result := range results {
		// Write the score.
		_, err = fmt.Fprintf(output, "%s", result.score.Hex())
		if err != nil {
			return
		}

		// Write its children.
		for _, child := range result.children {
			_, err = fmt.Fprintf(output, " %s", child.Hex())
			if err != nil {
				return
			}
		}

		// Add a line separator.
		_, err = fmt.Fprintf(output, "\n")
		if err != nil {
			return
		}
	}

	return
}

func runScanBlobs(args []string) {
	// Die on error.
	var err error
	defer func() {
		if err != nil {
			log.Fatalln(err)
		}
	}()

	// Open the output file.
	if *fOutputFile == "" {
		err = fmt.Errorf("You must set --output_file.")
		return
	}

	output, err := os.OpenFile(
		*fOutputFile,
		os.O_WRONLY|os.O_TRUNC|os.O_CREATE,
		0666)

	if err != nil {
		err = fmt.Errorf("OpenFile: %v", err)
		return
	}

	defer output.Close()

	// Grab dependencies.
	bucket := getBucket()
	crypter := getCrypter()

	// Allow parallelism.
	runtime.GOMAXPROCS(runtime.NumCPU())

	b := syncutil.NewBundle(context.Background())

	// List all of the scores in the bucket.
	scores := make(chan blob.Score, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(scores)
		err = listBlobs(ctx, bucket, scores)
		return
	})

	// Read the contents of the corresponding blobs in parallel, bounding how
	// hard we hammer GCS by bounding the parallelism.
	const readWorkers = 128
	var readWaitGroup sync.WaitGroup

	blobs := make(chan scoreAndContents, 10)
	for i := 0; i < readWorkers; i++ {
		readWaitGroup.Add(1)
		b.Add(func(ctx context.Context) (err error) {
			defer readWaitGroup.Done()
			err = readBlobs(ctx, bucket, scores, blobs)
			return
		})
	}

	go func() {
		readWaitGroup.Wait()
		close(blobs)
	}()

	// Verify the blob contents and summarize their children. Use one worker per
	// CPU.
	var verifyWaitGroup sync.WaitGroup

	results := make(chan verifiedScore, 100)
	for i := 0; i < runtime.NumCPU(); i++ {
		verifyWaitGroup.Add(1)
		b.Add(func(ctx context.Context) (err error) {
			defer verifyWaitGroup.Done()
			err = verifyScores(ctx, crypter, blobs, results)
			return
		})
	}

	go func() {
		verifyWaitGroup.Wait()
		close(results)
	}()

	// Process results.
	b.Add(func(ctx context.Context) (err error) {
		err = writeResults(ctx, output, results)
		return
	})

	// Wait for everything to complete.
	err = b.Join()
	return
}
<commit_msg>Cut parallelism.<commit_after>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A command that scans all of the blobs in a GCS bucket, verifying their
// scores against their content.
//
// Output is space-separated lines of the form:
//
//     <score> <child score>*
//
// where <score> is a verified score, and the list is a list of scores of
// children for directories (and empty for files).

package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/crypto"
	"github.com/jacobsa/comeback/repr"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
)

var cmdScanBlobs = &Command{
	Name: "scan_blobs",
}

var fOutputFile = cmdScanBlobs.Flags.String(
	"output_file", "", "Path to scan_blobs output file. Will be truncated.")

func init() {
	cmdScanBlobs.Run = runScanBlobs // Break flag-related dependency loop.
}

type scoreAndContents struct {
	score    blob.Score
	contents []byte
}

type verifiedScore struct {
	score    blob.Score
	children []blob.Score
}

// List all scores in the GCS bucket into the channel. Do not close the
// channel.
func listBlobs(
	ctx context.Context,
	bucket gcs.Bucket,
	scores chan<- blob.Score) (err error) {
	req := &gcs.ListObjectsRequest{
		Prefix: blobKeyPrefix,
	}

	// List until we run out.
	for {
		// Fetch the next batch.
		var listing *gcs.Listing
		listing, err = bucket.ListObjects(ctx, req)
		if err != nil {
			err = fmt.Errorf("ListObjects: %v", err)
			return
		}

		// Transform to scores.
		var batch []blob.Score
		for _, o := range listing.Objects {
			var score blob.Score
			score, err = blob.ParseHexScore(strings.TrimPrefix(o.Name, blobKeyPrefix))
			if err != nil {
				err = fmt.Errorf("Parsing object name \"%s\": %v", o.Name, err)
				return
			}

			batch = append(batch, score)
		}

		// Feed out each score.
		for _, score := range batch {
			select {
			case scores <- score:

				// Cancelled?
			case <-ctx.Done():
				err = ctx.Err()
				return
			}
		}

		// Are we done?
		if listing.ContinuationToken == "" {
			break
		}

		req.ContinuationToken = listing.ContinuationToken
	}

	return
}

func readAndClose(rc io.ReadCloser) (d []byte, err error) {
	// Read.
	d, err = ioutil.ReadAll(rc)
	if err != nil {
		rc.Close()
		err = fmt.Errorf("ReadAll: %v", err)
		return
	}

	// Close.
	err = rc.Close()
	if err != nil {
		err = fmt.Errorf("Close: %v", err)
		return
	}

	return
}

func readBlob(
	ctx context.Context,
	bucket gcs.Bucket,
	score blob.Score) (blob scoreAndContents, err error) {
	// Fill in the score.
	blob.score = score

	// Obtain a reader.
	req := &gcs.ReadObjectRequest{
		Name: blobKeyPrefix + score.Hex(),
	}

	rc, err := bucket.NewReader(ctx, req)
	if err != nil {
		err = fmt.Errorf("NewReader: %v", err)
		return
	}

	// Consume it.
	blob.contents, err = readAndClose(rc)
	if err != nil {
		err = fmt.Errorf("readAndClose: %v", err)
		return
	}

	return
}

func readBlobWithRetryLoop(
	ctx context.Context,
	bucket gcs.Bucket,
	score blob.Score) (blob scoreAndContents, err error) {
	const maxRetries = 5
	for n := 0; n < maxRetries; n++ {
		blob, err = readBlob(ctx, bucket, score)
		if err == nil {
			return
		}

		time.Sleep(500 * time.Millisecond)
	}

	return
}

// Read the contents of blobs specified on the incoming channel. Do not close
// the outgoing channel.
func readBlobs(
	ctx context.Context,
	bucket gcs.Bucket,
	scores <-chan blob.Score,
	blobs chan<- scoreAndContents) (err error) {
	for score := range scores {
		var blob scoreAndContents

		// Read the contents for this score.
		blob, err = readBlobWithRetryLoop(ctx, bucket, score)
		if err != nil {
			err = fmt.Errorf("readBlob: %v", err)
			return
		}

		select {
		case blobs <- blob:
		case <-ctx.Done():
			err = ctx.Err()
			return
		}
	}

	return
}

func scoresEqual(a, b blob.Score) bool {
	for i := 0; i < blob.ScoreLength; i++ {
		if a[i] != b[i] {
			return false
		}
	}

	return true
}

// Parse the blob ciphertext as a directory if appropriate, returning a list of
// children. If not a directory, return the empty list.
func unmarshalBlob(
	crypter crypto.Crypter,
	b []byte) (children []blob.Score, err error) {
	// Attempt to decrypt.
	b, err = crypter.Decrypt(b)
	if err != nil {
		err = fmt.Errorf("Decrypt: %v", err)
		return
	}

	// If this is a file, simply make sure it is in a legal format.
	if !repr.IsDir(b) {
		_, err = repr.UnmarshalFile(b)
		if err != nil {
			err = fmt.Errorf("UnmarshalFile: %v", err)
			return
		}

		return
	}

	// Otherwise, parse it as a directory.
	entries, err := repr.UnmarshalDir(b)
	if err != nil {
		err = fmt.Errorf("UnmarshalDir: %v", err)
		return
	}

	// Pull out the child scores.
	for _, entry := range entries {
		for _, score := range entry.Scores {
			children = append(children, score)
		}
	}

	return
}

// Verify the contents of the incoming blobs. Do not close the outgoing
// channel.
func verifyScores(
	ctx context.Context,
	crypter crypto.Crypter,
	blobs <-chan scoreAndContents,
	results chan<- verifiedScore) (err error) {
	for b := range blobs {
		result := verifiedScore{
			score: b.score,
		}

		// Make sure the score matches.
		computed := blob.ComputeScore(b.contents)
		if !scoresEqual(b.score, computed) {
			err = fmt.Errorf(
				"Score mismatch: %v vs. %v",
				b.score.Hex(),
				computed.Hex())

			return
		}

		// Parse the blob.
		result.children, err = unmarshalBlob(crypter, b.contents)
		if err != nil {
			err = fmt.Errorf("unmarshalBlob: %v", err)
			return
		}

		// Write out the result.
		select {
		case results <- result:
		case <-ctx.Done():
			err = ctx.Err()
			return
		}
	}

	return
}

// Write results to stdout.
func writeResults(
	ctx context.Context,
	output io.Writer,
	results <-chan verifiedScore) (err error) {
	// Log also to stdout.
	output = io.MultiWriter(output, os.Stdout)

	// Process each result.
	for result := range results {
		// Write the score.
		_, err = fmt.Fprintf(output, "%s", result.score.Hex())
		if err != nil {
			return
		}

		// Write its children.
		for _, child := range result.children {
			_, err = fmt.Fprintf(output, " %s", child.Hex())
			if err != nil {
				return
			}
		}

		// Add a line separator.
		_, err = fmt.Fprintf(output, "\n")
		if err != nil {
			return
		}
	}

	return
}

func runScanBlobs(args []string) {
	// Die on error.
	var err error
	defer func() {
		if err != nil {
			log.Fatalln(err)
		}
	}()

	// Open the output file.
	if *fOutputFile == "" {
		err = fmt.Errorf("You must set --output_file.")
		return
	}

	output, err := os.OpenFile(
		*fOutputFile,
		os.O_WRONLY|os.O_TRUNC|os.O_CREATE,
		0666)

	if err != nil {
		err = fmt.Errorf("OpenFile: %v", err)
		return
	}

	defer output.Close()

	// Grab dependencies.
	bucket := getBucket()
	crypter := getCrypter()

	// Allow parallelism.
	runtime.GOMAXPROCS(runtime.NumCPU())

	b := syncutil.NewBundle(context.Background())

	// List all of the scores in the bucket.
	scores := make(chan blob.Score, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(scores)
		err = listBlobs(ctx, bucket, scores)
		return
	})

	// Read the contents of the corresponding blobs in parallel, bounding how
	// hard we hammer GCS by bounding the parallelism.
	const readWorkers = 8
	var readWaitGroup sync.WaitGroup

	blobs := make(chan scoreAndContents, 10)
	for i := 0; i < readWorkers; i++ {
		readWaitGroup.Add(1)
		b.Add(func(ctx context.Context) (err error) {
			defer readWaitGroup.Done()
			err = readBlobs(ctx, bucket, scores, blobs)
			return
		})
	}

	go func() {
		readWaitGroup.Wait()
		close(blobs)
	}()

	// Verify the blob contents and summarize their children. Use one worker per
	// CPU.
	var verifyWaitGroup sync.WaitGroup

	results := make(chan verifiedScore, 100)
	for i := 0; i < runtime.NumCPU(); i++ {
		verifyWaitGroup.Add(1)
		b.Add(func(ctx context.Context) (err error) {
			defer verifyWaitGroup.Done()
			err = verifyScores(ctx, crypter, blobs, results)
			return
		})
	}

	go func() {
		verifyWaitGroup.Wait()
		close(results)
	}()

	// Process results.
	b.Add(func(ctx context.Context) (err error) {
		err = writeResults(ctx, output, results)
		return
	})

	// Wait for everything to complete.
	err = b.Join()
	return
}
<|endoftext|>


<commit_before>package main

const build = "3d154c548fcd9b5f03824f12d27f245d9febaaac"
<commit_msg>update build sequence<commit_after>package main

const build = "4cab16135e23f592f3382593ae6dfa89c57b350e"
<|endoftext|>


<commit_before>package pm

import (
	"archive/tar"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"
)

func Build(m *Metadata, outputDir string) error {
	tarGzPath := filepath.Join(outputDir, fmt.Sprintf("%s.tar.gz", m.PackageName()))
	tgz, err := os.Create(tarGzPath)
	if err != nil {
		return fmt.Errorf("pm: error creating %q: %v", tarGzPath, err)
	}

	gzw := gzip.NewWriter(tgz)
	defer gzw.Close()

	tarw := tar.NewWriter(gzw)
	defer tarw.Close()

	// Add the metadata file.
	if err := writeMetadata(tarw, m); err != nil {
		return err
	}

	// Add the binaries.
	for _, binPath := range m.Binaries {
		if err := tarAddBinary(tarw, binPath); err != nil {
			return err
		}
	}

	// Add resources (extra files).
	for baseDir, globs := range m.Resources {
		for _, g := range globs {
			resourceGlob := filepath.Join(baseDir, g)
			matches, err := filepath.Glob(resourceGlob)
			if err != nil {
				return fmt.Errorf("pm: bad globbing pattern: %q", resourceGlob)
			}

			for _, match := range matches {
				if err := writeResource(tarw, match); err != nil {
					return err
				}
			}
		}
	}

	return nil
}

func writeResource(tw *tar.Writer, resourcePath string) error {
	log.Println("adding resource", resourcePath)

	f, err := os.Open(resourcePath)
	if err != nil {
		return fmt.Errorf("pm: error opening resource %q: %v", resourcePath, err)
	}
	defer f.Close()

	fileInfo, err := f.Stat()
	if err != nil {
		return fmt.Errorf("pm: error getting info for file %q: %v", resourcePath, err)
	}

	hdr, err := tar.FileInfoHeader(fileInfo, "")
	if err != nil {
		return fmt.Errorf("pm: error creating tar header for file %q: %v", resourcePath, err)
	}
	hdr.Name = strings.TrimLeft(filepath.Clean(resourcePath), "./")
	hdr.Uid = 0
	hdr.Gid = 0
	hdr.Mode = 0444

	if err := tw.WriteHeader(hdr); err != nil {
		return fmt.Errorf("pm: error writing resource tar header for %q: %v", resourcePath, err)
	}

	if _, err := io.Copy(tw, f); err != nil {
		return fmt.Errorf("pm: error writing resource %q to archive: %v", resourcePath, err)
	}

	return nil
}

func writeMetadata(tw *tar.Writer, m *Metadata) error {
	log.Println("adding metadata")

	b, err := json.MarshalIndent(m, "", "\t")
	if err != nil {
		return fmt.Errorf("pm: error marshaling metadata: %v", err)
	}

	now := time.Now()

	hdr := &tar.Header{
		Name:       "metadata.json",
		Size:       int64(len(b)),
		ModTime:    now,
		AccessTime: now,
		ChangeTime: now,
		Mode:       0444,
	}

	if err := tw.WriteHeader(hdr); err != nil {
		return fmt.Errorf("pm: error writing metadata tar header: %v", err)
	}

	if _, err := tw.Write(b); err != nil {
		return fmt.Errorf("pm: error writing metadata: %v", err)
	}

	return nil
}

func tarAddBinary(tw *tar.Writer, binPath string) error {
	log.Println("adding binary", binPath)

	bf, err := os.Open(binPath)
	if err != nil {
		return fmt.Errorf("pm: error opening binary file %q: %v", binPath, err)
	}
	defer bf.Close()

	info, err := bf.Stat()
	if err != nil {
		return fmt.Errorf("pm: error getting stat info on binary file %q: %v", binPath, err)
	}

	var hdr *tar.Header
	if hdr, err = tar.FileInfoHeader(info, ""); err != nil {
		return fmt.Errorf("pm: error creating header for binary file %q: %v", binPath, err)
	}
	hdr.Name = strings.TrimLeft(filepath.Clean(binPath), "./")
	hdr.Uid = 0
	hdr.Gid = 0
	hdr.Mode = 0555

	if err := tw.WriteHeader(hdr); err != nil {
		return fmt.Errorf("pm: error writing tar header for %q: %v", binPath, err)
	}

	if _, err := io.Copy(tw, bf); err != nil {
		return fmt.Errorf("pm: error writing binary file %q: %v", binPath, err)
	}

	return nil
}
<commit_msg>pm: Pretty up the pathnames in the .tar.gz file<commit_after>package pm

import (
	"archive/tar"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"time"
)

func Build(m *Metadata, outputDir string) error {
	tarGzPath := filepath.Join(outputDir, fmt.Sprintf("%s.tar.gz", m.PackageName()))
	tgz, err := os.Create(tarGzPath)
	if err != nil {
		return fmt.Errorf("pm: error creating %q: %v", tarGzPath, err)
	}

	gzw := gzip.NewWriter(tgz)
	defer gzw.Close()

	tarw := tar.NewWriter(gzw)
	defer tarw.Close()

	// Add the metadata file.
	if err := writeMetadata(tarw, m); err != nil {
		return err
	}

	// Add the binaries.
	for _, binPath := range m.Binaries {
		if err := tarAddBinary(tarw, m, binPath); err != nil {
			return err
		}
	}

	// Add resources (extra files).
	for baseDir, globs := range m.Resources {
		for _, g := range globs {
			resourceGlob := filepath.Join(baseDir, g)
			matches, err := filepath.Glob(resourceGlob)
			if err != nil {
				return fmt.Errorf("pm: bad globbing pattern: %q", resourceGlob)
			}

			for _, match := range matches {
				if err := writeResource(tarw, m, match); err != nil {
					return err
				}
			}
		}
	}

	return nil
}

func writeResource(tw *tar.Writer, m *Metadata, resourcePath string) error {
	log.Println("adding resource", resourcePath)

	f, err := os.Open(resourcePath)
	if err != nil {
		return fmt.Errorf("pm: error opening resource %q: %v", resourcePath, err)
	}
	defer f.Close()

	fileInfo, err := f.Stat()
	if err != nil {
		return fmt.Errorf("pm: error getting info for file %q: %v", resourcePath, err)
	}

	hdr, err := tar.FileInfoHeader(fileInfo, "")
	if err != nil {
		return fmt.Errorf("pm: error creating tar header for file %q: %v", resourcePath, err)
	}

	name := filepath.Join(
		fmt.Sprintf("%s-%s", m.Name, m.Version),
		filepath.Clean(resourcePath),
	)
	hdr.Name = name
	hdr.Uid = 0
	hdr.Gid = 0
	hdr.Mode = 0444

	if err := tw.WriteHeader(hdr); err != nil {
		return fmt.Errorf("pm: error writing resource tar header for %q: %v", resourcePath, err)
	}

	if _, err := io.Copy(tw, f); err != nil {
		return fmt.Errorf("pm: error writing resource %q to archive: %v", resourcePath, err)
	}

	return nil
}

func writeMetadata(tw *tar.Writer, m *Metadata) error {
	log.Println("adding metadata")

	b, err := json.MarshalIndent(m, "", "\t")
	if err != nil {
		return fmt.Errorf("pm: error marshaling metadata: %v", err)
	}

	now := time.Now()

	name := filepath.Join(fmt.Sprintf("%s-%s", m.Name, m.Version), "metadata.json")
	hdr := &tar.Header{
		Name:       name,
		Size:       int64(len(b)),
		ModTime:    now,
		AccessTime: now,
		ChangeTime: now,
		Mode:       0444,
	}

	if err := tw.WriteHeader(hdr); err != nil {
		return fmt.Errorf("pm: error writing metadata tar header: %v", err)
	}

	if _, err := tw.Write(b); err != nil {
		return fmt.Errorf("pm: error writing metadata: %v", err)
	}

	return nil
}

func tarAddBinary(tw *tar.Writer, m *Metadata, binPath string) error {
	log.Println("adding binary", binPath)

	bf, err := os.Open(binPath)
	if err != nil {
		return fmt.Errorf("pm: error opening binary file %q: %v", binPath, err)
	}
	defer bf.Close()

	info, err := bf.Stat()
	if err != nil {
		return fmt.Errorf("pm: error getting stat info on binary file %q: %v", binPath, err)
	}

	var hdr *tar.Header
	if hdr, err = tar.FileInfoHeader(info, ""); err != nil {
		return fmt.Errorf("pm: error creating header for binary file %q: %v", binPath, err)
	}

	name := filepath.Join(
		fmt.Sprintf("%s-%s", m.Name, m.Version),
		filepath.Clean(binPath),
	)
	hdr.Name = name
	hdr.Uid = 0
	hdr.Gid = 0
	hdr.Mode = 0555

	if err := tw.WriteHeader(hdr); err != nil {
		return fmt.Errorf("pm: error writing tar header for %q: %v", binPath, err)
	}

	if _, err := io.Copy(tw, bf); err != nil {
		return fmt.Errorf("pm: error writing binary file %q: %v", binPath, err)
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Code generated by informer-gen. DO NOT EDIT.

package informers

import (
	reflect "reflect"
	sync "sync"
	time "time"

	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	runtime "k8s.io/apimachinery/pkg/runtime"
	schema "k8s.io/apimachinery/pkg/runtime/schema"
	admissionregistration "k8s.io/client-go/informers/admissionregistration"
	apps "k8s.io/client-go/informers/apps"
	autoscaling "k8s.io/client-go/informers/autoscaling"
	batch "k8s.io/client-go/informers/batch"
	certificates "k8s.io/client-go/informers/certificates"
	core "k8s.io/client-go/informers/core"
	events "k8s.io/client-go/informers/events"
	extensions "k8s.io/client-go/informers/extensions"
	internalinterfaces "k8s.io/client-go/informers/internalinterfaces"
	networking "k8s.io/client-go/informers/networking"
	policy "k8s.io/client-go/informers/policy"
	rbac "k8s.io/client-go/informers/rbac"
	scheduling "k8s.io/client-go/informers/scheduling"
	settings "k8s.io/client-go/informers/settings"
	storage "k8s.io/client-go/informers/storage"
	kubernetes "k8s.io/client-go/kubernetes"
	cache "k8s.io/client-go/tools/cache"
)

type sharedInformerFactory struct {
	client           kubernetes.Interface
	namespace        string
	tweakListOptions internalinterfaces.TweakListOptionsFunc
	lock             sync.Mutex
	defaultResync    time.Duration

	informers map[reflect.Type]cache.SharedIndexInformer
	// startedInformers is used for tracking which informers have been started.
	// This allows Start() to be called multiple times safely.
	startedInformers map[reflect.Type]bool
}

// NewSharedInformerFactory constructs a new instance of sharedInformerFactory
func NewSharedInformerFactory(client kubernetes.Interface, defaultResync time.Duration) SharedInformerFactory {
	return NewFilteredSharedInformerFactory(client, defaultResync, v1.NamespaceAll, nil)
}

// NewFilteredSharedInformerFactory constructs a new instance of sharedInformerFactory.
// Listers obtained via this SharedInformerFactory will be subject to the same filters
// as specified here.
func NewFilteredSharedInformerFactory(client kubernetes.Interface, defaultResync time.Duration, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerFactory {
	return &sharedInformerFactory{
		client:           client,
		namespace:        namespace,
		tweakListOptions: tweakListOptions,
		defaultResync:    defaultResync,
		informers:        make(map[reflect.Type]cache.SharedIndexInformer),
		startedInformers: make(map[reflect.Type]bool),
	}
}

// Start initializes all requested informers.
func (f *sharedInformerFactory) Start(stopCh <-chan struct{}) {
	f.lock.Lock()
	defer f.lock.Unlock()

	for informerType, informer := range f.informers {
		if !f.startedInformers[informerType] {
			go informer.Run(stopCh)
			f.startedInformers[informerType] = true
		}
	}
}

// WaitForCacheSync waits for all started informers' cache were synced.
func (f *sharedInformerFactory) WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool {
	informers := func() map[reflect.Type]cache.SharedIndexInformer {
		f.lock.Lock()
		defer f.lock.Unlock()

		informers := map[reflect.Type]cache.SharedIndexInformer{}
		for informerType, informer := range f.informers {
			if f.startedInformers[informerType] {
				informers[informerType] = informer
			}
		}
		return informers
	}()

	res := map[reflect.Type]bool{}
	for informType, informer := range informers {
		res[informType] = cache.WaitForCacheSync(stopCh, informer.HasSynced)
	}
	return res
}

// InternalInformerFor returns the SharedIndexInformer for obj using an internal
// client.
func (f *sharedInformerFactory) InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer {
	f.lock.Lock()
	defer f.lock.Unlock()

	informerType := reflect.TypeOf(obj)
	informer, exists := f.informers[informerType]
	if exists {
		return informer
	}
	informer = newFunc(f.client, f.defaultResync)
	f.informers[informerType] = informer

	return informer
}

// SharedInformerFactory provides shared informers for resources in all known
// API group versions.
type SharedInformerFactory interface {
	internalinterfaces.SharedInformerFactory
	ForResource(resource schema.GroupVersionResource) (GenericInformer, error)
	WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool

	Admissionregistration() admissionregistration.Interface
	Apps() apps.Interface
	Autoscaling() autoscaling.Interface
	Batch() batch.Interface
	Certificates() certificates.Interface
	Core() core.Interface
	Events() events.Interface
	Extensions() extensions.Interface
	Networking() networking.Interface
	Policy() policy.Interface
	Rbac() rbac.Interface
	Scheduling() scheduling.Interface
	Settings() settings.Interface
	Storage() storage.Interface
}

func (f *sharedInformerFactory) Admissionregistration() admissionregistration.Interface {
	return admissionregistration.New(f, f.namespace, f.tweakListOptions)
}

func (f *sharedInformerFactory) Apps() apps.Interface {
	return apps.New(f, f.namespace, f.tweakListOptions)
}

func (f *sharedInformerFactory) Autoscaling() autoscaling.Interface {
	return autoscaling.New(f, f.namespace, f.tweakListOptions)
}

func (f *sharedInformerFactory) Batch() batch.Interface {
	return batch.New(f, f.namespace, f.tweakListOptions)
}

func (f *sharedInformerFactory) Certificates() certificates.Interface {
	return certificates.New(f, f.namespace, f.tweakListOptions)
}

func (f *sharedInformerFactory) Core() core.Interface {
	return core.New(f, f.namespace, f.tweakListOptions)
}

func (f *sharedInformerFactory) Events() events.Interface {
	return events.New(f, f.namespace, f.tweakListOptions)
}

func (f *sharedInformerFactory) Extensions() extensions.Interface {
	return extensions.New(f, f.namespace, f.tweakListOptions)
}

func (f *sharedInformerFactory) Networking() networking.Interface {
	return networking.New(f, f.namespace, f.tweakListOptions)
}

func (f *sharedInformerFactory) Policy() policy.Interface {
	return policy.New(f, f.namespace, f.tweakListOptions)
}

func (f *sharedInformerFactory) Rbac() rbac.Interface {
	return rbac.New(f, f.namespace, f.tweakListOptions)
}

func (f *sharedInformerFactory) Scheduling() scheduling.Interface {
	return scheduling.New(f, f.namespace, f.tweakListOptions)
}

func (f *sharedInformerFactory) Settings() settings.Interface {
	return settings.New(f, f.namespace, f.tweakListOptions)
}

func (f *sharedInformerFactory) Storage() storage.Interface {
	return storage.New(f, f.namespace, f.tweakListOptions)
}
<commit_msg>Run hack/update-codegen.sh<commit_after>/*
Copyright The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Code generated by informer-gen. DO NOT EDIT.

package informers

import (
	reflect "reflect"
	sync "sync"
	time "time"

	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	runtime "k8s.io/apimachinery/pkg/runtime"
	schema "k8s.io/apimachinery/pkg/runtime/schema"
	admissionregistration "k8s.io/client-go/informers/admissionregistration"
	apps "k8s.io/client-go/informers/apps"
	autoscaling "k8s.io/client-go/informers/autoscaling"
	batch "k8s.io/client-go/informers/batch"
	certificates "k8s.io/client-go/informers/certificates"
	core "k8s.io/client-go/informers/core"
	events "k8s.io/client-go/informers/events"
	extensions "k8s.io/client-go/informers/extensions"
	internalinterfaces "k8s.io/client-go/informers/internalinterfaces"
	networking "k8s.io/client-go/informers/networking"
	policy "k8s.io/client-go/informers/policy"
	rbac "k8s.io/client-go/informers/rbac"
	scheduling "k8s.io/client-go/informers/scheduling"
	settings "k8s.io/client-go/informers/settings"
	storage "k8s.io/client-go/informers/storage"
	kubernetes "k8s.io/client-go/kubernetes"
	cache "k8s.io/client-go/tools/cache"
)

// SharedInformerOption defines the functional option type for SharedInformerFactory.
type SharedInformerOption func(*sharedInformerFactory) *sharedInformerFactory

type sharedInformerFactory struct {
	client           kubernetes.Interface
	namespace        string
	tweakListOptions internalinterfaces.TweakListOptionsFunc
	lock             sync.Mutex
	defaultResync    time.Duration
	customResync     map[reflect.Type]time.Duration

	informers map[reflect.Type]cache.SharedIndexInformer
	// startedInformers is used for tracking which informers have been started.
	// This allows Start() to be called multiple times safely.
	startedInformers map[reflect.Type]bool
}

// WithCustomResyncConfig sets a custom resync period for the specified informer types.
func WithCustomResyncConfig(resyncConfig map[v1.Object]time.Duration) SharedInformerOption {
	return func(factory *sharedInformerFactory) *sharedInformerFactory {
		for k, v := range resyncConfig {
			factory.customResync[reflect.TypeOf(k)] = v
		}
		return factory
	}
}

// WithTweakListOptions sets a custom filter on all listers of the configured SharedInformerFactory.
func WithTweakListOptions(tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerOption {
	return func(factory *sharedInformerFactory) *sharedInformerFactory {
		factory.tweakListOptions = tweakListOptions
		return factory
	}
}

// WithNamespace limits the SharedInformerFactory to the specified namespace.
func WithNamespace(namespace string) SharedInformerOption {
	return func(factory *sharedInformerFactory) *sharedInformerFactory {
		factory.namespace = namespace
		return factory
	}
}

// NewSharedInformerFactory constructs a new instance of sharedInformerFactory for all namespaces.
func NewSharedInformerFactory(client kubernetes.Interface, defaultResync time.Duration) SharedInformerFactory {
	return NewSharedInformerFactoryWithOptions(client, defaultResync)
}

// NewFilteredSharedInformerFactory constructs a new instance of sharedInformerFactory.
// Listers obtained via this SharedInformerFactory will be subject to the same filters
// as specified here.
// Deprecated: Please use NewSharedInformerFactoryWithOptions instead
func NewFilteredSharedInformerFactory(client kubernetes.Interface, defaultResync time.Duration, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerFactory {
	return NewSharedInformerFactoryWithOptions(client, defaultResync, WithNamespace(namespace), WithTweakListOptions(tweakListOptions))
}

// NewSharedInformerFactoryWithOptions constructs a new instance of a SharedInformerFactory with additional options.
func NewSharedInformerFactoryWithOptions(client kubernetes.Interface, defaultResync time.Duration, options ...SharedInformerOption) SharedInformerFactory {
	factory := &sharedInformerFactory{
		client:           client,
		namespace:        v1.NamespaceAll,
		defaultResync:    defaultResync,
		informers:        make(map[reflect.Type]cache.SharedIndexInformer),
		startedInformers: make(map[reflect.Type]bool),
		customResync:     make(map[reflect.Type]time.Duration),
	}

	// Apply all options
	for _, opt := range options {
		factory = opt(factory)
	}

	return factory
}

// Start initializes all requested informers.
func (f *sharedInformerFactory) Start(stopCh <-chan struct{}) {
	f.lock.Lock()
	defer f.lock.Unlock()

	for informerType, informer := range f.informers {
		if !f.startedInformers[informerType] {
			go informer.Run(stopCh)
			f.startedInformers[informerType] = true
		}
	}
}

// WaitForCacheSync waits for all started informers' cache were synced.
func (f *sharedInformerFactory) WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool {
	informers := func() map[reflect.Type]cache.SharedIndexInformer {
		f.lock.Lock()
		defer f.lock.Unlock()

		informers := map[reflect.Type]cache.SharedIndexInformer{}
		for informerType, informer := range f.informers {
			if f.startedInformers[informerType] {
				informers[informerType] = informer
			}
		}
		return informers
	}()

	res := map[reflect.Type]bool{}
	for informType, informer := range informers {
		res[informType] = cache.WaitForCacheSync(stopCh, informer.HasSynced)
	}
	return res
}

// InternalInformerFor returns the SharedIndexInformer for obj using an internal
// client.
func (f *sharedInformerFactory) InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer {
	f.lock.Lock()
	defer f.lock.Unlock()

	informerType := reflect.TypeOf(obj)
	informer, exists := f.informers[informerType]
	if exists {
		return informer
	}

	resyncPeriod, exists := f.customResync[informerType]
	if !exists {
		resyncPeriod = f.defaultResync
	}

	informer = newFunc(f.client, resyncPeriod)
	f.informers[informerType] = informer

	return informer
}

// SharedInformerFactory provides shared informers for resources in all known
// API group versions.
type SharedInformerFactory interface {
	internalinterfaces.SharedInformerFactory
	ForResource(resource schema.GroupVersionResource) (GenericInformer, error)
	WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool

	Admissionregistration() admissionregistration.Interface
	Apps() apps.Interface
	Autoscaling() autoscaling.Interface
	Batch() batch.Interface
	Certificates() certificates.Interface
	Core() core.Interface
	Events() events.Interface
	Extensions() extensions.Interface
	Networking() networking.Interface
	Policy() policy.Interface
	Rbac() rbac.Interface
	Scheduling() scheduling.Interface
	Settings() settings.Interface
	Storage() storage.Interface
}

func (f *sharedInformerFactory) Admissionregistration() admissionregistration.Interface {
	return admissionregistration.New(f, f.namespace, f.tweakListOptions)
}

func (f *sharedInformerFactory) Apps() apps.Interface {
	return apps.New(f, f.namespace, f.tweakListOptions)
}

func (f *sharedInformerFactory) Autoscaling() autoscaling.Interface {
	return autoscaling.New(f, f.namespace, f.tweakListOptions)
}

func (f *sharedInformerFactory) Batch() batch.Interface {
	return batch.New(f, f.namespace, f.tweakListOptions)
}

func (f *sharedInformerFactory) Certificates() certificates.Interface {
	return certificates.New(f, f.namespace, f.tweakListOptions)
}

func (f *sharedInformerFactory) Core() core.Interface {
	return core.New(f, f.namespace, f.tweakListOptions)
}

func (f *sharedInformerFactory) Events() events.Interface {
	return events.New(f, f.namespace, f.tweakListOptions)
}

func (f *sharedInformerFactory) Extensions() extensions.Interface {
	return extensions.New(f, f.namespace, f.tweakListOptions)
}

func (f *sharedInformerFactory) Networking() networking.Interface {
	return networking.New(f, f.namespace, f.tweakListOptions)
}

func (f *sharedInformerFactory) Policy() policy.Interface {
	return policy.New(f, f.namespace, f.tweakListOptions)
}

func (f *sharedInformerFactory) Rbac() rbac.Interface {
	return rbac.New(f, f.namespace, f.tweakListOptions)
}

func (f *sharedInformerFactory) Scheduling() scheduling.Interface {
	return scheduling.New(f, f.namespace, f.tweakListOptions)
}

func (f *sharedInformerFactory) Settings() settings.Interface {
	return settings.New(f, f.namespace, f.tweakListOptions)
}

func (f *sharedInformerFactory) Storage() storage.Interface {
	return storage.New(f, f.namespace, f.tweakListOptions)
}
<|endoftext|>


<commit_before>package integration_test

import (
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"runtime"

	integration "github.com/cloudfoundry/bosh-bootloader/integration-test"
	"github.com/cloudfoundry/bosh-bootloader/integration-test/actors"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Stack Migration", func() {
	var (
		bblStack     actors.BBL
		bblTerraform actors.BBL
		aws          actors.AWS
		boshcli      actors.BOSHCLI
		state        integration.State

		f *os.File
	)

	BeforeEach(func() {
		var err error
		configuration, err := integration.LoadConfig()
		Expect(err).NotTo(HaveOccurred())

		var bblBinaryLocation string
		if runtime.GOOS == "darwin" {
			bblBinaryLocation = "https://www.github.com/cloudfoundry/bosh-bootloader/releases/download/v3.2.4/bbl-v3.2.4_osx"
		} else {
			bblBinaryLocation = "https://www.github.com/cloudfoundry/bosh-bootloader/releases/download/v3.2.4/bbl-v3.2.4_linux_x86-64"
		}

		resp, err := http.Get(bblBinaryLocation)
		Expect(err).NotTo(HaveOccurred())

		f, err = ioutil.TempFile("", "")
		Expect(err).NotTo(HaveOccurred())

		_, err = io.Copy(f, resp.Body)
		Expect(err).NotTo(HaveOccurred())

		err = os.Chmod(f.Name(), 0700)
		Expect(err).NotTo(HaveOccurred())

		bblStack = actors.NewBBL(configuration.StateFileDir, f.Name(), configuration, "stack-migration-env")
		bblTerraform = actors.NewBBL(configuration.StateFileDir, pathToBBL, configuration, "stack-migration-env")
		aws = actors.NewAWS(configuration)
		boshcli = actors.NewBOSHCLI()
		state = integration.NewState(configuration.StateFileDir)
	})

	AfterEach(func() {
		bblTerraform.Destroy()

		err := os.Remove(f.Name())
		Expect(err).NotTo(HaveOccurred())
	})

	It("is able to bbl up idempotently with a director", func() {
		var (
			stackName       string
			directorAddress string
			caCertPath      string
		)

		By("bbl'ing up with cloudformation", func() {
			bblStack.Up(actors.AWSIAAS, []string{"--name", bblStack.PredefinedEnvID()})
		})

		By("verifying the stack exists", func() {
			stackName = state.StackName()
			Expect(aws.StackExists(stackName)).To(BeTrue())
		})

		By("verifying the director exists", func() {
			directorAddress = bblStack.DirectorAddress()
			caCertPath = bblStack.SaveDirectorCA()

			exists, err := boshcli.DirectorExists(directorAddress, caCertPath)
			Expect(err).NotTo(HaveOccurred())
			Expect(exists).To(BeTrue())
		})

		By("migrating to terraform with latest bbl", func() {
			bblTerraform.Up(actors.AWSIAAS, []string{})
		})

		By("verifying the stack doesn't exists", func() {
			Expect(aws.StackExists(stackName)).To(BeFalse())
		})

		By("verifying the director still exists", func() {
			exists, err := boshcli.DirectorExists(directorAddress, caCertPath)
			Expect(err).NotTo(HaveOccurred())
			Expect(exists).To(BeTrue())
		})
	})
})
<commit_msg>macos more tolerant than linux when writing to open file<commit_after>package integration_test

import (
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"runtime"

	integration "github.com/cloudfoundry/bosh-bootloader/integration-test"
	"github.com/cloudfoundry/bosh-bootloader/integration-test/actors"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Stack Migration", func() {
	var (
		bblStack     actors.BBL
		bblTerraform actors.BBL
		aws          actors.AWS
		boshcli      actors.BOSHCLI
		state        integration.State

		f *os.File
	)

	BeforeEach(func() {
		var err error
		configuration, err := integration.LoadConfig()
		Expect(err).NotTo(HaveOccurred())

		var bblBinaryLocation string
		if runtime.GOOS == "darwin" {
			bblBinaryLocation = "https://www.github.com/cloudfoundry/bosh-bootloader/releases/download/v3.2.4/bbl-v3.2.4_osx"
		} else {
			bblBinaryLocation = "https://www.github.com/cloudfoundry/bosh-bootloader/releases/download/v3.2.4/bbl-v3.2.4_linux_x86-64"
		}

		resp, err := http.Get(bblBinaryLocation)
		Expect(err).NotTo(HaveOccurred())

		f, err = ioutil.TempFile("", "")
		Expect(err).NotTo(HaveOccurred())

		_, err = io.Copy(f, resp.Body)
		Expect(err).NotTo(HaveOccurred())

		err = os.Chmod(f.Name(), 0700)
		Expect(err).NotTo(HaveOccurred())

		err = f.Close()
		Expect(err).NotTo(HaveOccurred())

		bblStack = actors.NewBBL(configuration.StateFileDir, f.Name(), configuration, "stack-migration-env")
		bblTerraform = actors.NewBBL(configuration.StateFileDir, pathToBBL, configuration, "stack-migration-env")
		aws = actors.NewAWS(configuration)
		boshcli = actors.NewBOSHCLI()
		state = integration.NewState(configuration.StateFileDir)
	})

	AfterEach(func() {
		bblTerraform.Destroy()

		err := os.Remove(f.Name())
		Expect(err).NotTo(HaveOccurred())
	})

	It("is able to bbl up idempotently with a director", func() {
		var (
			stackName       string
			directorAddress string
			caCertPath      string
		)

		By("bbl'ing up with cloudformation", func() {
			bblStack.Up(actors.AWSIAAS, []string{"--name", bblStack.PredefinedEnvID()})
		})

		By("verifying the stack exists", func() {
			stackName = state.StackName()
			Expect(aws.StackExists(stackName)).To(BeTrue())
		})

		By("verifying the director exists", func() {
			directorAddress = bblStack.DirectorAddress()
			caCertPath = bblStack.SaveDirectorCA()

			exists, err := boshcli.DirectorExists(directorAddress, caCertPath)
			Expect(err).NotTo(HaveOccurred())
			Expect(exists).To(BeTrue())
		})

		By("migrating to terraform with latest bbl", func() {
			bblTerraform.Up(actors.AWSIAAS, []string{})
		})

		By("verifying the stack doesn't exists", func() {
			Expect(aws.StackExists(stackName)).To(BeFalse())
		})

		By("verifying the director still exists", func() {
			exists, err := boshcli.DirectorExists(directorAddress, caCertPath)
			Expect(err).NotTo(HaveOccurred())
			Expect(exists).To(BeTrue())
		})
	})
})
<|endoftext|>


<commit_before>/*
 * Minio Cloud Storage, (C) 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"path/filepath"
	"strings"
	"sync"

	"github.com/minio/minio/pkg/mimedb"
)

const (
	multipartSuffix   = ".minio.multipart"
	multipartMetaFile = "00000" + multipartSuffix
	formatConfigFile  = "format.json"
)

// xlObjects - Implements fs object layer.
type xlObjects struct {
	storage            StorageAPI
	listObjectMap      map[listParams][]*treeWalker
	listObjectMapMutex *sync.Mutex
}

// isValidFormat - validates input arguments with backend 'format.json'
func isValidFormat(storage StorageAPI, exportPaths ...string) bool {
	// Load saved XL format.json and validate.
	xl, err := loadFormatXL(storage)
	if err != nil {
		log.Errorf("loadFormatXL failed with %s", err)
		return false
	}
	if xl.Version != "1" {
		log.Errorf("Unsupported XL backend format found [%s]", xl.Version)
		return false
	}
	if len(exportPaths) != len(xl.Disks) {
		log.Errorf("Number of disks %d passed at the command-line did not match the backend format %d", len(exportPaths), len(xl.Disks))
		return false
	}
	for index, disk := range xl.Disks {
		if exportPaths[index] != disk {
			log.Errorf("Invalid order of disks detected %s. Required order is %s.", exportPaths, xl.Disks)
			return false
		}
	}
	return true
}

// newXLObjects - initialize new xl object layer.
func newXLObjects(exportPaths ...string) (ObjectLayer, error) {
	storage, err := newXL(exportPaths...)
	if err != nil {
		log.Errorf("newXL failed with %s", err)
		return nil, err
	}

	// Initialize object layer - like creating minioMetaBucket,
	// cleaning up tmp files etc.
	initObjectLayer(storage)

	err = checkFormat(storage)
	if err != nil {
		if err == errFileNotFound {
			// Save new XL format.
			errSave := saveFormatXL(storage, &xlFormat{
				Version: "1",
				Disks:   exportPaths,
			})
			if errSave != nil {
				log.Errorf("saveFormatXL failed with %s", errSave)
				return nil, errSave
			}
		} else {
			log.Errorf("Unable to check backend format %s", err)
			return nil, err
		}
	}

	// Validate if format exists and input arguments are validated
	// with backend format.
	if !isValidFormat(storage, exportPaths...) {
		return nil, fmt.Errorf("Command-line arguments %s is not valid.", exportPaths)
	}

	// Return successfully initialized object layer.
	return xlObjects{
		storage:            storage,
		listObjectMap:      make(map[listParams][]*treeWalker),
		listObjectMapMutex: &sync.Mutex{},
	}, nil
}

/// Bucket operations

// MakeBucket - make a bucket.
func (xl xlObjects) MakeBucket(bucket string) error {
	return makeBucket(xl.storage, bucket)
}

// GetBucketInfo - get bucket info.
func (xl xlObjects) GetBucketInfo(bucket string) (BucketInfo, error) {
	return getBucketInfo(xl.storage, bucket)
}

// ListBuckets - list buckets.
func (xl xlObjects) ListBuckets() ([]BucketInfo, error) {
	return listBuckets(xl.storage)
}

// DeleteBucket - delete a bucket.
func (xl xlObjects) DeleteBucket(bucket string) error {
	return deleteBucket(xl.storage, bucket)
}

/// Object Operations

// GetObject - get an object.
func (xl xlObjects) GetObject(bucket, object string, startOffset int64) (io.ReadCloser, error) {
	// Verify if bucket is valid.
	if !IsValidBucketName(bucket) {
		return nil, BucketNameInvalid{Bucket: bucket}
	}
	if !isBucketExist(xl.storage, bucket) {
		return nil, BucketNotFound{Bucket: bucket}
	}
	// Verify if object is valid.
	if !IsValidObjectName(object) {
		return nil, ObjectNameInvalid{Bucket: bucket, Object: object}
	}
	if ok, err := isMultipartObject(xl.storage, bucket, object); err != nil {
		return nil, toObjectErr(err, bucket, object)
	} else if !ok {
		if _, err = xl.storage.StatFile(bucket, object); err == nil {
			var reader io.ReadCloser
			reader, err = xl.storage.ReadFile(bucket, object, startOffset)
			if err != nil {
				return nil, toObjectErr(err, bucket, object)
			}
			return reader, nil
		}
		return nil, toObjectErr(err, bucket, object)
	}
	fileReader, fileWriter := io.Pipe()
	info, err := getMultipartObjectInfo(xl.storage, bucket, object)
	if err != nil {
		return nil, toObjectErr(err, bucket, object)
	}
	partIndex, offset, err := info.GetPartNumberOffset(startOffset)
	if err != nil {
		return nil, toObjectErr(err, bucket, object)
	}
	go func() {
		for ; partIndex < len(info.Parts); partIndex++ {
			part := info.Parts[partIndex]
			r, err := xl.storage.ReadFile(bucket, pathJoin(object, partNumToPartFileName(part.PartNumber)), offset)
			if err != nil {
				fileWriter.CloseWithError(err)
				return
			}
			if _, err := io.Copy(fileWriter, r); err != nil {
				fileWriter.CloseWithError(err)
				return
			}
		}
		fileWriter.Close()
	}()
	return fileReader, nil
}

// Return the partsInfo of a special multipart object.
func getMultipartObjectInfo(storage StorageAPI, bucket, object string) (info MultipartObjectInfo, err error) {
	offset := int64(0)
	r, err := storage.ReadFile(bucket, pathJoin(object, multipartMetaFile), offset)
	if err != nil {
		return MultipartObjectInfo{}, err
	}
	decoder := json.NewDecoder(r)
	err = decoder.Decode(&info)
	if err != nil {
		return MultipartObjectInfo{}, err
	}
	return info, nil
}

// GetObjectInfo - get object info.
func (xl xlObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) {
	// Verify if bucket is valid.
	if !IsValidBucketName(bucket) {
		return ObjectInfo{}, BucketNameInvalid{Bucket: bucket}
	}
	// Check whether the bucket exists.
	if !isBucketExist(xl.storage, bucket) {
		return ObjectInfo{}, BucketNotFound{Bucket: bucket}
	}
	// Verify if object is valid.
	if !IsValidObjectName(object) {
		return ObjectInfo{}, ObjectNameInvalid{Bucket: bucket, Object: object}
	}
	fi, err := xl.storage.StatFile(bucket, object)
	if err != nil {
		if err != errFileNotFound {
			return ObjectInfo{}, toObjectErr(err, bucket, object)
		}
		var info MultipartObjectInfo
		info, err = getMultipartObjectInfo(xl.storage, bucket, object)
		if err != nil {
			return ObjectInfo{}, toObjectErr(err, bucket, object)
		}
		fi.Size = info.Size
		fi.ModTime = info.ModTime
		fi.MD5Sum = info.MD5Sum
	}
	contentType := "application/octet-stream"
	if objectExt := filepath.Ext(object); objectExt != "" {
		content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]
		if ok {
			contentType = content.ContentType
		}
	}
	return ObjectInfo{
		Bucket:      bucket,
		Name:        object,
		ModTime:     fi.ModTime,
		Size:        fi.Size,
		IsDir:       fi.Mode.IsDir(),
		ContentType: contentType,
		MD5Sum:      fi.MD5Sum,
	}, nil
}

// PutObject - create an object.
func (xl xlObjects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (string, error) {
	return putObjectCommon(xl.storage, bucket, object, size, data, metadata)
}

// isMultipartObject - verifies if an object is special multipart file.
func isMultipartObject(storage StorageAPI, bucket, object string) (bool, error) {
	_, err := storage.StatFile(bucket, pathJoin(object, multipartMetaFile))
	if err != nil {
		if err == errFileNotFound {
			return false, nil
		}
		return false, err
	}
	return true, nil
}

func (xl xlObjects) DeleteObject(bucket, object string) error {
	// Verify if bucket is valid.
	if !IsValidBucketName(bucket) {
		return BucketNameInvalid{Bucket: bucket}
	}
	if !isBucketExist(xl.storage, bucket) {
		return BucketNotFound{Bucket: bucket}
	}
	if !IsValidObjectName(object) {
		return ObjectNameInvalid{Bucket: bucket, Object: object}
	}
	// Verify if the object is a multipart object.
	if ok, err := isMultipartObject(xl.storage, bucket, object); err != nil {
		return toObjectErr(err, bucket, object)
	} else if !ok {
		if err = xl.storage.DeleteFile(bucket, object); err != nil {
			return toObjectErr(err, bucket, object)
		}
		return nil
	}
	// Get parts info.
	info, err := getMultipartObjectInfo(xl.storage, bucket, object)
	if err != nil {
		return toObjectErr(err, bucket, object)
	}
	// Range through all files and delete it.
	var wg = &sync.WaitGroup{}
	var errs = make([]error, len(info.Parts))
	for index, part := range info.Parts {
		wg.Add(1)
		// Start deleting parts in routine.
		go func(index int, part MultipartPartInfo) {
			defer wg.Done()
			partFileName := partNumToPartFileName(part.PartNumber)
			errs[index] = xl.storage.DeleteFile(bucket, pathJoin(object, partFileName))
		}(index, part)
	}
	// Wait for all the deletes to finish.
	wg.Wait()
	// Loop through and validate if any errors, return back the first
	// error occurred.
	for index, err := range errs {
		if err != nil {
			partFileName := partNumToPartFileName(info.Parts[index].PartNumber)
			return toObjectErr(err, bucket, pathJoin(object, partFileName))
		}
	}
	err = xl.storage.DeleteFile(bucket, pathJoin(object, multipartMetaFile))
	if err != nil {
		return toObjectErr(err, bucket, object)
	}
	return nil
}

// ListObjects - list all objects at prefix, delimited by '/'.
func (xl xlObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
	return listObjectsCommon(xl, bucket, prefix, marker, delimiter, maxKeys)
}
<commit_msg>XL/GetObject: offset should be reset to 0 after reading first part. (#1580) (#1581)<commit_after>/*
 * Minio Cloud Storage, (C) 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"path/filepath"
	"strings"
	"sync"

	"github.com/minio/minio/pkg/mimedb"
)

const (
	multipartSuffix   = ".minio.multipart"
	multipartMetaFile = "00000" + multipartSuffix
	formatConfigFile  = "format.json"
)

// xlObjects - Implements fs object layer.
type xlObjects struct {
	storage            StorageAPI
	listObjectMap      map[listParams][]*treeWalker
	listObjectMapMutex *sync.Mutex
}

// isValidFormat - validates input arguments with backend 'format.json'
func isValidFormat(storage StorageAPI, exportPaths ...string) bool {
	// Load saved XL format.json and validate.
	xl, err := loadFormatXL(storage)
	if err != nil {
		log.Errorf("loadFormatXL failed with %s", err)
		return false
	}
	if xl.Version != "1" {
		log.Errorf("Unsupported XL backend format found [%s]", xl.Version)
		return false
	}
	if len(exportPaths) != len(xl.Disks) {
		log.Errorf("Number of disks %d passed at the command-line did not match the backend format %d", len(exportPaths), len(xl.Disks))
		return false
	}
	for index, disk := range xl.Disks {
		if exportPaths[index] != disk {
			log.Errorf("Invalid order of disks detected %s. Required order is %s.", exportPaths, xl.Disks)
			return false
		}
	}
	return true
}

// newXLObjects - initialize new xl object layer.
func newXLObjects(exportPaths ...string) (ObjectLayer, error) {
	storage, err := newXL(exportPaths...)
	if err != nil {
		log.Errorf("newXL failed with %s", err)
		return nil, err
	}

	// Initialize object layer - like creating minioMetaBucket,
	// cleaning up tmp files etc.
	initObjectLayer(storage)

	err = checkFormat(storage)
	if err != nil {
		if err == errFileNotFound {
			// Save new XL format.
			errSave := saveFormatXL(storage, &xlFormat{
				Version: "1",
				Disks:   exportPaths,
			})
			if errSave != nil {
				log.Errorf("saveFormatXL failed with %s", errSave)
				return nil, errSave
			}
		} else {
			log.Errorf("Unable to check backend format %s", err)
			return nil, err
		}
	}

	// Validate if format exists and input arguments are validated
	// with backend format.
	if !isValidFormat(storage, exportPaths...) {
		return nil, fmt.Errorf("Command-line arguments %s is not valid.", exportPaths)
	}

	// Return successfully initialized object layer.
	return xlObjects{
		storage:            storage,
		listObjectMap:      make(map[listParams][]*treeWalker),
		listObjectMapMutex: &sync.Mutex{},
	}, nil
}

/// Bucket operations

// MakeBucket - make a bucket.
func (xl xlObjects) MakeBucket(bucket string) error {
	return makeBucket(xl.storage, bucket)
}

// GetBucketInfo - get bucket info.
func (xl xlObjects) GetBucketInfo(bucket string) (BucketInfo, error) {
	return getBucketInfo(xl.storage, bucket)
}

// ListBuckets - list buckets.
func (xl xlObjects) ListBuckets() ([]BucketInfo, error) {
	return listBuckets(xl.storage)
}

// DeleteBucket - delete a bucket.
func (xl xlObjects) DeleteBucket(bucket string) error {
	return deleteBucket(xl.storage, bucket)
}

/// Object Operations

// GetObject - get an object.
func (xl xlObjects) GetObject(bucket, object string, startOffset int64) (io.ReadCloser, error) {
	// Verify if bucket is valid.
	if !IsValidBucketName(bucket) {
		return nil, BucketNameInvalid{Bucket: bucket}
	}
	if !isBucketExist(xl.storage, bucket) {
		return nil, BucketNotFound{Bucket: bucket}
	}
	// Verify if object is valid.
	if !IsValidObjectName(object) {
		return nil, ObjectNameInvalid{Bucket: bucket, Object: object}
	}
	if ok, err := isMultipartObject(xl.storage, bucket, object); err != nil {
		return nil, toObjectErr(err, bucket, object)
	} else if !ok {
		if _, err = xl.storage.StatFile(bucket, object); err == nil {
			var reader io.ReadCloser
			reader, err = xl.storage.ReadFile(bucket, object, startOffset)
			if err != nil {
				return nil, toObjectErr(err, bucket, object)
			}
			return reader, nil
		}
		return nil, toObjectErr(err, bucket, object)
	}
	fileReader, fileWriter := io.Pipe()
	info, err := getMultipartObjectInfo(xl.storage, bucket, object)
	if err != nil {
		return nil, toObjectErr(err, bucket, object)
	}
	partIndex, offset, err := info.GetPartNumberOffset(startOffset)
	if err != nil {
		return nil, toObjectErr(err, bucket, object)
	}
	go func() {
		for ; partIndex < len(info.Parts); partIndex++ {
			part := info.Parts[partIndex]
			r, err := xl.storage.ReadFile(bucket, pathJoin(object, partNumToPartFileName(part.PartNumber)), offset)
			if err != nil {
				fileWriter.CloseWithError(err)
				return
			}
			// Reset offset to 0 as it would be non-0 only for the first loop if startOffset is non-0.
			offset = 0
			if _, err := io.Copy(fileWriter, r); err != nil {
				fileWriter.CloseWithError(err)
				return
			}
		}
		fileWriter.Close()
	}()
	return fileReader, nil
}

// Return the partsInfo of a special multipart object.
func getMultipartObjectInfo(storage StorageAPI, bucket, object string) (info MultipartObjectInfo, err error) {
	offset := int64(0)
	r, err := storage.ReadFile(bucket, pathJoin(object, multipartMetaFile), offset)
	if err != nil {
		return MultipartObjectInfo{}, err
	}
	decoder := json.NewDecoder(r)
	err = decoder.Decode(&info)
	if err != nil {
		return MultipartObjectInfo{}, err
	}
	return info, nil
}

// GetObjectInfo - get object info.
func (xl xlObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) {
	// Verify if bucket is valid.
	if !IsValidBucketName(bucket) {
		return ObjectInfo{}, BucketNameInvalid{Bucket: bucket}
	}
	// Check whether the bucket exists.
	if !isBucketExist(xl.storage, bucket) {
		return ObjectInfo{}, BucketNotFound{Bucket: bucket}
	}
	// Verify if object is valid.
	if !IsValidObjectName(object) {
		return ObjectInfo{}, ObjectNameInvalid{Bucket: bucket, Object: object}
	}
	fi, err := xl.storage.StatFile(bucket, object)
	if err != nil {
		if err != errFileNotFound {
			return ObjectInfo{}, toObjectErr(err, bucket, object)
		}
		var info MultipartObjectInfo
		info, err = getMultipartObjectInfo(xl.storage, bucket, object)
		if err != nil {
			return ObjectInfo{}, toObjectErr(err, bucket, object)
		}
		fi.Size = info.Size
		fi.ModTime = info.ModTime
		fi.MD5Sum = info.MD5Sum
	}
	contentType := "application/octet-stream"
	if objectExt := filepath.Ext(object); objectExt != "" {
		content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]
		if ok {
			contentType = content.ContentType
		}
	}
	return ObjectInfo{
		Bucket:      bucket,
		Name:        object,
		ModTime:     fi.ModTime,
		Size:        fi.Size,
		IsDir:       fi.Mode.IsDir(),
		ContentType: contentType,
		MD5Sum:      fi.MD5Sum,
	}, nil
}

// PutObject - create an object.
func (xl xlObjects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (string, error) {
	return putObjectCommon(xl.storage, bucket, object, size, data, metadata)
}

// isMultipartObject - verifies if an object is special multipart file.
func isMultipartObject(storage StorageAPI, bucket, object string) (bool, error) {
	_, err := storage.StatFile(bucket, pathJoin(object, multipartMetaFile))
	if err != nil {
		if err == errFileNotFound {
			return false, nil
		}
		return false, err
	}
	return true, nil
}

func (xl xlObjects) DeleteObject(bucket, object string) error {
	// Verify if bucket is valid.
	if !IsValidBucketName(bucket) {
		return BucketNameInvalid{Bucket: bucket}
	}
	if !isBucketExist(xl.storage, bucket) {
		return BucketNotFound{Bucket: bucket}
	}
	if !IsValidObjectName(object) {
		return ObjectNameInvalid{Bucket: bucket, Object: object}
	}
	// Verify if the object is a multipart object.
	if ok, err := isMultipartObject(xl.storage, bucket, object); err != nil {
		return toObjectErr(err, bucket, object)
	} else if !ok {
		if err = xl.storage.DeleteFile(bucket, object); err != nil {
			return toObjectErr(err, bucket, object)
		}
		return nil
	}
	// Get parts info.
	info, err := getMultipartObjectInfo(xl.storage, bucket, object)
	if err != nil {
		return toObjectErr(err, bucket, object)
	}
	// Range through all files and delete it.
	var wg = &sync.WaitGroup{}
	var errs = make([]error, len(info.Parts))
	for index, part := range info.Parts {
		wg.Add(1)
		// Start deleting parts in routine.
		go func(index int, part MultipartPartInfo) {
			defer wg.Done()
			partFileName := partNumToPartFileName(part.PartNumber)
			errs[index] = xl.storage.DeleteFile(bucket, pathJoin(object, partFileName))
		}(index, part)
	}
	// Wait for all the deletes to finish.
	wg.Wait()
	// Loop through and validate if any errors, return back the first
	// error occurred.
	for index, err := range errs {
		if err != nil {
			partFileName := partNumToPartFileName(info.Parts[index].PartNumber)
			return toObjectErr(err, bucket, pathJoin(object, partFileName))
		}
	}
	err = xl.storage.DeleteFile(bucket, pathJoin(object, multipartMetaFile))
	if err != nil {
		return toObjectErr(err, bucket, object)
	}
	return nil
}

// ListObjects - list all objects at prefix, delimited by '/'.
func (xl xlObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
	return listObjectsCommon(xl, bucket, prefix, marker, delimiter, maxKeys)
}
<|endoftext|>


<commit_before>package push

import (
	"code.cloudfoundry.org/cli/integration/helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("the experimental warning", func() {
	var (
		appName string
	)

	BeforeEach(func() {
		appName = helpers.PrefixedRandomName("app")
	})

	It("displays the experimental warning", func() {
		session := helpers.CF(PushCommandName, appName)
		Eventually(session.Err).Should(Say("This command is in EXPERIMENTAL stage and may change without notice"))
		Eventually(session).Should(Exit())
	})
})
<commit_msg>🐞 V7 push is not tested for experimental warning<commit_after><|endoftext|>