<commit_before>// $G $D/$F.go && $L $F.$A && ./$A.out

// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Tests verifying the semantics of the select statement
// for basic empty/non-empty cases.

package main

import "time"

const always = "function did not"
const never = "function did"

func unreachable() {
	panic("control flow shouldn't reach here")
}

// Calls f and verifies that f always/never panics depending on signal.
func testPanic(signal string, f func()) {
	defer func() {
		s := never
		if recover() != nil {
			s = always  // f panicked
		}
		if s != signal {
			panic(signal + " panic")
		}
	}()
	f()
}

// Calls f and empirically verifies that f always/never blocks depending on signal.
func testBlock(signal string, f func()) {
	c := make(chan string)
	go func() {
		f()
		c <- never // f didn't block
	}()
	go func() {
		time.Sleep(1e8) // 0.1s seems plenty long
		c <- always     // f blocked always
	}()
	if <-c != signal {
		panic(signal + " block")
	}
}

func main() {
	const async = 1 // asynchronous channels
	var nilch chan int

	// sending/receiving from a nil channel outside a select panics
	testPanic(always, func() {
		nilch <- 7
	})
	testPanic(always, func() {
		<-nilch
	})

	// sending/receiving from a nil channel inside a select never panics
	testPanic(never, func() {
		select {
		case nilch <- 7:
			unreachable()
		default:
		}
	})
	testPanic(never, func() {
		select {
		case <-nilch:
			unreachable()
		default:
		}
	})

	// sending to an async channel with free buffer space never blocks
	testBlock(never, func() {
		ch := make(chan int, async)
		ch <- 7
	})

	// receiving from a non-ready channel always blocks
	testBlock(always, func() {
		ch := make(chan int)
		<-ch
	})

	// TODO(gri) remove this if once 6g accepts empty selects
	enabled := false
	if enabled {
		// empty selects always block
		testBlock(always, func() {
			select {
			case <-make(chan int): // remove this once 6g accepts empty selects
			}
		})

		// selects with only nil channels always block
		testBlock(always, func() {
			select {
			case <-nilch:
				unreachable()
			}
		})
		testBlock(always, func() {
			select {
			case nilch <- 7:
				unreachable()
			}
		})
		testBlock(always, func() {
			select {
			case <-nilch:
				unreachable()
			case nilch <- 7:
				unreachable()
			}
		})
	}

	// selects with non-ready non-nil channels always block
	testBlock(always, func() {
		ch := make(chan int)
		select {
		case <-ch:
			unreachable()
		}
	})

	// selects with default cases don't block
	testBlock(never, func() {
		select {
		default:
		}
	})
	testBlock(never, func() {
		select {
		case <-nilch:
			unreachable()
		default:
		}
	})
	testBlock(never, func() {
		select {
		case nilch <- 7:
			unreachable()
		default:
		}
	})

	// selects with ready channels don't block
	testBlock(never, func() {
		ch := make(chan int, async)
		select {
		case ch <- 7:
		default:
			unreachable()
		}
	})
	testBlock(never, func() {
		ch := make(chan int, async)
		ch <- 7
		select {
		case <-ch:
		default:
			unreachable()
		}
	})
}
<commit_msg>channel tests: added a couple of tests with closed channels<commit_after>// $G $D/$F.go && $L $F.$A && ./$A.out

// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Tests verifying the semantics of the select statement
// for basic empty/non-empty cases.

package main

import "time"

const always = "function did not"
const never = "function did"

func unreachable() {
	panic("control flow shouldn't reach here")
}

// Calls f and verifies that f always/never panics depending on signal.
func testPanic(signal string, f func()) {
	defer func() {
		s := never
		if recover() != nil {
			s = always // f panicked
		}
		if s != signal {
			panic(signal + " panic")
		}
	}()
	f()
}

// Calls f and empirically verifies that f always/never blocks depending on signal.
func testBlock(signal string, f func()) {
	c := make(chan string)
	go func() {
		f()
		c <- never // f didn't block
	}()
	go func() {
		time.Sleep(1e8) // 0.1s seems plenty long
		c <- always     // f blocked always
	}()
	if <-c != signal {
		panic(signal + " block")
	}
}

func main() {
	const async = 1 // asynchronous channels
	var nilch chan int
	closedch := make(chan int)
	close(closedch)

	// sending/receiving from a nil channel outside a select panics
	testPanic(always, func() {
		nilch <- 7
	})
	testPanic(always, func() {
		<-nilch
	})

	// sending/receiving from a nil channel inside a select never panics
	testPanic(never, func() {
		select {
		case nilch <- 7:
			unreachable()
		default:
		}
	})
	testPanic(never, func() {
		select {
		case <-nilch:
			unreachable()
		default:
		}
	})

	// sending to an async channel with free buffer space never blocks
	testBlock(never, func() {
		ch := make(chan int, async)
		ch <- 7
	})

	// receiving (a small number of times) from a closed channel never blocks
	testBlock(never, func() {
		for i := 0; i < 10; i++ {
			if <-closedch != 0 {
				panic("expected zero value when reading from closed channel")
			}
		}
	})

	// sending (a small number of times) to a closed channel is not specified
	// but the current implementation doesn't block: test that different
	// implementations behave the same
	testBlock(never, func() {
		for i := 0; i < 10; i++ {
			closedch <- 7
		}
	})

	// receiving from a non-ready channel always blocks
	testBlock(always, func() {
		ch := make(chan int)
		<-ch
	})

	// TODO(gri) remove this if once 6g accepts empty selects
	enabled := false
	if enabled {
		// empty selects always block
		testBlock(always, func() {
			select {
			case <-make(chan int): // remove this once 6g accepts empty selects
			}
		})

		// selects with only nil channels always block
		testBlock(always, func() {
			select {
			case <-nilch:
				unreachable()
			}
		})
		testBlock(always, func() {
			select {
			case nilch <- 7:
				unreachable()
			}
		})
		testBlock(always, func() {
			select {
			case <-nilch:
				unreachable()
			case nilch <- 7:
				unreachable()
			}
		})
	}

	// selects with non-ready non-nil channels always block
	testBlock(always, func() {
		ch := make(chan int)
		select {
		case <-ch:
			unreachable()
		}
	})

	// selects with default cases don't block
	testBlock(never, func() {
		select {
		default:
		}
	})
	testBlock(never, func() {
		select {
		case <-nilch:
			unreachable()
		default:
		}
	})
	testBlock(never, func() {
		select {
		case nilch <- 7:
			unreachable()
		default:
		}
	})

	// selects with ready channels don't block
	testBlock(never, func() {
		ch := make(chan int, async)
		select {
		case ch <- 7:
		default:
			unreachable()
		}
	})
	testBlock(never, func() {
		ch := make(chan int, async)
		ch <- 7
		select {
		case <-ch:
		default:
			unreachable()
		}
	})

	// selects with closed channels don't block
	testBlock(never, func() {
		select {
		case <-closedch:
		}
	})
	testBlock(never, func() {
		select {
		case closedch <- 7:
		}
	})
}
<|endoftext|>


<commit_before>package director

import (
	// "fmt"
	"time"
)

const (
	FOREVER = -1
	ONCE    = 1
)

// A Looper is used in place of a direct call to "for {}" and implements some
// controls over how the loop will be run. The Loop() function is the main
// call used by dependant routines. Common patterns like Quit and Done
// channels are easily implemented in a Looper.
type Looper interface {
	Loop(fn func() error)
	Wait() error
	Done(err error)
	Quit()
}

// A TimedLooper is a Looper that runs on a timed schedule, using a Timer
// underneath. It also implements Quit and Done channels to allow external
// routines to more easily control and synchronize the loop.
//
// If you pass in a DoneChan at creation time, it will send a nil on the
// channel when the loop has completed successfully or an error if the loop
// resulted in an error condition.
type TimedLooper struct {
	Count     int
	Interval  time.Duration
	DoneChan  chan error
	quitChan  chan bool
	Immediate bool
}

func NewTimedLooper(count int, interval time.Duration, done chan error) *TimedLooper {
	return &TimedLooper{
		Count:     count,
		Interval:  interval,
		DoneChan:  done,
		quitChan:  make(chan bool),
		Immediate: false,
	}
}

// Same as a TimedLooper, except it will execute an iteration of the loop
// immediately after calling on Loop() (as opposed to waiting until the tick)
func NewImmediateTimedLooper(count int, interval time.Duration, done chan error) *TimedLooper {
	return &TimedLooper{
		Count:     count,
		Interval:  interval,
		DoneChan:  done,
		quitChan:  make(chan bool),
		Immediate: true,
	}
}

func (l *TimedLooper) Wait() error {
	return <-l.DoneChan
}

// Signal a dependant routine that we're done with our work
func (l *TimedLooper) Done(err error) {
	if l.DoneChan != nil {
		l.DoneChan <- err
	}
}

// The main method of the Looper. This call takes a function with a single
// return value, an error. If the error is nil, the Looper will run the
// next iteration. If it's an error, it will not run the next iteration,
// will clean up any internals that need to be, and will invoke done().
func (l *TimedLooper) Loop(fn func() error) {
	i := 0

	var stop bool

	stopFunc := func(err error) {
		l.Done(err)
		stop = true
	}

	runIteration := func() {
		err := fn()
		if err != nil {
			stopFunc(err)
			return
		}

		// We have to make sure not to increment if we started
		// at -1 otherwise we quit on maxint rollover.
		if l.Count != FOREVER {
			i = i + 1
			if i >= l.Count {
				stopFunc(nil)
				return
			}
		}

		select {
		case <-l.quitChan:
			stopFunc(nil)
			return
		default:
		}
	}

	// Immediatelly run our function if we've been instantiated via
	// NewImmediateTimedLooper
	if l.Immediate {
		runIteration()
	}

	ticker := time.NewTicker(l.Interval)
	defer ticker.Stop()

	for range ticker.C {
		if stop {
			break
		}

		runIteration()
	}
}

// Quit() signals to the Looper to not run the next iteration and to
// call done() and return as quickly as possible. It is does not
// intervene between iterations.
func (l *TimedLooper) Quit() {
	go func() {
		l.quitChan <- true
	}()
}

// A FreeLooper is like a TimedLooper but doesn't wait between iterations.
type FreeLooper struct {
	Count    int
	DoneChan chan error
	quitChan chan bool
}

func NewFreeLooper(count int, done chan error) *FreeLooper {
	return &FreeLooper{
		Count:    count,
		DoneChan: done,
		quitChan: make(chan bool),
	}
}

func (l *FreeLooper) Wait() error {
	return <-l.DoneChan
}

// This is used internally, but can also be used by controlling routines
// to signal that a job is completed. The FreeLooper doesn's support its
// use outside the internals.
func (l *FreeLooper) Done(err error) {
	if l.DoneChan != nil {
		l.DoneChan <- err
	}
}

func (l *FreeLooper) Loop(fn func() error) {
	i := 0

	for {
		err := fn()
		if err != nil {
			l.Done(err)
			return
		}

		// We have to make sure not to increment if we started
		// at -1 otherwise we quit on maxint rollover.
		if l.Count != FOREVER {
			i = i + 1
			if i >= l.Count {
				l.Done(nil)
				return
			}
		}

		select {
		case <-l.quitChan:
			l.Done(nil)
			return
		default:
		}
	}
}

// Quit() signals to the Looper to not run the next iteration and to
// call Done() and return as quickly as possible. It is does not
// intervene between iterations. It is a non-blocking operation.
func (l *FreeLooper) Quit() {
	go func() {
		l.quitChan <- true
	}()
}
<commit_msg>Make quit() exit even while waiting for next tick<commit_after>package director

import (
	// "fmt"
	"time"
)

const (
	FOREVER = -1
	ONCE    = 1
)

// A Looper is used in place of a direct call to "for {}" and implements some
// controls over how the loop will be run. The Loop() function is the main
// call used by dependant routines. Common patterns like Quit and Done
// channels are easily implemented in a Looper.
type Looper interface {
	Loop(fn func() error)
	Wait() error
	Done(err error)
	Quit()
}

// A TimedLooper is a Looper that runs on a timed schedule, using a Timer
// underneath. It also implements Quit and Done channels to allow external
// routines to more easily control and synchronize the loop.
//
// If you pass in a DoneChan at creation time, it will send a nil on the
// channel when the loop has completed successfully or an error if the loop
// resulted in an error condition.
type TimedLooper struct {
	Count     int
	Interval  time.Duration
	DoneChan  chan error
	quitChan  chan bool
	Immediate bool
}

func NewTimedLooper(count int, interval time.Duration, done chan error) *TimedLooper {
	return &TimedLooper{
		Count:     count,
		Interval:  interval,
		DoneChan:  done,
		quitChan:  make(chan bool),
		Immediate: false,
	}
}

// Same as a TimedLooper, except it will execute an iteration of the loop
// immediately after calling on Loop() (as opposed to waiting until the tick)
func NewImmediateTimedLooper(count int, interval time.Duration, done chan error) *TimedLooper {
	return &TimedLooper{
		Count:     count,
		Interval:  interval,
		DoneChan:  done,
		quitChan:  make(chan bool),
		Immediate: true,
	}
}

func (l *TimedLooper) Wait() error {
	return <-l.DoneChan
}

// Signal a dependant routine that we're done with our work
func (l *TimedLooper) Done(err error) {
	if l.DoneChan != nil {
		l.DoneChan <- err
	}
}

// The main method of the Looper. This call takes a function with a single
// return value, an error. If the error is nil, the Looper will run the
// next iteration. If it's an error, it will not run the next iteration,
// will clean up any internals that need to be, and will invoke done().
func (l *TimedLooper) Loop(fn func() error) {
	i := 0

	var stop bool

	stopFunc := func(err error) {
		l.Done(err)
		stop = true
	}

	runIteration := func() {
		err := fn()
		if err != nil {
			stopFunc(err)
			return
		}

		// We have to make sure not to increment if we started
		// at -1 otherwise we quit on maxint rollover.
		if l.Count != FOREVER {
			i = i + 1
			if i >= l.Count {
				stopFunc(nil)
				return
			}
		}

		select {
		case <-l.quitChan:
			stopFunc(nil)
			return
		default:
		}
	}

	// Immediatelly run our function if we've been instantiated via
	// NewImmediateTimedLooper
	if l.Immediate {
		runIteration()
	}

	ticker := time.NewTicker(l.Interval)
	defer ticker.Stop()
	for {
		select {
		case <-ticker.C:
			runIteration()
		case <-l.quitChan:
			stopFunc(nil)
			return
		}
	}
}

// Quit() signals to the Looper to not run the next iteration and to
// call done() and return as quickly as possible. It is does not
// intervene between iterations.
func (l *TimedLooper) Quit() {
	go func() {
		l.quitChan <- true
	}()
}

// A FreeLooper is like a TimedLooper but doesn't wait between iterations.
type FreeLooper struct {
	Count    int
	DoneChan chan error
	quitChan chan bool
}

func NewFreeLooper(count int, done chan error) *FreeLooper {
	return &FreeLooper{
		Count:    count,
		DoneChan: done,
		quitChan: make(chan bool),
	}
}

func (l *FreeLooper) Wait() error {
	return <-l.DoneChan
}

// This is used internally, but can also be used by controlling routines
// to signal that a job is completed. The FreeLooper doesn's support its
// use outside the internals.
func (l *FreeLooper) Done(err error) {
	if l.DoneChan != nil {
		l.DoneChan <- err
	}
}

func (l *FreeLooper) Loop(fn func() error) {
	i := 0

	for {
		err := fn()
		if err != nil {
			l.Done(err)
			return
		}

		// We have to make sure not to increment if we started
		// at -1 otherwise we quit on maxint rollover.
		if l.Count != FOREVER {
			i = i + 1
			if i >= l.Count {
				l.Done(nil)
				return
			}
		}

		select {
		case <-l.quitChan:
			l.Done(nil)
			return
		default:
		}
	}
}

// Quit() signals to the Looper to not run the next iteration and to
// call Done() and return as quickly as possible. It is does not
// intervene between iterations. It is a non-blocking operation.
func (l *FreeLooper) Quit() {
	go func() {
		l.quitChan <- true
	}()
}
<|endoftext|>


<commit_before>package sftp

import (
	"crypto/rand"
	"encoding/hex"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"

	"github.com/juju/errors"
	"github.com/pkg/sftp"
	"github.com/restic/restic/backend"
)

const (
	tempfileRandomSuffixLength = 10
)

type SFTP struct {
	c *sftp.Client
	p string

	cmd *exec.Cmd
}

func startClient(program string, args ...string) (*SFTP, error) {
	// Connect to a remote host and request the sftp subsystem via the 'ssh'
	// command.  This assumes that passwordless login is correctly configured.
	cmd := exec.Command(program, args...)

	// send errors from ssh to stderr
	cmd.Stderr = os.Stderr

	// ignore signals sent to the parent (e.g. SIGINT)
	cmd.SysProcAttr = ignoreSigIntProcAttr()

	// get stdin and stdout
	wr, err := cmd.StdinPipe()
	if err != nil {
		log.Fatal(err)
	}
	rd, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}

	// start the process
	if err := cmd.Start(); err != nil {
		log.Fatal(err)
	}

	// open the SFTP session
	client, err := sftp.NewClientPipe(rd, wr)
	if err != nil {
		log.Fatal(err)
	}

	return &SFTP{c: client, cmd: cmd}, nil
}

// Open opens an sftp backend. When the command is started via
// exec.Command, it is expected to speak sftp on stdin/stdout. The backend
// is expected at the given path.
func Open(dir string, program string, args ...string) (*SFTP, error) {
	sftp, err := startClient(program, args...)
	if err != nil {
		return nil, err
	}

	// test if all necessary dirs and files are there
	items := []string{
		dir,
		Join(dir, backend.Paths.Data),
		Join(dir, backend.Paths.Snapshots),
		Join(dir, backend.Paths.Index),
		Join(dir, backend.Paths.Locks),
		Join(dir, backend.Paths.Keys),
		Join(dir, backend.Paths.Temp),
	}
	for _, d := range items {
		if _, err := sftp.c.Lstat(d); err != nil {
			return nil, fmt.Errorf("%s does not exist", d)
		}
	}

	sftp.p = dir
	return sftp, nil
}

// Create creates all the necessary files and directories for a new sftp
// backend at dir. Afterwards a new config blob should be created.
func Create(dir string, program string, args ...string) (*SFTP, error) {
	sftp, err := startClient(program, args...)
	if err != nil {
		return nil, err
	}

	dirs := []string{
		dir,
		Join(dir, backend.Paths.Data),
		Join(dir, backend.Paths.Snapshots),
		Join(dir, backend.Paths.Index),
		Join(dir, backend.Paths.Locks),
		Join(dir, backend.Paths.Keys),
		Join(dir, backend.Paths.Temp),
	}

	// test if config file already exists
	_, err = sftp.c.Lstat(Join(dir, backend.Paths.Config))
	if err == nil {
		return nil, errors.New("config file already exists")
	}

	// create paths for data, refs and temp blobs
	for _, d := range dirs {
		err = sftp.mkdirAll(d, backend.Modes.Dir)
		if err != nil {
			return nil, err
		}
	}

	err = sftp.c.Close()
	if err != nil {
		return nil, err
	}

	err = sftp.cmd.Wait()
	if err != nil {
		return nil, err
	}

	// open backend
	return Open(dir, program, args...)
}

// Location returns this backend's location (the directory name).
func (r *SFTP) Location() string {
	return r.p
}

// Return temp directory in correct directory for this backend.
func (r *SFTP) tempFile() (string, *sftp.File, error) {
	// choose random suffix
	buf := make([]byte, tempfileRandomSuffixLength)
	_, err := io.ReadFull(rand.Reader, buf)
	if err != nil {
		return "", nil, errors.Annotatef(err,
			"unable to read %d random bytes for tempfile name",
			tempfileRandomSuffixLength)
	}

	// construct tempfile name
	name := Join(r.p, backend.Paths.Temp, "temp-"+hex.EncodeToString(buf))

	// create file in temp dir
	f, err := r.c.Create(name)
	if err != nil {
		return "", nil, errors.Annotatef(err, "creating tempfile %q failed", name)
	}

	return name, f, nil
}

func (r *SFTP) mkdirAll(dir string, mode os.FileMode) error {
	// check if directory already exists
	fi, err := r.c.Lstat(dir)
	if err == nil {
		if fi.IsDir() {
			return nil
		}

		return fmt.Errorf("mkdirAll(%s): entry exists but is not a directory", dir)
	}

	// create parent directories
	errMkdirAll := r.mkdirAll(filepath.Dir(dir), backend.Modes.Dir)

	// create directory
	errMkdir := r.c.Mkdir(dir)

	// test if directory was created successfully
	fi, err = r.c.Lstat(dir)
	if err != nil {
		// return previous errors
		return fmt.Errorf("mkdirAll(%s): unable to create directories: %v, %v", dir, errMkdirAll, errMkdir)
	}

	if !fi.IsDir() {
		return fmt.Errorf("mkdirAll(%s): entry exists but is not a directory", dir)
	}

	// set mode
	return r.c.Chmod(dir, mode)
}

// Rename temp file to final name according to type and name.
func (r *SFTP) renameFile(oldname string, t backend.Type, name string) error {
	filename := r.filename(t, name)

	// create directories if necessary
	if t == backend.Data {
		err := r.mkdirAll(filepath.Dir(filename), backend.Modes.Dir)
		if err != nil {
			return err
		}
	}

	// test if new file exists
	if _, err := r.c.Lstat(filename); err == nil {
		return fmt.Errorf("Close(): file %v already exists", filename)
	}

	err := r.c.Rename(oldname, filename)
	if err != nil {
		return err
	}

	// set mode to read-only
	fi, err := r.c.Lstat(filename)
	if err != nil {
		return err
	}

	return r.c.Chmod(filename, fi.Mode()&os.FileMode(^uint32(0222)))
}

type sftpBlob struct {
	f        *sftp.File
	tempname string
	size     uint
	closed   bool
	backend  *SFTP
}

func (sb *sftpBlob) Finalize(t backend.Type, name string) error {
	if sb.closed {
		return errors.New("Close() called on closed file")
	}
	sb.closed = true

	err := sb.f.Close()
	if err != nil {
		return fmt.Errorf("sftp: file.Close: %v", err)
	}

	// rename file
	err = sb.backend.renameFile(sb.tempname, t, name)
	if err != nil {
		return fmt.Errorf("sftp: renameFile: %v", err)
	}

	return nil
}

func (sb *sftpBlob) Write(p []byte) (int, error) {
	n, err := sb.f.Write(p)
	sb.size += uint(n)
	return n, err
}

func (sb *sftpBlob) Size() uint {
	return sb.size
}

// Create creates a new Blob. The data is available only after Finalize()
// has been called on the returned Blob.
func (r *SFTP) Create() (backend.Blob, error) {
	// TODO: make sure that tempfile is removed upon error

	// create tempfile in backend
	filename, file, err := r.tempFile()
	if err != nil {
		return nil, errors.Annotate(err, "create tempfile")
	}

	blob := sftpBlob{
		f:        file,
		tempname: filename,
		backend:  r,
	}

	return &blob, nil
}

func Join(parts ...string) string {
	return filepath.Clean(strings.Join(parts, "/"))
}

// Construct path for given backend.Type and name.
func (r *SFTP) filename(t backend.Type, name string) string {
	if t == backend.Config {
		return Join(r.p, "config")
	}

	return Join(r.dirname(t, name), name)
}

// Construct directory for given backend.Type.
func (r *SFTP) dirname(t backend.Type, name string) string {
	var n string
	switch t {
	case backend.Data:
		n = backend.Paths.Data
		if len(name) > 2 {
			n = Join(n, name[:2])
		}
	case backend.Snapshot:
		n = backend.Paths.Snapshots
	case backend.Index:
		n = backend.Paths.Index
	case backend.Lock:
		n = backend.Paths.Locks
	case backend.Key:
		n = backend.Paths.Keys
	}
	return Join(r.p, n)
}

// Get returns a reader that yields the content stored under the given
// name. The reader should be closed after draining it.
func (r *SFTP) Get(t backend.Type, name string) (io.ReadCloser, error) {
	// try to open file
	file, err := r.c.Open(r.filename(t, name))
	if err != nil {
		return nil, err
	}

	return file, nil
}

// GetReader returns an io.ReadCloser for the Blob with the given name of
// type t at offset and length. If length is 0, the reader reads until EOF.
func (r *SFTP) GetReader(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) {
	f, err := r.c.Open(r.filename(t, name))
	if err != nil {
		return nil, err
	}

	if offset > 0 {
		_, err = f.Seek(int64(offset), 0)
		if err != nil {
			return nil, err
		}
	}

	if length == 0 {
		return f, nil
	}

	return backend.LimitReadCloser(f, int64(length)), nil
}

// Test returns true if a blob of the given type and name exists in the backend.
func (r *SFTP) Test(t backend.Type, name string) (bool, error) {
	_, err := r.c.Lstat(r.filename(t, name))
	if err != nil {
		if _, ok := err.(*sftp.StatusError); ok {
			return false, nil
		}

		return false, err
	}

	return true, nil
}

// Remove removes the content stored at name.
func (r *SFTP) Remove(t backend.Type, name string) error {
	return r.c.Remove(r.filename(t, name))
}

// List returns a channel that yields all names of blobs of type t. A
// goroutine is started for this. If the channel done is closed, sending
// stops.
func (r *SFTP) List(t backend.Type, done <-chan struct{}) <-chan string {
	ch := make(chan string)

	go func() {
		defer close(ch)

		if t == backend.Data {
			// read first level
			basedir := r.dirname(t, "")

			list1, err := r.c.ReadDir(basedir)
			if err != nil {
				return
			}

			dirs := make([]string, 0, len(list1))
			for _, d := range list1 {
				dirs = append(dirs, d.Name())
			}

			sort.Strings(dirs)

			// read files
			for _, dir := range dirs {
				entries, err := r.c.ReadDir(Join(basedir, dir))
				if err != nil {
					continue
				}

				items := make([]string, 0, len(entries))
				for _, entry := range entries {
					items = append(items, entry.Name())
				}

				sort.Strings(items)

				for _, file := range items {
					select {
					case ch <- file:
					case <-done:
						return
					}
				}
			}
		} else {
			entries, err := r.c.ReadDir(r.dirname(t, ""))
			if err != nil {
				return
			}

			items := make([]string, 0, len(entries))
			for _, entry := range entries {
				items = append(items, entry.Name())
			}

			sort.Strings(items)

			for _, file := range items {
				select {
				case ch <- file:
				case <-done:
					return
				}
			}
		}
	}()

	return ch

}

// Close closes the sftp connection and terminates the underlying command.
func (s *SFTP) Close() error {
	if s == nil {
		return nil
	}

	s.c.Close()

	if err := s.cmd.Process.Kill(); err != nil {
		return err
	}

	return s.cmd.Wait()
}
<commit_msg>sftp: implement open with config<commit_after>package sftp

import (
	"crypto/rand"
	"encoding/hex"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"

	"github.com/juju/errors"
	"github.com/pkg/sftp"
	"github.com/restic/restic/backend"
)

const (
	tempfileRandomSuffixLength = 10
)

type SFTP struct {
	c *sftp.Client
	p string

	cmd *exec.Cmd
}

func startClient(program string, args ...string) (*SFTP, error) {
	// Connect to a remote host and request the sftp subsystem via the 'ssh'
	// command.  This assumes that passwordless login is correctly configured.
	cmd := exec.Command(program, args...)

	// send errors from ssh to stderr
	cmd.Stderr = os.Stderr

	// ignore signals sent to the parent (e.g. SIGINT)
	cmd.SysProcAttr = ignoreSigIntProcAttr()

	// get stdin and stdout
	wr, err := cmd.StdinPipe()
	if err != nil {
		log.Fatal(err)
	}
	rd, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}

	// start the process
	if err := cmd.Start(); err != nil {
		log.Fatal(err)
	}

	// open the SFTP session
	client, err := sftp.NewClientPipe(rd, wr)
	if err != nil {
		log.Fatal(err)
	}

	return &SFTP{c: client, cmd: cmd}, nil
}

// Open opens an sftp backend. When the command is started via
// exec.Command, it is expected to speak sftp on stdin/stdout. The backend
// is expected at the given path.
func Open(dir string, program string, args ...string) (*SFTP, error) {
	sftp, err := startClient(program, args...)
	if err != nil {
		return nil, err
	}

	// test if all necessary dirs and files are there
	items := []string{
		dir,
		Join(dir, backend.Paths.Data),
		Join(dir, backend.Paths.Snapshots),
		Join(dir, backend.Paths.Index),
		Join(dir, backend.Paths.Locks),
		Join(dir, backend.Paths.Keys),
		Join(dir, backend.Paths.Temp),
	}
	for _, d := range items {
		if _, err := sftp.c.Lstat(d); err != nil {
			return nil, fmt.Errorf("%s does not exist", d)
		}
	}

	sftp.p = dir
	return sftp, nil
}

func buildSSHCommand(cfg Config) []string {
	args := []string{cfg.Host}
	if cfg.User != "" {
		args = append(args, "-l")
		args = append(args, cfg.User)
	}
	args = append(args, "-s")
	args = append(args, "sftp")
	return args
}

// OpenWithConfig opens an sftp backend as described by the config by running
// "ssh" with the appropiate arguments.
func OpenWithConfig(cfg Config) (*SFTP, error) {
	return Open(cfg.Dir, "ssh", buildSSHCommand(cfg)...)
}

// Create creates all the necessary files and directories for a new sftp
// backend at dir. Afterwards a new config blob should be created.
func Create(dir string, program string, args ...string) (*SFTP, error) {
	sftp, err := startClient(program, args...)
	if err != nil {
		return nil, err
	}

	dirs := []string{
		dir,
		Join(dir, backend.Paths.Data),
		Join(dir, backend.Paths.Snapshots),
		Join(dir, backend.Paths.Index),
		Join(dir, backend.Paths.Locks),
		Join(dir, backend.Paths.Keys),
		Join(dir, backend.Paths.Temp),
	}

	// test if config file already exists
	_, err = sftp.c.Lstat(Join(dir, backend.Paths.Config))
	if err == nil {
		return nil, errors.New("config file already exists")
	}

	// create paths for data, refs and temp blobs
	for _, d := range dirs {
		err = sftp.mkdirAll(d, backend.Modes.Dir)
		if err != nil {
			return nil, err
		}
	}

	err = sftp.c.Close()
	if err != nil {
		return nil, err
	}

	err = sftp.cmd.Wait()
	if err != nil {
		return nil, err
	}

	// open backend
	return Open(dir, program, args...)
}

// CreateWithConfig creates an sftp backend as described by the config by running
// "ssh" with the appropiate arguments.
func CreateWithConfig(cfg Config) (*SFTP, error) {
	return Create(cfg.Dir, "ssh", buildSSHCommand(cfg)...)
}

// Location returns this backend's location (the directory name).
func (r *SFTP) Location() string {
	return r.p
}

// Return temp directory in correct directory for this backend.
func (r *SFTP) tempFile() (string, *sftp.File, error) {
	// choose random suffix
	buf := make([]byte, tempfileRandomSuffixLength)
	_, err := io.ReadFull(rand.Reader, buf)
	if err != nil {
		return "", nil, errors.Annotatef(err,
			"unable to read %d random bytes for tempfile name",
			tempfileRandomSuffixLength)
	}

	// construct tempfile name
	name := Join(r.p, backend.Paths.Temp, "temp-"+hex.EncodeToString(buf))

	// create file in temp dir
	f, err := r.c.Create(name)
	if err != nil {
		return "", nil, errors.Annotatef(err, "creating tempfile %q failed", name)
	}

	return name, f, nil
}

func (r *SFTP) mkdirAll(dir string, mode os.FileMode) error {
	// check if directory already exists
	fi, err := r.c.Lstat(dir)
	if err == nil {
		if fi.IsDir() {
			return nil
		}

		return fmt.Errorf("mkdirAll(%s): entry exists but is not a directory", dir)
	}

	// create parent directories
	errMkdirAll := r.mkdirAll(filepath.Dir(dir), backend.Modes.Dir)

	// create directory
	errMkdir := r.c.Mkdir(dir)

	// test if directory was created successfully
	fi, err = r.c.Lstat(dir)
	if err != nil {
		// return previous errors
		return fmt.Errorf("mkdirAll(%s): unable to create directories: %v, %v", dir, errMkdirAll, errMkdir)
	}

	if !fi.IsDir() {
		return fmt.Errorf("mkdirAll(%s): entry exists but is not a directory", dir)
	}

	// set mode
	return r.c.Chmod(dir, mode)
}

// Rename temp file to final name according to type and name.
func (r *SFTP) renameFile(oldname string, t backend.Type, name string) error {
	filename := r.filename(t, name)

	// create directories if necessary
	if t == backend.Data {
		err := r.mkdirAll(filepath.Dir(filename), backend.Modes.Dir)
		if err != nil {
			return err
		}
	}

	// test if new file exists
	if _, err := r.c.Lstat(filename); err == nil {
		return fmt.Errorf("Close(): file %v already exists", filename)
	}

	err := r.c.Rename(oldname, filename)
	if err != nil {
		return err
	}

	// set mode to read-only
	fi, err := r.c.Lstat(filename)
	if err != nil {
		return err
	}

	return r.c.Chmod(filename, fi.Mode()&os.FileMode(^uint32(0222)))
}

type sftpBlob struct {
	f        *sftp.File
	tempname string
	size     uint
	closed   bool
	backend  *SFTP
}

func (sb *sftpBlob) Finalize(t backend.Type, name string) error {
	if sb.closed {
		return errors.New("Close() called on closed file")
	}
	sb.closed = true

	err := sb.f.Close()
	if err != nil {
		return fmt.Errorf("sftp: file.Close: %v", err)
	}

	// rename file
	err = sb.backend.renameFile(sb.tempname, t, name)
	if err != nil {
		return fmt.Errorf("sftp: renameFile: %v", err)
	}

	return nil
}

func (sb *sftpBlob) Write(p []byte) (int, error) {
	n, err := sb.f.Write(p)
	sb.size += uint(n)
	return n, err
}

func (sb *sftpBlob) Size() uint {
	return sb.size
}

// Create creates a new Blob. The data is available only after Finalize()
// has been called on the returned Blob.
func (r *SFTP) Create() (backend.Blob, error) {
	// TODO: make sure that tempfile is removed upon error

	// create tempfile in backend
	filename, file, err := r.tempFile()
	if err != nil {
		return nil, errors.Annotate(err, "create tempfile")
	}

	blob := sftpBlob{
		f:        file,
		tempname: filename,
		backend:  r,
	}

	return &blob, nil
}

func Join(parts ...string) string {
	return filepath.Clean(strings.Join(parts, "/"))
}

// Construct path for given backend.Type and name.
func (r *SFTP) filename(t backend.Type, name string) string {
	if t == backend.Config {
		return Join(r.p, "config")
	}

	return Join(r.dirname(t, name), name)
}

// Construct directory for given backend.Type.
func (r *SFTP) dirname(t backend.Type, name string) string {
	var n string
	switch t {
	case backend.Data:
		n = backend.Paths.Data
		if len(name) > 2 {
			n = Join(n, name[:2])
		}
	case backend.Snapshot:
		n = backend.Paths.Snapshots
	case backend.Index:
		n = backend.Paths.Index
	case backend.Lock:
		n = backend.Paths.Locks
	case backend.Key:
		n = backend.Paths.Keys
	}
	return Join(r.p, n)
}

// Get returns a reader that yields the content stored under the given
// name. The reader should be closed after draining it.
func (r *SFTP) Get(t backend.Type, name string) (io.ReadCloser, error) {
	// try to open file
	file, err := r.c.Open(r.filename(t, name))
	if err != nil {
		return nil, err
	}

	return file, nil
}

// GetReader returns an io.ReadCloser for the Blob with the given name of
// type t at offset and length. If length is 0, the reader reads until EOF.
func (r *SFTP) GetReader(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) {
	f, err := r.c.Open(r.filename(t, name))
	if err != nil {
		return nil, err
	}

	if offset > 0 {
		_, err = f.Seek(int64(offset), 0)
		if err != nil {
			return nil, err
		}
	}

	if length == 0 {
		return f, nil
	}

	return backend.LimitReadCloser(f, int64(length)), nil
}

// Test returns true if a blob of the given type and name exists in the backend.
func (r *SFTP) Test(t backend.Type, name string) (bool, error) {
	_, err := r.c.Lstat(r.filename(t, name))
	if err != nil {
		if _, ok := err.(*sftp.StatusError); ok {
			return false, nil
		}

		return false, err
	}

	return true, nil
}

// Remove removes the content stored at name.
func (r *SFTP) Remove(t backend.Type, name string) error {
	return r.c.Remove(r.filename(t, name))
}

// List returns a channel that yields all names of blobs of type t. A
// goroutine is started for this. If the channel done is closed, sending
// stops.
func (r *SFTP) List(t backend.Type, done <-chan struct{}) <-chan string {
	ch := make(chan string)

	go func() {
		defer close(ch)

		if t == backend.Data {
			// read first level
			basedir := r.dirname(t, "")

			list1, err := r.c.ReadDir(basedir)
			if err != nil {
				return
			}

			dirs := make([]string, 0, len(list1))
			for _, d := range list1 {
				dirs = append(dirs, d.Name())
			}

			sort.Strings(dirs)

			// read files
			for _, dir := range dirs {
				entries, err := r.c.ReadDir(Join(basedir, dir))
				if err != nil {
					continue
				}

				items := make([]string, 0, len(entries))
				for _, entry := range entries {
					items = append(items, entry.Name())
				}

				sort.Strings(items)

				for _, file := range items {
					select {
					case ch <- file:
					case <-done:
						return
					}
				}
			}
		} else {
			entries, err := r.c.ReadDir(r.dirname(t, ""))
			if err != nil {
				return
			}

			items := make([]string, 0, len(entries))
			for _, entry := range entries {
				items = append(items, entry.Name())
			}

			sort.Strings(items)

			for _, file := range items {
				select {
				case ch <- file:
				case <-done:
					return
				}
			}
		}
	}()

	return ch

}

// Close closes the sftp connection and terminates the underlying command.
func (s *SFTP) Close() error {
	if s == nil {
		return nil
	}

	s.c.Close()

	if err := s.cmd.Process.Kill(); err != nil {
		return err
	}

	return s.cmd.Wait()
}
<|endoftext|>


<commit_before>package alita

import (
	"bytes"
	"reflect"
	"testing"
)

func testAlign(t *testing.T, a *Aligner, src, dst []byte) {
	w := bytes.NewBuffer(make([]byte, 0))
	a.SetOutput(w)

	r := bytes.NewReader(src)
	if err := a.ReadAll(r); err != nil {
		t.Errorf("ReadAll(%q) returns err; want nil", src)
	}
	if err := a.Flush(); err != nil {
		t.Errorf("Flush(%q) returns err; want nil", src)
	}

	actual := w.Bytes()
	expect := dst
	if !reflect.DeepEqual(actual, expect) {
		t.Errorf("got:\n%swant:\n%s", actual, expect)
	}
}

type AlignTest struct {
	src []byte
	dst []byte
}

var indexTestsAlignSimple = []AlignTest{
	{[]byte(`
`[1:]), []byte(`
`[1:])},

	{[]byte(`
a
bb
ccc
`[1:]), []byte(`
a
bb
ccc
`[1:])},

	{[]byte(`
1 100
10 10
100 1
`[1:]), []byte(`
1   100
10  10
100 1
`[1:])},

	{[]byte(`
1 10 100
10 100 1
100 1 10
`[1:]), []byte(`
1   10  100
10  100 1
100 1   10
`[1:])},

	{[]byte(`
1 10 100 1000
10 100 1000 1
100 1000 1 10
1000 1 10 100
`[1:]), []byte(`
1    10   100  1000
10   100  1000 1
100  1000 1    10
1000 1    10   100
`[1:])},

	{[]byte(`
(x == ABC) ? "abc" :
(x == DEFG) ? "defg" :
(x == HIJKL) ? "hijkl" : "???");
`[1:]), []byte(`
(x == ABC)   ? "abc"   :
(x == DEFG)  ? "defg"  :
(x == HIJKL) ? "hijkl" : "???");
`[1:])},

	{[]byte(`
one two three four five
six seven eight nine ten
eleven twelve thirteen fourteen fifteen
`[1:]), []byte(`
one    two    three    four     five
six    seven  eight    nine     ten
eleven twelve thirteen fourteen fifteen
`[1:])},
}

func TestAlignSimple(t *testing.T) {
	for _, test := range indexTestsAlignSimple {
		a := NewAligner(nil)
		testAlign(t, a, test.src, test.dst)
	}
}

type AlignDelimiterTest struct {
	delim string
	src   []byte
	dst   []byte
}

var indexTestsAlignFixedTest = []AlignDelimiterTest{
	{`=`, []byte(`
a =  1
 bbb = 10
ccccc = 100
`[1:]), []byte(`
a     = 1
bbb   = 10
ccccc = 100
`[1:])},

	{`=`, []byte(`
[user]
name=        Tom
age  =17
userid = 10001
`[1:]), []byte(`
[user]
name   = Tom
age    = 17
userid = 10001
`[1:])},

	{`=`, []byte(`
name=Tom
age=17
`[1:]), []byte(`
name = Tom
age  = 17
`[1:])},

	{`<<`, []byte(`
cout    <<    "9 * 2 = "<<9 * 2 << endl;
cout << "9 / 2 = "<<9 / 2 << ".." << 9 % 2<< endl;
`[1:]), []byte(`
cout << "9 * 2 = " << 9 * 2 << endl;
cout << "9 / 2 = " << 9 / 2 << ".."  << 9 % 2 << endl;
`[1:])},

	{`<<`, []byte(`
cin << x;
cin      << y;
cout << "this is x=" << x;
cout << "but y=" << y << "is not";
`[1:]), []byte(`
cin  << x;
cin  << y;
cout << "this is x=" << x;
cout << "but y="     << y  << "is not";
`[1:])},

	{`:`, []byte(`
one:two:three
four:five:six
seven:eight:nine
`[1:]), []byte(`
one   : two   : three
four  : five  : six
seven : eight : nine
`[1:])},

	{`:=`, []byte(`
aa:=bb:=cc:=1;
a:=b:=c:=1;
aaa:=bbb:=ccc:=1;
`[1:]), []byte(`
aa  := bb  := cc  := 1;
a   := b   := c   := 1;
aaa := bbb := ccc := 1;
`[1:])},

	{`=`, []byte(`
aa=bb=cc=1;
a=b=c=1;
aaa=bbb=ccc=1;
`[1:]), []byte(`
aa  = bb  = cc  = 1;
a   = b   = c   = 1;
aaa = bbb = ccc = 1;
`[1:])},

	{`=`, []byte(`
a =  1
 bbb = 10
ccccc = 100
a =  2
 bbb = 20
ccccc = 200
a =  3
 bbb = 30
ccccc = 300
a =  4
 bbb = 40
ccccc = 400
`[1:]), []byte(`
a     = 1
bbb   = 10
ccccc = 100
a     = 2
bbb   = 20
ccccc = 200
a     = 3
bbb   = 30
ccccc = 300
a     = 4
bbb   = 40
ccccc = 400
`[1:])},

	{`=`, []byte(`
a = 1

bbb = 10

ccccc = 100
`[1:]), []byte(`
a     = 1

bbb   = 10

ccccc = 100
`[1:])},

	{`＝`, []byte(`
あ ＝  壱
あいう ＝ 壱十
あいうえお ＝ 壱十百
`[1:]), []byte(`
あ         ＝ 壱
あいう     ＝ 壱十
あいうえお ＝ 壱十百
`[1:])},
}

func TestAlignFixed(t *testing.T) {
	for _, test := range indexTestsAlignFixedTest {
		a := NewAligner(nil)
		if err := a.Delimiter.Set(test.delim); err != nil {
			t.Errorf("Set(%q) returns %q; want nil",
				test.delim, err)
		}
		testAlign(t, a, test.src, test.dst)
	}
}

var indexTestsAlignRegexpTest = []AlignDelimiterTest{
	{`=+>`, []byte(`
a=>b ===>  c
c ==>    d ==>e
f===> g =>   h
`[1:]), []byte(`
a =>   b ===> c
c ==>  d ==>  e
f ===> g =>   h
`[1:])},

	{`[:/]+`, []byte(`
https://github.com/vim-scripts/Align
https://github.com/h1mesuke/vim-alignta
https://github.com/kusabashira/alita
`[1:]), []byte(`
https :// github.com / vim-scripts / Align
https :// github.com / h1mesuke    / vim-alignta
https :// github.com / kusabashira / alita
`[1:])},

	{`\\$`, []byte(`
one \
two three \
four five six \
seven \\ \
eight \nine \
ten \
`[1:]), []byte(`
one           \
two three     \
four five six \
seven \\      \
eight \nine   \
ten           \
`[1:])},

	{`(&|\\\\)`, []byte(`
one&two&three\\ \hline
four&five&six\\
seven&eight&nine\\
`[1:]), []byte(`
one   & two   & three \\ \hline
four  & five  & six   \\
seven & eight & nine  \\
`[1:])},

	{`\d+`, []byte(`
a \d\+ 1 aaaaa
bbb \d\+ 10 bbb
ccccc \d\+ 100 c
`[1:]), []byte(`
a \d\+     1   aaaaa
bbb \d\+   10  bbb
ccccc \d\+ 100 c
`[1:])},

	{`\d+`, []byte(`
a \d\+ 1 \u\+ AAAAA a
bbb \d\+ 10 \u\+ BBB b
ccccc \d\+ 100 \u\+ C c
`[1:]), []byte(`
a \d\+     1   \u\+ AAAAA a
bbb \d\+   10  \u\+ BBB b
ccccc \d\+ 100 \u\+ C c
`[1:])},
}

func TestAlignRegexp(t *testing.T) {
	for _, test := range indexTestsAlignRegexpTest {
		a := NewAligner(nil)
		a.Delimiter.UseRegexp = true
		if err := a.Delimiter.Set(test.delim); err != nil {
			t.Errorf("Set(%q) returns %q; want nil",
				test.delim, err)
		}
		testAlign(t, a, test.src, test.dst)
	}
}
<commit_msg>Add test for aligne with margin or justify<commit_after>package alita

import (
	"bytes"
	"reflect"
	"testing"
)

func testAlign(t *testing.T, a *Aligner, src, dst []byte) {
	w := bytes.NewBuffer(make([]byte, 0))
	a.SetOutput(w)

	r := bytes.NewReader(src)
	if err := a.ReadAll(r); err != nil {
		t.Errorf("ReadAll(%q) returns err; want nil", src)
	}
	if err := a.Flush(); err != nil {
		t.Errorf("Flush(%q) returns err; want nil", src)
	}

	actual := w.Bytes()
	expect := dst
	if !reflect.DeepEqual(actual, expect) {
		t.Errorf("got:\n%swant:\n%s", actual, expect)
	}
}

type AlignTest struct {
	src []byte
	dst []byte
}

var indexTestsAlignSimple = []AlignTest{
	{[]byte(`
`[1:]), []byte(`
`[1:])},

	{[]byte(`
a
bb
ccc
`[1:]), []byte(`
a
bb
ccc
`[1:])},

	{[]byte(`
1 100
10 10
100 1
`[1:]), []byte(`
1   100
10  10
100 1
`[1:])},

	{[]byte(`
1 10 100
10 100 1
100 1 10
`[1:]), []byte(`
1   10  100
10  100 1
100 1   10
`[1:])},

	{[]byte(`
1 10 100 1000
10 100 1000 1
100 1000 1 10
1000 1 10 100
`[1:]), []byte(`
1    10   100  1000
10   100  1000 1
100  1000 1    10
1000 1    10   100
`[1:])},

	{[]byte(`
(x == ABC) ? "abc" :
(x == DEFG) ? "defg" :
(x == HIJKL) ? "hijkl" : "???");
`[1:]), []byte(`
(x == ABC)   ? "abc"   :
(x == DEFG)  ? "defg"  :
(x == HIJKL) ? "hijkl" : "???");
`[1:])},

	{[]byte(`
one two three four five
six seven eight nine ten
eleven twelve thirteen fourteen fifteen
`[1:]), []byte(`
one    two    three    four     five
six    seven  eight    nine     ten
eleven twelve thirteen fourteen fifteen
`[1:])},
}

func TestAlignSimple(t *testing.T) {
	for _, test := range indexTestsAlignSimple {
		a := NewAligner(nil)
		testAlign(t, a, test.src, test.dst)
	}
}

type AlignDelimiterTest struct {
	delim string
	src   []byte
	dst   []byte
}

var indexTestsAlignFixedTest = []AlignDelimiterTest{
	{`=`, []byte(`
a =  1
 bbb = 10
ccccc = 100
`[1:]), []byte(`
a     = 1
bbb   = 10
ccccc = 100
`[1:])},

	{`=`, []byte(`
[user]
name=        Tom
age  =17
userid = 10001
`[1:]), []byte(`
[user]
name   = Tom
age    = 17
userid = 10001
`[1:])},

	{`=`, []byte(`
name=Tom
age=17
`[1:]), []byte(`
name = Tom
age  = 17
`[1:])},

	{`<<`, []byte(`
cout    <<    "9 * 2 = "<<9 * 2 << endl;
cout << "9 / 2 = "<<9 / 2 << ".." << 9 % 2<< endl;
`[1:]), []byte(`
cout << "9 * 2 = " << 9 * 2 << endl;
cout << "9 / 2 = " << 9 / 2 << ".."  << 9 % 2 << endl;
`[1:])},

	{`<<`, []byte(`
cin << x;
cin      << y;
cout << "this is x=" << x;
cout << "but y=" << y << "is not";
`[1:]), []byte(`
cin  << x;
cin  << y;
cout << "this is x=" << x;
cout << "but y="     << y  << "is not";
`[1:])},

	{`:`, []byte(`
one:two:three
four:five:six
seven:eight:nine
`[1:]), []byte(`
one   : two   : three
four  : five  : six
seven : eight : nine
`[1:])},

	{`:=`, []byte(`
aa:=bb:=cc:=1;
a:=b:=c:=1;
aaa:=bbb:=ccc:=1;
`[1:]), []byte(`
aa  := bb  := cc  := 1;
a   := b   := c   := 1;
aaa := bbb := ccc := 1;
`[1:])},

	{`=`, []byte(`
aa=bb=cc=1;
a=b=c=1;
aaa=bbb=ccc=1;
`[1:]), []byte(`
aa  = bb  = cc  = 1;
a   = b   = c   = 1;
aaa = bbb = ccc = 1;
`[1:])},

	{`=`, []byte(`
a =  1
 bbb = 10
ccccc = 100
a =  2
 bbb = 20
ccccc = 200
a =  3
 bbb = 30
ccccc = 300
a =  4
 bbb = 40
ccccc = 400
`[1:]), []byte(`
a     = 1
bbb   = 10
ccccc = 100
a     = 2
bbb   = 20
ccccc = 200
a     = 3
bbb   = 30
ccccc = 300
a     = 4
bbb   = 40
ccccc = 400
`[1:])},

	{`=`, []byte(`
a = 1

bbb = 10

ccccc = 100
`[1:]), []byte(`
a     = 1

bbb   = 10

ccccc = 100
`[1:])},

	{`＝`, []byte(`
あ ＝  壱
あいう ＝ 壱十
あいうえお ＝ 壱十百
`[1:]), []byte(`
あ         ＝ 壱
あいう     ＝ 壱十
あいうえお ＝ 壱十百
`[1:])},
}

func TestAlignFixed(t *testing.T) {
	for _, test := range indexTestsAlignFixedTest {
		a := NewAligner(nil)
		if err := a.Delimiter.Set(test.delim); err != nil {
			t.Errorf("Set(%q) returns %q; want nil",
				test.delim, err)
		}
		testAlign(t, a, test.src, test.dst)
	}
}

var indexTestsAlignRegexpTest = []AlignDelimiterTest{
	{`=+>`, []byte(`
a=>b ===>  c
c ==>    d ==>e
f===> g =>   h
`[1:]), []byte(`
a =>   b ===> c
c ==>  d ==>  e
f ===> g =>   h
`[1:])},

	{`[:/]+`, []byte(`
https://github.com/vim-scripts/Align
https://github.com/h1mesuke/vim-alignta
https://github.com/kusabashira/alita
`[1:]), []byte(`
https :// github.com / vim-scripts / Align
https :// github.com / h1mesuke    / vim-alignta
https :// github.com / kusabashira / alita
`[1:])},

	{`\\$`, []byte(`
one \
two three \
four five six \
seven \\ \
eight \nine \
ten \
`[1:]), []byte(`
one           \
two three     \
four five six \
seven \\      \
eight \nine   \
ten           \
`[1:])},

	{`(&|\\\\)`, []byte(`
one&two&three\\ \hline
four&five&six\\
seven&eight&nine\\
`[1:]), []byte(`
one   & two   & three \\ \hline
four  & five  & six   \\
seven & eight & nine  \\
`[1:])},

	{`\d+`, []byte(`
a \d\+ 1 aaaaa
bbb \d\+ 10 bbb
ccccc \d\+ 100 c
`[1:]), []byte(`
a \d\+     1   aaaaa
bbb \d\+   10  bbb
ccccc \d\+ 100 c
`[1:])},

	{`\d+`, []byte(`
a \d\+ 1 \u\+ AAAAA a
bbb \d\+ 10 \u\+ BBB b
ccccc \d\+ 100 \u\+ C c
`[1:]), []byte(`
a \d\+     1   \u\+ AAAAA a
bbb \d\+   10  \u\+ BBB b
ccccc \d\+ 100 \u\+ C c
`[1:])},
}

func TestAlignRegexp(t *testing.T) {
	for _, test := range indexTestsAlignRegexpTest {
		a := NewAligner(nil)
		a.Delimiter.UseRegexp = true
		if err := a.Delimiter.Set(test.delim); err != nil {
			t.Errorf("Set(%q) returns %q; want nil",
				test.delim, err)
		}
		testAlign(t, a, test.src, test.dst)
	}
}

type AlignMarginTest struct {
	margin string
	delim  string
	src    []byte
	dst    []byte
}

var indexTestsAlignMarginTest = []AlignMarginTest{
	{`1:0`, `=`, []byte(`
`[1:]), []byte(`
`[1:])},

	{`0:1`, `=`, []byte(`
name=Tom
age=17
`[1:]), []byte(`
name= Tom
age = 17
`[1:])},

	{`3:2`, `=`, []byte(`
name=Tom
age=17
`[1:]), []byte(`
name   =  Tom
age    =  17
`[1:])},

	{`2`, `=`, []byte(`
name=Tom
age=17
`[1:]), []byte(`
name  =  Tom
age   =  17
`[1:])},

	{`0`, `=`, []byte(`
name=Tom
age=17
`[1:]), []byte(`
name=Tom
age =17
`[1:])},

	{`1:3`, `=`, []byte(`
a=bbb=ccccc
aaa=b=ccc
`[1:]), []byte(`
a   =   bbb =   ccccc
aaa =   b   =   ccc
`[1:])},

	{`0:2`, `=`, []byte(`
a=bbb=ccccc
aaa=b=ccc
`[1:]), []byte(`
a  =  bbb=  ccccc
aaa=  b  =  ccc
`[1:])},
}

func TestAlignMargin(t *testing.T) {
	for _, test := range indexTestsAlignMarginTest {
		a := NewAligner(nil)
		if err := a.Delimiter.Set(test.delim); err != nil {
			t.Errorf("Set(%q) returns %q; want nil",
				test.delim, err)
		}
		if err := a.Margin.Set(test.margin); err != nil {
			t.Errorf("Set(%q) returns %q; want nil",
				test.delim, err)
		}
		testAlign(t, a, test.src, test.dst)
	}
}

type AlignJustifyTest struct {
	justfy string
	delim  string
	src    []byte
	dst    []byte
}

var indexTestsAlignJustify = []AlignJustifyTest{
	{`l`, `=`, []byte(`
a = bbbbb =  c = ddddd =  e = fffff =  1
 aaa = bbb = ccc =  ddd = eee = fff = 10
aaaaa =  b = ccccc = d = eeeee =  f = 100
`[1:]), []byte(`
a     = bbbbb = c     = ddddd = e     = fffff = 1
aaa   = bbb   = ccc   = ddd   = eee   = fff   = 10
aaaaa = b     = ccccc = d     = eeeee = f     = 100
`[1:])},

	{`r`, `=`, []byte(`
a = bbbbb =  c = ddddd =  e = fffff =  1
 aaa = bbb = ccc =  ddd = eee = fff = 10
aaaaa =  b = ccccc = d = eeeee =  f = 100
`[1:]), []byte(`
    a = bbbbb =     c = ddddd =     e = fffff =   1
  aaa =   bbb =   ccc =   ddd =   eee =   fff =  10
aaaaa =     b = ccccc =     d = eeeee =     f = 100
`[1:])},

	{`rl`, `=`, []byte(`
a = bbbbb =  c = ddddd =  e = fffff =  1
 aaa = bbb = ccc =  ddd = eee = fff = 10
aaaaa =  b = ccccc = d = eeeee =  f = 100
`[1:]), []byte(`
    a = bbbbb = c     = ddddd = e     = fffff = 1
  aaa = bbb   = ccc   = ddd   = eee   = fff   = 10
aaaaa = b     = ccccc = d     = eeeee = f     = 100
`[1:])},

	{`rllcc`, `=`, []byte(`
a = bbbbb =  c = ddddd =  e = fffff =  1
 aaa = bbb = ccc =  ddd = eee = fff = 10
aaaaa =  b = ccccc = d = eeeee =  f = 100
`[1:]), []byte(`
    a = bbbbb =   c   = ddddd =   e   = fffff =  1
  aaa = bbb   =  ccc  = ddd   =  eee  = fff   = 10
aaaaa = b     = ccccc = d     = eeeee = f     = 100
`[1:])},
}

func TestAlignJustify(t *testing.T) {
	for _, test := range indexTestsAlignJustify {
		a := NewAligner(nil)
		if err := a.Delimiter.Set(test.delim); err != nil {
			t.Errorf("Set(%q) returns %q; want nil",
				test.delim, err)
		}
		if err := a.Padding.Set(test.justfy); err != nil {
			t.Errorf("Set(%q) returns %q; want nil",
				test.delim, err)
		}
		testAlign(t, a, test.src, test.dst)
	}
}
<|endoftext|>


<commit_before>package zookeeper

import (
        "time"
        "strings"

	zk "github.com/samuel/go-zookeeper/zk"
)

// Client provides a wrapper around the zookeeper client
type Client struct{
    client *zk.Conn;
}

func NewZookeeperClient(machines []string) (*Client, error) {
        c, _, err := zk.Connect(machines, time.Second) //*10)
        if err != nil {
		panic(err)
	}
        return &Client{c}, nil
}

func node_walk(prefix string, c *Client, vars map[string]string ) error{
    l,stat,err:= c.client.Children(prefix);
        if err != nil {
                 return err
        }

    if (stat.NumChildren == 0) {
        b,_, err:= c.client.Get(prefix);
        if err != nil {
                 return err
        }
        vars[prefix]=string(b)

    } else {
    for _, key := range l {
       s := prefix+"/"+key
       _, stat, err := c.client.Exists(s);
        if err != nil {
                 return err
        }
        if (stat.NumChildren == 0) {
             b,_, err:= c.client.Get(s);
        if err != nil {
                 return err
        }
             vars[s]=string(b)
        } else {
          node_walk(s,c,vars)
        }
    }
   }
   return nil
}

func (c *Client) GetValues(keys []string) (map[string]string, error) {
    vars := make(map[string]string)
    for _, v := range keys {
        v= strings.Replace(v,"/*","",-1);
        _, _, err := c.client.Exists(v);
        if err != nil {
	         return vars, err
	}
          if (v == "/" ) {v=""}
          err=node_walk(v,c,vars)
		if err != nil {
			return vars, err
		}
     }
    return vars, nil
}

// WatchPrefix is not yet implemented. There's a WIP.
// Since zookeeper doesn't handle recursive watch, we need to create a *lot* of watches.
// Implementation should take care of this.
// A good start is bamboo
// URL https://github.com/QubitProducts/bamboo/blob/master/qzk/qzk.go
// We also need to encourage users to set prefix and add a flag to enale support for "" prefix (aka "/")
//

func (c *Client) WatchPrefix(prefix string, waitIndex uint64, stopChan chan bool) (uint64, error) {
	<-stopChan
	return 0, nil
}

<commit_msg>Fmt file<commit_after>package zookeeper

import (
	"strings"
	"time"

	zk "github.com/samuel/go-zookeeper/zk"
)

// Client provides a wrapper around the zookeeper client
type Client struct {
	client *zk.Conn
}

func NewZookeeperClient(machines []string) (*Client, error) {
	c, _, err := zk.Connect(machines, time.Second) //*10)
	if err != nil {
		panic(err)
	}
	return &Client{c}, nil
}

func node_walk(prefix string, c *Client, vars map[string]string) error {
	l, stat, err := c.client.Children(prefix)
	if err != nil {
		return err
	}

	if stat.NumChildren == 0 {
		b, _, err := c.client.Get(prefix)
		if err != nil {
			return err
		}
		vars[prefix] = string(b)

	} else {
		for _, key := range l {
			s := prefix + "/" + key
			_, stat, err := c.client.Exists(s)
			if err != nil {
				return err
			}
			if stat.NumChildren == 0 {
				b, _, err := c.client.Get(s)
				if err != nil {
					return err
				}
				vars[s] = string(b)
			} else {
				node_walk(s, c, vars)
			}
		}
	}
	return nil
}

func (c *Client) GetValues(keys []string) (map[string]string, error) {
	vars := make(map[string]string)
	for _, v := range keys {
		v = strings.Replace(v, "/*", "", -1)
		_, _, err := c.client.Exists(v)
		if err != nil {
			return vars, err
		}
		if v == "/" {
			v = ""
		}
		err = node_walk(v, c, vars)
		if err != nil {
			return vars, err
		}
	}
	return vars, nil
}

// WatchPrefix is not yet implemented. There's a WIP.
// Since zookeeper doesn't handle recursive watch, we need to create a *lot* of watches.
// Implementation should take care of this.
// A good start is bamboo
// URL https://github.com/QubitProducts/bamboo/blob/master/qzk/qzk.go
// We also need to encourage users to set prefix and add a flag to enale support for "" prefix (aka "/")
//

func (c *Client) WatchPrefix(prefix string, waitIndex uint64, stopChan chan bool) (uint64, error) {
	<-stopChan
	return 0, nil
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backup_test

import (
	"bytes"
	"errors"
	"github.com/jacobsa/comeback/backup"
	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/blob/mock"
	"github.com/jacobsa/comeback/fs/mock"
	. "github.com/jacobsa/oglematchers"
	"github.com/jacobsa/oglemock"
	. "github.com/jacobsa/ogletest"
	"os"
	"testing"
)

func TestFileRestorer(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type closeSnoopingBuffer struct {
	bytes.Buffer

	closeError error
	closeCalled bool
}

func (b *closeSnoopingBuffer) Close() error {
	b.closeCalled = true
	return b.closeError
}

type FileRestorerTest struct {
	blobStore mock_blob.MockStore
	fileSystem mock_fs.MockFileSystem
	file closeSnoopingBuffer

	fileRestorer backup.FileRestorer

	scores []blob.Score
	path string
	perms os.FileMode

	err    error
}

func init() { RegisterTestSuite(&FileRestorerTest{}) }

func (t *FileRestorerTest) SetUp(i *TestInfo) {
	var err error

	// Create dependencies.
	t.blobStore = mock_blob.NewMockStore(i.MockController, "blobStore")
	t.fileSystem = mock_fs.NewMockFileSystem(i.MockController, "fileSystem")

	// Create restorer.
	t.fileRestorer, err = backup.NewFileRestorer(t.blobStore, t.fileSystem)
	AssertEq(nil, err)
}

func (t *FileRestorerTest) call() {
	t.err = t.fileRestorer.RestoreFile(t.scores, t.path, t.perms)
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *FileRestorerTest) CallsCreateFile() {
	t.path = "taco"
	t.perms = 0612

	// File system
	ExpectCall(t.fileSystem, "CreateFile")("taco", 0612).
		WillOnce(oglemock.Return(nil, errors.New("")))

	// Call
	t.call()
}

func (t *FileRestorerTest) CreateFileReturnsError() {
	// File system
	ExpectCall(t.fileSystem, "CreateFile")(Any(), Any()).
		WillOnce(oglemock.Return(nil, errors.New("taco")))

	// Call
	t.call()

	ExpectThat(t.err, Error(HasSubstr("CreateFile")))
	ExpectThat(t.err, Error(HasSubstr("taco")))
}

func (t *FileRestorerTest) NoBlobs() {
	// File system
	ExpectCall(t.fileSystem, "CreateFile")(Any(), Any()).
		WillOnce(oglemock.Return(&t.file, nil))

	// Call
	t.call()
	AssertEq(nil, t.err)

	ExpectTrue(t.file.closeCalled)
	ExpectThat(t.file.Bytes(), ElementsAre())
}

func (t *FileRestorerTest) CallsBlobStore() {
	t.scores = []blob.Score{
		blob.ComputeScore([]byte("foo")),
		blob.ComputeScore([]byte("bar")),
		blob.ComputeScore([]byte("baz")),
	}

	// File system
	ExpectCall(t.fileSystem, "CreateFile")(Any(), Any()).
		WillOnce(oglemock.Return(&t.file, nil))

	// Blob store
	ExpectCall(t.blobStore, "Load")(DeepEquals(t.scores[0])).
		WillOnce(oglemock.Return([]byte{}, nil))

	ExpectCall(t.blobStore, "Load")(DeepEquals(t.scores[1])).
		WillOnce(oglemock.Return([]byte{}, nil))

	ExpectCall(t.blobStore, "Load")(DeepEquals(t.scores[2])).
		WillOnce(oglemock.Return([]byte{}, nil))

	// Call
	t.call()
}

func (t *FileRestorerTest) BlobStoreReturnsErrorForOneCall() {
	ExpectEq("TODO", "")
}

func (t *FileRestorerTest) WritesBlobs() {
	ExpectEq("TODO", "")
}

func (t *FileRestorerTest) WriteReturnsErrorForOneCall() {
	ExpectEq("TODO", "")
}

func (t *FileRestorerTest) CallsClose() {
	ExpectEq("TODO", "")
}

func (t *FileRestorerTest) CloseReturnsError() {
	ExpectEq("TODO", "")
}

func (t *FileRestorerTest) EverythingSucceeds() {
	ExpectEq("TODO", "")
}
<commit_msg>FileRestorerTest.BlobStoreReturnsErrorForOneCall<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backup_test

import (
	"bytes"
	"errors"
	"github.com/jacobsa/comeback/backup"
	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/blob/mock"
	"github.com/jacobsa/comeback/fs/mock"
	. "github.com/jacobsa/oglematchers"
	"github.com/jacobsa/oglemock"
	. "github.com/jacobsa/ogletest"
	"os"
	"testing"
)

func TestFileRestorer(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type closeSnoopingBuffer struct {
	bytes.Buffer

	closeError error
	closeCalled bool
}

func (b *closeSnoopingBuffer) Close() error {
	b.closeCalled = true
	return b.closeError
}

type FileRestorerTest struct {
	blobStore mock_blob.MockStore
	fileSystem mock_fs.MockFileSystem
	file closeSnoopingBuffer

	fileRestorer backup.FileRestorer

	scores []blob.Score
	path string
	perms os.FileMode

	err    error
}

func init() { RegisterTestSuite(&FileRestorerTest{}) }

func (t *FileRestorerTest) SetUp(i *TestInfo) {
	var err error

	// Create dependencies.
	t.blobStore = mock_blob.NewMockStore(i.MockController, "blobStore")
	t.fileSystem = mock_fs.NewMockFileSystem(i.MockController, "fileSystem")

	// Create restorer.
	t.fileRestorer, err = backup.NewFileRestorer(t.blobStore, t.fileSystem)
	AssertEq(nil, err)
}

func (t *FileRestorerTest) call() {
	t.err = t.fileRestorer.RestoreFile(t.scores, t.path, t.perms)
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *FileRestorerTest) CallsCreateFile() {
	t.path = "taco"
	t.perms = 0612

	// File system
	ExpectCall(t.fileSystem, "CreateFile")("taco", 0612).
		WillOnce(oglemock.Return(nil, errors.New("")))

	// Call
	t.call()
}

func (t *FileRestorerTest) CreateFileReturnsError() {
	// File system
	ExpectCall(t.fileSystem, "CreateFile")(Any(), Any()).
		WillOnce(oglemock.Return(nil, errors.New("taco")))

	// Call
	t.call()

	ExpectThat(t.err, Error(HasSubstr("CreateFile")))
	ExpectThat(t.err, Error(HasSubstr("taco")))
}

func (t *FileRestorerTest) NoBlobs() {
	// File system
	ExpectCall(t.fileSystem, "CreateFile")(Any(), Any()).
		WillOnce(oglemock.Return(&t.file, nil))

	// Call
	t.call()
	AssertEq(nil, t.err)

	ExpectTrue(t.file.closeCalled)
	ExpectThat(t.file.Bytes(), ElementsAre())
}

func (t *FileRestorerTest) CallsBlobStore() {
	t.scores = []blob.Score{
		blob.ComputeScore([]byte("foo")),
		blob.ComputeScore([]byte("bar")),
		blob.ComputeScore([]byte("baz")),
	}

	// File system
	ExpectCall(t.fileSystem, "CreateFile")(Any(), Any()).
		WillOnce(oglemock.Return(&t.file, nil))

	// Blob store
	ExpectCall(t.blobStore, "Load")(DeepEquals(t.scores[0])).
		WillOnce(oglemock.Return([]byte{}, nil))

	ExpectCall(t.blobStore, "Load")(DeepEquals(t.scores[1])).
		WillOnce(oglemock.Return([]byte{}, nil))

	ExpectCall(t.blobStore, "Load")(DeepEquals(t.scores[2])).
		WillOnce(oglemock.Return([]byte{}, nil))

	// Call
	t.call()
}

func (t *FileRestorerTest) BlobStoreReturnsErrorForOneCall() {
	t.scores = []blob.Score{
		blob.ComputeScore([]byte("foo")),
		blob.ComputeScore([]byte("bar")),
		blob.ComputeScore([]byte("baz")),
	}

	// File system
	ExpectCall(t.fileSystem, "CreateFile")(Any(), Any()).
		WillOnce(oglemock.Return(&t.file, nil))

	// Blob store
	ExpectCall(t.blobStore, "Load")(Any()).
		WillOnce(oglemock.Return([]byte{}, nil)).
		WillOnce(oglemock.Return(nil, errors.New("taco")))

	// Call
	t.call()

	ExpectTrue(t.file.closeCalled)
	ExpectThat(t.err, Error(HasSubstr("Load")))
	ExpectThat(t.err, Error(HasSubstr("taco")))
}

func (t *FileRestorerTest) WritesBlobs() {
	ExpectEq("TODO", "")
}

func (t *FileRestorerTest) WriteReturnsErrorForOneCall() {
	ExpectEq("TODO", "")
}

func (t *FileRestorerTest) CallsClose() {
	ExpectEq("TODO", "")
}

func (t *FileRestorerTest) CloseReturnsError() {
	ExpectEq("TODO", "")
}

func (t *FileRestorerTest) EverythingSucceeds() {
	ExpectEq("TODO", "")
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2013, 2014 Conformal Systems LLC <info@conformal.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"

	"github.com/conformal/btcnet"
	"github.com/conformal/btcwire"
)

// networkDir returns the directory name of a network directory to hold account
// files.
func networkDir(net *btcnet.Params) string {
	netname := net.Name

	// For now, we must always name the testnet data directory as "testnet"
	// and not "testnet3" or any other version, as the btcnet testnet3
	// paramaters will likely be switched to being named "testnet3" in the
	// future.  This is done to future proof that change, and an upgrade
	// plan to move the testnet3 data directory can be worked out later.
	if net.Net == btcwire.TestNet3 {
		netname = "testnet"
	}

	return filepath.Join(cfg.DataDir, netname)
}

// tmpNetworkDir returns the temporary directory name for a given network.
func tmpNetworkDir(net *btcnet.Params) string {
	return networkDir(net) + "_tmp"
}

// freshDir creates a new directory specified by path if it does not
// exist.  If the directory already exists, all files contained in the
// directory are removed.
func freshDir(path string) error {
	if err := checkCreateDir(path); err != nil {
		return err
	}

	// Remove all files in the directory.
	fd, err := os.Open(path)
	if err != nil {
		return err
	}
	defer func() {
		if err := fd.Close(); err != nil {
			log.Warnf("Cannot close directory: %v", err)
		}
	}()
	names, err := fd.Readdirnames(0)
	if err != nil {
		return err
	}
	for _, name := range names {
		if err := os.RemoveAll(name); err != nil {
			return err
		}
	}

	return nil
}

// checkCreateDir checks that the path exists and is a directory.
// If path does not exist, it is created.
func checkCreateDir(path string) error {
	if fi, err := os.Stat(path); err != nil {
		if os.IsNotExist(err) {
			// Attempt data directory creation
			if err = os.MkdirAll(path, 0700); err != nil {
				return fmt.Errorf("cannot create directory: %s", err)
			}
		} else {
			return fmt.Errorf("error checking directory: %s", err)
		}
	} else {
		if !fi.IsDir() {
			return fmt.Errorf("path '%s' is not a directory", path)
		}
	}

	return nil
}

// accountFilename returns the filepath of an account file given the
// filename suffix ("wallet.bin", or "tx.bin"), account name and the
// network directory holding the file.
func accountFilename(suffix, account, netdir string) string {
	if account == "" {
		// default account
		return filepath.Join(netdir, suffix)
	}

	// non-default account
	return filepath.Join(netdir, fmt.Sprintf("%v-%v", account, suffix))
}

// syncSchedule references the account files which have been
// scheduled to be written and the directory to write to.
type syncSchedule struct {
	dir     string
	wallets map[*Account]struct{}
	txs     map[*Account]struct{}
}

func newSyncSchedule(dir string) *syncSchedule {
	s := &syncSchedule{
		dir:     dir,
		wallets: make(map[*Account]struct{}),
		txs:     make(map[*Account]struct{}),
	}
	return s
}

// flushAccount writes all scheduled account files to disk for
// a single account and removes them from the schedule.
func (s *syncSchedule) flushAccount(a *Account) error {
	if _, ok := s.txs[a]; ok {
		if err := a.writeTxStore(s.dir); err != nil {
			return err
		}
		delete(s.txs, a)
	}
	if _, ok := s.wallets[a]; ok {
		if err := a.writeWallet(s.dir); err != nil {
			return err
		}
		delete(s.wallets, a)
	}

	return nil
}

// flush writes all scheduled account files and removes each
// from the schedule.
func (s *syncSchedule) flush() error {
	for a := range s.txs {
		if err := a.writeTxStore(s.dir); err != nil {
			return err
		}
		delete(s.txs, a)
	}

	for a := range s.wallets {
		if err := a.writeWallet(s.dir); err != nil {
			return err
		}
		delete(s.wallets, a)
	}

	return nil
}

type flushAccountRequest struct {
	a   *Account
	err chan error
}

type writeBatchRequest struct {
	a   []*Account
	err chan error
}

type exportRequest struct {
	dir string
	a   *Account
	err chan error
}

// DiskSyncer manages all disk write operations for a collection of accounts.
type DiskSyncer struct {
	// Flush scheduled account writes.
	flushAccount chan *flushAccountRequest

	// Schedule file writes for an account.
	scheduleWallet  chan *Account
	scheduleTxStore chan *Account

	// Write a collection of accounts all at once.
	writeBatch chan *writeBatchRequest

	// Write an account export.
	exportAccount chan *exportRequest

	// Account manager for this DiskSyncer.  This is only
	// needed to grab the account manager semaphore.
	am *AccountManager

	quit     chan struct{}
	shutdown chan struct{}
}

// NewDiskSyncer creates a new DiskSyncer.
func NewDiskSyncer(am *AccountManager) *DiskSyncer {
	return &DiskSyncer{
		flushAccount:    make(chan *flushAccountRequest),
		scheduleWallet:  make(chan *Account),
		scheduleTxStore: make(chan *Account),
		writeBatch:      make(chan *writeBatchRequest),
		exportAccount:   make(chan *exportRequest),
		am:              am,
		quit:            make(chan struct{}),
		shutdown:        make(chan struct{}),
	}
}

// Start starts the goroutines required to run the DiskSyncer.
func (ds *DiskSyncer) Start() {
	go ds.handler()
}

func (ds *DiskSyncer) Stop() {
	close(ds.quit)
}

func (ds *DiskSyncer) WaitForShutdown() {
	<-ds.shutdown
}

// handler runs the disk syncer.  It manages a set of "dirty" account files
// which must be written to disk, and synchronizes all writes in a single
// goroutine.  Periodic flush operations may be signaled by an AccountManager.
//
// This never returns and is should be called from a new goroutine.
func (ds *DiskSyncer) handler() {
	netdir := networkDir(activeNet.Params)
	if err := checkCreateDir(netdir); err != nil {
		log.Errorf("Unable to create or write to account directory: %v", err)
	}
	tmpnetdir := tmpNetworkDir(activeNet.Params)

	const wait = 10 * time.Second
	var timer <-chan time.Time
	var sem chan struct{}
	schedule := newSyncSchedule(netdir)
out:
	for {
		select {
		case <-sem: // Now have exclusive access of the account manager
			err := schedule.flush()
			if err != nil {
				log.Errorf("Cannot write accounts: %v", err)
			}

			timer = nil

			// Do not grab semaphore again until another flush is needed.
			sem = nil

			// Release semaphore.
			ds.am.bsem <- struct{}{}

		case <-timer:
			// Grab AccountManager semaphore when ready so flush can occur.
			sem = ds.am.bsem

		case fr := <-ds.flushAccount:
			fr.err <- schedule.flushAccount(fr.a)

		case a := <-ds.scheduleWallet:
			schedule.wallets[a] = struct{}{}
			if timer == nil {
				timer = time.After(wait)
			}

		case a := <-ds.scheduleTxStore:
			schedule.txs[a] = struct{}{}
			if timer == nil {
				timer = time.After(wait)
			}

		case sr := <-ds.writeBatch:
			err := batchWriteAccounts(sr.a, tmpnetdir, netdir)
			if err == nil {
				// All accounts have been synced, old schedule
				// can be discarded.
				schedule = newSyncSchedule(netdir)
				timer = nil
			}
			sr.err <- err

		case er := <-ds.exportAccount:
			a := er.a
			dir := er.dir
			er.err <- a.writeAll(dir)

		case <-ds.quit:
			err := schedule.flush()
			if err != nil {
				log.Errorf("Cannot write accounts: %v", err)
			}
			break out
		}
	}
	close(ds.shutdown)
}

// FlushAccount writes all scheduled account files to disk for a single
// account.
func (ds *DiskSyncer) FlushAccount(a *Account) error {
	err := make(chan error)
	ds.flushAccount <- &flushAccountRequest{a: a, err: err}
	return <-err
}

// ScheduleWalletWrite schedules an account's wallet to be written to disk.
func (ds *DiskSyncer) ScheduleWalletWrite(a *Account) {
	ds.scheduleWallet <- a
}

// ScheduleTxStoreWrite schedules an account's transaction store to be
// written to disk.
func (ds *DiskSyncer) ScheduleTxStoreWrite(a *Account) {
	ds.scheduleTxStore <- a
}

// WriteBatch safely replaces all account files in the network directory
// with new files created from all accounts in a.
func (ds *DiskSyncer) WriteBatch(a []*Account) error {
	err := make(chan error)
	ds.writeBatch <- &writeBatchRequest{
		a:   a,
		err: err,
	}
	return <-err
}

// ExportAccount writes all account files for a to a new directory.
func (ds *DiskSyncer) ExportAccount(a *Account, dir string) error {
	err := make(chan error)
	er := &exportRequest{
		dir: dir,
		a:   a,
		err: err,
	}
	ds.exportAccount <- er
	return <-err
}

func batchWriteAccounts(accts []*Account, tmpdir, netdir string) error {
	if err := freshDir(tmpdir); err != nil {
		return err
	}
	for _, a := range accts {
		if err := a.writeAll(tmpdir); err != nil {
			return err
		}
	}
	// This is technically NOT an atomic operation, but at startup, if the
	// network directory is missing but the temporary network directory
	// exists, the temporary is moved before accounts are opened.
	if err := os.RemoveAll(netdir); err != nil {
		return err
	}
	if err := Rename(tmpdir, netdir); err != nil {
		return err
	}
	return nil
}

func (a *Account) writeAll(dir string) error {
	if err := a.writeTxStore(dir); err != nil {
		return err
	}
	if err := a.writeWallet(dir); err != nil {
		return err
	}
	return nil
}

func (a *Account) writeWallet(dir string) error {
	wfilepath := accountFilename("wallet.bin", a.name, dir)
	_, filename := filepath.Split(wfilepath)
	tmpfile, err := ioutil.TempFile(dir, filename)
	if err != nil {
		return err
	}

	if _, err = a.Wallet.WriteTo(tmpfile); err != nil {
		return err
	}

	tmppath := tmpfile.Name()
	if err := tmpfile.Close(); err != nil {
		log.Warnf("Cannot close temporary wallet file: %v", err)
	}

	return Rename(tmppath, wfilepath)
}

func (a *Account) writeTxStore(dir string) error {
	txfilepath := accountFilename("tx.bin", a.name, dir)
	_, filename := filepath.Split(txfilepath)
	tmpfile, err := ioutil.TempFile(dir, filename)
	if err != nil {
		return err
	}

	if _, err = a.TxStore.WriteTo(tmpfile); err != nil {
		return err
	}

	tmppath := tmpfile.Name()
	if err := tmpfile.Close(); err != nil {
		log.Warnf("Cannot close temporary txstore file: %v", err)
	}

	return Rename(tmppath, txfilepath)
}
<commit_msg>Sync temporary files before closing and renaming.<commit_after>/*
 * Copyright (c) 2013, 2014 Conformal Systems LLC <info@conformal.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"

	"github.com/conformal/btcnet"
	"github.com/conformal/btcwire"
)

// networkDir returns the directory name of a network directory to hold account
// files.
func networkDir(net *btcnet.Params) string {
	netname := net.Name

	// For now, we must always name the testnet data directory as "testnet"
	// and not "testnet3" or any other version, as the btcnet testnet3
	// paramaters will likely be switched to being named "testnet3" in the
	// future.  This is done to future proof that change, and an upgrade
	// plan to move the testnet3 data directory can be worked out later.
	if net.Net == btcwire.TestNet3 {
		netname = "testnet"
	}

	return filepath.Join(cfg.DataDir, netname)
}

// tmpNetworkDir returns the temporary directory name for a given network.
func tmpNetworkDir(net *btcnet.Params) string {
	return networkDir(net) + "_tmp"
}

// freshDir creates a new directory specified by path if it does not
// exist.  If the directory already exists, all files contained in the
// directory are removed.
func freshDir(path string) error {
	if err := checkCreateDir(path); err != nil {
		return err
	}

	// Remove all files in the directory.
	fd, err := os.Open(path)
	if err != nil {
		return err
	}
	defer func() {
		if err := fd.Close(); err != nil {
			log.Warnf("Cannot close directory: %v", err)
		}
	}()
	names, err := fd.Readdirnames(0)
	if err != nil {
		return err
	}
	for _, name := range names {
		if err := os.RemoveAll(name); err != nil {
			return err
		}
	}

	return nil
}

// checkCreateDir checks that the path exists and is a directory.
// If path does not exist, it is created.
func checkCreateDir(path string) error {
	if fi, err := os.Stat(path); err != nil {
		if os.IsNotExist(err) {
			// Attempt data directory creation
			if err = os.MkdirAll(path, 0700); err != nil {
				return fmt.Errorf("cannot create directory: %s", err)
			}
		} else {
			return fmt.Errorf("error checking directory: %s", err)
		}
	} else {
		if !fi.IsDir() {
			return fmt.Errorf("path '%s' is not a directory", path)
		}
	}

	return nil
}

// accountFilename returns the filepath of an account file given the
// filename suffix ("wallet.bin", or "tx.bin"), account name and the
// network directory holding the file.
func accountFilename(suffix, account, netdir string) string {
	if account == "" {
		// default account
		return filepath.Join(netdir, suffix)
	}

	// non-default account
	return filepath.Join(netdir, fmt.Sprintf("%v-%v", account, suffix))
}

// syncSchedule references the account files which have been
// scheduled to be written and the directory to write to.
type syncSchedule struct {
	dir     string
	wallets map[*Account]struct{}
	txs     map[*Account]struct{}
}

func newSyncSchedule(dir string) *syncSchedule {
	s := &syncSchedule{
		dir:     dir,
		wallets: make(map[*Account]struct{}),
		txs:     make(map[*Account]struct{}),
	}
	return s
}

// flushAccount writes all scheduled account files to disk for
// a single account and removes them from the schedule.
func (s *syncSchedule) flushAccount(a *Account) error {
	if _, ok := s.txs[a]; ok {
		if err := a.writeTxStore(s.dir); err != nil {
			return err
		}
		delete(s.txs, a)
	}
	if _, ok := s.wallets[a]; ok {
		if err := a.writeWallet(s.dir); err != nil {
			return err
		}
		delete(s.wallets, a)
	}

	return nil
}

// flush writes all scheduled account files and removes each
// from the schedule.
func (s *syncSchedule) flush() error {
	for a := range s.txs {
		if err := a.writeTxStore(s.dir); err != nil {
			return err
		}
		delete(s.txs, a)
	}

	for a := range s.wallets {
		if err := a.writeWallet(s.dir); err != nil {
			return err
		}
		delete(s.wallets, a)
	}

	return nil
}

type flushAccountRequest struct {
	a   *Account
	err chan error
}

type writeBatchRequest struct {
	a   []*Account
	err chan error
}

type exportRequest struct {
	dir string
	a   *Account
	err chan error
}

// DiskSyncer manages all disk write operations for a collection of accounts.
type DiskSyncer struct {
	// Flush scheduled account writes.
	flushAccount chan *flushAccountRequest

	// Schedule file writes for an account.
	scheduleWallet  chan *Account
	scheduleTxStore chan *Account

	// Write a collection of accounts all at once.
	writeBatch chan *writeBatchRequest

	// Write an account export.
	exportAccount chan *exportRequest

	// Account manager for this DiskSyncer.  This is only
	// needed to grab the account manager semaphore.
	am *AccountManager

	quit     chan struct{}
	shutdown chan struct{}
}

// NewDiskSyncer creates a new DiskSyncer.
func NewDiskSyncer(am *AccountManager) *DiskSyncer {
	return &DiskSyncer{
		flushAccount:    make(chan *flushAccountRequest),
		scheduleWallet:  make(chan *Account),
		scheduleTxStore: make(chan *Account),
		writeBatch:      make(chan *writeBatchRequest),
		exportAccount:   make(chan *exportRequest),
		am:              am,
		quit:            make(chan struct{}),
		shutdown:        make(chan struct{}),
	}
}

// Start starts the goroutines required to run the DiskSyncer.
func (ds *DiskSyncer) Start() {
	go ds.handler()
}

func (ds *DiskSyncer) Stop() {
	close(ds.quit)
}

func (ds *DiskSyncer) WaitForShutdown() {
	<-ds.shutdown
}

// handler runs the disk syncer.  It manages a set of "dirty" account files
// which must be written to disk, and synchronizes all writes in a single
// goroutine.  Periodic flush operations may be signaled by an AccountManager.
//
// This never returns and is should be called from a new goroutine.
func (ds *DiskSyncer) handler() {
	netdir := networkDir(activeNet.Params)
	if err := checkCreateDir(netdir); err != nil {
		log.Errorf("Unable to create or write to account directory: %v", err)
	}
	tmpnetdir := tmpNetworkDir(activeNet.Params)

	const wait = 10 * time.Second
	var timer <-chan time.Time
	var sem chan struct{}
	schedule := newSyncSchedule(netdir)
out:
	for {
		select {
		case <-sem: // Now have exclusive access of the account manager
			err := schedule.flush()
			if err != nil {
				log.Errorf("Cannot write accounts: %v", err)
			}

			timer = nil

			// Do not grab semaphore again until another flush is needed.
			sem = nil

			// Release semaphore.
			ds.am.bsem <- struct{}{}

		case <-timer:
			// Grab AccountManager semaphore when ready so flush can occur.
			sem = ds.am.bsem

		case fr := <-ds.flushAccount:
			fr.err <- schedule.flushAccount(fr.a)

		case a := <-ds.scheduleWallet:
			schedule.wallets[a] = struct{}{}
			if timer == nil {
				timer = time.After(wait)
			}

		case a := <-ds.scheduleTxStore:
			schedule.txs[a] = struct{}{}
			if timer == nil {
				timer = time.After(wait)
			}

		case sr := <-ds.writeBatch:
			err := batchWriteAccounts(sr.a, tmpnetdir, netdir)
			if err == nil {
				// All accounts have been synced, old schedule
				// can be discarded.
				schedule = newSyncSchedule(netdir)
				timer = nil
			}
			sr.err <- err

		case er := <-ds.exportAccount:
			a := er.a
			dir := er.dir
			er.err <- a.writeAll(dir)

		case <-ds.quit:
			err := schedule.flush()
			if err != nil {
				log.Errorf("Cannot write accounts: %v", err)
			}
			break out
		}
	}
	close(ds.shutdown)
}

// FlushAccount writes all scheduled account files to disk for a single
// account.
func (ds *DiskSyncer) FlushAccount(a *Account) error {
	err := make(chan error)
	ds.flushAccount <- &flushAccountRequest{a: a, err: err}
	return <-err
}

// ScheduleWalletWrite schedules an account's wallet to be written to disk.
func (ds *DiskSyncer) ScheduleWalletWrite(a *Account) {
	ds.scheduleWallet <- a
}

// ScheduleTxStoreWrite schedules an account's transaction store to be
// written to disk.
func (ds *DiskSyncer) ScheduleTxStoreWrite(a *Account) {
	ds.scheduleTxStore <- a
}

// WriteBatch safely replaces all account files in the network directory
// with new files created from all accounts in a.
func (ds *DiskSyncer) WriteBatch(a []*Account) error {
	err := make(chan error)
	ds.writeBatch <- &writeBatchRequest{
		a:   a,
		err: err,
	}
	return <-err
}

// ExportAccount writes all account files for a to a new directory.
func (ds *DiskSyncer) ExportAccount(a *Account, dir string) error {
	err := make(chan error)
	er := &exportRequest{
		dir: dir,
		a:   a,
		err: err,
	}
	ds.exportAccount <- er
	return <-err
}

func batchWriteAccounts(accts []*Account, tmpdir, netdir string) error {
	if err := freshDir(tmpdir); err != nil {
		return err
	}
	for _, a := range accts {
		if err := a.writeAll(tmpdir); err != nil {
			return err
		}
	}
	// This is technically NOT an atomic operation, but at startup, if the
	// network directory is missing but the temporary network directory
	// exists, the temporary is moved before accounts are opened.
	if err := os.RemoveAll(netdir); err != nil {
		return err
	}
	if err := Rename(tmpdir, netdir); err != nil {
		return err
	}
	return nil
}

func (a *Account) writeAll(dir string) error {
	if err := a.writeTxStore(dir); err != nil {
		return err
	}
	if err := a.writeWallet(dir); err != nil {
		return err
	}
	return nil
}

func (a *Account) writeWallet(dir string) error {
	wfilepath := accountFilename("wallet.bin", a.name, dir)
	_, filename := filepath.Split(wfilepath)
	tmpfile, err := ioutil.TempFile(dir, filename)
	if err != nil {
		return err
	}

	if _, err = a.Wallet.WriteTo(tmpfile); err != nil {
		return err
	}

	if err := tmpfile.Sync(); err != nil {
		return err
	}

	tmppath := tmpfile.Name()
	if err := tmpfile.Close(); err != nil {
		log.Warnf("Cannot close temporary wallet file: %v", err)
	}

	return Rename(tmppath, wfilepath)
}

func (a *Account) writeTxStore(dir string) error {
	txfilepath := accountFilename("tx.bin", a.name, dir)
	_, filename := filepath.Split(txfilepath)
	tmpfile, err := ioutil.TempFile(dir, filename)
	if err != nil {
		return err
	}

	if _, err = a.TxStore.WriteTo(tmpfile); err != nil {
		return err
	}

	if err := tmpfile.Sync(); err != nil {
		return err
	}

	tmppath := tmpfile.Name()
	if err := tmpfile.Close(); err != nil {
		log.Warnf("Cannot close temporary txstore file: %v", err)
	}

	return Rename(tmppath, txfilepath)
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"

	. "github.com/onsi/ginkgo"
)

var (
	root = absOrDie(filepath.Clean(filepath.Join(path.Base(os.Args[0]), "..")))
)

var _ = Describe("Shell", func() {

	defer GinkgoRecover()
	// Slurp up all the tests in hack/e2e-suite
	bashE2ERoot := filepath.Join(root, "hack/e2e-suite")
	files, err := ioutil.ReadDir(bashE2ERoot)
	if err != nil {
		Fail(fmt.Sprintf("Error reading test suites from %v %v", bashE2ERoot, err.Error()))
	}

	for _, file := range files {
		fileName := file.Name() // Make a copy
		It(fmt.Sprintf("tests that %v passes", fileName), func() {
			// A number of scripts only work on gce
			if !providerIs("gce", "gke") {
				By(fmt.Sprintf("Skipping Shell test %s, which is only supported for provider gce and gke (not %s)",
					fileName, testContext.Provider))
				return
			}
			runCmdTest(filepath.Join(bashE2ERoot, fileName))
		})
	}
})

func absOrDie(path string) string {
	out, err := filepath.Abs(path)
	if err != nil {
		panic(err)
	}
	return out
}

// Runs the given cmd test.
func runCmdTest(path string) {
	By(fmt.Sprintf("Running %v", path))
	cmd := exec.Command(path)
	cmd.Stdout = bytes.NewBuffer(nil)
	cmd.Stderr = cmd.Stdout

	if err := cmd.Run(); err != nil {
		Fail(fmt.Sprintf("Error running %v:\nCommand output:\n%v\n", cmd, cmd.Stdout))
		return
	}
	return
}
<commit_msg>Fix path construction bug in shell e2e<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os/exec"
	"path/filepath"

	. "github.com/onsi/ginkgo"
)

var _ = Describe("Shell", func() {
	bashE2ERoot := filepath.Join(testContext.RepoRoot, "hack/e2e-suite")

	// Slurp up all the tests in hack/e2e-suite
	files, err := ioutil.ReadDir(bashE2ERoot)
	if err != nil {
		Fail(fmt.Sprintf("Error reading test suites from %v %v", bashE2ERoot, err.Error()))
	}

	for _, file := range files {
		fileName := file.Name() // Make a copy
		It(fmt.Sprintf("tests that %v passes", fileName), func() {
			// A number of scripts only work on gce
			if !providerIs("gce", "gke") {
				By(fmt.Sprintf("Skipping Shell test %s, which is only supported for provider gce and gke (not %s)",
					fileName, testContext.Provider))
				return
			}
			runCmdTest(filepath.Join(bashE2ERoot, fileName))
		})
	}
})

func absOrDie(path string) string {
	out, err := filepath.Abs(path)
	if err != nil {
		panic(err)
	}
	return out
}

// Runs the given cmd test.
func runCmdTest(path string) {
	By(fmt.Sprintf("Running %v", path))
	cmd := exec.Command(path)
	cmd.Stdout = bytes.NewBuffer(nil)
	cmd.Stderr = cmd.Stdout

	if err := cmd.Run(); err != nil {
		Fail(fmt.Sprintf("Error running %v:\nCommand output:\n%v\n", cmd, cmd.Stdout))
		return
	}
	return
}
<|endoftext|>


<commit_before>package test

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"

	"github.com/Originate/git-town/src/command"
	"github.com/kballard/go-shellquote"
	"github.com/pkg/errors"
)

// ShellRunner runs shell commands in the given directory, using a customizable environment.
// Possible customizations:
// - Temporarily override certain shell commands with mock implementations.
//   Temporary mocks are only valid for the next command being run.
type ShellRunner struct {

	// workingDir contains the directory path in which this runner runs.
	workingDir string

	// homeDir contains the path that contains the global Git configuration.
	homeDir string

	// tempShellOverrideDirDir contains the directory path that stores the mock shell command implementations.
	// This variable is populated when shell overrides are being set.
	// An empty string indicates that no shell overrides have been set.
	tempShellOverridesDir string
}

// NewShellRunner provides a new ShellRunner instance that executes in the given directory.
func NewShellRunner(workingDir string, homeDir string) ShellRunner {
	return ShellRunner{workingDir: workingDir, homeDir: homeDir}
}

// AddTempShellOverride temporarily mocks the shell command with the given name
// with the given Bash script.
func (runner *ShellRunner) AddTempShellOverride(name, content string) error {
	if !runner.hasTempShellOverrides() {
		err := runner.createTempShellOverridesDir()
		if err != nil {
			return errors.Wrap(err, "cannot create temp shell overrides dir")
		}
	}
	return ioutil.WriteFile(runner.tempShellOverrideFilePath(name), []byte(content), 0744)
}

// tempShellOverrideFilePath provides the full file path where to store a temp shell command with the given name.
func (runner *ShellRunner) tempShellOverrideFilePath(shellOverrideFilename string) string {
	return path.Join(runner.tempShellOverridesDir, shellOverrideFilename)
}

// RemoveTempShellOverrides removes all custom shell overrides.
func (runner *ShellRunner) RemoveTempShellOverrides() {
	os.RemoveAll(runner.tempShellOverridesDir)
	runner.tempShellOverridesDir = ""
}

// createTempShellOverridesDir creates the folder that will contain the temp shell overrides.
// It is safe to call this method multiple times.
func (runner *ShellRunner) createTempShellOverridesDir() error {
	var err error
	runner.tempShellOverridesDir, err = ioutil.TempDir("", "")
	return err
}

// hasTempShellOverrides indicates whether there are temp shell overrides for the next command.
func (runner *ShellRunner) hasTempShellOverrides() bool {
	return runner.tempShellOverridesDir != ""
}

// Run runs the given command with the given arguments
// in this ShellRunner's directory.
// Shell overrides will be used and removed when done.
func (runner *ShellRunner) Run(name string, arguments ...string) (output string, err error) {
	// create an environment with the temp shell overrides directory added to the PATH
	customEnv := os.Environ()

	// set HOME to the given global directory so that Git puts the global configuration there.
	for i := range customEnv {
		if strings.HasPrefix(customEnv[i], "HOME=") {
			customEnv[i] = fmt.Sprintf("HOME=%s", runner.homeDir)
		}
	}

	// enable shell overrides
	if runner.hasTempShellOverrides() {
		for i := range customEnv {
			if strings.HasPrefix(customEnv[i], "PATH=") {
				parts := strings.SplitN(customEnv[i], "=", 2)
				parts[1] = runner.tempShellOverridesDir + ":" + parts[1]
				customEnv[i] = strings.Join(parts, "=")
				break
			}
		}
		defer runner.RemoveTempShellOverrides()
	}

	// run the command inside the custom environment
	outcome := command.RunDirEnv(runner.workingDir, customEnv, name, arguments...)
	if Debug {
		fmt.Println(path.Base(runner.workingDir), ">", name, strings.Join(arguments, " "))
		fmt.Println(outcome.Output())
	}
	return outcome.Output(), outcome.Err()
}

// RunString runs the given command (including possible arguments)
// in this ShellRunner's directory.
// Shell overrides will be used and removed when done.
//
// The current implementation splits the string by space
// and therefore only works for simple commands without quoted arguments.
func (runner *ShellRunner) RunString(command string) (output string, err error) {
	parts, err := shellquote.Split(command)
	if err != nil {
		return "", errors.Wrapf(err, "cannot split command: %q", command)
	}
	command, args := parts[0], parts[1:]
	return runner.Run(command, args...)
}

// RunMany runs all given commands in current directory.
// Commands are provided as a list of argv-style strings.
// Shell overrides apply for the first command only.
// Failed commands abort immediately with the encountered error.
func (runner *ShellRunner) RunMany(commands [][]string) error {
	for _, argv := range commands {
		command, args := argv[0], argv[1:]
		output, err := runner.Run(command, args...)
		if err != nil {
			return errors.Wrapf(err, "error running command %q: %s", argv, output)
		}
	}
	return nil
}
<commit_msg>Remove obsolete docs (#1264)<commit_after>package test

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"

	"github.com/Originate/git-town/src/command"
	"github.com/kballard/go-shellquote"
	"github.com/pkg/errors"
)

// ShellRunner runs shell commands in the given directory, using a customizable environment.
// Possible customizations:
// - Temporarily override certain shell commands with mock implementations.
//   Temporary mocks are only valid for the next command being run.
type ShellRunner struct {

	// workingDir contains the directory path in which this runner runs.
	workingDir string

	// homeDir contains the path that contains the global Git configuration.
	homeDir string

	// tempShellOverrideDirDir contains the directory path that stores the mock shell command implementations.
	// This variable is populated when shell overrides are being set.
	// An empty string indicates that no shell overrides have been set.
	tempShellOverridesDir string
}

// NewShellRunner provides a new ShellRunner instance that executes in the given directory.
func NewShellRunner(workingDir string, homeDir string) ShellRunner {
	return ShellRunner{workingDir: workingDir, homeDir: homeDir}
}

// AddTempShellOverride temporarily mocks the shell command with the given name
// with the given Bash script.
func (runner *ShellRunner) AddTempShellOverride(name, content string) error {
	if !runner.hasTempShellOverrides() {
		err := runner.createTempShellOverridesDir()
		if err != nil {
			return errors.Wrap(err, "cannot create temp shell overrides dir")
		}
	}
	return ioutil.WriteFile(runner.tempShellOverrideFilePath(name), []byte(content), 0744)
}

// tempShellOverrideFilePath provides the full file path where to store a temp shell command with the given name.
func (runner *ShellRunner) tempShellOverrideFilePath(shellOverrideFilename string) string {
	return path.Join(runner.tempShellOverridesDir, shellOverrideFilename)
}

// RemoveTempShellOverrides removes all custom shell overrides.
func (runner *ShellRunner) RemoveTempShellOverrides() {
	os.RemoveAll(runner.tempShellOverridesDir)
	runner.tempShellOverridesDir = ""
}

// createTempShellOverridesDir creates the folder that will contain the temp shell overrides.
// It is safe to call this method multiple times.
func (runner *ShellRunner) createTempShellOverridesDir() error {
	var err error
	runner.tempShellOverridesDir, err = ioutil.TempDir("", "")
	return err
}

// hasTempShellOverrides indicates whether there are temp shell overrides for the next command.
func (runner *ShellRunner) hasTempShellOverrides() bool {
	return runner.tempShellOverridesDir != ""
}

// Run runs the given command with the given arguments
// in this ShellRunner's directory.
// Shell overrides will be used and removed when done.
func (runner *ShellRunner) Run(name string, arguments ...string) (output string, err error) {
	// create an environment with the temp shell overrides directory added to the PATH
	customEnv := os.Environ()

	// set HOME to the given global directory so that Git puts the global configuration there.
	for i := range customEnv {
		if strings.HasPrefix(customEnv[i], "HOME=") {
			customEnv[i] = fmt.Sprintf("HOME=%s", runner.homeDir)
		}
	}

	// enable shell overrides
	if runner.hasTempShellOverrides() {
		for i := range customEnv {
			if strings.HasPrefix(customEnv[i], "PATH=") {
				parts := strings.SplitN(customEnv[i], "=", 2)
				parts[1] = runner.tempShellOverridesDir + ":" + parts[1]
				customEnv[i] = strings.Join(parts, "=")
				break
			}
		}
		defer runner.RemoveTempShellOverrides()
	}

	// run the command inside the custom environment
	outcome := command.RunDirEnv(runner.workingDir, customEnv, name, arguments...)
	if Debug {
		fmt.Println(path.Base(runner.workingDir), ">", name, strings.Join(arguments, " "))
		fmt.Println(outcome.Output())
	}
	return outcome.Output(), outcome.Err()
}

// RunString runs the given command (including possible arguments)
// in this ShellRunner's directory.
// Shell overrides will be used and removed when done.
func (runner *ShellRunner) RunString(command string) (output string, err error) {
	parts, err := shellquote.Split(command)
	if err != nil {
		return "", errors.Wrapf(err, "cannot split command: %q", command)
	}
	command, args := parts[0], parts[1:]
	return runner.Run(command, args...)
}

// RunMany runs all given commands in current directory.
// Commands are provided as a list of argv-style strings.
// Shell overrides apply for the first command only.
// Failed commands abort immediately with the encountered error.
func (runner *ShellRunner) RunMany(commands [][]string) error {
	for _, argv := range commands {
		command, args := argv[0], argv[1:]
		output, err := runner.Run(command, args...)
		if err != nil {
			return errors.Wrapf(err, "error running command %q: %s", argv, output)
		}
	}
	return nil
}
<|endoftext|>