<commit_before>package rockredis

import (
	"errors"
	"fmt"
	"github.com/absolute8511/ZanRedisDB/common"
	"github.com/absolute8511/gorocksdb"
	"io"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

func GetCheckpointDir(term uint64, index uint64) string {
	return fmt.Sprintf("%016x-%016x", term, index)
}

type RockConfig struct {
	DataDir          string
	DefaultReadOpts  *gorocksdb.ReadOptions
	DefaultWriteOpts *gorocksdb.WriteOptions
}

func NewRockConfig() *RockConfig {
	c := &RockConfig{
		DefaultReadOpts:  gorocksdb.NewDefaultReadOptions(),
		DefaultWriteOpts: gorocksdb.NewDefaultWriteOptions(),
	}
	c.DefaultReadOpts.SetVerifyChecksums(false)
	return c
}

type RockDB struct {
	cfg              *RockConfig
	eng              *gorocksdb.DB
	dbOpts           *gorocksdb.Options
	defaultWriteOpts *gorocksdb.WriteOptions
	defaultReadOpts  *gorocksdb.ReadOptions
	wb               *gorocksdb.WriteBatch
	quit             chan struct{}
	wg               sync.WaitGroup
	backupC          chan *BackupInfo
}

func OpenRockDB(cfg *RockConfig) (*RockDB, error) {
	if len(cfg.DataDir) == 0 {
		return nil, errors.New("config error")
	}

	os.MkdirAll(cfg.DataDir, common.DIR_PERM)
	// options need be adjust due to using hdd or sdd, please reference
	// https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide
	bbto := gorocksdb.NewDefaultBlockBasedTableOptions()
	// use large block to reduce index block size for hdd
	// if using ssd, should use the default value
	bbto.SetBlockSize(1024 * 16)
	// should about 20% less than host RAM
	// http://smalldatum.blogspot.com/2016/09/tuning-rocksdb-block-cache.html
	bbto.SetBlockCache(gorocksdb.NewLRUCache(1024 * 1024 * 1024))
	// for hdd , we nee cache index and filter blocks
	bbto.SetCacheIndexAndFilterBlocks(true)
	filter := gorocksdb.NewBloomFilter(10)
	bbto.SetFilterPolicy(filter)
	opts := gorocksdb.NewDefaultOptions()
	opts.SetBlockBasedTableFactory(bbto)
	opts.SetCreateIfMissing(true)
	opts.SetMaxOpenFiles(-1)
	// keep level0_file_num_compaction_trigger * write_buffer_size = max_bytes_for_level_base to minimize write amplification
	opts.SetWriteBufferSize(1024 * 1024 * 128)
	opts.SetMaxWriteBufferNumber(8)
	opts.SetLevel0FileNumCompactionTrigger(4)
	opts.SetMaxBytesForLevelBase(1024 * 1024 * 1024 * 2)
	opts.SetMinWriteBufferNumberToMerge(2)
	opts.SetTargetFileSizeBase(1024 * 1024 * 128)
	opts.SetMaxBackgroundFlushes(2)
	opts.SetMaxBackgroundCompactions(4)
	opts.SetMinLevelToCompress(3)
	// we use table, so we use prefix seek feature
	opts.SetPrefixExtractor(gorocksdb.NewFixedPrefixTransform(3))
	opts.SetMemtablePrefixBloomSizeRatio(0.1)
	opts.EnableStatistics()
	opts.SetMaxLogFileSize(1024 * 1024 * 32)
	opts.SetLogFileTimeToRoll(3600 * 24 * 3)
	// https://github.com/facebook/mysql-5.6/wiki/my.cnf-tuning
	// rate limiter need to reduce the compaction io

	db := &RockDB{
		cfg:              cfg,
		dbOpts:           opts,
		defaultReadOpts:  cfg.DefaultReadOpts,
		defaultWriteOpts: cfg.DefaultWriteOpts,
		wb:               gorocksdb.NewWriteBatch(),
		backupC:          make(chan *BackupInfo),
		quit:             make(chan struct{}),
	}
	eng, err := gorocksdb.OpenDb(opts, db.GetDataDir())
	if err != nil {
		return nil, err
	}
	db.eng = eng
	os.MkdirAll(db.GetBackupDir(), common.DIR_PERM)

	db.wg.Add(1)
	go func() {
		defer db.wg.Done()
		db.backupLoop()
	}()
	return db, nil
}

func GetBackupDir(base string) string {
	return path.Join(base, "rocksdb_backup")
}

func (r *RockDB) GetBackupBase() string {
	return r.cfg.DataDir
}

func (r *RockDB) GetBackupDir() string {
	return GetBackupDir(r.cfg.DataDir)
}

func (r *RockDB) GetDataDir() string {
	return path.Join(r.cfg.DataDir, "rocksdb")
}

func (r *RockDB) reOpen() error {
	var err error
	r.eng, err = gorocksdb.OpenDb(r.dbOpts, r.GetDataDir())
	return err
}

func (r *RockDB) CompactRange() {
	var rg gorocksdb.Range
	r.eng.CompactRange(rg)
}

func (r *RockDB) Close() {
	close(r.quit)
	r.wg.Wait()
	if r.defaultReadOpts != nil {
		r.defaultReadOpts.Destroy()
	}
	if r.defaultWriteOpts != nil {
		r.defaultWriteOpts.Destroy()
	}
	if r.eng != nil {
		r.eng.Close()
	}
}

func (r *RockDB) SetPerfLevel(level int) {
	// TODO:
}

func (r *RockDB) GetStatistics() string {
	return r.dbOpts.GetStatistics()
}

func (r *RockDB) GetInternalStatus() map[string]interface{} {
	status := make(map[string]interface{})
	bbt := r.dbOpts.GetBlockBasedTableFactory()
	if bbt != nil {
		bc := bbt.GetBlockCache()
		if bc != nil {
			status["block-cache-usage"] = bc.GetUsage()
			status["block-cache-pinned-usage"] = bc.GetPinnedUsage()
		}
	}

	memStr := r.eng.GetProperty("rocksdb.estimate-table-readers-mem")
	status["estimate-table-readers-mem"] = memStr
	memStr = r.eng.GetProperty("rocksdb.cur-size-all-mem-tables")
	status["cur-size-all-mem-tables"] = memStr
	memStr = r.eng.GetProperty("rocksdb.cur-size-active-mem-table")
	status["cur-size-active-mem-tables"] = memStr
	return status
}

func (r *RockDB) GetInternalPropertyStatus(p string) string {
	return r.eng.GetProperty(p)
}

func (r *RockDB) ReadRange(sKey, eKey []byte, maxNum int) chan common.KVRecord {
	retChan := make(chan common.KVRecord, 32)
	go func() {
		it := NewDBRangeLimitIterator(r.eng, sKey, eKey, RangeClose, 0, maxNum, false)
		defer it.Close()
		for it = it; it.Valid(); it.Next() {
			key := it.Key()
			value := it.Value()
			retChan <- common.KVRecord{Key: key, Value: value}
		}
		close(retChan)
	}()
	return retChan
}

type BackupInfo struct {
	backupDir string
	started   chan struct{}
	done      chan struct{}
	rsp       []byte
	err       error
}

func newBackupInfo(dir string) *BackupInfo {
	return &BackupInfo{
		backupDir: dir,
		started:   make(chan struct{}),
		done:      make(chan struct{}),
	}
}

func (self *BackupInfo) WaitReady() {
	select {
	case <-self.started:
	case <-self.done:
	}
}

func (self *BackupInfo) GetResult() ([]byte, error) {
	select {
	case <-self.done:
	}
	return self.rsp, self.err
}

func (r *RockDB) backupLoop() {
	for {
		select {
		case rsp, ok := <-r.backupC:
			if !ok {
				return
			}

			func() {
				defer close(rsp.done)
				log.Printf("begin backup to:%v \n", rsp.backupDir)
				start := time.Now()
				ck, err := gorocksdb.NewCheckpoint(r.eng)
				if err != nil {
					log.Printf("init checkpoint failed: %v", err)
					rsp.err = err
					return
				}
				_, err := os.Stat(rsp.backupDir)
				if !os.IsNotExist(err) {
					log.Printf("checkpoint exist: %v, remove it", rsp.backupDir)
					os.RemoveAll(rsp.backupDir)
				}
				time.AfterFunc(time.Second*2, func() {
					close(rsp.started)
				})
				err = ck.Save(rsp.backupDir)
				if err != nil {
					log.Printf("save checkpoint failed: %v", err)
					rsp.err = err
					return
				}
				cost := time.Now().Sub(start)
				log.Printf("backup done (cost %v), check point to: %v\n", cost.String(), rsp.backupDir)
				// TODO: purge some old checkpoint
				rsp.rsp = []byte(rsp.backupDir)
			}()
		case <-r.quit:
			return
		}
	}
}

func (r *RockDB) Backup(term uint64, index uint64) *BackupInfo {
	fname := GetCheckpointDir(term, index)
	checkpointDir := path.Join(r.GetBackupDir(), fname)
	bi := newBackupInfo(checkpointDir)
	select {
	case r.backupC <- bi:
	default:
		return nil
	}
	return bi
}

func (r *RockDB) IsLocalBackupOK(term uint64, index uint64) (bool, error) {
	backupDir := r.GetBackupDir()
	checkpointDir := GetCheckpointDir(term, index)
	ro := *r.dbOpts
	ro.SetCreateIfMissing(false)
	db, err := gorocksdb.OpenDb(&ro, path.Join(backupDir, checkpointDir))
	if err != nil {
		log.Printf("checkpoint open failed: %v", err)
		return false, err
	}
	db.Close()
	return true, nil
}

func copyFile(src, dst string, override bool) error {
	sfi, err := os.Stat(src)
	if err != nil {
		return err
	}
	if !sfi.Mode().IsRegular() {
		return fmt.Errorf("copyfile: non-regular source file %v (%v)", sfi.Name(), sfi.Mode().String())
	}
	_, err = os.Stat(dst)
	if err != nil {
		if !os.IsNotExist(err) {
			return err
		}
	} else {
		if !override {
			return nil
		}
	}
	in, err := os.Open(src)
	if err != nil {
		return err
	}
	defer in.Close()
	out, err := os.Create(dst)
	if err != nil {
		return err
	}
	_, err = io.Copy(out, in)
	if err != nil {
		out.Close()
		return err
	}
	err = out.Sync()
	if err != nil {
		out.Close()
		return err
	}
	return out.Close()
}

func (r *RockDB) Restore(term uint64, index uint64) error {
	// TODO: maybe write meta (snap term and index) and check the meta data in the backup
	backupDir := r.GetBackupDir()
	hasBackup, _ := r.IsLocalBackupOK(term, index)
	if !hasBackup {
		return errors.New("no backup for restore")
	}

	checkpointDir := GetCheckpointDir(term, index)
	start := time.Now()
	log.Printf("begin restore from checkpoint: %v\n", checkpointDir)
	r.eng.Close()
	// 1. remove all files in current db except sst files
	// 2. get the list of sst in checkpoint
	// 3. remove all the sst files not in the checkpoint list
	// 4. copy all files from checkpoint to current db and do not override sst
	matchName := path.Join(r.GetDataDir(), "*")
	nameList, err := filepath.Glob(matchName)
	if err != nil {
		log.Printf("list files failed:  %v\n", err)
		return err
	}
	ckNameList, err := filepath.Glob(path.Join(backupDir, checkpointDir, "*"))
	if err != nil {
		log.Printf("list checkpoint files failed:  %v\n", err)
		return err
	}
	ckSstNameMap := make(map[string]bool)
	for _, fn := range ckNameList {
		if strings.HasSuffix(fn, ".sst") {
			ckSstNameMap[fn] = true
		}
	}

	for _, fn := range nameList {
		shortName := path.Base(fn)
		if strings.HasPrefix(shortName, "LOG") {
			continue
		}
		if strings.HasSuffix(fn, ".sst") {
			if _, ok := ckSstNameMap[fn]; ok {
				log.Printf("keeping sst file: %v", fn)
				continue
			}
		}
		log.Printf("removing: %v", fn)
		os.RemoveAll(fn)
	}
	for _, fn := range ckNameList {
		dst := path.Join(r.GetDataDir(), path.Base(fn))
		err := copyFile(fn, dst, false)
		if err != nil {
			log.Printf("copy %v to %v failed: %v", fn, dst, err)
			return err
		} else {
			log.Printf("copy %v to %v done", fn, dst, err)
		}
	}

	err = r.reOpen()
	log.Printf("restore done, cost: %v\n", time.Now().Sub(start))
	if err != nil {
		log.Printf("reopen the restored db failed:  %v\n", err)
	}
	return err
}

func (r *RockDB) ClearBackup(term uint64, index uint64) error {
	backupDir := r.GetBackupDir()
	checkpointDir := GetCheckpointDir(term, index)
	return os.RemoveAll(path.Join(backupDir, checkpointDir))
}
<commit_msg>check in read only mode<commit_after>package rockredis

import (
	"errors"
	"fmt"
	"github.com/absolute8511/ZanRedisDB/common"
	"github.com/absolute8511/gorocksdb"
	"io"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

func GetCheckpointDir(term uint64, index uint64) string {
	return fmt.Sprintf("%016x-%016x", term, index)
}

type RockConfig struct {
	DataDir          string
	DefaultReadOpts  *gorocksdb.ReadOptions
	DefaultWriteOpts *gorocksdb.WriteOptions
}

func NewRockConfig() *RockConfig {
	c := &RockConfig{
		DefaultReadOpts:  gorocksdb.NewDefaultReadOptions(),
		DefaultWriteOpts: gorocksdb.NewDefaultWriteOptions(),
	}
	c.DefaultReadOpts.SetVerifyChecksums(false)
	return c
}

type RockDB struct {
	cfg              *RockConfig
	eng              *gorocksdb.DB
	dbOpts           *gorocksdb.Options
	defaultWriteOpts *gorocksdb.WriteOptions
	defaultReadOpts  *gorocksdb.ReadOptions
	wb               *gorocksdb.WriteBatch
	quit             chan struct{}
	wg               sync.WaitGroup
	backupC          chan *BackupInfo
}

func OpenRockDB(cfg *RockConfig) (*RockDB, error) {
	if len(cfg.DataDir) == 0 {
		return nil, errors.New("config error")
	}

	os.MkdirAll(cfg.DataDir, common.DIR_PERM)
	// options need be adjust due to using hdd or sdd, please reference
	// https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide
	bbto := gorocksdb.NewDefaultBlockBasedTableOptions()
	// use large block to reduce index block size for hdd
	// if using ssd, should use the default value
	bbto.SetBlockSize(1024 * 16)
	// should about 20% less than host RAM
	// http://smalldatum.blogspot.com/2016/09/tuning-rocksdb-block-cache.html
	bbto.SetBlockCache(gorocksdb.NewLRUCache(1024 * 1024 * 1024))
	// for hdd , we nee cache index and filter blocks
	bbto.SetCacheIndexAndFilterBlocks(true)
	filter := gorocksdb.NewBloomFilter(10)
	bbto.SetFilterPolicy(filter)
	opts := gorocksdb.NewDefaultOptions()
	opts.SetBlockBasedTableFactory(bbto)
	opts.SetCreateIfMissing(true)
	opts.SetMaxOpenFiles(-1)
	// keep level0_file_num_compaction_trigger * write_buffer_size = max_bytes_for_level_base to minimize write amplification
	opts.SetWriteBufferSize(1024 * 1024 * 128)
	opts.SetMaxWriteBufferNumber(8)
	opts.SetLevel0FileNumCompactionTrigger(4)
	opts.SetMaxBytesForLevelBase(1024 * 1024 * 1024 * 2)
	opts.SetMinWriteBufferNumberToMerge(2)
	opts.SetTargetFileSizeBase(1024 * 1024 * 128)
	opts.SetMaxBackgroundFlushes(2)
	opts.SetMaxBackgroundCompactions(4)
	opts.SetMinLevelToCompress(3)
	// we use table, so we use prefix seek feature
	opts.SetPrefixExtractor(gorocksdb.NewFixedPrefixTransform(3))
	opts.SetMemtablePrefixBloomSizeRatio(0.1)
	opts.EnableStatistics()
	opts.SetMaxLogFileSize(1024 * 1024 * 32)
	opts.SetLogFileTimeToRoll(3600 * 24 * 3)
	// https://github.com/facebook/mysql-5.6/wiki/my.cnf-tuning
	// rate limiter need to reduce the compaction io

	db := &RockDB{
		cfg:              cfg,
		dbOpts:           opts,
		defaultReadOpts:  cfg.DefaultReadOpts,
		defaultWriteOpts: cfg.DefaultWriteOpts,
		wb:               gorocksdb.NewWriteBatch(),
		backupC:          make(chan *BackupInfo),
		quit:             make(chan struct{}),
	}
	eng, err := gorocksdb.OpenDb(opts, db.GetDataDir())
	if err != nil {
		return nil, err
	}
	db.eng = eng
	os.MkdirAll(db.GetBackupDir(), common.DIR_PERM)

	db.wg.Add(1)
	go func() {
		defer db.wg.Done()
		db.backupLoop()
	}()
	return db, nil
}

func GetBackupDir(base string) string {
	return path.Join(base, "rocksdb_backup")
}

func (r *RockDB) GetBackupBase() string {
	return r.cfg.DataDir
}

func (r *RockDB) GetBackupDir() string {
	return GetBackupDir(r.cfg.DataDir)
}

func (r *RockDB) GetDataDir() string {
	return path.Join(r.cfg.DataDir, "rocksdb")
}

func (r *RockDB) reOpen() error {
	var err error
	r.eng, err = gorocksdb.OpenDb(r.dbOpts, r.GetDataDir())
	return err
}

func (r *RockDB) CompactRange() {
	var rg gorocksdb.Range
	r.eng.CompactRange(rg)
}

func (r *RockDB) Close() {
	close(r.quit)
	r.wg.Wait()
	if r.defaultReadOpts != nil {
		r.defaultReadOpts.Destroy()
	}
	if r.defaultWriteOpts != nil {
		r.defaultWriteOpts.Destroy()
	}
	if r.eng != nil {
		r.eng.Close()
	}
}

func (r *RockDB) SetPerfLevel(level int) {
	// TODO:
}

func (r *RockDB) GetStatistics() string {
	return r.dbOpts.GetStatistics()
}

func (r *RockDB) GetInternalStatus() map[string]interface{} {
	status := make(map[string]interface{})
	bbt := r.dbOpts.GetBlockBasedTableFactory()
	if bbt != nil {
		bc := bbt.GetBlockCache()
		if bc != nil {
			status["block-cache-usage"] = bc.GetUsage()
			status["block-cache-pinned-usage"] = bc.GetPinnedUsage()
		}
	}

	memStr := r.eng.GetProperty("rocksdb.estimate-table-readers-mem")
	status["estimate-table-readers-mem"] = memStr
	memStr = r.eng.GetProperty("rocksdb.cur-size-all-mem-tables")
	status["cur-size-all-mem-tables"] = memStr
	memStr = r.eng.GetProperty("rocksdb.cur-size-active-mem-table")
	status["cur-size-active-mem-tables"] = memStr
	return status
}

func (r *RockDB) GetInternalPropertyStatus(p string) string {
	return r.eng.GetProperty(p)
}

func (r *RockDB) ReadRange(sKey, eKey []byte, maxNum int) chan common.KVRecord {
	retChan := make(chan common.KVRecord, 32)
	go func() {
		it := NewDBRangeLimitIterator(r.eng, sKey, eKey, RangeClose, 0, maxNum, false)
		defer it.Close()
		for it = it; it.Valid(); it.Next() {
			key := it.Key()
			value := it.Value()
			retChan <- common.KVRecord{Key: key, Value: value}
		}
		close(retChan)
	}()
	return retChan
}

type BackupInfo struct {
	backupDir string
	started   chan struct{}
	done      chan struct{}
	rsp       []byte
	err       error
}

func newBackupInfo(dir string) *BackupInfo {
	return &BackupInfo{
		backupDir: dir,
		started:   make(chan struct{}),
		done:      make(chan struct{}),
	}
}

func (self *BackupInfo) WaitReady() {
	select {
	case <-self.started:
	case <-self.done:
	}
}

func (self *BackupInfo) GetResult() ([]byte, error) {
	select {
	case <-self.done:
	}
	return self.rsp, self.err
}

func (r *RockDB) backupLoop() {
	for {
		select {
		case rsp, ok := <-r.backupC:
			if !ok {
				return
			}

			func() {
				defer close(rsp.done)
				log.Printf("begin backup to:%v \n", rsp.backupDir)
				start := time.Now()
				ck, err := gorocksdb.NewCheckpoint(r.eng)
				if err != nil {
					log.Printf("init checkpoint failed: %v", err)
					rsp.err = err
					return
				}
				_, err = os.Stat(rsp.backupDir)
				if !os.IsNotExist(err) {
					log.Printf("checkpoint exist: %v, remove it", rsp.backupDir)
					os.RemoveAll(rsp.backupDir)
				}
				time.AfterFunc(time.Second*2, func() {
					close(rsp.started)
				})
				err = ck.Save(rsp.backupDir)
				if err != nil {
					log.Printf("save checkpoint failed: %v", err)
					rsp.err = err
					return
				}
				cost := time.Now().Sub(start)
				log.Printf("backup done (cost %v), check point to: %v\n", cost.String(), rsp.backupDir)
				// TODO: purge some old checkpoint
				rsp.rsp = []byte(rsp.backupDir)
			}()
		case <-r.quit:
			return
		}
	}
}

func (r *RockDB) Backup(term uint64, index uint64) *BackupInfo {
	fname := GetCheckpointDir(term, index)
	checkpointDir := path.Join(r.GetBackupDir(), fname)
	bi := newBackupInfo(checkpointDir)
	select {
	case r.backupC <- bi:
	default:
		return nil
	}
	return bi
}

func (r *RockDB) IsLocalBackupOK(term uint64, index uint64) (bool, error) {
	backupDir := r.GetBackupDir()
	checkpointDir := GetCheckpointDir(term, index)
	ro := *r.dbOpts
	ro.SetCreateIfMissing(false)
	db, err := gorocksdb.OpenDbForReadOnly(&ro, path.Join(backupDir, checkpointDir), false)
	if err != nil {
		log.Printf("checkpoint open failed: %v", err)
		return false, err
	}
	db.Close()
	return true, nil
}

func copyFile(src, dst string, override bool) error {
	sfi, err := os.Stat(src)
	if err != nil {
		return err
	}
	if !sfi.Mode().IsRegular() {
		return fmt.Errorf("copyfile: non-regular source file %v (%v)", sfi.Name(), sfi.Mode().String())
	}
	_, err = os.Stat(dst)
	if err != nil {
		if !os.IsNotExist(err) {
			return err
		}
	} else {
		if !override {
			return nil
		}
	}
	in, err := os.Open(src)
	if err != nil {
		return err
	}
	defer in.Close()
	out, err := os.Create(dst)
	if err != nil {
		return err
	}
	_, err = io.Copy(out, in)
	if err != nil {
		out.Close()
		return err
	}
	err = out.Sync()
	if err != nil {
		out.Close()
		return err
	}
	return out.Close()
}

func (r *RockDB) Restore(term uint64, index uint64) error {
	// TODO: maybe write meta (snap term and index) and check the meta data in the backup
	backupDir := r.GetBackupDir()
	hasBackup, _ := r.IsLocalBackupOK(term, index)
	if !hasBackup {
		return errors.New("no backup for restore")
	}

	checkpointDir := GetCheckpointDir(term, index)
	start := time.Now()
	log.Printf("begin restore from checkpoint: %v\n", checkpointDir)
	r.eng.Close()
	// 1. remove all files in current db except sst files
	// 2. get the list of sst in checkpoint
	// 3. remove all the sst files not in the checkpoint list
	// 4. copy all files from checkpoint to current db and do not override sst
	matchName := path.Join(r.GetDataDir(), "*")
	nameList, err := filepath.Glob(matchName)
	if err != nil {
		log.Printf("list files failed:  %v\n", err)
		return err
	}
	ckNameList, err := filepath.Glob(path.Join(backupDir, checkpointDir, "*"))
	if err != nil {
		log.Printf("list checkpoint files failed:  %v\n", err)
		return err
	}
	ckSstNameMap := make(map[string]bool)
	for _, fn := range ckNameList {
		if strings.HasSuffix(fn, ".sst") {
			ckSstNameMap[fn] = true
		}
	}

	for _, fn := range nameList {
		shortName := path.Base(fn)
		if strings.HasPrefix(shortName, "LOG") {
			continue
		}
		if strings.HasSuffix(fn, ".sst") {
			if _, ok := ckSstNameMap[fn]; ok {
				log.Printf("keeping sst file: %v", fn)
				continue
			}
		}
		log.Printf("removing: %v", fn)
		os.RemoveAll(fn)
	}
	for _, fn := range ckNameList {
		dst := path.Join(r.GetDataDir(), path.Base(fn))
		err := copyFile(fn, dst, false)
		if err != nil {
			log.Printf("copy %v to %v failed: %v", fn, dst, err)
			return err
		} else {
			log.Printf("copy %v to %v done", fn, dst)
		}
	}

	err = r.reOpen()
	log.Printf("restore done, cost: %v\n", time.Now().Sub(start))
	if err != nil {
		log.Printf("reopen the restored db failed:  %v\n", err)
	}
	return err
}

func (r *RockDB) ClearBackup(term uint64, index uint64) error {
	backupDir := r.GetBackupDir()
	checkpointDir := GetCheckpointDir(term, index)
	return os.RemoveAll(path.Join(backupDir, checkpointDir))
}
<|endoftext|>


<commit_before>package hashicorp

import (
	"fmt"
	"io"
	"time"

	"golang.org/x/net/context"

	"github.com/Sirupsen/logrus"
	hraft "github.com/hashicorp/raft"
	"github.com/relab/raft"
	"github.com/relab/raft/commonpb"
)

type future struct {
	apply hraft.ApplyFuture
	index hraft.IndexFuture
	res   chan raft.Result
}

func (f *future) ResultCh() <-chan raft.Result {
	go func() {
		confChange := false
		var g hraft.Future = f.apply
		if g == nil {
			confChange = true
			g = f.index
		}
		err := g.Error()

		if err != nil {
			f.res <- raft.Result{
				Value: err,
			}
			return
		}

		if !confChange {
			f.res <- raft.Result{
				Index: f.apply.Index(),
				Value: f.apply.Response(),
			}
			return
		}
		f.res <- raft.Result{
			Index: f.index.Index(),
			Value: &commonpb.ReconfResponse{
				Status: commonpb.ReconfOK,
			},
		}
	}()

	return f.res
}

// Wrapper wraps a hashicorp/raft.Raft and implements relab/raft.Raft.
type Wrapper struct {
	n       *hraft.Raft
	sm      raft.StateMachine
	servers []hraft.Server
	logger  logrus.FieldLogger
}

func NewRaft(logger logrus.FieldLogger,
	sm raft.StateMachine, cfg *hraft.Config, servers []hraft.Server, trans hraft.Transport,
	logs hraft.LogStore, stable hraft.StableStore, snaps hraft.SnapshotStore,
	enabled []uint64,
) *Wrapper {
	w := &Wrapper{
		sm:      sm,
		servers: servers,
		logger:  logger,
	}

	node, err := hraft.NewRaft(cfg, w, logs, stable, snaps, trans)
	if err != nil {
		panic(err)
	}

	voters := make([]hraft.Server, len(enabled))

	for i, id := range enabled {
		voters[i] = servers[id-1]
	}

	f := node.BootstrapCluster(hraft.Configuration{Servers: voters})
	if err := f.Error(); err != nil {
		panic(err)
	}

	w.n = node

	return w
}

func (w *Wrapper) ProposeCmd(ctx context.Context, req []byte) (raft.Future, error) {
	deadline, _ := ctx.Deadline()
	timeout := time.Until(deadline)
	f := w.n.Apply(req, timeout)
	ff := &future{f, nil, make(chan raft.Result, 1)}

	return ff, nil
}

func (w *Wrapper) ReadCmd(context.Context, []byte) (raft.Future, error) {
	panic("ReadCmd not implemented")
}

func (w *Wrapper) ProposeConf(ctx context.Context, req *commonpb.ReconfRequest) (raft.Future, error) {
	deadline, _ := ctx.Deadline()
	timeout := time.Until(deadline)
	server := w.servers[req.ServerID-1]
	f := w.n.AddVoter(server.ID, server.Address, 0, timeout)
	ff := &future{nil, f, make(chan raft.Result, 1)}

	return ff, nil
}

func (w *Wrapper) Apply(logentry *hraft.Log) interface{} {
	rmetrics.commitIndex.Set(float64(logentry.Index))

	switch logentry.Type {
	case hraft.LogCommand:
		res := w.sm.Apply(&commonpb.Entry{
			Term:      logentry.Term,
			Index:     logentry.Index,
			EntryType: commonpb.EntryNormal,
			Data:      logentry.Data,
		})
		return res
	}

	panic(fmt.Sprintf("no case for logtype: %v", logentry.Type))
}

func (w *Wrapper) Snapshot() (hraft.FSMSnapshot, error) { return &snapStore{}, nil }
func (w *Wrapper) Restore(io.ReadCloser) error          { return nil }

type snapStore struct{}

func (s *snapStore) Persist(sink hraft.SnapshotSink) error { return nil }
func (s *snapStore) Release()                              {}
<commit_msg>hashicorp/raft.go: Refactor future creation<commit_after>package hashicorp

import (
	"fmt"
	"io"
	"time"

	"golang.org/x/net/context"

	"github.com/Sirupsen/logrus"
	hraft "github.com/hashicorp/raft"
	"github.com/relab/raft"
	"github.com/relab/raft/commonpb"
)

type future struct {
	apply hraft.ApplyFuture
	index hraft.IndexFuture
	res   chan raft.Result
}

func (f *future) ResultCh() <-chan raft.Result {
	go func() {
		confChange := false
		var g hraft.Future = f.apply
		if g == nil {
			confChange = true
			g = f.index
		}
		err := g.Error()

		if err != nil {
			f.res <- raft.Result{
				Value: err,
			}
			return
		}

		if !confChange {
			f.res <- raft.Result{
				Index: f.apply.Index(),
				Value: f.apply.Response(),
			}
			return
		}
		f.res <- raft.Result{
			Index: f.index.Index(),
			Value: &commonpb.ReconfResponse{
				Status: commonpb.ReconfOK,
			},
		}
	}()

	return f.res
}

// Wrapper wraps a hashicorp/raft.Raft and implements relab/raft.Raft.
type Wrapper struct {
	n       *hraft.Raft
	sm      raft.StateMachine
	servers []hraft.Server
	logger  logrus.FieldLogger
}

func NewRaft(logger logrus.FieldLogger,
	sm raft.StateMachine, cfg *hraft.Config, servers []hraft.Server, trans hraft.Transport,
	logs hraft.LogStore, stable hraft.StableStore, snaps hraft.SnapshotStore,
	enabled []uint64,
) *Wrapper {
	w := &Wrapper{
		sm:      sm,
		servers: servers,
		logger:  logger,
	}

	node, err := hraft.NewRaft(cfg, w, logs, stable, snaps, trans)
	if err != nil {
		panic(err)
	}

	voters := make([]hraft.Server, len(enabled))

	for i, id := range enabled {
		voters[i] = servers[id-1]
	}

	f := node.BootstrapCluster(hraft.Configuration{Servers: voters})
	if err := f.Error(); err != nil {
		panic(err)
	}

	w.n = node

	return w
}

func (w *Wrapper) ProposeCmd(ctx context.Context, req []byte) (raft.Future, error) {
	deadline, _ := ctx.Deadline()
	timeout := time.Until(deadline)
	ff := &future{res: make(chan raft.Result, 1)}
	ff.apply = w.n.Apply(req, timeout)

	return ff, nil
}

func (w *Wrapper) ReadCmd(context.Context, []byte) (raft.Future, error) {
	panic("ReadCmd not implemented")
}

func (w *Wrapper) ProposeConf(ctx context.Context, req *commonpb.ReconfRequest) (raft.Future, error) {
	deadline, _ := ctx.Deadline()
	timeout := time.Until(deadline)
	server := w.servers[req.ServerID-1]
	f := w.n.AddVoter(server.ID, server.Address, 0, timeout)
	ff := &future{nil, f, make(chan raft.Result, 1)}

	return ff, nil
}

func (w *Wrapper) Apply(logentry *hraft.Log) interface{} {
	rmetrics.commitIndex.Set(float64(logentry.Index))

	switch logentry.Type {
	case hraft.LogCommand:
		res := w.sm.Apply(&commonpb.Entry{
			Term:      logentry.Term,
			Index:     logentry.Index,
			EntryType: commonpb.EntryNormal,
			Data:      logentry.Data,
		})
		return res
	}

	panic(fmt.Sprintf("no case for logtype: %v", logentry.Type))
}

func (w *Wrapper) Snapshot() (hraft.FSMSnapshot, error) { return &snapStore{}, nil }
func (w *Wrapper) Restore(io.ReadCloser) error          { return nil }

type snapStore struct{}

func (s *snapStore) Persist(sink hraft.SnapshotSink) error { return nil }
func (s *snapStore) Release()                              {}
<|endoftext|>


<commit_before>package runtime

import (
	"context"
	"fmt"
	"io"
	"os"
	"strings"
	"time"

	"sync"

	"github.com/pkg/errors"
	"github.com/taskcluster/taskcluster-worker/runtime/client"
	"github.com/taskcluster/taskcluster-worker/runtime/ioext"

	"gopkg.in/djherbis/stream.v1"
)

// ErrLogNotClosed represents an invalid attempt to extract a log
// while it is still open.
var ErrLogNotClosed = errors.New("Log is still open")

// TaskStatus represents the current status of the task.
type TaskStatus string // TODO: (jonasfj) TaskContext shouldn't track status

// Enumerate task status to aid life-cycle decision making
// Use strings for benefit of simple logging/reporting
const (
	Aborted   TaskStatus = "Aborted" // TODO: (jonasfj) Don't distinguish between cancel/abort
	Cancelled TaskStatus = "Cancelled"
	Succeeded TaskStatus = "Succeeded"
	Failed    TaskStatus = "Failed"
	Errored   TaskStatus = "Errored"
	Claimed   TaskStatus = "Claimed"
	Reclaimed TaskStatus = "Reclaimed"
)

// The TaskInfo struct exposes generic properties from a task definition.
//
// Note, do not be tempted to add task definition or status here in its entirety
// as it can encourage undesired behaviors.  Instead only the data necessary
// should be exposed and nothing more.  One such anti-pattern could be for a
// plugin to look at task.extra instead of adding data to task.payload.
type TaskInfo struct {
	TaskID   string
	RunID    int
	Created  time.Time
	Deadline time.Time
	Expires  time.Time
	Scopes   []string
	Task     interface{} // task definition in map[string]interface{} types..
}

// The TaskContext exposes generic properties and functionality related to a
// task that is currently being executed.
//
// This context is used to ensure that every component both engines and plugins
// that operates on a task have access to some common information about the
// task. This includes log drains, per-task credentials, generic task
// properties, and abortion notifications.
type TaskContext struct {
	TaskInfo
	logStream   *stream.Stream
	logLocation string // Absolute path to log file
	logClosed   bool
	mu          sync.RWMutex
	queue       client.Queue
	status      TaskStatus
	done        chan struct{}
	authorizer  client.Authorizer
	clientID    string
	accessToken string
	certificate string
}

// TaskContextController exposes logic for controlling the TaskContext.
//
// Spliting this out from TaskContext ensures that engines and plugins doesn't
// accidentally Dispose() the TaskContext.
type TaskContextController struct {
	*TaskContext
}

// NewTaskContext creates a TaskContext and associated TaskContextController
func NewTaskContext(tempLogFile string, task TaskInfo) (*TaskContext, *TaskContextController, error) {
	logStream, err := stream.New(tempLogFile)
	if err != nil {
		return nil, nil, errors.Wrap(err, "failed to create temporary file for storing log")
	}
	ctx := &TaskContext{
		logStream:   logStream,
		logLocation: tempLogFile,
		TaskInfo:    task,
		done:        make(chan struct{}),
	}
	ctx.authorizer = client.NewAuthorizer(func() (string, string, string, error) {
		ctx.mu.RLock()
		defer ctx.mu.RUnlock()

		if ctx.clientID == "" || ctx.accessToken == "" {
			panic(errors.New("TaskContext doesn't have clientID and accessToken"))
		}
		return ctx.clientID, ctx.accessToken, ctx.certificate, nil
	})
	return ctx, &TaskContextController{ctx}, nil
}

// CloseLog will close the log so no more messages can be written.
func (c *TaskContextController) CloseLog() error {
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.logClosed {
		return nil
	}

	debug("closing log on TaskContext")
	c.logClosed = true
	return c.logStream.Close()
}

// Dispose will clean-up all resources held by the TaskContext
func (c *TaskContextController) Dispose() error {
	debug("disposing TaskContext")
	return c.logStream.Remove()
}

// SetQueueClient will set a client for the TaskCluster Queue.  This client
// can then be used by others that have access to the task context and require
// interaction with the queue.
func (c *TaskContextController) SetQueueClient(client client.Queue) {
	c.mu.Lock()
	defer c.mu.Unlock()

	c.queue = client
}

// Queue will return a client for the TaskCluster Queue.  This client
// is useful for plugins that require interactions with the queue, such as creating
// artifacts.
func (c *TaskContext) Queue() client.Queue {
	c.mu.RLock()
	defer c.mu.RUnlock()
	// TODO: Remove this method when client library has been rewritten to consume
	// 			 a Authorizor implementation
	return c.queue
}

// Authorizer can sign requests with temporary credentials associated with the
// task.
//
// Notice, when blindly forwarding requests task.scopes should be set as
// authorizedScopes, otherwise artifact upload and resolution will possible.
func (c *TaskContext) Authorizer() client.Authorizer {
	return c.authorizer
}

// SetCredentials is used to provide the task-specific temporary credentials,
// and update these whenever they change.
func (c *TaskContextController) SetCredentials(clientID, accessToken, certificate string) {
	c.mu.Lock()
	defer c.mu.Unlock()

	c.clientID = clientID
	c.accessToken = accessToken
	c.certificate = certificate
}

// Deadline returns empty time and false, this is implemented to satisfy
// context.Context.
func (c *TaskContext) Deadline() (deadline time.Time, ok bool) {
	return time.Time{}, false
}

// Done returns a channel that is closed when to TaskContext is aborted or
// canceled.
//
// Implemented in compliance with context.Context.
func (c *TaskContext) Done() <-chan struct{} {
	return c.done
}

// Err returns context.Canceled, if task as canceled or aborted.
//
// Implemented in compliance with context.Context.
func (c *TaskContext) Err() error {
	// NOTE: This method is implemented to support the context.Context interface
	//       and may not return anything but context.Canceled or
	//       context.DeadlineExceeded.
	if c.status == Aborted || c.status == Cancelled {
		return context.Canceled
	}
	return nil
}

// Value returns nil, this is implemented to satisfy context.Context
func (c *TaskContext) Value(key interface{}) interface{} {
	return nil
}

// Abort sets the status to aborted
func (c *TaskContext) Abort() {
	// TODO: (jonasfj): Remove this method TaskContext
	// TODO (garndt): add abort/cancel channels for plugins to listen on
	c.mu.Lock()
	c.status = Aborted
	select {
	case <-c.done:
	default:
		close(c.done)
	}
	c.mu.Unlock()
}

// IsAborted returns true if the current status is Aborted
func (c *TaskContext) IsAborted() bool {
	c.mu.RLock()
	defer c.mu.RUnlock()
	return c.status == Aborted
}

// Cancel sets the status to cancelled
func (c *TaskContext) Cancel() {
	// TODO: (jonasfj): Remove this method TaskContext, add to TaskContextController
	c.mu.Lock()
	c.status = Cancelled
	select {
	case <-c.done:
	default:
		close(c.done)
	}
	c.mu.Unlock()
}

// IsCancelled returns true if the current status is Cancelled
func (c *TaskContext) IsCancelled() bool {
	c.mu.RLock()
	defer c.mu.RUnlock()
	return c.status == Cancelled
}

// Log writes a log message from the worker
//
// These log messages will be prefixed "[taskcluster]" so it's easy to see to
// that they are worker logs.
func (c *TaskContext) Log(a ...interface{}) {
	c.log("[taskcluster] ", a...)
}

// LogError writes a log error message from the worker
//
// These log messages will be prefixed "[taskcluster:error]" so it's easy to see to
// that they are worker logs.  These errors are also easy to grep from the logs in
// case of failure.
func (c *TaskContext) LogError(a ...interface{}) {
	c.log("[taskcluster:error] ", a...)
}

func (c *TaskContext) log(prefix string, a ...interface{}) {
	a = append([]interface{}{prefix}, a...)
	_, err := fmt.Fprintln(c.logStream, a...)
	if err != nil {
		_ = err //TODO: Forward this to the system log, it's not a critical error
	}
}

// LogDrain returns a drain to which log message can be written.
//
// Users should note that multiple writers are writing to this drain
// concurrently, and it is recommend that writers write in chunks of one line.
func (c *TaskContext) LogDrain() io.Writer {
	return c.logStream
}

// NewLogReader returns a ReadCloser that reads the log from the start as the
// log is written.
//
// Calls to Read() on the resulting ReadCloser are blocking. They will return
// when data is written or EOF is reached.
//
// Consumers should ensure the ReadCloser is closed before discarding it.
func (c *TaskContext) NewLogReader() (io.ReadCloser, error) {
	return c.logStream.NextReader()
}

// ExtractLog returns an IO object to read the log.
func (c *TaskContext) ExtractLog() (ioext.ReadSeekCloser, error) {
	c.mu.Lock()
	defer c.mu.Unlock()

	if !c.logClosed {
		return nil, ErrLogNotClosed
	}

	file, err := os.Open(c.logLocation)
	if err != nil {
		return nil, err
	}

	return file, nil
}

// HasScopes returns true, if task.scopes covers one of the scopeSets given
func (c *TaskContext) HasScopes(scopeSets ...[]string) bool {
	for _, scopes := range scopeSets {
		satisfied := true
		for _, required := range scopes {
			satisfied = false
			for _, scope := range c.Scopes {
				if required == scope || strings.HasSuffix(scope, "*") && strings.HasPrefix(required, scope[0:len(scope)-1]) {
					satisfied = true
					break
				}
			}
			if !satisfied {
				break
			}
		}
		if satisfied {
			return true
		}
	}
	return false
}
<commit_msg>Fixed race condition on error variable, due to missing lock<commit_after>package runtime

import (
	"context"
	"fmt"
	"io"
	"os"
	"strings"
	"time"

	"sync"

	"github.com/pkg/errors"
	"github.com/taskcluster/taskcluster-worker/runtime/client"
	"github.com/taskcluster/taskcluster-worker/runtime/ioext"

	"gopkg.in/djherbis/stream.v1"
)

// ErrLogNotClosed represents an invalid attempt to extract a log
// while it is still open.
var ErrLogNotClosed = errors.New("Log is still open")

// TaskStatus represents the current status of the task.
type TaskStatus string // TODO: (jonasfj) TaskContext shouldn't track status

// Enumerate task status to aid life-cycle decision making
// Use strings for benefit of simple logging/reporting
const (
	Aborted   TaskStatus = "Aborted" // TODO: (jonasfj) Don't distinguish between cancel/abort
	Cancelled TaskStatus = "Cancelled"
	Succeeded TaskStatus = "Succeeded"
	Failed    TaskStatus = "Failed"
	Errored   TaskStatus = "Errored"
	Claimed   TaskStatus = "Claimed"
	Reclaimed TaskStatus = "Reclaimed"
)

// The TaskInfo struct exposes generic properties from a task definition.
//
// Note, do not be tempted to add task definition or status here in its entirety
// as it can encourage undesired behaviors.  Instead only the data necessary
// should be exposed and nothing more.  One such anti-pattern could be for a
// plugin to look at task.extra instead of adding data to task.payload.
type TaskInfo struct {
	TaskID   string
	RunID    int
	Created  time.Time
	Deadline time.Time
	Expires  time.Time
	Scopes   []string
	Task     interface{} // task definition in map[string]interface{} types..
}

// The TaskContext exposes generic properties and functionality related to a
// task that is currently being executed.
//
// This context is used to ensure that every component both engines and plugins
// that operates on a task have access to some common information about the
// task. This includes log drains, per-task credentials, generic task
// properties, and abortion notifications.
type TaskContext struct {
	TaskInfo
	logStream   *stream.Stream
	logLocation string // Absolute path to log file
	logClosed   bool
	mu          sync.RWMutex
	queue       client.Queue
	status      TaskStatus
	done        chan struct{}
	authorizer  client.Authorizer
	clientID    string
	accessToken string
	certificate string
}

// TaskContextController exposes logic for controlling the TaskContext.
//
// Spliting this out from TaskContext ensures that engines and plugins doesn't
// accidentally Dispose() the TaskContext.
type TaskContextController struct {
	*TaskContext
}

// NewTaskContext creates a TaskContext and associated TaskContextController
func NewTaskContext(tempLogFile string, task TaskInfo) (*TaskContext, *TaskContextController, error) {
	logStream, err := stream.New(tempLogFile)
	if err != nil {
		return nil, nil, errors.Wrap(err, "failed to create temporary file for storing log")
	}
	ctx := &TaskContext{
		logStream:   logStream,
		logLocation: tempLogFile,
		TaskInfo:    task,
		done:        make(chan struct{}),
	}
	ctx.authorizer = client.NewAuthorizer(func() (string, string, string, error) {
		ctx.mu.RLock()
		defer ctx.mu.RUnlock()

		if ctx.clientID == "" || ctx.accessToken == "" {
			panic(errors.New("TaskContext doesn't have clientID and accessToken"))
		}
		return ctx.clientID, ctx.accessToken, ctx.certificate, nil
	})
	return ctx, &TaskContextController{ctx}, nil
}

// CloseLog will close the log so no more messages can be written.
func (c *TaskContextController) CloseLog() error {
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.logClosed {
		return nil
	}

	debug("closing log on TaskContext")
	c.logClosed = true
	return c.logStream.Close()
}

// Dispose will clean-up all resources held by the TaskContext
func (c *TaskContextController) Dispose() error {
	debug("disposing TaskContext")
	return c.logStream.Remove()
}

// SetQueueClient will set a client for the TaskCluster Queue.  This client
// can then be used by others that have access to the task context and require
// interaction with the queue.
func (c *TaskContextController) SetQueueClient(client client.Queue) {
	c.mu.Lock()
	defer c.mu.Unlock()

	c.queue = client
}

// Queue will return a client for the TaskCluster Queue.  This client
// is useful for plugins that require interactions with the queue, such as creating
// artifacts.
func (c *TaskContext) Queue() client.Queue {
	c.mu.RLock()
	defer c.mu.RUnlock()
	// TODO: Remove this method when client library has been rewritten to consume
	// 			 a Authorizor implementation
	return c.queue
}

// Authorizer can sign requests with temporary credentials associated with the
// task.
//
// Notice, when blindly forwarding requests task.scopes should be set as
// authorizedScopes, otherwise artifact upload and resolution will possible.
func (c *TaskContext) Authorizer() client.Authorizer {
	return c.authorizer
}

// SetCredentials is used to provide the task-specific temporary credentials,
// and update these whenever they change.
func (c *TaskContextController) SetCredentials(clientID, accessToken, certificate string) {
	c.mu.Lock()
	defer c.mu.Unlock()

	c.clientID = clientID
	c.accessToken = accessToken
	c.certificate = certificate
}

// Deadline returns empty time and false, this is implemented to satisfy
// context.Context.
func (c *TaskContext) Deadline() (deadline time.Time, ok bool) {
	return time.Time{}, false
}

// Done returns a channel that is closed when to TaskContext is aborted or
// canceled.
//
// Implemented in compliance with context.Context.
func (c *TaskContext) Done() <-chan struct{} {
	return c.done
}

// Err returns context.Canceled, if task as canceled or aborted.
//
// Implemented in compliance with context.Context.
func (c *TaskContext) Err() error {
	// NOTE: This method is implemented to support the context.Context interface
	//       and may not return anything but context.Canceled or
	//       context.DeadlineExceeded.
	c.mu.RLock()
	defer c.mu.RUnlock()
	if c.status == Aborted || c.status == Cancelled {
		return context.Canceled
	}
	return nil
}

// Value returns nil, this is implemented to satisfy context.Context
func (c *TaskContext) Value(key interface{}) interface{} {
	return nil
}

// Abort sets the status to aborted
func (c *TaskContext) Abort() {
	// TODO: (jonasfj): Remove this method TaskContext
	// TODO (garndt): add abort/cancel channels for plugins to listen on
	c.mu.Lock()
	defer c.mu.Unlock()
	c.status = Aborted
	select {
	case <-c.done:
	default:
		close(c.done)
	}
}

// IsAborted returns true if the current status is Aborted
func (c *TaskContext) IsAborted() bool {
	c.mu.RLock()
	defer c.mu.RUnlock()
	return c.status == Aborted
}

// Cancel sets the status to cancelled
func (c *TaskContext) Cancel() {
	// TODO: (jonasfj): Remove this method TaskContext, add to TaskContextController
	c.mu.Lock()
	c.status = Cancelled
	select {
	case <-c.done:
	default:
		close(c.done)
	}
	c.mu.Unlock()
}

// IsCancelled returns true if the current status is Cancelled
func (c *TaskContext) IsCancelled() bool {
	c.mu.RLock()
	defer c.mu.RUnlock()
	return c.status == Cancelled
}

// Log writes a log message from the worker
//
// These log messages will be prefixed "[taskcluster]" so it's easy to see to
// that they are worker logs.
func (c *TaskContext) Log(a ...interface{}) {
	c.log("[taskcluster] ", a...)
}

// LogError writes a log error message from the worker
//
// These log messages will be prefixed "[taskcluster:error]" so it's easy to see to
// that they are worker logs.  These errors are also easy to grep from the logs in
// case of failure.
func (c *TaskContext) LogError(a ...interface{}) {
	c.log("[taskcluster:error] ", a...)
}

func (c *TaskContext) log(prefix string, a ...interface{}) {
	a = append([]interface{}{prefix}, a...)
	_, err := fmt.Fprintln(c.logStream, a...)
	if err != nil {
		_ = err //TODO: Forward this to the system log, it's not a critical error
	}
}

// LogDrain returns a drain to which log message can be written.
//
// Users should note that multiple writers are writing to this drain
// concurrently, and it is recommend that writers write in chunks of one line.
func (c *TaskContext) LogDrain() io.Writer {
	return c.logStream
}

// NewLogReader returns a ReadCloser that reads the log from the start as the
// log is written.
//
// Calls to Read() on the resulting ReadCloser are blocking. They will return
// when data is written or EOF is reached.
//
// Consumers should ensure the ReadCloser is closed before discarding it.
func (c *TaskContext) NewLogReader() (io.ReadCloser, error) {
	return c.logStream.NextReader()
}

// ExtractLog returns an IO object to read the log.
func (c *TaskContext) ExtractLog() (ioext.ReadSeekCloser, error) {
	c.mu.Lock()
	defer c.mu.Unlock()

	if !c.logClosed {
		return nil, ErrLogNotClosed
	}

	file, err := os.Open(c.logLocation)
	if err != nil {
		return nil, err
	}

	return file, nil
}

// HasScopes returns true, if task.scopes covers one of the scopeSets given
func (c *TaskContext) HasScopes(scopeSets ...[]string) bool {
	for _, scopes := range scopeSets {
		satisfied := true
		for _, required := range scopes {
			satisfied = false
			for _, scope := range c.Scopes {
				if required == scope || strings.HasSuffix(scope, "*") && strings.HasPrefix(required, scope[0:len(scope)-1]) {
					satisfied = true
					break
				}
			}
			if !satisfied {
				break
			}
		}
		if satisfied {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package shell

import (
	"flag"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/util"
	"io"
	"math"
	"time"

	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
)

func init() {
	Commands = append(Commands, &commandS3CleanUploads{})
}

type commandS3CleanUploads struct {
}

func (c *commandS3CleanUploads) Name() string {
	return "s3.clean.uploads"
}

func (c *commandS3CleanUploads) Help() string {
	return `clean up stale multipart uploads

	Example:
		s3.clean.uploads -replication 001

`
}

func (c *commandS3CleanUploads) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	bucketCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
	uploadedTimeAgo := bucketCommand.Duration("timeAgo", 24*time.Hour, "created time before now. \"1.5h\" or \"2h45m\". Valid time units are \"m\", \"h\"")
	if err = bucketCommand.Parse(args); err != nil {
		return nil
	}

	var filerBucketsPath string
	filerBucketsPath, err = readFilerBucketsPath(commandEnv)
	if err != nil {
		return fmt.Errorf("read buckets: %v", err)
	}

	var buckets []string
	err = filer_pb.List(commandEnv, filerBucketsPath, "", func(entry *filer_pb.Entry, isLast bool) error {
		buckets = append(buckets, entry.Name)
		return nil
	}, "", false, math.MaxUint32)
	if err != nil {
		return fmt.Errorf("list buckets under %v: %v", filerBucketsPath, err)
	}

	for _, bucket := range buckets {
		c.cleanupUploads(commandEnv, writer, filerBucketsPath, bucket, *uploadedTimeAgo)
	}

	return err

}

func (c *commandS3CleanUploads) cleanupUploads(commandEnv *CommandEnv, writer io.Writer, filerBucketsPath string, bucket string, timeAgo time.Duration) error {
	uploadsDir := filerBucketsPath + "/" + bucket + "/.uploads"
	var staleUploads []string
	now := time.Now()
	err := filer_pb.List(commandEnv, uploadsDir, "", func(entry *filer_pb.Entry, isLast bool) error {
		ctime := time.Unix(entry.Attributes.Crtime, 0)
		if ctime.Add(timeAgo).Before(now) {
			staleUploads = append(staleUploads, entry.Name)
		}
		return nil
	}, "", false, math.MaxUint32)
	if err != nil {
		return fmt.Errorf("list uploads under %v: %v", uploadsDir, err)
	}

	for _, staleUpload := range staleUploads {
		deleteUrl := fmt.Sprintf("http://%s:%d%s/%s?recursive=true&ignoreRecursiveError=true", commandEnv.option.FilerHost, commandEnv.option.FilerPort, uploadsDir, staleUpload)
		fmt.Fprintf(writer, "purge %s\n", deleteUrl)

		err = util.Delete(deleteUrl, "")
		if err != nil {
			return fmt.Errorf("purge %s/%s: %v", uploadsDir, staleUpload, err)
		}
	}

	return nil

}
<commit_msg>correct help message<commit_after>package shell

import (
	"flag"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/util"
	"io"
	"math"
	"time"

	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
)

func init() {
	Commands = append(Commands, &commandS3CleanUploads{})
}

type commandS3CleanUploads struct {
}

func (c *commandS3CleanUploads) Name() string {
	return "s3.clean.uploads"
}

func (c *commandS3CleanUploads) Help() string {
	return `clean up stale multipart uploads

	Example:
		s3.clean.uploads -timeAgo 1.5h

`
}

func (c *commandS3CleanUploads) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	bucketCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
	uploadedTimeAgo := bucketCommand.Duration("timeAgo", 24*time.Hour, "created time before now. \"1.5h\" or \"2h45m\". Valid time units are \"m\", \"h\"")
	if err = bucketCommand.Parse(args); err != nil {
		return nil
	}

	var filerBucketsPath string
	filerBucketsPath, err = readFilerBucketsPath(commandEnv)
	if err != nil {
		return fmt.Errorf("read buckets: %v", err)
	}

	var buckets []string
	err = filer_pb.List(commandEnv, filerBucketsPath, "", func(entry *filer_pb.Entry, isLast bool) error {
		buckets = append(buckets, entry.Name)
		return nil
	}, "", false, math.MaxUint32)
	if err != nil {
		return fmt.Errorf("list buckets under %v: %v", filerBucketsPath, err)
	}

	for _, bucket := range buckets {
		c.cleanupUploads(commandEnv, writer, filerBucketsPath, bucket, *uploadedTimeAgo)
	}

	return err

}

func (c *commandS3CleanUploads) cleanupUploads(commandEnv *CommandEnv, writer io.Writer, filerBucketsPath string, bucket string, timeAgo time.Duration) error {
	uploadsDir := filerBucketsPath + "/" + bucket + "/.uploads"
	var staleUploads []string
	now := time.Now()
	err := filer_pb.List(commandEnv, uploadsDir, "", func(entry *filer_pb.Entry, isLast bool) error {
		ctime := time.Unix(entry.Attributes.Crtime, 0)
		if ctime.Add(timeAgo).Before(now) {
			staleUploads = append(staleUploads, entry.Name)
		}
		return nil
	}, "", false, math.MaxUint32)
	if err != nil {
		return fmt.Errorf("list uploads under %v: %v", uploadsDir, err)
	}

	for _, staleUpload := range staleUploads {
		deleteUrl := fmt.Sprintf("http://%s:%d%s/%s?recursive=true&ignoreRecursiveError=true", commandEnv.option.FilerHost, commandEnv.option.FilerPort, uploadsDir, staleUpload)
		fmt.Fprintf(writer, "purge %s\n", deleteUrl)

		err = util.Delete(deleteUrl, "")
		if err != nil {
			return fmt.Errorf("purge %s/%s: %v", uploadsDir, staleUpload, err)
		}
	}

	return nil

}
<|endoftext|>


<commit_before>// Copyright 2012 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"syscall"
)

const (
	/*
	 * IOCTL commands --- we will commandeer 0x4C ('L')
	 */
	LOOP_SET_CAPACITY = 0x4C07
	LOOP_CHANGE_FD    = 0x4C06
	LOOP_GET_STATUS64 = 0x4C05
	LOOP_SET_STATUS64 = 0x4C04
	LOOP_GET_STATUS   = 0x4C03
	LOOP_SET_STATUS   = 0x4C02
	LOOP_CLR_FD       = 0x4C01
	LOOP_SET_FD       = 0x4C00
	LO_NAME_SIZE      = 64
	LO_KEY_SIZE       = 32
	/* /dev/loop-control interface */
	LOOP_CTL_ADD      = 0x4C80
	LOOP_CTL_REMOVE   = 0x4C81
	LOOP_CTL_GET_FREE = 0x4C82
	SYS_ioctl         = 16
)

const tcz = "/tinycorelinux.net/5.x/x86/tcz"

var l = log.New(os.Stdout, "tcz: ", 0)

// consider making this a goroutine which pushes the string down the channel.
func findloop() (name string, err error) {
	cfd, err := syscall.Open("/dev/loop-control", syscall.O_RDWR, 0)
	if err != nil {
		log.Fatalf("/dev/loop-control: %v", err)
	}

	a, b, errno := syscall.Syscall(SYS_ioctl, uintptr(cfd), LOOP_CTL_GET_FREE, 0)
	if errno != 0 {
		log.Fatalf("ioctl: %v\n", err)
	}
	log.Printf("a %v b %v err %v\n", a, b, err)
	name = fmt.Sprintf("/dev/loop%d", a)
	return name, nil
}
func linkone(p string, i os.FileInfo, err error) error {
	l.Printf("symtree: p %v\n", p)
	if err != nil {
		return err
	}

	// the tree of symlinks starts at /tcz
	packagel := filepath.SplitList(p)
	// surely there's a better way.
	n := append([]string{"/"}, packagel[2:]...)
	to := path.Join(n...)

	l.Printf("symtree: symlink %v to %v\n", p, to)
	return os.Symlink(p, to)
}
func main() {
	if len(os.Args) < 2 {
		os.Exit(1)
	}
	cmdName := os.Args[1]

	if err := os.MkdirAll(tcz, 0600); err != nil {
		l.Fatal(err)
	}

	packagePath := path.Join("/tcz", cmdName)
	if err := os.MkdirAll(packagePath, 0600); err != nil {
		l.Fatal(err)
	}

	// path.Join doesn't quite work here.
	filepath := path.Join(tcz, cmdName+".tcz")
	cmd := "http:/" + filepath

	resp, err := http.Get(cmd)
	if err != nil {
		l.Fatalf("Get of %v failed: %v\n", cmd, err)
	}
	defer resp.Body.Close()

	if resp.Status != "200 OK" {
		l.Fatalf("Not OK! %v\n", resp.Status)
	}

	l.Printf("resp %v err %v\n", resp, err)
	// we've to the whole tcz in resp.Body.
	// First, save it to /tcz/name
	f, err := os.Create(filepath)
	if err != nil {
		l.Fatal("Create of :%v: failed: %v\n", filepath, err)
	} else {
		l.Printf("created %v f %v\n", filepath, f)
	}

	if c, err := io.Copy(f, resp.Body); err != nil {
		l.Fatal(err)
	} else {
		/* OK, these are compressed tars ... */
		l.Printf("c %v err %v\n", c, err)
	}
	/* now walk it, and fetch anything we need. */
	name, err := findloop()
	l.Printf("findloop gets %v err %v\n", name, err)

	//ffd := syscall.Open("backing-file", syscall.O_RDONLY)
	//lfd := syscall.Open(loopname, syscall.O_RDONLY)
	/* now mount it. The convention is the mount is in /tcz/packagename */
	if err := syscall.Mount(filepath, packagePath, "squashfs", 0, ""); err != nil {
		l.Fatalf("Mount %s on %s: %v\n", filepath, packagePath, err)
	}
}
<commit_msg>Now with squashfs mounting!<commit_after>// Copyright 2012 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"syscall"
)

const (
	/*
	 * IOCTL commands --- we will commandeer 0x4C ('L')
	 */
	LOOP_SET_CAPACITY = 0x4C07
	LOOP_CHANGE_FD    = 0x4C06
	LOOP_GET_STATUS64 = 0x4C05
	LOOP_SET_STATUS64 = 0x4C04
	LOOP_GET_STATUS   = 0x4C03
	LOOP_SET_STATUS   = 0x4C02
	LOOP_CLR_FD       = 0x4C01
	LOOP_SET_FD       = 0x4C00
	LO_NAME_SIZE      = 64
	LO_KEY_SIZE       = 32
	/* /dev/loop-control interface */
	LOOP_CTL_ADD      = 0x4C80
	LOOP_CTL_REMOVE   = 0x4C81
	LOOP_CTL_GET_FREE = 0x4C82
	SYS_ioctl         = 16
)

const tcz = "/tinycorelinux.net/5.x/x86/tcz"

var l = log.New(os.Stdout, "tcz: ", 0)

// consider making this a goroutine which pushes the string down the channel.
func findloop() (name string, err error) {
	cfd, err := syscall.Open("/dev/loop-control", syscall.O_RDWR, 0)
	if err != nil {
		log.Fatalf("/dev/loop-control: %v", err)
	}
	defer syscall.Close(cfd)
	a, b, errno := syscall.Syscall(SYS_ioctl, uintptr(cfd), LOOP_CTL_GET_FREE, 0)
	if errno != 0 {
		log.Fatalf("ioctl: %v\n", err)
	}
	log.Printf("a %v b %v err %v\n", a, b, err)
	name = fmt.Sprintf("/dev/loop%d", a)
	return name, nil
}
func linkone(p string, i os.FileInfo, err error) error {
	l.Printf("symtree: p %v\n", p)
	if err != nil {
		return err
	}

	// the tree of symlinks starts at /tcz
	packagel := filepath.SplitList(p)
	// surely there's a better way.
	n := append([]string{"/"}, packagel[2:]...)
	to := path.Join(n...)

	l.Printf("symtree: symlink %v to %v\n", p, to)
	return os.Symlink(p, to)
}
func main() {
	if len(os.Args) < 2 {
		os.Exit(1)
	}
	cmdName := os.Args[1]

	if err := os.MkdirAll(tcz, 0600); err != nil {
		l.Fatal(err)
	}

	packagePath := path.Join("/tcz", cmdName)
	if err := os.MkdirAll(packagePath, 0600); err != nil {
		l.Fatal(err)
	}

	// path.Join doesn't quite work here.
	filepath := path.Join(tcz, cmdName+".tcz")
	if _, err := os.Stat(filepath); err != nil {
		cmd := "http:/" + filepath

		resp, err := http.Get(cmd)
		if err != nil {
			l.Fatalf("Get of %v failed: %v\n", cmd, err)
		}
		defer resp.Body.Close()

		if resp.Status != "200 OK" {
			l.Fatalf("Not OK! %v\n", resp.Status)
		}

		l.Printf("resp %v err %v\n", resp, err)
		// we've to the whole tcz in resp.Body.
		// First, save it to /tcz/name
		f, err := os.Create(filepath)
		if err != nil {
			l.Fatal("Create of :%v: failed: %v\n", filepath, err)
		} else {
			l.Printf("created %v f %v\n", filepath, f)
		}

		if c, err := io.Copy(f, resp.Body); err != nil {
			l.Fatal(err)
		} else {
			/* OK, these are compressed tars ... */
			l.Printf("c %v err %v\n", c, err)
		}
		f.Close()
	}
	/* now walk it, and fetch anything we need. */
	loopname, err := findloop()
	if err != nil {
		l.Fatal(err)
	}
	l.Printf("findloop gets %v err %v\n", loopname, err)

	ffd, err := syscall.Open(filepath, syscall.O_RDONLY, 0)
	lfd, err := syscall.Open(loopname, syscall.O_RDONLY, 0)
	log.Printf("ffd %v lfd %v\n", ffd, lfd)
	a, b, errno := syscall.Syscall(SYS_ioctl, uintptr(lfd), LOOP_SET_FD, uintptr(ffd))
	if errno != 0 {
		log.Fatalf("loop set fd ioctl: %v, %v, %v\n", a, b, errno)
	}
	/* now mount it. The convention is the mount is in /tcz/packagename */
	if err := syscall.Mount(loopname, packagePath, "squashfs", syscall.MS_MGC_VAL|syscall.MS_RDONLY, ""); err != nil {
		l.Fatalf("Mount %s on %s: %v\n", loopname, packagePath, err)
	}
}
<|endoftext|>


<commit_before>package contractor

import (
	"path/filepath"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

// These interfaces define the HostDB's dependencies. Using the smallest
// interface possible makes it easier to mock these dependencies in testing.
type (
	consensusSet interface {
		ConsensusSetSubscribe(modules.ConsensusSetSubscriber, modules.ConsensusChangeID) error
		Synced() bool
		Unsubscribe(modules.ConsensusSetSubscriber)
	}
	// In order to restrict the modules.TransactionBuilder interface, we must
	// provide a shim to bridge the gap between modules.Wallet and
	// transactionBuilder.
	walletShim interface {
		NextAddress() (types.UnlockConditions, error)
		StartTransaction() modules.TransactionBuilder
	}
	wallet interface {
		NextAddress() (types.UnlockConditions, error)
		StartTransaction() transactionBuilder
	}
	transactionBuilder interface {
		AddArbitraryData([]byte) uint64
		AddFileContract(types.FileContract) uint64
		AddMinerFee(types.Currency) uint64
		AddParents([]types.Transaction)
		AddSiacoinInput(types.SiacoinInput) uint64
		AddSiacoinOutput(types.SiacoinOutput) uint64
		AddTransactionSignature(types.TransactionSignature) uint64
		Drop()
		FundSiacoins(types.Currency) error
		Sign(bool) ([]types.Transaction, error)
		View() (types.Transaction, []types.Transaction)
		ViewAdded() (parents, coins, funds, signatures []int)
	}
	transactionPool interface {
		AcceptTransactionSet([]types.Transaction) error
		FeeEstimation() (min types.Currency, max types.Currency)
	}

	hostDB interface {
		AllHosts() []modules.HostDBEntry
		ActiveHosts() []modules.HostDBEntry
		Host(types.SiaPublicKey) (modules.HostDBEntry, bool)
		RandomHosts(n int, exclude []types.SiaPublicKey) []modules.HostDBEntry
	}

	persister interface {
		save(contractorPersist) error
		update(...journalUpdate) error
		load(*contractorPersist) error
		Close() error
	}
)

// Because wallet is not directly compatible with modules.Wallet (wrong
// type signature for StartTransaction), we must provide a bridge type.
type walletBridge struct {
	w walletShim
}

func (ws *walletBridge) NextAddress() (types.UnlockConditions, error) { return ws.w.NextAddress() }
func (ws *walletBridge) StartTransaction() transactionBuilder         { return ws.w.StartTransaction() }

// stdPersist implements the persister interface via the journal type. The
// filename required by these functions is internal to stdPersist.
type stdPersist struct {
	journal  *journal
	filename string
}

func (p *stdPersist) save(data contractorPersist) error {
	if p.journal == nil {
		var err error
		p.journal, err = newJournal(p.filename, data)
		return err
	}
	return p.journal.checkpoint(data)
}

func (p *stdPersist) update(us ...journalUpdate) error {
	return p.journal.update(us)
}

func (p *stdPersist) load(data *contractorPersist) error {
	var err error
	p.journal, err = openJournal(p.filename, data)
	if err != nil {
		// Try loading old persist.
		err = loadv110persist(filepath.Dir(p.filename), data)
		if err != nil {
			return err
		}
		p.journal, err = newJournal(p.filename, *data)
	}
	return err
}

func (p stdPersist) Close() error {
	return p.journal.Close()
}

func newPersist(dir string) *stdPersist {
	return &stdPersist{
		filename: filepath.Join(dir, "contractor.journal"),
	}
}
<commit_msg>fix journal checkpointing (fix for TestRenterAllowance failure)<commit_after>package contractor

import (
	"path/filepath"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

// These interfaces define the HostDB's dependencies. Using the smallest
// interface possible makes it easier to mock these dependencies in testing.
type (
	consensusSet interface {
		ConsensusSetSubscribe(modules.ConsensusSetSubscriber, modules.ConsensusChangeID) error
		Synced() bool
		Unsubscribe(modules.ConsensusSetSubscriber)
	}
	// In order to restrict the modules.TransactionBuilder interface, we must
	// provide a shim to bridge the gap between modules.Wallet and
	// transactionBuilder.
	walletShim interface {
		NextAddress() (types.UnlockConditions, error)
		StartTransaction() modules.TransactionBuilder
	}
	wallet interface {
		NextAddress() (types.UnlockConditions, error)
		StartTransaction() transactionBuilder
	}
	transactionBuilder interface {
		AddArbitraryData([]byte) uint64
		AddFileContract(types.FileContract) uint64
		AddMinerFee(types.Currency) uint64
		AddParents([]types.Transaction)
		AddSiacoinInput(types.SiacoinInput) uint64
		AddSiacoinOutput(types.SiacoinOutput) uint64
		AddTransactionSignature(types.TransactionSignature) uint64
		Drop()
		FundSiacoins(types.Currency) error
		Sign(bool) ([]types.Transaction, error)
		View() (types.Transaction, []types.Transaction)
		ViewAdded() (parents, coins, funds, signatures []int)
	}
	transactionPool interface {
		AcceptTransactionSet([]types.Transaction) error
		FeeEstimation() (min types.Currency, max types.Currency)
	}

	hostDB interface {
		AllHosts() []modules.HostDBEntry
		ActiveHosts() []modules.HostDBEntry
		Host(types.SiaPublicKey) (modules.HostDBEntry, bool)
		RandomHosts(n int, exclude []types.SiaPublicKey) []modules.HostDBEntry
	}

	persister interface {
		save(contractorPersist) error
		update(...journalUpdate) error
		load(*contractorPersist) error
		Close() error
	}
)

// Because wallet is not directly compatible with modules.Wallet (wrong
// type signature for StartTransaction), we must provide a bridge type.
type walletBridge struct {
	w walletShim
}

func (ws *walletBridge) NextAddress() (types.UnlockConditions, error) { return ws.w.NextAddress() }
func (ws *walletBridge) StartTransaction() transactionBuilder         { return ws.w.StartTransaction() }

// stdPersist implements the persister interface via the journal type. The
// filename required by these functions is internal to stdPersist.
type stdPersist struct {
	journal  *journal
	filename string
}

func (p *stdPersist) save(data contractorPersist) error {
	if p.journal == nil {
		var err error
		p.journal, err = newJournal(p.filename, data)
		if err != nil {
			return err
		}
	}
	return p.journal.checkpoint(data)
}

func (p *stdPersist) update(us ...journalUpdate) error {
	return p.journal.update(us)
}

func (p *stdPersist) load(data *contractorPersist) error {
	var err error
	p.journal, err = openJournal(p.filename, data)
	if err != nil {
		// Try loading old persist.
		err = loadv110persist(filepath.Dir(p.filename), data)
		if err != nil {
			return err
		}
		p.journal, err = newJournal(p.filename, *data)
	}
	return err
}

func (p stdPersist) Close() error {
	return p.journal.Close()
}

func newPersist(dir string) *stdPersist {
	return &stdPersist{
		filename: filepath.Join(dir, "contractor.journal"),
	}
}
<|endoftext|>


<commit_before>package main

import (
	"consumer"
	"flag"
	"log"
	"time"
)

func main() {
	var ini = flag.String("config", "./simple.ini", "The configuration file to use.")

	c, err := consumer.Create(*ini)
	if err != nil {
		log.Fatalf("Unable to create consumer. Error: %v", err)
	}
	err = c.Consume(worker)
	if err != nil {
		log.Fatalf("Unable to consume messages. Error: %v", err)
	}
}

// Example worker function
// Just prints to the logs and sleeps between receiving messages.
func worker(msg *consumer.Message) {
	log.Printf(
		"Message received Exchange: %s, RoutingKey: %s, Body: %s",
		msg.Exchange,
		msg.RoutingKey,
		msg.Body)

	msg.Ack(true)
	time.Sleep(1000 * time.Millisecond)
	log.Print("sleep complete")
}
<commit_msg>Add missing flag.Parse() call.<commit_after>package main

import (
	"consumer"
	"flag"
	"log"
	"time"
)

func main() {
	var ini = flag.String("config", "./simple.ini", "The configuration file to use.")
	flag.Parse()

	c, err := consumer.Create(*ini)
	if err != nil {
		log.Fatalf("Unable to create consumer. Error: %v", err)
	}
	err = c.Consume(worker)
	if err != nil {
		log.Fatalf("Unable to consume messages. Error: %v", err)
	}
}

// Example worker function
// Just prints to the logs and sleeps between receiving messages.
func worker(msg *consumer.Message) {
	log.Printf(
		"Message received Exchange: %s, RoutingKey: %s, Body: %s",
		msg.Exchange,
		msg.RoutingKey,
		msg.Body)

	msg.Ack(true)
	time.Sleep(1000 * time.Millisecond)
	log.Print("sleep complete")
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package persistence_test

import (
	"fmt"
	"sort"

	"github.com/juju/errors"
	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"
	"gopkg.in/mgo.v2/bson"
	"gopkg.in/mgo.v2/txn"

	"github.com/juju/juju/workload"
	"github.com/juju/juju/workload/persistence"
)

var _ = gc.Suite(&workloadsPersistenceSuite{})

type workloadsPersistenceSuite struct {
	persistence.BaseSuite
}

func (s *workloadsPersistenceSuite) TestTrackOkay(c *gc.C) {
	wl := s.NewWorkload("docker", "workloadA/workloadA-xyz")

	wp := s.NewPersistence()
	id := "f47ac10b-58cc-4372-a567-0e02b2c3d479"
	okay, err := wp.Track(id, wl)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(okay, jc.IsTrue)
	s.Stub.CheckCallNames(c, "Run")
	s.State.CheckOps(c, [][]txn.Op{{
		{
			C:      "workloads",
			Id:     "workload#a-unit/0#f47ac10b-58cc-4372-a567-0e02b2c3d479",
			Assert: txn.DocMissing,
			Insert: &persistence.WorkloadDoc{
				DocID:  "workload#a-unit/0#f47ac10b-58cc-4372-a567-0e02b2c3d479",
				UnitID: "a-unit/0",

				Name: "workloadA",
				Type: "docker",

				PluginID:       "workloadA-xyz",
				OriginalStatus: "running",

				PluginStatus: "running",
			},
		},
	}})
}

func (s *workloadsPersistenceSuite) TestTrackAlreadyExists(c *gc.C) {
	id := "f47ac10b-58cc-4372-a567-0e02b2c3d479"

	wl := s.NewWorkload("docker", "workloadA/workloadA-xyz")
	s.SetDoc(id, wl)
	s.Stub.SetErrors(txn.ErrAborted)

	wp := s.NewPersistence()
	okay, err := wp.Track(id, wl)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(okay, jc.IsFalse)
	s.Stub.CheckCallNames(c, "Run")
	s.State.CheckOps(c, [][]txn.Op{{
		{
			C:      "workloads",
			Id:     "workload#a-unit/0#f47ac10b-58cc-4372-a567-0e02b2c3d479",
			Assert: txn.DocMissing,
			Insert: &persistence.WorkloadDoc{
				DocID:  "workload#a-unit/0#f47ac10b-58cc-4372-a567-0e02b2c3d479",
				UnitID: "a-unit/0",

				Name: "workloadA",
				Type: "docker",

				PluginID:       "workloadA-xyz",
				OriginalStatus: "running",

				PluginStatus: "running",
			},
		},
	}})
}

func (s *workloadsPersistenceSuite) TestTrackFailed(c *gc.C) {
	id := "f47ac10b-58cc-4372-a567-0e02b2c3d479"
	failure := errors.Errorf("<failed!>")
	s.Stub.SetErrors(failure)
	wl := s.NewWorkload("docker", "workloadA")

	pp := s.NewPersistence()
	_, err := pp.Track(id, wl)

	c.Check(errors.Cause(err), gc.Equals, failure)
}

func (s *workloadsPersistenceSuite) TestSetStatusOkay(c *gc.C) {
	id := "f47ac10b-58cc-4372-a567-0e02b2c3d479"
	wl := s.NewWorkload("docker", "workloadA/workloadA-xyz")
	s.SetDoc(id, wl)

	pp := s.NewPersistence()
	okay, err := pp.SetStatus(id, workload.StateRunning)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(okay, jc.IsTrue)
	s.Stub.CheckCallNames(c, "Run")
	s.State.CheckOps(c, [][]txn.Op{{
		{
			C:      "workloads",
			Id:     "workload#a-unit/0#f47ac10b-58cc-4372-a567-0e02b2c3d479",
			Assert: txn.DocExists,
			Update: bson.D{
				{"$set", bson.D{
					{"state", workload.StateRunning},
					{"status", workload.StateRunning},
					{"pluginstatus", workload.StateRunning},
				}},
			},
		},
	}})
}

func (s *workloadsPersistenceSuite) TestSetStatusMissing(c *gc.C) {
	id := "f47ac10b-58cc-4372-a567-0e02b2c3d479"
	s.Stub.SetErrors(txn.ErrAborted)

	pp := s.NewPersistence()
	okay, err := pp.SetStatus(id, workload.StateRunning)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(okay, jc.IsFalse)
	s.Stub.CheckCallNames(c, "Run")
	s.State.CheckOps(c, [][]txn.Op{{
		{
			C:      "workloads",
			Id:     "workload#a-unit/0#f47ac10b-58cc-4372-a567-0e02b2c3d479",
			Assert: txn.DocExists,
			Update: bson.D{
				{"$set", bson.D{
					{"state", workload.StateRunning},
					{"status", workload.StateRunning},
					{"pluginstatus", workload.StateRunning},
				}},
			},
		},
	}})
}

func (s *workloadsPersistenceSuite) TestSetStatusFailed(c *gc.C) {
	id := "f47ac10b-58cc-4372-a567-0e02b2c3d479"
	wl := s.NewWorkload("docker", "workloadA/workloadA-xyz")
	s.SetDoc(id, wl)
	failure := errors.Errorf("<failed!>")
	s.Stub.SetErrors(failure)

	pp := s.NewPersistence()
	_, err := pp.SetStatus(id, workload.StateRunning)

	c.Check(errors.Cause(err), gc.Equals, failure)
}

func (s *workloadsPersistenceSuite) TestListOkay(c *gc.C) {
	id := "f47ac10b-58cc-4372-a567-0e02b2c3d479"
	wl := s.NewWorkload("docker", "workloadA/xyz")
	s.SetDoc(id, wl)
	other := s.NewWorkload("docker", "workloadB/abc")
	s.SetDoc("f47ac10b-58cc-4372-a567-0e02b2c3d480", other)

	pp := s.NewPersistence()
	workloads, missing, err := pp.List(id)
	c.Assert(err, jc.ErrorIsNil)

	s.Stub.CheckCallNames(c, "All")
	s.State.CheckNoOps(c)
	c.Check(workloads, jc.DeepEquals, []workload.Info{wl})
	c.Check(missing, gc.HasLen, 0)
}

func (s *workloadsPersistenceSuite) TestListSomeMissing(c *gc.C) {
	id := "f47ac10b-58cc-4372-a567-0e02b2c3d479"
	wl := s.NewWorkload("docker", "workloadB/abc")
	s.SetDoc(id, wl)
	other := s.NewWorkload("docker", "workloadA/xyz")
	s.SetDoc("f47ac10b-58cc-4372-a567-0e02b2c3d480", other)

	missingID := "f47ac10b-58cc-4372-a567-0e02b2c3d481"
	pp := s.NewPersistence()
	workloads, missing, err := pp.List(id, missingID)
	c.Assert(err, jc.ErrorIsNil)

	s.Stub.CheckCallNames(c, "All")
	s.State.CheckNoOps(c)
	c.Check(workloads, jc.DeepEquals, []workload.Info{wl})
	c.Check(missing, jc.DeepEquals, []string{missingID})
}

func (s *workloadsPersistenceSuite) TestListEmpty(c *gc.C) {
	id := "f47ac10b-58cc-4372-a567-0e02b2c3d479"
	pp := s.NewPersistence()
	workloads, missing, err := pp.List(id)
	c.Assert(err, jc.ErrorIsNil)

	s.Stub.CheckCallNames(c, "All")
	s.State.CheckNoOps(c)
	c.Check(workloads, gc.HasLen, 0)
	c.Check(missing, jc.DeepEquals, []string{id})
}

func (s *workloadsPersistenceSuite) TestListFailure(c *gc.C) {
	failure := errors.Errorf("<failed!>")
	s.Stub.SetErrors(failure)

	pp := s.NewPersistence()
	_, _, err := pp.List()

	c.Check(errors.Cause(err), gc.Equals, failure)
}

func (s *workloadsPersistenceSuite) TestListAllOkay(c *gc.C) {
	existing := s.NewWorkloads("docker", "workloadA/xyz", "workloadB/abc")
	for i, wl := range existing {
		s.SetDoc(fmt.Sprintf("%d", i), wl)
	}

	pp := s.NewPersistence()
	workloads, err := pp.ListAll()
	c.Assert(err, jc.ErrorIsNil)

	s.Stub.CheckCallNames(c, "All")
	s.State.CheckNoOps(c)
	sort.Sort(byName(workloads))
	sort.Sort(byName(existing))
	c.Check(workloads, jc.DeepEquals, existing)
}

func (s *workloadsPersistenceSuite) TestListAllEmpty(c *gc.C) {
	pp := s.NewPersistence()
	workloads, err := pp.ListAll()
	c.Assert(err, jc.ErrorIsNil)

	s.Stub.CheckCallNames(c, "All")
	s.State.CheckNoOps(c)
	c.Check(workloads, gc.HasLen, 0)
}

type byName []workload.Info

func (b byName) Len() int           { return len(b) }
func (b byName) Less(i, j int) bool { return b[i].Name < b[j].Name }
func (b byName) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }

func (s *workloadsPersistenceSuite) TestListAllFailed(c *gc.C) {
	failure := errors.Errorf("<failed!>")
	s.Stub.SetErrors(failure)

	pp := s.NewPersistence()
	_, err := pp.ListAll()

	c.Check(errors.Cause(err), gc.Equals, failure)
}

func (s *workloadsPersistenceSuite) TestUntrackOkay(c *gc.C) {
	id := "f47ac10b-58cc-4372-a567-0e02b2c3d479"
	wl := s.NewWorkload("docker", "workloadA/xyz")
	s.SetDoc(id, wl)

	pp := s.NewPersistence()
	found, err := pp.Untrack(id)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(found, jc.IsTrue)
	s.Stub.CheckCallNames(c, "Run")
	s.State.CheckOps(c, [][]txn.Op{{
		{
			C:      "workloads",
			Id:     "workload#a-unit/0#f47ac10b-58cc-4372-a567-0e02b2c3d479",
			Assert: txn.DocExists,
			Remove: true,
		},
	}})
}

func (s *workloadsPersistenceSuite) TestUntrackMissing(c *gc.C) {
	id := "f47ac10b-58cc-4372-a567-0e02b2c3d479"
	s.Stub.SetErrors(txn.ErrAborted)

	pp := s.NewPersistence()
	found, err := pp.Untrack(id)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(found, jc.IsFalse)
	s.Stub.CheckCallNames(c, "Run")
	s.State.CheckOps(c, [][]txn.Op{{
		{
			C:      "workloads",
			Id:     "workload#a-unit/0#f47ac10b-58cc-4372-a567-0e02b2c3d479",
			Assert: txn.DocExists,
			Remove: true,
		},
	}})
}

func (s *workloadsPersistenceSuite) TestUntrackFailed(c *gc.C) {
	id := "f47ac10b-58cc-4372-a567-0e02b2c3d479"
	failure := errors.Errorf("<failed!>")
	s.Stub.SetErrors(failure)

	pp := s.NewPersistence()
	_, err := pp.Untrack(id)

	c.Check(errors.Cause(err), gc.Equals, failure)
}
<commit_msg>Fix some tests.<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package persistence_test

import (
	"fmt"
	"sort"

	"github.com/juju/errors"
	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"
	"gopkg.in/mgo.v2/bson"
	"gopkg.in/mgo.v2/txn"

	"github.com/juju/juju/workload"
	"github.com/juju/juju/workload/persistence"
)

var _ = gc.Suite(&workloadsPersistenceSuite{})

type workloadsPersistenceSuite struct {
	persistence.BaseSuite
}

func (s *workloadsPersistenceSuite) TestTrackOkay(c *gc.C) {
	wl := s.NewWorkload("docker", "workloadA/workloadA-xyz")

	wp := s.NewPersistence()
	id := "f47ac10b-58cc-4372-a567-0e02b2c3d479"
	okay, err := wp.Track(id, wl)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(okay, jc.IsTrue)
	s.Stub.CheckCallNames(c, "All", "Run")
	s.State.CheckOps(c, [][]txn.Op{{
		{
			C:      "workloads",
			Id:     "workload#a-unit/0#f47ac10b-58cc-4372-a567-0e02b2c3d479",
			Assert: txn.DocMissing,
			Insert: &persistence.WorkloadDoc{
				DocID:  "workload#a-unit/0#f47ac10b-58cc-4372-a567-0e02b2c3d479",
				UnitID: "a-unit/0",

				Name: "workloadA",
				Type: "docker",

				PluginID:       "workloadA-xyz",
				OriginalStatus: "running",

				PluginStatus: "running",
			},
		},
	}})
}

func (s *workloadsPersistenceSuite) TestTrackAlreadyExists(c *gc.C) {
	id := "f47ac10b-58cc-4372-a567-0e02b2c3d479"

	wl := s.NewWorkload("docker", "workloadA/workloadA-xyz")
	s.SetDoc(id, wl)
	s.Stub.SetErrors(nil, txn.ErrAborted)

	wp := s.NewPersistence()
	okay, err := wp.Track(id, wl)

	c.Check(okay, jc.IsFalse)
	c.Check(err, jc.Satisfies, errors.IsAlreadyExists)
	s.Stub.CheckCallNames(c, "All")
	s.State.CheckOps(c, nil)
}

func (s *workloadsPersistenceSuite) TestTrackFailed(c *gc.C) {
	id := "f47ac10b-58cc-4372-a567-0e02b2c3d479"
	failure := errors.Errorf("<failed!>")
	s.Stub.SetErrors(nil, failure)
	wl := s.NewWorkload("docker", "workloadA")

	pp := s.NewPersistence()
	_, err := pp.Track(id, wl)

	c.Check(errors.Cause(err), gc.Equals, failure)
	s.Stub.CheckCallNames(c, "All", "Run")
}

func (s *workloadsPersistenceSuite) TestSetStatusOkay(c *gc.C) {
	id := "f47ac10b-58cc-4372-a567-0e02b2c3d479"
	wl := s.NewWorkload("docker", "workloadA/workloadA-xyz")
	s.SetDoc(id, wl)

	pp := s.NewPersistence()
	okay, err := pp.SetStatus(id, workload.StateRunning)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(okay, jc.IsTrue)
	s.Stub.CheckCallNames(c, "Run")
	s.State.CheckOps(c, [][]txn.Op{{
		{
			C:      "workloads",
			Id:     "workload#a-unit/0#f47ac10b-58cc-4372-a567-0e02b2c3d479",
			Assert: txn.DocExists,
			Update: bson.D{
				{"$set", bson.D{
					{"state", workload.StateRunning},
					{"status", workload.StateRunning},
					{"pluginstatus", workload.StateRunning},
				}},
			},
		},
	}})
}

func (s *workloadsPersistenceSuite) TestSetStatusMissing(c *gc.C) {
	id := "f47ac10b-58cc-4372-a567-0e02b2c3d479"
	s.Stub.SetErrors(txn.ErrAborted)

	pp := s.NewPersistence()
	okay, err := pp.SetStatus(id, workload.StateRunning)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(okay, jc.IsFalse)
	s.Stub.CheckCallNames(c, "Run")
	s.State.CheckOps(c, [][]txn.Op{{
		{
			C:      "workloads",
			Id:     "workload#a-unit/0#f47ac10b-58cc-4372-a567-0e02b2c3d479",
			Assert: txn.DocExists,
			Update: bson.D{
				{"$set", bson.D{
					{"state", workload.StateRunning},
					{"status", workload.StateRunning},
					{"pluginstatus", workload.StateRunning},
				}},
			},
		},
	}})
}

func (s *workloadsPersistenceSuite) TestSetStatusFailed(c *gc.C) {
	id := "f47ac10b-58cc-4372-a567-0e02b2c3d479"
	wl := s.NewWorkload("docker", "workloadA/workloadA-xyz")
	s.SetDoc(id, wl)
	failure := errors.Errorf("<failed!>")
	s.Stub.SetErrors(failure)

	pp := s.NewPersistence()
	_, err := pp.SetStatus(id, workload.StateRunning)

	c.Check(errors.Cause(err), gc.Equals, failure)
}

func (s *workloadsPersistenceSuite) TestListOkay(c *gc.C) {
	id := "f47ac10b-58cc-4372-a567-0e02b2c3d479"
	wl := s.NewWorkload("docker", "workloadA/xyz")
	s.SetDoc(id, wl)
	other := s.NewWorkload("docker", "workloadB/abc")
	s.SetDoc("f47ac10b-58cc-4372-a567-0e02b2c3d480", other)

	pp := s.NewPersistence()
	workloads, missing, err := pp.List(id)
	c.Assert(err, jc.ErrorIsNil)

	s.Stub.CheckCallNames(c, "All")
	s.State.CheckNoOps(c)
	c.Check(workloads, jc.DeepEquals, []workload.Info{wl})
	c.Check(missing, gc.HasLen, 0)
}

func (s *workloadsPersistenceSuite) TestListSomeMissing(c *gc.C) {
	id := "f47ac10b-58cc-4372-a567-0e02b2c3d479"
	wl := s.NewWorkload("docker", "workloadB/abc")
	s.SetDoc(id, wl)
	other := s.NewWorkload("docker", "workloadA/xyz")
	s.SetDoc("f47ac10b-58cc-4372-a567-0e02b2c3d480", other)

	missingID := "f47ac10b-58cc-4372-a567-0e02b2c3d481"
	pp := s.NewPersistence()
	workloads, missing, err := pp.List(id, missingID)
	c.Assert(err, jc.ErrorIsNil)

	s.Stub.CheckCallNames(c, "All")
	s.State.CheckNoOps(c)
	c.Check(workloads, jc.DeepEquals, []workload.Info{wl})
	c.Check(missing, jc.DeepEquals, []string{missingID})
}

func (s *workloadsPersistenceSuite) TestListEmpty(c *gc.C) {
	id := "f47ac10b-58cc-4372-a567-0e02b2c3d479"
	pp := s.NewPersistence()
	workloads, missing, err := pp.List(id)
	c.Assert(err, jc.ErrorIsNil)

	s.Stub.CheckCallNames(c, "All")
	s.State.CheckNoOps(c)
	c.Check(workloads, gc.HasLen, 0)
	c.Check(missing, jc.DeepEquals, []string{id})
}

func (s *workloadsPersistenceSuite) TestListFailure(c *gc.C) {
	failure := errors.Errorf("<failed!>")
	s.Stub.SetErrors(failure)

	pp := s.NewPersistence()
	_, _, err := pp.List()

	c.Check(errors.Cause(err), gc.Equals, failure)
}

func (s *workloadsPersistenceSuite) TestListAllOkay(c *gc.C) {
	existing := s.NewWorkloads("docker", "workloadA/xyz", "workloadB/abc")
	for i, wl := range existing {
		s.SetDoc(fmt.Sprintf("%d", i), wl)
	}

	pp := s.NewPersistence()
	workloads, err := pp.ListAll()
	c.Assert(err, jc.ErrorIsNil)

	s.Stub.CheckCallNames(c, "All")
	s.State.CheckNoOps(c)
	sort.Sort(byName(workloads))
	sort.Sort(byName(existing))
	c.Check(workloads, jc.DeepEquals, existing)
}

func (s *workloadsPersistenceSuite) TestListAllEmpty(c *gc.C) {
	pp := s.NewPersistence()
	workloads, err := pp.ListAll()
	c.Assert(err, jc.ErrorIsNil)

	s.Stub.CheckCallNames(c, "All")
	s.State.CheckNoOps(c)
	c.Check(workloads, gc.HasLen, 0)
}

type byName []workload.Info

func (b byName) Len() int           { return len(b) }
func (b byName) Less(i, j int) bool { return b[i].Name < b[j].Name }
func (b byName) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }

func (s *workloadsPersistenceSuite) TestListAllFailed(c *gc.C) {
	failure := errors.Errorf("<failed!>")
	s.Stub.SetErrors(failure)

	pp := s.NewPersistence()
	_, err := pp.ListAll()

	c.Check(errors.Cause(err), gc.Equals, failure)
}

func (s *workloadsPersistenceSuite) TestUntrackOkay(c *gc.C) {
	id := "f47ac10b-58cc-4372-a567-0e02b2c3d479"
	wl := s.NewWorkload("docker", "workloadA/xyz")
	s.SetDoc(id, wl)

	pp := s.NewPersistence()
	found, err := pp.Untrack(id)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(found, jc.IsTrue)
	s.Stub.CheckCallNames(c, "Run")
	s.State.CheckOps(c, [][]txn.Op{{
		{
			C:      "workloads",
			Id:     "workload#a-unit/0#f47ac10b-58cc-4372-a567-0e02b2c3d479",
			Assert: txn.DocExists,
			Remove: true,
		},
	}})
}

func (s *workloadsPersistenceSuite) TestUntrackMissing(c *gc.C) {
	id := "f47ac10b-58cc-4372-a567-0e02b2c3d479"
	s.Stub.SetErrors(txn.ErrAborted)

	pp := s.NewPersistence()
	found, err := pp.Untrack(id)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(found, jc.IsFalse)
	s.Stub.CheckCallNames(c, "Run")
	s.State.CheckOps(c, [][]txn.Op{{
		{
			C:      "workloads",
			Id:     "workload#a-unit/0#f47ac10b-58cc-4372-a567-0e02b2c3d479",
			Assert: txn.DocExists,
			Remove: true,
		},
	}})
}

func (s *workloadsPersistenceSuite) TestUntrackFailed(c *gc.C) {
	id := "f47ac10b-58cc-4372-a567-0e02b2c3d479"
	failure := errors.Errorf("<failed!>")
	s.Stub.SetErrors(failure)

	pp := s.NewPersistence()
	_, err := pp.Untrack(id)

	c.Check(errors.Cause(err), gc.Equals, failure)
}
<|endoftext|>