<commit_before>// Copyright 2016 Elliott Polk. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package pgp

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"io/ioutil"

	"golang.org/x/crypto/openpgp"
	"golang.org/x/crypto/openpgp/armor"
)

const PGPMessageType string = "PGP MESSAGE"

//  Encrypt takes a key and text which attempts to encode using the OpenPGP
//  symmetrical encryption.
func Encrypt(key, text []byte) ([]byte, error) {
	buf := new(bytes.Buffer)

	encodeWriter, err := armor.Encode(buf, PGPMessageType, nil)
	if err != nil {
		return nil, err
	}

	ptxtWriter, err := openpgp.SymmetricallyEncrypt(encodeWriter, key, nil, nil)
	if err != nil {
		return nil, err
	}

	fmt.Fprintf(ptxtWriter, string(text))

	ptxtWriter.Close()
	encodeWriter.Close()

	return []byte(base64.StdEncoding.EncodeToString(buf.Bytes())), nil
}

//  Decrypt expects an OpenPGP encoded ciphertext, returning the decrypted results
//  of the cipher text using the provided token.
func Decrypt(token, ciphertxt []byte) ([]byte, error) {
	block, err := armor.Decode(bytes.NewBuffer(ciphertxt))
	if err != nil {
		return nil, err
	}

	readTick := 0
	details, err := openpgp.ReadMessage(block.Body, nil, func(k []openpgp.Key, s bool) ([]byte, error) {
		// 	temporary hack since this will be called several times when a given
		//	token is not valid.
		//	TODO :: review openpgp source for more info
		if readTick > 100 {
			return token, fmt.Errorf("invalid token provided")
		}

		readTick++
		return token, nil
	}, nil)

	if err != nil {
		return nil, err
	}

	plaintxt, err := ioutil.ReadAll(details.UnverifiedBody)
	if err != nil {
		fmt.Println(string(plaintxt))
		return nil, err
	}

	return plaintxt, nil
}
<commit_msg>adds base64 DecodeString for consistency with Encrypt<commit_after>// Copyright 2016 Elliott Polk. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package pgp

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"io/ioutil"

	"golang.org/x/crypto/openpgp"
	"golang.org/x/crypto/openpgp/armor"
)

const PGPMessageType string = "PGP MESSAGE"

//  Encrypt takes a key and text which attempts to encode using the OpenPGP
//  symmetrical encryption.
func Encrypt(key, text []byte) ([]byte, error) {
	buf := new(bytes.Buffer)

	encodeWriter, err := armor.Encode(buf, PGPMessageType, nil)
	if err != nil {
		return nil, err
	}

	ptxtWriter, err := openpgp.SymmetricallyEncrypt(encodeWriter, key, nil, nil)
	if err != nil {
		return nil, err
	}

	fmt.Fprintf(ptxtWriter, string(text))

	ptxtWriter.Close()
	encodeWriter.Close()

	return []byte(base64.StdEncoding.EncodeToString(buf.Bytes())), nil
}

//  Decrypt expects an OpenPGP encoded ciphertext, returning the decrypted results
//  of the cipher text using the provided token.
func Decrypt(token, ciphertxt []byte) ([]byte, error) {
	decoded, err := base64.StdEncoding.DecodeString(string(ciphertxt))
	if err != nil {
		return nil, err
	}

	block, err := armor.Decode(bytes.NewBuffer(decoded))
	if err != nil {
		return nil, err
	}

	readTick := 0
	details, err := openpgp.ReadMessage(block.Body, nil, func(k []openpgp.Key, s bool) ([]byte, error) {
		// 	temporary hack since this will be called several times when a given
		//	token is not valid.
		//	TODO :: review openpgp source for more info
		if readTick > 100 {
			return token, fmt.Errorf("invalid token provided")
		}

		readTick++
		return token, nil
	}, nil)

	if err != nil {
		return nil, err
	}

	plaintxt, err := ioutil.ReadAll(details.UnverifiedBody)
	if err != nil {
		fmt.Println(string(plaintxt))
		return nil, err
	}

	return plaintxt, nil
}
<|endoftext|>


<commit_before>package stitch

import (
	"context"
	"fmt"
	"sort"
	"strings"
	"sync"

	. "github.com/polydawn/go-errcat"

	"go.polydawn.net/go-timeless-api"
	"go.polydawn.net/go-timeless-api/rio"
	"go.polydawn.net/rio/cache"
	"go.polydawn.net/rio/config"
	"go.polydawn.net/rio/fs"
	"go.polydawn.net/rio/fs/osfs"
	"go.polydawn.net/rio/fsOp"
	"go.polydawn.net/rio/stitch/placer"
)

/*
	Struct to gather the args for a single rio.Unpack func call.
	(The context object and monitors are handled in a different band.)

	Note the similar name to a structure in the go-timeless-api packages;
	this one is not serializable, is internal, and
	contains the literal set of warehouses already resolved,
	as well as the path inline rather than in a map key, so we can sort slices.
*/
type UnpackSpec struct {
	Path       fs.AbsolutePath
	WareID     api.WareID
	Filters    api.FilesetFilters
	Warehouses []api.WarehouseAddr
}

// Cast slices to this type to sort by target path (which is effectively mountability order).
type UnpackSpecByPath []UnpackSpec

func (a UnpackSpecByPath) Len() int           { return len(a) }
func (a UnpackSpecByPath) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a UnpackSpecByPath) Less(i, j int) bool { return a[i].Path.String() < a[j].Path.String() }

type unpackResult struct {
	Path  fs.AbsolutePath // cache path or mount source path
	Error error
}

type Assembler struct {
	cache          fs.FS
	unpackTool     rio.UnpackFunc
	placerTool     placer.Placer
	fillerDirProps fs.Metadata
}

func NewAssembler(unpackTool rio.UnpackFunc) (*Assembler, error) {
	placerTool, err := placer.GetMountPlacer()
	if err != nil {
		return nil, err
	}
	return &Assembler{
		cache:      osfs.New(config.GetCacheBasePath()),
		unpackTool: unpackTool,
		placerTool: placerTool,
		fillerDirProps: fs.Metadata{
			Type: fs.Type_Dir, Perms: 0755, Uid: 0, Gid: 0, Mtime: fs.DefaultAtime,
		},
	}, nil
}

func (a *Assembler) Run(ctx context.Context, targetFs fs.FS, parts []UnpackSpec) (func() error, error) {
	sort.Sort(UnpackSpecByPath(parts))

	// Fan out materialization into cache paths.
	unpackResults := make([]unpackResult, len(parts))
	var wg sync.WaitGroup
	wg.Add(len(parts))
	for i, part := range parts {
		go func(i int, part UnpackSpec) {
			defer wg.Done()
			res := &unpackResults[i]
			// If it's a mount, shortcut.
			if part.WareID.Type == "mount" {
				res.Path, res.Error = fs.ParseAbsolutePath(part.WareID.Hash)
				return
			}
			// Unpack with placement=none to populate cache.
			resultWareID, err := a.unpackTool(
				ctx, // TODO fork em out
				part.WareID,
				"-",
				part.Filters,
				rio.Placement_None,
				part.Warehouses,
				rio.Monitor{},
			)
			// Yield the cache path.
			res.Path = config.GetCacheBasePath().Join(cache.ShelfFor(resultWareID))
			res.Error = err
			// TODO if any error, fan out cancellations
		}(i, part)
	}
	wg.Wait()
	// Yield up any errors from individual unpacks.
	for _, result := range unpackResults {
		if result.Error != nil {
			return nil, result.Error
		}
	}

	// Zip up all placements, in order.
	//  Parent dirs are made as necessary along the way.
	hk := &housekeeping{}
	for i, part := range parts {
		path := part.Path.CoerceRelative()

		// Ensure parent dirs.
		for _, parentPath := range path.Dir().Split() {
			target, isSymlink, err := targetFs.Readlink(parentPath)
			if isSymlink {
				return nil, fs.NewBreakoutError(
					targetFs.BasePath(),
					path,
					parentPath,
					target,
				)
			} else if err == nil {
				continue
			} else if Category(err) == fs.ErrNotExists {
				// Make the parent dir if it does not exist.
				a.fillerDirProps.Name = parentPath
				// Could be cleaner: this PlaceFile call rechecks the symlink thing, but it's the shortest call for "make all props right plz".
				if err := fsOp.PlaceFile(targetFs, a.fillerDirProps, nil, false); err != nil {
					return nil, err
				}
			} else {
				// Halt assembly attempt for any unhandlable errors that come up during parent path establishment.
				return nil, err
			}
		}

		// Invoke placer.
		//  Accumulate the individual cleanup funcs into a mega func we'll return.
		//  If errors occur during any placement, fire the cleanups so far before returning.
		var janitor placer.Janitor
		var err error
		switch part.WareID.Type {
		case "mount":
			janitor, err = placer.BindPlacer(unpackResults[i].Path, part.Path, false)
		default:
			janitor, err = a.placerTool(unpackResults[i].Path, part.Path, false)
		}
		if err != nil {
			hk.Teardown()
			return nil, err
		}
		hk.append(janitor)
	}
	return hk.Teardown, nil
}

type housekeeping struct {
	CleanupStack []placer.Janitor
}

func (hk *housekeeping) append(janitor placer.Janitor) {
	hk.CleanupStack = append(hk.CleanupStack, janitor)
}

func (hk housekeeping) Teardown() error {
	progress := make([]string, len(hk.CleanupStack))
	var firstError error
	for i := len(hk.CleanupStack) - 1; i >= 0; i-- {
		janitor := hk.CleanupStack[i]
		if firstError != nil && !janitor.AlwaysTry() {
			progress[i] = "\tskipped: " + janitor.Description()
			continue
		}
		err := hk.CleanupStack[i].Teardown()
		if err != nil {
			if firstError == nil {
				firstError = err
			}
			progress[i] = "\tfailed:  " + janitor.Description()
			continue
		}
		progress[i] = "\tsuccess: " + janitor.Description()
	}
	if firstError != nil {
		// Keep the category of the first one, but also fold in
		//  the string of everything that did or did not get cleaned up.
		cleanupReport := strings.Join(progress, "\n")
		firstError = ErrorDetailed(
			Category(firstError),
			fmt.Sprintf("%s.  The following cleanups were attempted:\n%s", firstError, cleanupReport),
			map[string]string{"cleanupReport": cleanupReport},
		)
	}
	return firstError
}
<commit_msg>stitch: add safety check for inputs under mounts.<commit_after>package stitch

import (
	"context"
	"fmt"
	"sort"
	"strings"
	"sync"

	. "github.com/polydawn/go-errcat"

	"go.polydawn.net/go-timeless-api"
	"go.polydawn.net/go-timeless-api/rio"
	"go.polydawn.net/rio/cache"
	"go.polydawn.net/rio/config"
	"go.polydawn.net/rio/fs"
	"go.polydawn.net/rio/fs/osfs"
	"go.polydawn.net/rio/fsOp"
	"go.polydawn.net/rio/stitch/placer"
)

/*
	Struct to gather the args for a single rio.Unpack func call.
	(The context object and monitors are handled in a different band.)

	Note the similar name to a structure in the go-timeless-api packages;
	this one is not serializable, is internal, and
	contains the literal set of warehouses already resolved,
	as well as the path inline rather than in a map key, so we can sort slices.
*/
type UnpackSpec struct {
	Path       fs.AbsolutePath
	WareID     api.WareID
	Filters    api.FilesetFilters
	Warehouses []api.WarehouseAddr
}

// Cast slices to this type to sort by target path (which is effectively mountability order).
type UnpackSpecByPath []UnpackSpec

func (a UnpackSpecByPath) Len() int           { return len(a) }
func (a UnpackSpecByPath) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a UnpackSpecByPath) Less(i, j int) bool { return a[i].Path.String() < a[j].Path.String() }

type unpackResult struct {
	Path  fs.AbsolutePath // cache path or mount source path
	Error error
}

type Assembler struct {
	cache          fs.FS
	unpackTool     rio.UnpackFunc
	placerTool     placer.Placer
	fillerDirProps fs.Metadata
}

func NewAssembler(unpackTool rio.UnpackFunc) (*Assembler, error) {
	placerTool, err := placer.GetMountPlacer()
	if err != nil {
		return nil, err
	}
	return &Assembler{
		cache:      osfs.New(config.GetCacheBasePath()),
		unpackTool: unpackTool,
		placerTool: placerTool,
		fillerDirProps: fs.Metadata{
			Type: fs.Type_Dir, Perms: 0755, Uid: 0, Gid: 0, Mtime: fs.DefaultAtime,
		},
	}, nil
}

func (a *Assembler) Run(ctx context.Context, targetFs fs.FS, parts []UnpackSpec) (func() error, error) {
	sort.Sort(UnpackSpecByPath(parts))

	// Unpacking either wares or more mounts into paths under mounts is seriously illegal.
	//  It's a massive footgun, entirely strange, and just No.
	//  Doing it into paths under other wares is fine because it's not *leaving* our zone.
	var mounts map[fs.AbsolutePath]struct{}
	for _, part := range parts {
		for mount := range mounts {
			if strings.HasPrefix(part.Path.String(), mount.String()) {
				return nil, Errorf(rio.ErrAssemblyInvalid, "invalid inputs config: "+
					"cannot stitch additional inputs under a mount (%q is under mount at %q)",
					part.Path, mount)
			}
		}
		// If this one is a mount, mark it for the rest.
		//  (Paths under it must come after it, due to the sort.)
		if part.WareID.Type == "mount" {
			mounts[part.Path] = struct{}{}
		}
	}

	// Fan out materialization into cache paths.
	unpackResults := make([]unpackResult, len(parts))
	var wg sync.WaitGroup
	wg.Add(len(parts))
	for i, part := range parts {
		go func(i int, part UnpackSpec) {
			defer wg.Done()
			res := &unpackResults[i]
			// If it's a mount, shortcut.
			if part.WareID.Type == "mount" {
				res.Path, res.Error = fs.ParseAbsolutePath(part.WareID.Hash)
				return
			}
			// Unpack with placement=none to populate cache.
			resultWareID, err := a.unpackTool(
				ctx, // TODO fork em out
				part.WareID,
				"-",
				part.Filters,
				rio.Placement_None,
				part.Warehouses,
				rio.Monitor{},
			)
			// Yield the cache path.
			res.Path = config.GetCacheBasePath().Join(cache.ShelfFor(resultWareID))
			res.Error = err
			// TODO if any error, fan out cancellations
		}(i, part)
	}
	wg.Wait()
	// Yield up any errors from individual unpacks.
	for _, result := range unpackResults {
		if result.Error != nil {
			return nil, result.Error
		}
	}

	// Zip up all placements, in order.
	//  Parent dirs are made as necessary along the way.
	hk := &housekeeping{}
	for i, part := range parts {
		path := part.Path.CoerceRelative()

		// Ensure parent dirs.
		for _, parentPath := range path.Dir().Split() {
			target, isSymlink, err := targetFs.Readlink(parentPath)
			if isSymlink {
				// Future hackers: if you ever try to make this check cleverer,
				//  also make sure you include a check for host mount crossings.
				return nil, fs.NewBreakoutError(
					targetFs.BasePath(),
					path,
					parentPath,
					target,
				)
			} else if err == nil {
				continue
			} else if Category(err) == fs.ErrNotExists {
				// Make the parent dir if it does not exist.
				a.fillerDirProps.Name = parentPath
				// Could be cleaner: this PlaceFile call rechecks the symlink thing, but it's the shortest call for "make all props right plz".
				if err := fsOp.PlaceFile(targetFs, a.fillerDirProps, nil, false); err != nil {
					return nil, err
				}
			} else {
				// Halt assembly attempt for any unhandlable errors that come up during parent path establishment.
				return nil, err
			}
		}

		// Invoke placer.
		//  Accumulate the individual cleanup funcs into a mega func we'll return.
		//  If errors occur during any placement, fire the cleanups so far before returning.
		var janitor placer.Janitor
		var err error
		switch part.WareID.Type {
		case "mount":
			janitor, err = placer.BindPlacer(unpackResults[i].Path, part.Path, false)
		default:
			janitor, err = a.placerTool(unpackResults[i].Path, part.Path, false)
		}
		if err != nil {
			hk.Teardown()
			return nil, err
		}
		hk.append(janitor)
	}
	return hk.Teardown, nil
}

type housekeeping struct {
	CleanupStack []placer.Janitor
}

func (hk *housekeeping) append(janitor placer.Janitor) {
	hk.CleanupStack = append(hk.CleanupStack, janitor)
}

func (hk housekeeping) Teardown() error {
	progress := make([]string, len(hk.CleanupStack))
	var firstError error
	for i := len(hk.CleanupStack) - 1; i >= 0; i-- {
		janitor := hk.CleanupStack[i]
		if firstError != nil && !janitor.AlwaysTry() {
			progress[i] = "\tskipped: " + janitor.Description()
			continue
		}
		err := hk.CleanupStack[i].Teardown()
		if err != nil {
			if firstError == nil {
				firstError = err
			}
			progress[i] = "\tfailed:  " + janitor.Description()
			continue
		}
		progress[i] = "\tsuccess: " + janitor.Description()
	}
	if firstError != nil {
		// Keep the category of the first one, but also fold in
		//  the string of everything that did or did not get cleaned up.
		cleanupReport := strings.Join(progress, "\n")
		firstError = ErrorDetailed(
			Category(firstError),
			fmt.Sprintf("%s.  The following cleanups were attempted:\n%s", firstError, cleanupReport),
			map[string]string{"cleanupReport": cleanupReport},
		)
	}
	return firstError
}
<|endoftext|>


<commit_before>// +build linux

package storage

import (
	"errors"
	"os"
	"syscall"
	"time"

	"github.com/golang/glog"
)

type wal struct {
	fd   int
	mode string
}

func openWriteAheadFile(filename string, mode string, size int) (wal, error) {
	var WAL wal
	WAL.mode = mode
	var err error

	switch mode {
	case "osync":
		WAL.fd, err = syscall.Open(filename, syscall.O_SYNC|os.O_WRONLY|os.O_CREATE, 0666)
	case "dsync":
		WAL.fd, err = syscall.Open(filename, syscall.O_DSYNC|os.O_WRONLY|os.O_CREATE, 0666)
	case "direct":
		WAL.fd, err = syscall.Open(filename, syscall.O_DIRECT|os.O_WRONLY|os.O_CREATE, 0666)
	case "none", "fsync":
		WAL.fd, err = syscall.Open(filename, os.O_WRONLY|os.O_CREATE, 0666)
	default:
		return WAL, errors.New("PersistenceMode not reconised")
	}
	if err != nil {
		return WAL, err
	}
	glog.Info("Opened file: ", filename)
	start, err := syscall.Seek(WAL.fd, 0, 2)
	if err != nil {
		return WAL, err
	}

	glog.Info("Starting file pointer ", start)
	err = syscall.Fallocate(WAL.fd, 0, 0, int64(size))
	if err != nil {
		return WAL, err
	}
	return WAL, nil
}

func (w wal) writeAhead(bytes []byte) error {
	startTime := time.Now()
	// TODO: remove hack
	n, err := syscall.Write(w.fd, bytes[:10])
	if err != nil {
		return err
	}
	if n != 10 {
		return errors.New("Short write")
	}
	delim := []byte("\n")
	n2, err := syscall.Write(w.fd, delim)
	if err != nil {
		return err
	}
	if n2 != len(delim) {
		return errors.New("Short write")
	}
	glog.V(1).Info(n+n2, " bytes written to persistent log in ", time.Since(startTime).String())
	if w.mode == "fsync" || w.mode == "direct" {
		err = syscall.Fdatasync(w.fd)
		if err != nil {
			return err
		}
		glog.V(1).Info(n+n2, " bytes synced to persistent log in ", time.Since(startTime).String())
	}
	slowDiskWarning(startTime, n+n2)
	return nil
}
<commit_msg>removing encoding hack<commit_after>// +build linux

package storage

import (
	"errors"
	"os"
	"syscall"
	"time"

	"github.com/golang/glog"
)

type wal struct {
	fd   int
	mode string
}

func openWriteAheadFile(filename string, mode string, size int) (wal, error) {
	var WAL wal
	WAL.mode = mode
	var err error

	switch mode {
	case "osync":
		WAL.fd, err = syscall.Open(filename, syscall.O_SYNC|os.O_WRONLY|os.O_CREATE, 0666)
	case "dsync":
		WAL.fd, err = syscall.Open(filename, syscall.O_DSYNC|os.O_WRONLY|os.O_CREATE, 0666)
	case "direct":
		WAL.fd, err = syscall.Open(filename, syscall.O_DIRECT|os.O_WRONLY|os.O_CREATE, 0666)
	case "none", "fsync":
		WAL.fd, err = syscall.Open(filename, os.O_WRONLY|os.O_CREATE, 0666)
	default:
		return WAL, errors.New("PersistenceMode not reconised")
	}
	if err != nil {
		return WAL, err
	}
	glog.Info("Opened file: ", filename)
	start, err := syscall.Seek(WAL.fd, 0, 2)
	if err != nil {
		return WAL, err
	}

	glog.Info("Starting file pointer ", start)
	err = syscall.Fallocate(WAL.fd, 0, 0, int64(size))
	if err != nil {
		return WAL, err
	}
	return WAL, nil
}

func (w wal) writeAhead(bytes []byte) error {
	startTime := time.Now()
	// TODO: remove hack
	n, err := syscall.Write(w.fd, bytes)
	if err != nil {
		return err
	}
	if n != len(bytes) {
		return errors.New("Short write")
	}
	delim := []byte("\n")
	n2, err := syscall.Write(w.fd, delim)
	if err != nil {
		return err
	}
	if n2 != len(delim) {
		return errors.New("Short write")
	}
	glog.V(1).Info(n+n2, " bytes written to persistent log in ", time.Since(startTime).String())
	if w.mode == "fsync" || w.mode == "direct" {
		err = syscall.Fdatasync(w.fd)
		if err != nil {
			return err
		}
		glog.V(1).Info(n+n2, " bytes synced to persistent log in ", time.Since(startTime).String())
	}
	slowDiskWarning(startTime, n+n2)
	return nil
}
<|endoftext|>


<commit_before>package traversal

import (
	"context"
	"sort"
	"sync/atomic"

	"github.com/anacrolix/multiless"
	"github.com/anacrolix/sync"

	"github.com/anacrolix/chansync"

	"github.com/anacrolix/dht/v2/int160"
	k_nearest_nodes "github.com/anacrolix/dht/v2/k-nearest-nodes"
	"github.com/anacrolix/dht/v2/krpc"
	"github.com/anacrolix/dht/v2/types"
)

type QueryResult struct {
	ResponseFrom *krpc.NodeInfo
	Nodes        []krpc.NodeInfo
}

type OperationInput struct {
	Target     krpc.ID
	Alpha      int
	K          int
	DoQuery    func(context.Context, krpc.NodeAddr) QueryResult
	NodeFilter func(krpc.NodeInfo) bool
}

type defaultsAppliedOperationInput OperationInput

func Start(input OperationInput) *Operation {
	herp := defaultsAppliedOperationInput(input)
	if herp.Alpha == 0 {
		herp.Alpha = 3
	}
	if herp.NodeFilter == nil {
		herp.NodeFilter = func(krpc.NodeInfo) bool {
			return true
		}
	}
	op := &Operation{
		targetInt160: input.Target.Int160(),
		input:        herp,
		queried:      make(map[addrString]struct{}),
		closest:      k_nearest_nodes.New(input.Target.Int160(), input.K),
	}
	go op.run()
	return op
}

type addrString string

type Operation struct {
	stats        Stats
	mu           sync.Mutex
	unqueried    []types.AddrMaybeId
	queried      map[addrString]struct{}
	closest      k_nearest_nodes.Type
	targetInt160 int160.T
	input        defaultsAppliedOperationInput
	outstanding  int
	cond         chansync.BroadcastCond
	stalled      chansync.LevelTrigger
	stopping     chansync.SetOnce
	stopped      chansync.SetOnce
}

func (op *Operation) Stats() *Stats {
	return &op.stats
}

func (op *Operation) Stop() {
	if op.stopping.Set() {
		go func() {
			defer op.stopped.Set()
			op.mu.Lock()
			defer op.mu.Unlock()
			for {
				if op.outstanding == 0 {
					break
				}
				cond := op.cond.Signaled()
				op.mu.Unlock()
				<-cond
				op.mu.Lock()
			}
		}()
	}
}

func (op *Operation) Stopped() chansync.Done {
	return op.stopped.Done()
}

func (op *Operation) Stalled() chansync.Active {
	return op.stalled.Active()
}

func (op *Operation) AddNodes(nodes []types.AddrMaybeId) {
	op.mu.Lock()
	defer op.mu.Unlock()
	for _, n := range nodes {
		if _, ok := op.queried[addrString(n.Addr.String())]; ok {
			continue
		}
		if ni := n.TryIntoNodeInfo(); ni != nil && !op.input.NodeFilter(*ni) {
			continue
		}
		op.unqueried = append(op.unqueried, n)
	}
	op.cond.Broadcast()
}

func (op *Operation) markQueried(addr krpc.NodeAddr) {
	op.queried[addrString(addr.String())] = struct{}{}
}

func (op *Operation) closestUnqueriedIndex() int {
	closest := 0
	for i, a := range op.unqueried {
		if a.CloserThan(op.unqueried[closest], op.targetInt160) {
			closest = i
		}
	}
	return closest

}

func (op *Operation) closestUnqueried() (ret types.AddrMaybeId) {
	//defer func() {
	//	spew.Dump("closest unqueried", ret)
	//}()
	return op.unqueried[op.closestUnqueriedIndex()]
}

func (op *Operation) popClosestUnqueried() types.AddrMaybeId {
	i := op.closestUnqueriedIndex()
	ret := op.unqueried[i]
	op.unqueried = append(op.unqueried[:i], op.unqueried[i+1:]...)
	return ret
}

func (op *Operation) haveQuery() bool {
	if len(op.unqueried) == 0 {
		return false
	}
	if !op.closest.Full() {
		return true
	}
	cu := op.closestUnqueried()
	if cu.Id == nil {
		return false
	}
	return cu.Id.Distance(op.targetInt160).Cmp(op.closest.Farthest().ID.Int160().Distance(op.targetInt160)) < 0
}

func (op *Operation) run() {
	op.mu.Lock()
	defer op.mu.Unlock()
	for {
		if op.stopping.IsSet() {
			return
		}
		for op.outstanding < op.input.Alpha && op.haveQuery() {
			op.startQuery()
		}
		var stalled chansync.Signal
		if (!op.haveQuery() || op.input.Alpha == 0) && op.outstanding == 0 {
			stalled = op.stalled.Signal()
		}
		queryCondSignaled := op.cond.Signaled()
		op.mu.Unlock()
		select {
		case stalled <- struct{}{}:
		case <-op.stopping.Done():
		case <-queryCondSignaled:
		}
		op.mu.Lock()
	}
}

func (op *Operation) sortNodesByClosest(nodes []krpc.NodeInfo) {
	sort.Slice(nodes, func(i, j int) bool {
		return multiless.New().Cmp(
			int160.Distance(int160.FromByteArray(nodes[i].ID), op.targetInt160).Cmp(
				int160.Distance(int160.FromByteArray(nodes[j].ID), op.targetInt160)),
		).Less()
	})
}

func (op *Operation) addClosest(node krpc.NodeInfo) {
	if !op.input.NodeFilter(node) {
		return
	}
	op.closest = op.closest.Push(k_nearest_nodes.Elem{
		Key:  node,
		Data: nil,
	})
}

func (op *Operation) startQuery() {
	a := op.popClosestUnqueried()
	op.markQueried(a.Addr)
	op.outstanding++
	go func() {
		defer func() {
			op.mu.Lock()
			defer op.mu.Unlock()
			op.outstanding--
			op.cond.Broadcast()
		}()
		//log.Printf("traversal querying %v", a)
		atomic.AddInt64(&op.stats.NumAddrsTried, 1)
		res := op.input.DoQuery(context.TODO(), a.Addr)
		if res.ResponseFrom != nil {
			func() {
				op.mu.Lock()
				defer op.mu.Unlock()
				atomic.AddInt64(&op.stats.NumResponses, 1)
				op.addClosest(*res.ResponseFrom)
			}()
		}
		op.AddNodes(func() (ret []types.AddrMaybeId) {
			for _, ni := range res.Nodes {
				id := int160.FromByteArray(ni.ID)
				ret = append(ret, types.AddrMaybeId{
					Addr: ni.Addr,
					Id:   &id,
				})
			}
			return
		}())
	}()
}
<commit_msg>Use a better container for unqueried nodes in the external traversal implementation<commit_after>package traversal

import (
	"context"
	"sync/atomic"

	"github.com/anacrolix/dht/v2/containers"
	"github.com/anacrolix/missinggo/v2/iter"
	"github.com/anacrolix/stm/stmutil"
	"github.com/anacrolix/sync"

	"github.com/anacrolix/chansync"

	"github.com/anacrolix/dht/v2/int160"
	k_nearest_nodes "github.com/anacrolix/dht/v2/k-nearest-nodes"
	"github.com/anacrolix/dht/v2/krpc"
	"github.com/anacrolix/dht/v2/types"
)

type QueryResult struct {
	ResponseFrom *krpc.NodeInfo
	Nodes        []krpc.NodeInfo
}

type OperationInput struct {
	Target     krpc.ID
	Alpha      int
	K          int
	DoQuery    func(context.Context, krpc.NodeAddr) QueryResult
	NodeFilter func(krpc.NodeInfo) bool
}

type defaultsAppliedOperationInput OperationInput

func Start(input OperationInput) *Operation {
	herp := defaultsAppliedOperationInput(input)
	if herp.Alpha == 0 {
		herp.Alpha = 3
	}
	if herp.NodeFilter == nil {
		herp.NodeFilter = func(krpc.NodeInfo) bool {
			return true
		}
	}
	targetInt160 := input.Target.Int160()
	op := &Operation{
		targetInt160: targetInt160,
		input:        herp,
		queried:      make(map[addrString]struct{}),
		closest:      k_nearest_nodes.New(targetInt160, input.K),
		unqueried:    containers.NewAddrMaybeIdsByDistance(targetInt160),
	}
	go op.run()
	return op
}

type addrString string

type Operation struct {
	stats        Stats
	mu           sync.Mutex
	unqueried    stmutil.Settish
	queried      map[addrString]struct{}
	closest      k_nearest_nodes.Type
	targetInt160 int160.T
	input        defaultsAppliedOperationInput
	outstanding  int
	cond         chansync.BroadcastCond
	stalled      chansync.LevelTrigger
	stopping     chansync.SetOnce
	stopped      chansync.SetOnce
}

func (op *Operation) Stats() *Stats {
	return &op.stats
}

func (op *Operation) Stop() {
	if op.stopping.Set() {
		go func() {
			defer op.stopped.Set()
			op.mu.Lock()
			defer op.mu.Unlock()
			for {
				if op.outstanding == 0 {
					break
				}
				cond := op.cond.Signaled()
				op.mu.Unlock()
				<-cond
				op.mu.Lock()
			}
		}()
	}
}

func (op *Operation) Stopped() chansync.Done {
	return op.stopped.Done()
}

func (op *Operation) Stalled() chansync.Active {
	return op.stalled.Active()
}

func (op *Operation) AddNodes(nodes []types.AddrMaybeId) {
	op.mu.Lock()
	defer op.mu.Unlock()
	for _, n := range nodes {
		if _, ok := op.queried[addrString(n.Addr.String())]; ok {
			continue
		}
		if ni := n.TryIntoNodeInfo(); ni != nil && !op.input.NodeFilter(*ni) {
			continue
		}
		op.unqueried = op.unqueried.Add(n)
	}
	op.cond.Broadcast()
}

func (op *Operation) markQueried(addr krpc.NodeAddr) {
	op.queried[addrString(addr.String())] = struct{}{}
}

func (op *Operation) closestUnqueried() (ret types.AddrMaybeId) {
	//defer func() {
	//	spew.Dump("closest unqueried", ret)
	//}()
	_ret, _ := iter.First(op.unqueried.Iter)
	return _ret.(types.AddrMaybeId)
}

func (op *Operation) popClosestUnqueried() types.AddrMaybeId {
	ret := op.closestUnqueried()
	op.unqueried = op.unqueried.Delete(ret)
	return ret
}

func (op *Operation) haveQuery() bool {
	if op.unqueried.Len() == 0 {
		return false
	}
	if !op.closest.Full() {
		return true
	}
	cu := op.closestUnqueried()
	if cu.Id == nil {
		return false
	}
	return cu.Id.Distance(op.targetInt160).Cmp(op.closest.Farthest().ID.Int160().Distance(op.targetInt160)) < 0
}

func (op *Operation) run() {
	op.mu.Lock()
	defer op.mu.Unlock()
	for {
		if op.stopping.IsSet() {
			return
		}
		for op.outstanding < op.input.Alpha && op.haveQuery() {
			op.startQuery()
		}
		var stalled chansync.Signal
		if (!op.haveQuery() || op.input.Alpha == 0) && op.outstanding == 0 {
			stalled = op.stalled.Signal()
		}
		queryCondSignaled := op.cond.Signaled()
		op.mu.Unlock()
		select {
		case stalled <- struct{}{}:
		case <-op.stopping.Done():
		case <-queryCondSignaled:
		}
		op.mu.Lock()
	}
}

func (op *Operation) addClosest(node krpc.NodeInfo) {
	if !op.input.NodeFilter(node) {
		return
	}
	op.closest = op.closest.Push(k_nearest_nodes.Elem{
		Key:  node,
		Data: nil,
	})
}

func (op *Operation) startQuery() {
	a := op.popClosestUnqueried()
	op.markQueried(a.Addr)
	op.outstanding++
	go func() {
		defer func() {
			op.mu.Lock()
			defer op.mu.Unlock()
			op.outstanding--
			op.cond.Broadcast()
		}()
		//log.Printf("traversal querying %v", a)
		atomic.AddInt64(&op.stats.NumAddrsTried, 1)
		res := op.input.DoQuery(context.TODO(), a.Addr)
		if res.ResponseFrom != nil {
			func() {
				op.mu.Lock()
				defer op.mu.Unlock()
				atomic.AddInt64(&op.stats.NumResponses, 1)
				op.addClosest(*res.ResponseFrom)
			}()
		}
		op.AddNodes(func() (ret []types.AddrMaybeId) {
			for _, ni := range res.Nodes {
				id := int160.FromByteArray(ni.ID)
				ret = append(ret, types.AddrMaybeId{
					Addr: ni.Addr,
					Id:   &id,
				})
			}
			return
		}())
	}()
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/elasticache"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
	"github.com/hashicorp/terraform/helper/acctest"
)

func TestAccAWSEcacheReplicationGroup(t *testing.T) {
	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSEcacheReplicationGroupDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccAWSEcacheReplicationGroupConfig,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSEcacheReplicationGroupExists("aws_elasticache_replication_group.bar"),
				),
			},
		},
	})
}

func testAccCheckAWSEcacheReplicationGroupDestroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).elasticacheconn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_elasticache_replication_group" {
			continue
		}
		res, err := conn.DescribeReplicationGroups(&elasticache.DescribeReplicationGroupsInput{
			ReplicationGroupId: aws.String(rs.Primary.ID),
		})
		if err != nil {
			return err
		}
		if len(res.ReplicationGroups) > 0 {
			return fmt.Errorf("still exist.")
		}
	}
	return nil
}

func testAccCheckAWSEcacheReplicationGroupExists(n string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		conn := testAccProvider.Meta().(*AWSClient).elasticacheconn
		res, err := conn.DescribeReplicationGroups(&elasticache.DescribeReplicationGroupsInput{
			ReplicationGroupId: aws.String(rs.Primary.ID),
		})

		if err != nil {
			return fmt.Errorf("CacheReplicationGroup error: %v", err)
		}

		if len(res.ReplicationGroups) != 1 ||
			*res.ReplicationGroups[0].ReplicationGroupId != rs.Primary.ID {
			return fmt.Errorf("Replication group not found")
		}
		log.Printf("[DEBUG] Rep group found")
		return nil
	}
}

var testAccAWSEcacheReplicationGroupConfig = fmt.Sprintf(`
resource "aws_elasticache_replication_group" "bar" {
    replication_group_id = "tf-repgrp-%03d"
    cache_node_type = "cache.m1.small"
    num_cache_clusters = 2
    description = "tf-test-replication-group-descr"
}
`, acctest.RandInt())
<commit_msg>Trying to get gofrmt to work...<commit_after>package aws

import (
	"fmt"
	"log"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/elasticache"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
	"github.com/hashicorp/terraform/helper/acctest"

)

func TestAccAWSEcacheReplicationGroup(t *testing.T) {
	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSEcacheReplicationGroupDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccAWSEcacheReplicationGroupConfig,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSEcacheReplicationGroupExists("aws_elasticache_replication_group.bar"),
				),
			},
		},
	})
}

func testAccCheckAWSEcacheReplicationGroupDestroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).elasticacheconn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_elasticache_replication_group" {
			continue
		}
		res, err := conn.DescribeReplicationGroups(&elasticache.DescribeReplicationGroupsInput{
			ReplicationGroupId: aws.String(rs.Primary.ID),
		})
		if err != nil {
			return err
		}
		if len(res.ReplicationGroups) > 0 {
			return fmt.Errorf("still exist.")
		}
	}
	return nil
}

func testAccCheckAWSEcacheReplicationGroupExists(n string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("Not found: %s", n)
		}

		conn := testAccProvider.Meta().(*AWSClient).elasticacheconn
		res, err := conn.DescribeReplicationGroups(&elasticache.DescribeReplicationGroupsInput{
			ReplicationGroupId: aws.String(rs.Primary.ID),
		})

		if err != nil {
			return fmt.Errorf("CacheReplicationGroup error: %v", err)
		}

		if len(res.ReplicationGroups) != 1 ||
			*res.ReplicationGroups[0].ReplicationGroupId != rs.Primary.ID {
			return fmt.Errorf("Replication group not found")
		}
		log.Printf("[DEBUG] Rep group found")
		return nil
	}
}

var testAccAWSEcacheReplicationGroupConfig = fmt.Sprintf(`
resource "aws_elasticache_replication_group" "bar" {
    replication_group_id = "tf-repgrp-%03d"
    cache_node_type = "cache.m1.small"
    num_cache_clusters = 2
    description = "tf-test-replication-group-descr"
}
`, acctest.RandInt())
<|endoftext|>


<commit_before>//  Copyright (c) 2015 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package firestorm

import (
	"bytes"
	"math/rand"
	"sort"
	"sync"

	"github.com/steveyen/gtreap"
	"github.com/willf/bitset"
)

type Compensator struct {
	inFlightMutex     sync.RWMutex
	maxRead           uint64
	inFlight          *gtreap.Treap
	deletedMutex      sync.RWMutex
	deletedDocNumbers *bitset.BitSet
}

func NewCompensator() *Compensator {
	rv := Compensator{
		inFlight:          gtreap.NewTreap(inFlightItemCompare),
		deletedDocNumbers: bitset.New(1000000),
	}
	return &rv
}

type Snapshot struct {
	maxRead           uint64
	inFlight          *gtreap.Treap
	deletedDocNumbers *bitset.BitSet
}

// returns which doc number is valid
// if none, then 0
func (s *Snapshot) Which(docID []byte, docNumList DocNumberList) uint64 {
	sort.Sort(docNumList)
	for _, docNum := range docNumList { // docNumList is sorted descending.
		if docNum > 0 && s.Valid(docID, docNum) {
			return docNum
		}
	}
	return 0
}

func (s *Snapshot) Valid(docID []byte, docNum uint64) bool {
	logger.Printf("checking validity of: '%s' - % x - %d", docID, docID, docNum)
	if docNum > s.maxRead {
		return false
	}
	logger.Printf("<= maxRead")
	inFlightVal := s.inFlight.Get(&InFlightItem{docID: docID})
	if inFlightVal != nil && inFlightVal.(*InFlightItem).docNum != docNum {
		return false
	}
	logger.Printf("not in flight")
	if s.deletedDocNumbers.Test(uint(docNum)) {
		return false
	}
	logger.Printf("not deleted")
	return true
}

func (c *Compensator) Mutate(docID []byte, docNum uint64) {
	c.inFlightMutex.Lock()
	defer c.inFlightMutex.Unlock()
	c.inFlight = c.inFlight.Upsert(&InFlightItem{docID: docID, docNum: docNum}, rand.Int())
	if docNum != 0 {
		c.maxRead = docNum
	}
}

func (c *Compensator) MutateBatch(inflightItems []*InFlightItem, lastDocNum uint64) {
	c.inFlightMutex.Lock()
	defer c.inFlightMutex.Unlock()
	for _, item := range inflightItems {
		c.inFlight = c.inFlight.Upsert(item, rand.Int())
	}
	c.maxRead = lastDocNum
}

func (c *Compensator) Migrate(docID []byte, docNum uint64, oldDocNums []uint64) {
	c.inFlightMutex.Lock()
	defer c.inFlightMutex.Unlock()
	c.deletedMutex.Lock()
	defer c.deletedMutex.Unlock()

	// clone deleted doc numbers and mutate
	if len(oldDocNums) > 0 {
		newDeletedDocNumbers := c.deletedDocNumbers.Clone()
		for _, oldDocNum := range oldDocNums {
			newDeletedDocNumbers.Set(uint(oldDocNum))
		}
		// update pointer
		c.deletedDocNumbers = newDeletedDocNumbers
	}

	// remove entry from in-flight if it still has same doc num
	val := c.inFlight.Get(&InFlightItem{docID: docID})
	if val != nil && val.(*InFlightItem).docNum == docNum {
		c.inFlight = c.inFlight.Delete(&InFlightItem{docID: docID})
	}
}

func (c *Compensator) GarbageCollect(docNums []uint64) {
	c.deletedMutex.Lock()
	defer c.deletedMutex.Unlock()

	for _, docNum := range docNums {
		c.deletedDocNumbers.Clear(uint(docNum))
	}
}

func (c *Compensator) Snapshot() *Snapshot {
	c.inFlightMutex.RLock()
	defer c.inFlightMutex.RUnlock()
	c.deletedMutex.RLock()
	defer c.deletedMutex.RUnlock()

	rv := Snapshot{
		maxRead:           c.maxRead,
		inFlight:          c.inFlight,
		deletedDocNumbers: c.deletedDocNumbers,
	}
	return &rv
}

func (c *Compensator) GarbageCount() uint64 {
	return uint64(c.deletedDocNumbers.Count())
}

//**************

type InFlightItem struct {
	docID  []byte
	docNum uint64
}

func inFlightItemCompare(a, b interface{}) int {
	return bytes.Compare(a.(*InFlightItem).docID, b.(*InFlightItem).docID)
}
<commit_msg>firestorm gtreap lookup once per snapshot docID<commit_after>//  Copyright (c) 2015 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package firestorm

import (
	"bytes"
	"math/rand"
	"sort"
	"sync"

	"github.com/steveyen/gtreap"
	"github.com/willf/bitset"
)

type Compensator struct {
	inFlightMutex     sync.RWMutex
	maxRead           uint64
	inFlight          *gtreap.Treap
	deletedMutex      sync.RWMutex
	deletedDocNumbers *bitset.BitSet
}

func NewCompensator() *Compensator {
	rv := Compensator{
		inFlight:          gtreap.NewTreap(inFlightItemCompare),
		deletedDocNumbers: bitset.New(1000000),
	}
	return &rv
}

type Snapshot struct {
	maxRead           uint64
	inFlight          *gtreap.Treap
	deletedDocNumbers *bitset.BitSet
}

// returns which doc number is valid
// if none, then 0
func (s *Snapshot) Which(docID []byte, docNumList DocNumberList) uint64 {
	inFlightVal := s.inFlight.Get(&InFlightItem{docID: docID})

	sort.Sort(docNumList) // Descending ordering.

	for _, docNum := range docNumList {
		if docNum > 0 && docNum <= s.maxRead &&
			(inFlightVal == nil || inFlightVal.(*InFlightItem).docNum == docNum) &&
			!s.deletedDocNumbers.Test(uint(docNum)) {
			return docNum
		}
	}
	return 0
}

func (s *Snapshot) Valid(docID []byte, docNum uint64) bool {
	logger.Printf("checking validity of: '%s' - % x - %d", docID, docID, docNum)
	if docNum > s.maxRead {
		return false
	}
	logger.Printf("<= maxRead")
	inFlightVal := s.inFlight.Get(&InFlightItem{docID: docID})
	if inFlightVal != nil && inFlightVal.(*InFlightItem).docNum != docNum {
		return false
	}
	logger.Printf("not in flight")
	if s.deletedDocNumbers.Test(uint(docNum)) {
		return false
	}
	logger.Printf("not deleted")
	return true
}

func (c *Compensator) Mutate(docID []byte, docNum uint64) {
	c.inFlightMutex.Lock()
	defer c.inFlightMutex.Unlock()
	c.inFlight = c.inFlight.Upsert(&InFlightItem{docID: docID, docNum: docNum}, rand.Int())
	if docNum != 0 {
		c.maxRead = docNum
	}
}

func (c *Compensator) MutateBatch(inflightItems []*InFlightItem, lastDocNum uint64) {
	c.inFlightMutex.Lock()
	defer c.inFlightMutex.Unlock()
	for _, item := range inflightItems {
		c.inFlight = c.inFlight.Upsert(item, rand.Int())
	}
	c.maxRead = lastDocNum
}

func (c *Compensator) Migrate(docID []byte, docNum uint64, oldDocNums []uint64) {
	c.inFlightMutex.Lock()
	defer c.inFlightMutex.Unlock()
	c.deletedMutex.Lock()
	defer c.deletedMutex.Unlock()

	// clone deleted doc numbers and mutate
	if len(oldDocNums) > 0 {
		newDeletedDocNumbers := c.deletedDocNumbers.Clone()
		for _, oldDocNum := range oldDocNums {
			newDeletedDocNumbers.Set(uint(oldDocNum))
		}
		// update pointer
		c.deletedDocNumbers = newDeletedDocNumbers
	}

	// remove entry from in-flight if it still has same doc num
	val := c.inFlight.Get(&InFlightItem{docID: docID})
	if val != nil && val.(*InFlightItem).docNum == docNum {
		c.inFlight = c.inFlight.Delete(&InFlightItem{docID: docID})
	}
}

func (c *Compensator) GarbageCollect(docNums []uint64) {
	c.deletedMutex.Lock()
	defer c.deletedMutex.Unlock()

	for _, docNum := range docNums {
		c.deletedDocNumbers.Clear(uint(docNum))
	}
}

func (c *Compensator) Snapshot() *Snapshot {
	c.inFlightMutex.RLock()
	defer c.inFlightMutex.RUnlock()
	c.deletedMutex.RLock()
	defer c.deletedMutex.RUnlock()

	rv := Snapshot{
		maxRead:           c.maxRead,
		inFlight:          c.inFlight,
		deletedDocNumbers: c.deletedDocNumbers,
	}
	return &rv
}

func (c *Compensator) GarbageCount() uint64 {
	return uint64(c.deletedDocNumbers.Count())
}

//**************

type InFlightItem struct {
	docID  []byte
	docNum uint64
}

func inFlightItemCompare(a, b interface{}) int {
	return bytes.Compare(a.(*InFlightItem).docID, b.(*InFlightItem).docID)
}
<|endoftext|>


<commit_before>package via

import (
	"compress/gzip"
	"fmt"
	"github.com/mrosset/gurl"
	"github.com/mrosset/util/console"
	"github.com/mrosset/util/file"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"sort"
)

var (
	client  = new(http.Client)
	verbose = false
	elog    = log.New(os.Stderr, "", log.Lshortfile)
	lfmt    = "%-20.20s %v\n"
	debug   = false
	expand  = os.ExpandEnv
	update  = false
	deps    = false
)

func Root(path string) {
	config.Root = path
}

func Verbose(b bool) {
	verbose = b
}

func Deps(b bool) {
	deps = b
}

func Update(b bool) {
	update = b
}

func Debug(b bool) {
	debug = b
}

func DownloadSrc(config *Config, plan *Plan) (err error) {
	if file.Exists(plan.SourcePath()) && !update {
		return nil
	}
	fmt.Printf(lfmt, "download", plan.NameVersion())
	eurl := plan.Expand().Url
	u, err := url.Parse(eurl)
	if err != nil {
		return err
	}
	switch u.Scheme {
	case "ftp":
		wget(cache.Sources(), eurl)
	case "http", "https":
		return gurl.Download(cache.Sources(), eurl)
	case "git":
		spath := filepath.Join(cache.Sources(), plan.Name)
		if err := Clone(spath, "https"+eurl[3:]); err != nil {
			elog.Println(err)
			return err
		}
	default:
		return fmt.Errorf("%s: URL scheme is not supported", u.Scheme)
	}
	return nil
}

// Stages the downloaded source in via's cache directory
// the stage only happens once unless BuilInStage is used
func Stage(config *Config, plan *Plan) (err error) {
	if plan.Url == "" || file.Exists(plan.GetStageDir()) {
		// nothing to stage
		return nil
	}
	fmt.Printf(lfmt, "stage", plan.NameVersion())
	u, err := url.Parse(plan.Expand().Url)
	if err != nil {
		elog.Println(err)
		return err
	}
	//FIXME: move this down to switch statement so avoid goto
	if u.Scheme == "git" {
		fmt.Println(plan.SourcePath())
		fmt.Println(plan.GetStageDir())
		if err := Clone(plan.GetStageDir(), plan.SourcePath()); err != nil {
			return err
		}
		goto patch
	}
	switch filepath.Ext(plan.SourceFile()) {
	case ".zip":
		unzip(cache.Stages(), plan.SourcePath())
	default:
		GNUUntar(cache.Stages(), plan.SourcePath())
	}
patch:
	fmt.Printf(lfmt, "patch", plan.NameVersion())
	if err := doCommands(config, join(cache.Stages(), plan.stageDir()), plan.Patch); err != nil {
		return err
	}
	return
}

// Calls each shell command in the plans Build field.
func Build(config *Config, plan *Plan) (err error) {
	var (
		build = plan.Build
	)
	for _, p := range plan.BuildDepends {
		if IsInstalled(config, p) {
			continue
		}
		dp, err := NewPlan(config, p)
		if err != nil {
			return err
		}
		if err := NewInstaller(config, dp).Install(); err != nil {
			return err
		}
	}
	// FIXME: flags should not be merged should have a ConfigFLags
	// and PlanFlags environment variable
	flags := append(config.Flags, plan.Flags...)
	os.MkdirAll(plan.BuildDir(), 0755)
	// Parent plan Build is run first this plans is added at the end.
	if plan.Inherit != "" {
		parent, _ := NewPlan(config, plan.Inherit)
		build = append(parent.Build, plan.Build...)
		flags = append(flags, parent.Flags...)
	}
	// FIXME: this should be set within exec.Cmd
	os.Setenv("SRCDIR", plan.GetStageDir())
	os.Setenv("Flags", expand(flags.String()))
	err = doCommands(config, plan.BuildDir(), build)
	if err != nil {
		return fmt.Errorf("%s in %s", err.Error(), plan.BuildDir())
	}
	return nil
}

func doCommands(config *Config, dir string, cmds []string) (err error) {
	for _, j := range cmds {
		cmd := &exec.Cmd{
			Path:   "/bin/bash",
			Args:   []string{"bash", "-c", j},
			Stdin:  os.Stdin,
			Stderr: os.Stderr,
			Dir:    dir,
			Env:    config.Getenv(),
		}
		if verbose {
			cmd.Stdout = os.Stdout
		}
		if debug {
			fmt.Println(config.Getenv())
			fmt.Println(cmd.Args)
		}
		err = cmd.Run()
		if err != nil {
			elog.Printf("%s: %s\n", j, err)
			return err
		}
	}
	return nil
}

func Package(config *Config, bdir string, plan *Plan) (err error) {
	var (
		pack = plan.Package
	)
	pdir := join(cache.Packages(), plan.NameVersion())
	if bdir == "" {
		bdir = join(cache.Builds(), plan.NameVersion())
	}
	if plan.BuildInStage {
		bdir = join(cache.Stages(), plan.stageDir())
	}
	if file.Exists(pdir) {
		err := os.RemoveAll(pdir)
		if err != nil {
			return err
		}
	}
	err = os.Mkdir(pdir, 0755)
	if err != nil {
		elog.Println(err)
		return err
	}
	os.Setenv("PKGDIR", pdir)
	if plan.Inherit != "" {
		parent, _ := NewPlan(config, plan.Inherit)
		pack = append(parent.Package, plan.Package...)
	}
	err = doCommands(config, bdir, pack)
	if err != nil {
		return err
	}
	for _, j := range plan.SubPackages {
		sub, err := NewPlan(config, j)
		if err != nil {
			return err
		}
		if err = Package(config, bdir, sub); err != nil {
			return err
		}
	}
	err = CreatePackage(config, plan)
	if err != nil {
		elog.Println(err)
		return (err)
	}
	plan.Cid, err = IpfsAdd(config, Path(plan.PackagePath()))
	if err != nil {
		return err
	}
	plan.IsRebuilt = true
	return plan.Save()
	/*
		err = CreatePackage(plan)
		if err != nil {
			return err
		}
		return Sign(plan)
	*/
}

func CreatePackage(config *Config, plan *Plan) (err error) {
	pfile := plan.PackagePath()
	os.MkdirAll(filepath.Dir(pfile), 0755)
	fd, err := os.Create(pfile)
	if err != nil {
		elog.Println(err)
		return err
	}
	defer fd.Close()
	gz := gzip.NewWriter(fd)
	defer gz.Close()
	return Tarball(gz, plan)
}

// Updates each plans Oid to the Oid of the tarball in publish git repo
// this function should never be used in production. It's used for making sure
// the plans Oid match the git repo's Oid
func SyncHashs(config *Config) {
	plans, _ := GetPlans()
	for _, p := range plans {
		if file.Exists(p.PackagePath()) {
			p.Cid, _ = HashOnly(config, Path(p.PackagePath()))
			p.Save()
			log.Println(p.Cid, p.Name)
		}
	}
}

func PostInstall(config *Config, plan *Plan) (err error) {
	return doCommands(config, "/", append(plan.PostInstall, config.PostInstall...))
}

func Remove(config *Config, name string) (err error) {
	if !IsInstalled(config, name) {
		err = fmt.Errorf("%s is not installed.", name)
		elog.Println(err)
		return err
	}

	man, err := ReadManifest(config, name)
	if err != nil {
		elog.Println(err)
		return err
	}
	for _, f := range man.Files {
		fpath := join(config.Root, f)
		if err := os.Remove(fpath); err != nil {
			elog.Println(err)
		}
	}

	return os.RemoveAll(join(config.DB.Installed(config), name))
}

func BuildDeps(config *Config, plan *Plan) (err error) {
	for _, d := range plan.Depends() {
		if IsInstalled(config, d) {
			continue
		}
		p, _ := NewPlan(config, d)
		if file.Exists(p.PackagePath()) {
			if err := NewInstaller(config, plan).Install(); err != nil {
				return err
			}
			continue
		}
		fmt.Println("building", d, "for", plan.NameVersion())
		err := BuildDeps(config, p)
		if err != nil {
			elog.Println(err)
			return err
		}
	}
	err = BuildSteps(config, plan)
	if err != nil {
		return err
	}
	return NewInstaller(config, plan).Install()
}

// Run all of the functions required to build a package
func BuildSteps(config *Config, plan *Plan) (err error) {
	if file.Exists(plan.PackageFile()) {
		return fmt.Errorf("package %s exists", plan.PackageFile())
	}
	if err := DownloadSrc(config, plan); err != nil {
		elog.Println(err)
		return err
	}
	if err := Stage(config, plan); err != nil {
		elog.Println(err)
		return err
	}
	fmt.Printf(lfmt, "build", plan.NameVersion())
	if err := Build(config, plan); err != nil {
		elog.Println(err)
		return err
	}
	fmt.Printf(lfmt, "package", plan.NameVersion())
	if err := Package(config, "", plan); err != nil {
		elog.Println(err)
		return err
	}
	return RepoCreate(config)
}

var (
	rexName   = regexp.MustCompile("[A-Za-z]+")
	rexTruple = regexp.MustCompile("[0-9]+.[0-9]+.[0-9]+")
	rexDouble = regexp.MustCompile("[0-9]+.[0-9]+")
)

// Creates a new plan from a given Url
func Create(url, group string) (err error) {
	var (
		xfile   = filepath.Base(url)
		name    = rexName.FindString(xfile)
		triple  = rexTruple.FindString(xfile)
		double  = rexDouble.FindString(xfile)
		version string
	)
	switch {
	case triple != "":
		version = triple
	case double != "":
		version = double
	default:
		return fmt.Errorf("regex fail for %s", xfile)
	}
	plan := &Plan{Name: name, Version: version, Url: url, Group: group}
	plan.Inherit = "gnu"
	if file.Exists(plan.Path()) {
		return fmt.Errorf("%s already exists", plan.Path())
	}
	return plan.Save()
}

func IsInstalled(config *Config, name string) bool {
	return file.Exists(join(config.DB.Installed(config), name))
}

func refactor(plan *Plan) {
	if len(plan.SubPackages) > 0 {
		for _, j := range plan.SubPackages {
			s, _ := NewPlan(config, j)
			if s.Version == plan.Version {
				continue
			}
			s.Version = plan.Version
			s.Save()
		}
	}
}

func Lint() (err error) {
	e, err := PlanFiles()
	if err != nil {
		return err
	}
	for _, j := range e {
		plan, err := ReadPath(config, j)
		if err != nil {
			err = fmt.Errorf("%s %s", j, err)
			elog.Println(err)
			return err
		}
		// If Group is empty, we can set it
		if plan.Group == "" {
			plan.Group = baseDir(j)
		}
		if verbose {
			console.Println("lint", plan.Name, plan.Version, plan.IsRebuilt)
		}
		sort.Strings(plan.SubPackages)
		sort.Strings(plan.Flags)
		sort.Strings(plan.Remove)
		sort.Strings(plan.AutoDepends)
		sort.Strings(plan.ManualDepends)
		sort.Strings(plan.BuildDepends)
		refactor(plan)
		err = plan.Save()
		if err != nil {
			elog.Println(err)
			return err
		}
	}
	console.Flush()
	return nil
}

func fatal(err error) {
	if err != nil {
		log.Fatal(err)
	}
}
func Clean(name string) error {
	plan, err := NewPlan(config, name)
	if err != nil {
		return err
	}
	fmt.Printf(lfmt, "clean", plan.NameVersion())
	dir := join(cache.Builds(), plan.NameVersion())
	if err = os.RemoveAll(dir); err != nil {
		return err
	}

	if plan.BuildInStage {
		dir = join(cache.Stages(), plan.stageDir())
		return os.RemoveAll(dir)
	}
	return nil
}

func PlanFiles() ([]string, error) {
	return filepath.Glob(join(config.Plans, "*", "*.json"))
}

func conflicts(config *Config, man *Plan) (errs []error) {
	for _, f := range man.Files {
		fpath := join(config.Root, f)
		if file.Exists(fpath) {
			errs = append(errs, fmt.Errorf("%s already exists.", f))
		}
	}
	return errs
}

func GetConfig() *Config {
	return config
}
<commit_msg>reset Cid before building, remove built tarball once it's in ipfs<commit_after>package via

import (
	"compress/gzip"
	"fmt"
	"github.com/mrosset/gurl"
	"github.com/mrosset/util/console"
	"github.com/mrosset/util/file"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"sort"
)

var (
	client  = new(http.Client)
	verbose = false
	elog    = log.New(os.Stderr, "", log.Lshortfile)
	lfmt    = "%-20.20s %v\n"
	debug   = false
	expand  = os.ExpandEnv
	update  = false
	deps    = false
)

func Root(path string) {
	config.Root = path
}

func Verbose(b bool) {
	verbose = b
}

func Deps(b bool) {
	deps = b
}

func Update(b bool) {
	update = b
}

func Debug(b bool) {
	debug = b
}

func DownloadSrc(config *Config, plan *Plan) (err error) {
	if file.Exists(plan.SourcePath()) && !update {
		return nil
	}
	fmt.Printf(lfmt, "download", plan.NameVersion())
	eurl := plan.Expand().Url
	u, err := url.Parse(eurl)
	if err != nil {
		return err
	}
	switch u.Scheme {
	case "ftp":
		wget(cache.Sources(), eurl)
	case "http", "https":
		return gurl.Download(cache.Sources(), eurl)
	case "git":
		spath := filepath.Join(cache.Sources(), plan.Name)
		if err := Clone(spath, "https"+eurl[3:]); err != nil {
			elog.Println(err)
			return err
		}
	default:
		return fmt.Errorf("%s: URL scheme is not supported", u.Scheme)
	}
	return nil
}

// Stages the downloaded source in via's cache directory
// the stage only happens once unless BuilInStage is used
func Stage(config *Config, plan *Plan) (err error) {
	if plan.Url == "" || file.Exists(plan.GetStageDir()) {
		// nothing to stage
		return nil
	}
	fmt.Printf(lfmt, "stage", plan.NameVersion())
	u, err := url.Parse(plan.Expand().Url)
	if err != nil {
		elog.Println(err)
		return err
	}
	//FIXME: move this down to switch statement so avoid goto
	if u.Scheme == "git" {
		fmt.Println(plan.SourcePath())
		fmt.Println(plan.GetStageDir())
		if err := Clone(plan.GetStageDir(), plan.SourcePath()); err != nil {
			return err
		}
		goto patch
	}
	switch filepath.Ext(plan.SourceFile()) {
	case ".zip":
		unzip(cache.Stages(), plan.SourcePath())
	default:
		GNUUntar(cache.Stages(), plan.SourcePath())
	}
patch:
	fmt.Printf(lfmt, "patch", plan.NameVersion())
	if err := doCommands(config, join(cache.Stages(), plan.stageDir()), plan.Patch); err != nil {
		return err
	}
	return
}

// Calls each shell command in the plans Build field.
func Build(config *Config, plan *Plan) (err error) {
	var (
		build = plan.Build
	)
	for _, p := range plan.BuildDepends {
		if IsInstalled(config, p) {
			continue
		}
		dp, err := NewPlan(config, p)
		if err != nil {
			return err
		}
		if err := NewInstaller(config, dp).Install(); err != nil {
			return err
		}
	}
	// FIXME: flags should not be merged should have a ConfigFLags
	// and PlanFlags environment variable
	flags := append(config.Flags, plan.Flags...)
	os.MkdirAll(plan.BuildDir(), 0755)
	// Parent plan Build is run first this plans is added at the end.
	if plan.Inherit != "" {
		parent, _ := NewPlan(config, plan.Inherit)
		build = append(parent.Build, plan.Build...)
		flags = append(flags, parent.Flags...)
	}
	// FIXME: this should be set within exec.Cmd
	os.Setenv("SRCDIR", plan.GetStageDir())
	os.Setenv("Flags", expand(flags.String()))
	err = doCommands(config, plan.BuildDir(), build)
	if err != nil {
		return fmt.Errorf("%s in %s", err.Error(), plan.BuildDir())
	}
	return nil
}

func doCommands(config *Config, dir string, cmds []string) (err error) {
	for _, j := range cmds {
		cmd := &exec.Cmd{
			Path:   "/bin/bash",
			Args:   []string{"bash", "-c", j},
			Stdin:  os.Stdin,
			Stderr: os.Stderr,
			Dir:    dir,
			Env:    config.Getenv(),
		}
		if verbose {
			cmd.Stdout = os.Stdout
		}
		if debug {
			fmt.Println(config.Getenv())
			fmt.Println(cmd.Args)
		}
		err = cmd.Run()
		if err != nil {
			elog.Printf("%s: %s\n", j, err)
			return err
		}
	}
	return nil
}

func Package(config *Config, bdir string, plan *Plan) (err error) {
	var (
		pack = plan.Package
	)
	// Remove plans Cid it's assumed we'll be creating a new one
	plan.Cid = ""
	defer os.Remove(plan.PackagePath())
	pdir := join(cache.Packages(), plan.NameVersion())
	if bdir == "" {
		bdir = join(cache.Builds(), plan.NameVersion())
	}
	if plan.BuildInStage {
		bdir = join(cache.Stages(), plan.stageDir())
	}
	if file.Exists(pdir) {
		err := os.RemoveAll(pdir)
		if err != nil {
			return err
		}
	}
	err = os.Mkdir(pdir, 0755)
	if err != nil {
		elog.Println(err)
		return err
	}
	os.Setenv("PKGDIR", pdir)
	if plan.Inherit != "" {
		parent, _ := NewPlan(config, plan.Inherit)
		pack = append(parent.Package, plan.Package...)
	}
	err = doCommands(config, bdir, pack)
	if err != nil {
		return err
	}
	for _, j := range plan.SubPackages {
		sub, err := NewPlan(config, j)
		if err != nil {
			return err
		}
		if err = Package(config, bdir, sub); err != nil {
			return err
		}
	}
	err = CreatePackage(config, plan)
	if err != nil {
		elog.Println(err)
		return (err)
	}
	plan.Cid, err = IpfsAdd(config, Path(plan.PackagePath()))
	if err != nil {
		return err
	}
	plan.IsRebuilt = true
	return plan.Save()
	/*
		err = CreatePackage(plan)
		if err != nil {
			return err
		}
		return Sign(plan)
	*/
}

func CreatePackage(config *Config, plan *Plan) (err error) {
	pfile := plan.PackagePath()
	os.MkdirAll(filepath.Dir(pfile), 0755)
	fd, err := os.Create(pfile)
	if err != nil {
		elog.Println(err)
		return err
	}
	defer fd.Close()
	gz := gzip.NewWriter(fd)
	defer gz.Close()
	return Tarball(gz, plan)
}

// Updates each plans Oid to the Oid of the tarball in publish git repo
// this function should never be used in production. It's used for making sure
// the plans Oid match the git repo's Oid
func SyncHashs(config *Config) {
	plans, _ := GetPlans()
	for _, p := range plans {
		if file.Exists(p.PackagePath()) {
			p.Cid, _ = HashOnly(config, Path(p.PackagePath()))
			p.Save()
			log.Println(p.Cid, p.Name)
		}
	}
}

func PostInstall(config *Config, plan *Plan) (err error) {
	return doCommands(config, "/", append(plan.PostInstall, config.PostInstall...))
}

func Remove(config *Config, name string) (err error) {
	if !IsInstalled(config, name) {
		err = fmt.Errorf("%s is not installed.", name)
		elog.Println(err)
		return err
	}

	man, err := ReadManifest(config, name)
	if err != nil {
		elog.Println(err)
		return err
	}
	for _, f := range man.Files {
		fpath := join(config.Root, f)
		if err := os.Remove(fpath); err != nil {
			elog.Println(err)
		}
	}

	return os.RemoveAll(join(config.DB.Installed(config), name))
}

func BuildDeps(config *Config, plan *Plan) (err error) {
	for _, d := range plan.Depends() {
		if IsInstalled(config, d) {
			continue
		}
		p, _ := NewPlan(config, d)
		if file.Exists(p.PackagePath()) {
			if err := NewInstaller(config, plan).Install(); err != nil {
				return err
			}
			continue
		}
		fmt.Println("building", d, "for", plan.NameVersion())
		err := BuildDeps(config, p)
		if err != nil {
			elog.Println(err)
			return err
		}
	}
	err = BuildSteps(config, plan)
	if err != nil {
		return err
	}
	return NewInstaller(config, plan).Install()
}

// Run all of the functions required to build a package
func BuildSteps(config *Config, plan *Plan) (err error) {
	if file.Exists(plan.PackageFile()) {
		return fmt.Errorf("package %s exists", plan.PackageFile())
	}
	if err := DownloadSrc(config, plan); err != nil {
		elog.Println(err)
		return err
	}
	if err := Stage(config, plan); err != nil {
		elog.Println(err)
		return err
	}
	fmt.Printf(lfmt, "build", plan.NameVersion())
	if err := Build(config, plan); err != nil {
		elog.Println(err)
		return err
	}
	fmt.Printf(lfmt, "package", plan.NameVersion())
	if err := Package(config, "", plan); err != nil {
		elog.Println(err)
		return err
	}
	return RepoCreate(config)
}

var (
	rexName   = regexp.MustCompile("[A-Za-z]+")
	rexTruple = regexp.MustCompile("[0-9]+.[0-9]+.[0-9]+")
	rexDouble = regexp.MustCompile("[0-9]+.[0-9]+")
)

// Creates a new plan from a given Url
func Create(url, group string) (err error) {
	var (
		xfile   = filepath.Base(url)
		name    = rexName.FindString(xfile)
		triple  = rexTruple.FindString(xfile)
		double  = rexDouble.FindString(xfile)
		version string
	)
	switch {
	case triple != "":
		version = triple
	case double != "":
		version = double
	default:
		return fmt.Errorf("regex fail for %s", xfile)
	}
	plan := &Plan{Name: name, Version: version, Url: url, Group: group}
	plan.Inherit = "gnu"
	if file.Exists(plan.Path()) {
		return fmt.Errorf("%s already exists", plan.Path())
	}
	return plan.Save()
}

func IsInstalled(config *Config, name string) bool {
	return file.Exists(join(config.DB.Installed(config), name))
}

func refactor(plan *Plan) {
	if len(plan.SubPackages) > 0 {
		for _, j := range plan.SubPackages {
			s, _ := NewPlan(config, j)
			if s.Version == plan.Version {
				continue
			}
			s.Version = plan.Version
			s.Save()
		}
	}
}

func Lint() (err error) {
	e, err := PlanFiles()
	if err != nil {
		return err
	}
	for _, j := range e {
		plan, err := ReadPath(config, j)
		if err != nil {
			err = fmt.Errorf("%s %s", j, err)
			elog.Println(err)
			return err
		}
		// If Group is empty, we can set it
		if plan.Group == "" {
			plan.Group = baseDir(j)
		}
		if verbose {
			console.Println("lint", plan.Name, plan.Version, plan.IsRebuilt)
		}
		sort.Strings(plan.SubPackages)
		sort.Strings(plan.Flags)
		sort.Strings(plan.Remove)
		sort.Strings(plan.AutoDepends)
		sort.Strings(plan.ManualDepends)
		sort.Strings(plan.BuildDepends)
		refactor(plan)
		err = plan.Save()
		if err != nil {
			elog.Println(err)
			return err
		}
	}
	console.Flush()
	return nil
}

func fatal(err error) {
	if err != nil {
		log.Fatal(err)
	}
}
func Clean(name string) error {
	plan, err := NewPlan(config, name)
	if err != nil {
		return err
	}
	fmt.Printf(lfmt, "clean", plan.NameVersion())
	dir := join(cache.Builds(), plan.NameVersion())
	if err = os.RemoveAll(dir); err != nil {
		return err
	}

	if plan.BuildInStage {
		dir = join(cache.Stages(), plan.stageDir())
		return os.RemoveAll(dir)
	}
	return nil
}

func PlanFiles() ([]string, error) {
	return filepath.Glob(join(config.Plans, "*", "*.json"))
}

func conflicts(config *Config, man *Plan) (errs []error) {
	for _, f := range man.Files {
		fpath := join(config.Root, f)
		if file.Exists(fpath) {
			errs = append(errs, fmt.Errorf("%s already exists.", f))
		}
	}
	return errs
}

func GetConfig() *Config {
	return config
}
<|endoftext|>


<commit_before>package action

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/gopasspw/gopass/internal/config"
	"github.com/gopasspw/gopass/internal/out"
	"github.com/gopasspw/gopass/internal/store/sub"
	"github.com/gopasspw/gopass/pkg/ctxutil"
	"github.com/gopasspw/gopass/pkg/fsutil"
	"github.com/muesli/goprogressbar"
	"github.com/urfave/cli/v2"
)

// Fsck checks the store integrity
func (s *Action) Fsck(c *cli.Context) error {
	ctx := ctxutil.WithGlobalFlags(c)
	if c.IsSet("decrypt") {
		ctx = sub.WithFsckDecrypt(ctx, c.Bool("decrypt"))
	}
	out.Print(ctx, "Checking store integrity ...")
	// make sure config is in the right place
	// we may have loaded it from one of the fallback locations
	if err := s.cfg.Save(); err != nil {
		return ExitError(ctx, ExitConfig, err, "failed to save config: %s", err)
	}

	// clean up any previous config locations
	oldCfg := filepath.Join(config.Homedir(), ".gopass.yml")
	if fsutil.IsFile(oldCfg) {
		if err := os.Remove(oldCfg); err != nil {
			out.Error(ctx, "Failed to remove old gopass config %s: %s", oldCfg, err)
		}
	}

	// display progress bar
	t, err := s.Store.Tree(ctx)
	if err != nil {
		return ExitError(ctx, ExitUnknown, err, "failed to list stores: %s", err)
	}

	pwList := t.List(0)

	bar := &goprogressbar.ProgressBar{
		Total: int64(len(pwList) * 2),
		Width: 120,
	}
	if out.IsHidden(ctx) {
		old := goprogressbar.Stdout
		goprogressbar.Stdout = ioutil.Discard
		defer func() {
			goprogressbar.Stdout = old
		}()
	}

	ctx = ctxutil.WithProgressCallback(ctx, func() {
		bar.Current++
		bar.Text = fmt.Sprintf("%d of %d objects checked", bar.Current, bar.Total)
		bar.LazyPrint()
	})
	ctx = out.AddPrefix(ctx, "\n")

	// the main work in done by the sub stores
	if err := s.Store.Fsck(ctx, c.Args().Get(0)); err != nil {
		return ExitError(ctx, ExitFsck, err, "fsck found errors: %s", err)
	}
	out.Print(ctx, "")
	return nil
}
<commit_msg>Make sure current item is never larger than total number (#1384)<commit_after>package action

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/gopasspw/gopass/internal/config"
	"github.com/gopasspw/gopass/internal/out"
	"github.com/gopasspw/gopass/internal/store/sub"
	"github.com/gopasspw/gopass/pkg/ctxutil"
	"github.com/gopasspw/gopass/pkg/fsutil"
	"github.com/muesli/goprogressbar"
	"github.com/urfave/cli/v2"
)

// Fsck checks the store integrity
func (s *Action) Fsck(c *cli.Context) error {
	ctx := ctxutil.WithGlobalFlags(c)
	if c.IsSet("decrypt") {
		ctx = sub.WithFsckDecrypt(ctx, c.Bool("decrypt"))
	}
	out.Print(ctx, "Checking store integrity ...")
	// make sure config is in the right place
	// we may have loaded it from one of the fallback locations
	if err := s.cfg.Save(); err != nil {
		return ExitError(ctx, ExitConfig, err, "failed to save config: %s", err)
	}

	// clean up any previous config locations
	oldCfg := filepath.Join(config.Homedir(), ".gopass.yml")
	if fsutil.IsFile(oldCfg) {
		if err := os.Remove(oldCfg); err != nil {
			out.Error(ctx, "Failed to remove old gopass config %s: %s", oldCfg, err)
		}
	}

	// display progress bar
	t, err := s.Store.Tree(ctx)
	if err != nil {
		return ExitError(ctx, ExitUnknown, err, "failed to list stores: %s", err)
	}

	pwList := t.List(0)

	bar := &goprogressbar.ProgressBar{
		Total: int64(len(pwList) * 2),
		Width: 120,
	}
	if out.IsHidden(ctx) {
		old := goprogressbar.Stdout
		goprogressbar.Stdout = ioutil.Discard
		defer func() {
			goprogressbar.Stdout = old
		}()
	}

	ctx = ctxutil.WithProgressCallback(ctx, func() {
		bar.Current++
		if bar.Current > bar.Total {
			bar.Total = bar.Current
		}
		bar.Text = fmt.Sprintf("%d of %d objects checked", bar.Current, bar.Total)
		bar.LazyPrint()
	})
	ctx = out.AddPrefix(ctx, "\n")

	// the main work in done by the sub stores
	if err := s.Store.Fsck(ctx, c.Args().Get(0)); err != nil {
		return ExitError(ctx, ExitFsck, err, "fsck found errors: %s", err)
	}
	out.Print(ctx, "")
	return nil
}
<|endoftext|>


<commit_before>// local build script file, similar to a makefile or collection of bash scripts in other projects

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"os"
	"os/exec"
	"strings"

	"github.com/urfave/cli/v2"
)

var packages = []string{"cli", "altsrc"}

func main() {
	app := cli.NewApp()

	app.Name = "builder"
	app.Usage = "Generates a new urfave/cli build!"

	app.Commands = cli.Commands{
		{
			Name:   "vet",
			Action: VetActionFunc,
		},
		{
			Name:   "test",
			Action: TestActionFunc,
		},
		{
			Name:   "gfmrun",
			Action: GfmrunActionFunc,
		},
		{
			Name:   "toc",
			Action: TocActionFunc,
		},
		{
			Name:   "check-binary-size",
			Action: checkBinarySizeActionFunc,
		},
	}

	err := app.Run(os.Args)
	if err != nil {
		log.Fatal(err)
	}
}

func runCmd(arg string, args ...string) error {
	cmd := exec.Command(arg, args...)

	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	return cmd.Run()
}

func VetActionFunc(_ *cli.Context) error {
	return runCmd("go", "vet")
}

func TestActionFunc(c *cli.Context) error {
	for _, pkg := range packages {
		var packageName string

		if pkg == "cli" {
			packageName = "github.com/urfave/cli/v2"
		} else {
			packageName = fmt.Sprintf("github.com/urfave/cli/v2/%s", pkg)
		}

		coverProfile := fmt.Sprintf("--coverprofile=%s.coverprofile", pkg)

		err := runCmd("go", "test", "-v", coverProfile, packageName)
		if err != nil {
			return err
		}
	}

	return testCleanup()
}

func testCleanup() error {
	var out bytes.Buffer

	for _, pkg := range packages {
		file, err := os.Open(fmt.Sprintf("%s.coverprofile", pkg))
		if err != nil {
			return err
		}

		b, err := ioutil.ReadAll(file)
		if err != nil {
			return err
		}

		out.Write(b)
		err = file.Close()
		if err != nil {
			return err
		}

		err = os.Remove(fmt.Sprintf("%s.coverprofile", pkg))
		if err != nil {
			return err
		}
	}

	outFile, err := os.Create("coverage.txt")
	if err != nil {
		return err
	}

	_, err = out.WriteTo(outFile)
	if err != nil {
		return err
	}

	err = outFile.Close()
	if err != nil {
		return err
	}

	return nil
}

func GfmrunActionFunc(c *cli.Context) error {
	filename := c.Args().Get(0)
	if filename == "" {
		filename = "README.md"
	}

	file, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer file.Close()

	var counter int
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		if strings.Contains(scanner.Text(), "package main") {
			counter++
		}
	}

	err = file.Close()
	if err != nil {
		return err
	}

	err = scanner.Err()
	if err != nil {
		return err
	}

	return runCmd("gfmrun", "-c", fmt.Sprint(counter), "-s", filename)
}

func TocActionFunc(c *cli.Context) error {
	filename := c.Args().Get(0)
	if filename == "" {
		filename = "README.md"
	}

	err := runCmd("markdown-toc", "-i", filename)
	if err != nil {
		return err
	}

	err = runCmd("git", "diff", "--exit-code")
	if err != nil {
		return err
	}

	return nil
}

// checkBinarySizeActionFunc checks the size of an example binary to ensure that we are keeping size down
// this was originally inspired by https://github.com/urfave/cli/issues/1055, and followed up on as a part
// of https://github.com/urfave/cli/issues/1057
func checkBinarySizeActionFunc(c *cli.Context) (err error) {
	const (
		cliSourceFilePath    = "./internal/example-cli/example-cli.go"
		cliBuiltFilePath     = "./internal/example-cli/built-example"
		helloSourceFilePath  = "./internal/example-hello-world/example-hello-world.go"
		helloBuiltFilePath   = "./internal/example-hello-world/built-example"
		desiredMinBinarySize = 2.1
		desiredMaxBinarySize = 2.6
		badNewsEmoji         = "🚨"
		goodNewsEmoji        = "✨"
		checksPassedEmoji    = "✅"
		mbStringFormatter    = "%.1fMB"
	)

	// get cli example size
	cliSize, err := getSize(cliSourceFilePath, cliBuiltFilePath)
	if err != nil {
		return err
	}

	// get hello world size
	helloSize, err := getSize(helloSourceFilePath, helloBuiltFilePath)
	if err != nil {
		return err
	}

	// The CLI size diff is the number we are interested in.
	// This tells us how much our CLI package contributes to the binary size.
	cliSizeDiff := cliSize - helloSize

	// get human readable size, in MB with one decimal place.
	// example output is: 35.2MB. (note: this simply an example)
	// that output is much easier to reason about than the `35223432`
	// that you would see output without the rounding
	fileSizeInMB := float64(cliSizeDiff) / float64(1000000)
	roundedFileSize := math.Round(fileSizeInMB*10) / 10
	roundedFileSizeString := fmt.Sprintf(mbStringFormatter, roundedFileSize)

	// check against bounds
	isLessThanDesiredMin := roundedFileSize < desiredMinBinarySize
	isMoreThanDesiredMax := roundedFileSize > desiredMaxBinarySize
	desiredMinSizeString := fmt.Sprintf(mbStringFormatter, desiredMinBinarySize)
	desiredMaxSizeString := fmt.Sprintf(mbStringFormatter, desiredMaxBinarySize)

	// show guidance
	fmt.Println(fmt.Sprintf("\n%s is the current binary size", roundedFileSizeString))
	// show guidance for min size
	if isLessThanDesiredMin {
		fmt.Println(fmt.Sprintf("  %s %s is the target min size", goodNewsEmoji, desiredMinSizeString))
		fmt.Println("") // visual spacing
		fmt.Println("     The binary is smaller than the target min size, which is great news!")
		fmt.Println("     That means that your changes are shrinking the binary size.")
		fmt.Println("     You'll want to go into ./internal/build/build.go and decrease")
		fmt.Println("     the desiredMinBinarySize, and also probably decrease the ")
		fmt.Println("     desiredMaxBinarySize by the same amount. That will ensure that")
		fmt.Println("     future PRs will enforce the newly shrunk binary sizes.")
		fmt.Println("") // visual spacing
		os.Exit(1)
	} else {
		fmt.Println(fmt.Sprintf("  %s %s is the target min size", checksPassedEmoji, desiredMinSizeString))
	}
	// show guidance for max size
	if isMoreThanDesiredMax {
		fmt.Println(fmt.Sprintf("  %s %s is the target max size", badNewsEmoji, desiredMaxSizeString))
		fmt.Println("") // visual spacing
		fmt.Println("     The binary is larger than the target max size.")
		fmt.Println("     That means that your changes are increasing the binary size.")
		fmt.Println("     The first thing you'll want to do is ask your yourself")
		fmt.Println("     Is this change worth increasing the binary size?")
		fmt.Println("     Larger binary sizes for this package can dissuade its use.")
		fmt.Println("     If this change is worth the increase, then we can up the")
		fmt.Println("     desired max binary size. To do that you'll want to go into")
		fmt.Println("     ./internal/build/build.go and increase the desiredMaxBinarySize,")
		fmt.Println("     and increase the desiredMinBinarySize by the same amount.")
		fmt.Println("") // visual spacing
		os.Exit(1)
	} else {
		fmt.Println(fmt.Sprintf("  %s %s is the target max size", checksPassedEmoji, desiredMaxSizeString))
	}

	return nil
}

func getSize(sourcePath string, builtPath string) (size int64, err error) {
	// build example binary
	err = runCmd("go", "build", "-o", builtPath, "-ldflags", "-s -w", sourcePath)
	if err != nil {
		fmt.Println("issue getting size for example binary")
		return 0, err
	}

	// get file info
	fileInfo, err := os.Stat(builtPath)
	if err != nil {
		fmt.Println("issue getting size for example binary")
		return 0, err
	}

	// size!
	size = fileInfo.Size()

	return size, nil
}
<commit_msg>move bounds<commit_after>// local build script file, similar to a makefile or collection of bash scripts in other projects

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"os"
	"os/exec"
	"strings"

	"github.com/urfave/cli/v2"
)

var packages = []string{"cli", "altsrc"}

func main() {
	app := cli.NewApp()

	app.Name = "builder"
	app.Usage = "Generates a new urfave/cli build!"

	app.Commands = cli.Commands{
		{
			Name:   "vet",
			Action: VetActionFunc,
		},
		{
			Name:   "test",
			Action: TestActionFunc,
		},
		{
			Name:   "gfmrun",
			Action: GfmrunActionFunc,
		},
		{
			Name:   "toc",
			Action: TocActionFunc,
		},
		{
			Name:   "check-binary-size",
			Action: checkBinarySizeActionFunc,
		},
	}

	err := app.Run(os.Args)
	if err != nil {
		log.Fatal(err)
	}
}

func runCmd(arg string, args ...string) error {
	cmd := exec.Command(arg, args...)

	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	return cmd.Run()
}

func VetActionFunc(_ *cli.Context) error {
	return runCmd("go", "vet")
}

func TestActionFunc(c *cli.Context) error {
	for _, pkg := range packages {
		var packageName string

		if pkg == "cli" {
			packageName = "github.com/urfave/cli/v2"
		} else {
			packageName = fmt.Sprintf("github.com/urfave/cli/v2/%s", pkg)
		}

		coverProfile := fmt.Sprintf("--coverprofile=%s.coverprofile", pkg)

		err := runCmd("go", "test", "-v", coverProfile, packageName)
		if err != nil {
			return err
		}
	}

	return testCleanup()
}

func testCleanup() error {
	var out bytes.Buffer

	for _, pkg := range packages {
		file, err := os.Open(fmt.Sprintf("%s.coverprofile", pkg))
		if err != nil {
			return err
		}

		b, err := ioutil.ReadAll(file)
		if err != nil {
			return err
		}

		out.Write(b)
		err = file.Close()
		if err != nil {
			return err
		}

		err = os.Remove(fmt.Sprintf("%s.coverprofile", pkg))
		if err != nil {
			return err
		}
	}

	outFile, err := os.Create("coverage.txt")
	if err != nil {
		return err
	}

	_, err = out.WriteTo(outFile)
	if err != nil {
		return err
	}

	err = outFile.Close()
	if err != nil {
		return err
	}

	return nil
}

func GfmrunActionFunc(c *cli.Context) error {
	filename := c.Args().Get(0)
	if filename == "" {
		filename = "README.md"
	}

	file, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer file.Close()

	var counter int
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		if strings.Contains(scanner.Text(), "package main") {
			counter++
		}
	}

	err = file.Close()
	if err != nil {
		return err
	}

	err = scanner.Err()
	if err != nil {
		return err
	}

	return runCmd("gfmrun", "-c", fmt.Sprint(counter), "-s", filename)
}

func TocActionFunc(c *cli.Context) error {
	filename := c.Args().Get(0)
	if filename == "" {
		filename = "README.md"
	}

	err := runCmd("markdown-toc", "-i", filename)
	if err != nil {
		return err
	}

	err = runCmd("git", "diff", "--exit-code")
	if err != nil {
		return err
	}

	return nil
}

// checkBinarySizeActionFunc checks the size of an example binary to ensure that we are keeping size down
// this was originally inspired by https://github.com/urfave/cli/issues/1055, and followed up on as a part
// of https://github.com/urfave/cli/issues/1057
func checkBinarySizeActionFunc(c *cli.Context) (err error) {
	const (
		cliSourceFilePath    = "./internal/example-cli/example-cli.go"
		cliBuiltFilePath     = "./internal/example-cli/built-example"
		helloSourceFilePath  = "./internal/example-hello-world/example-hello-world.go"
		helloBuiltFilePath   = "./internal/example-hello-world/built-example"
		desiredMinBinarySize = 2.0
		desiredMaxBinarySize = 2.3
		badNewsEmoji         = "🚨"
		goodNewsEmoji        = "✨"
		checksPassedEmoji    = "✅"
		mbStringFormatter    = "%.1fMB"
	)

	// get cli example size
	cliSize, err := getSize(cliSourceFilePath, cliBuiltFilePath)
	if err != nil {
		return err
	}

	// get hello world size
	helloSize, err := getSize(helloSourceFilePath, helloBuiltFilePath)
	if err != nil {
		return err
	}

	// The CLI size diff is the number we are interested in.
	// This tells us how much our CLI package contributes to the binary size.
	cliSizeDiff := cliSize - helloSize

	// get human readable size, in MB with one decimal place.
	// example output is: 35.2MB. (note: this simply an example)
	// that output is much easier to reason about than the `35223432`
	// that you would see output without the rounding
	fileSizeInMB := float64(cliSizeDiff) / float64(1000000)
	roundedFileSize := math.Round(fileSizeInMB*10) / 10
	roundedFileSizeString := fmt.Sprintf(mbStringFormatter, roundedFileSize)

	// check against bounds
	isLessThanDesiredMin := roundedFileSize < desiredMinBinarySize
	isMoreThanDesiredMax := roundedFileSize > desiredMaxBinarySize
	desiredMinSizeString := fmt.Sprintf(mbStringFormatter, desiredMinBinarySize)
	desiredMaxSizeString := fmt.Sprintf(mbStringFormatter, desiredMaxBinarySize)

	// show guidance
	fmt.Println(fmt.Sprintf("\n%s is the current binary size", roundedFileSizeString))
	// show guidance for min size
	if isLessThanDesiredMin {
		fmt.Println(fmt.Sprintf("  %s %s is the target min size", goodNewsEmoji, desiredMinSizeString))
		fmt.Println("") // visual spacing
		fmt.Println("     The binary is smaller than the target min size, which is great news!")
		fmt.Println("     That means that your changes are shrinking the binary size.")
		fmt.Println("     You'll want to go into ./internal/build/build.go and decrease")
		fmt.Println("     the desiredMinBinarySize, and also probably decrease the ")
		fmt.Println("     desiredMaxBinarySize by the same amount. That will ensure that")
		fmt.Println("     future PRs will enforce the newly shrunk binary sizes.")
		fmt.Println("") // visual spacing
		os.Exit(1)
	} else {
		fmt.Println(fmt.Sprintf("  %s %s is the target min size", checksPassedEmoji, desiredMinSizeString))
	}
	// show guidance for max size
	if isMoreThanDesiredMax {
		fmt.Println(fmt.Sprintf("  %s %s is the target max size", badNewsEmoji, desiredMaxSizeString))
		fmt.Println("") // visual spacing
		fmt.Println("     The binary is larger than the target max size.")
		fmt.Println("     That means that your changes are increasing the binary size.")
		fmt.Println("     The first thing you'll want to do is ask your yourself")
		fmt.Println("     Is this change worth increasing the binary size?")
		fmt.Println("     Larger binary sizes for this package can dissuade its use.")
		fmt.Println("     If this change is worth the increase, then we can up the")
		fmt.Println("     desired max binary size. To do that you'll want to go into")
		fmt.Println("     ./internal/build/build.go and increase the desiredMaxBinarySize,")
		fmt.Println("     and increase the desiredMinBinarySize by the same amount.")
		fmt.Println("") // visual spacing
		os.Exit(1)
	} else {
		fmt.Println(fmt.Sprintf("  %s %s is the target max size", checksPassedEmoji, desiredMaxSizeString))
	}

	return nil
}

func getSize(sourcePath string, builtPath string) (size int64, err error) {
	// build example binary
	err = runCmd("go", "build", "-o", builtPath, "-ldflags", "-s -w", sourcePath)
	if err != nil {
		fmt.Println("issue getting size for example binary")
		return 0, err
	}

	// get file info
	fileInfo, err := os.Stat(builtPath)
	if err != nil {
		fmt.Println("issue getting size for example binary")
		return 0, err
	}

	// size!
	size = fileInfo.Size()

	return size, nil
}
<|endoftext|>


<commit_before>// local build script file, similar to a makefile or collection of bash scripts in other projects

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"os"
	"os/exec"
	"strings"

	"github.com/urfave/cli/v2"
)

var packages = []string{"cli", "altsrc"}

func main() {
	app := cli.NewApp()

	app.Name = "builder"
	app.Usage = "Generates a new urfave/cli build!"

	app.Commands = cli.Commands{
		{
			Name:   "vet",
			Action: VetActionFunc,
		},
		{
			Name:   "test",
			Action: TestActionFunc,
		},
		{
			Name:   "gfmrun",
			Action: GfmrunActionFunc,
		},
		{
			Name:   "toc",
			Action: TocActionFunc,
		},
		{
			Name:   "check-binary-size",
			Action: checkBinarySizeActionFunc,
		},
	}

	err := app.Run(os.Args)
	if err != nil {
		log.Fatal(err)
	}
}

func runCmd(arg string, args ...string) error {
	cmd := exec.Command(arg, args...)

	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	return cmd.Run()
}

func VetActionFunc(_ *cli.Context) error {
	return runCmd("go", "vet")
}

func TestActionFunc(c *cli.Context) error {
	for _, pkg := range packages {
		var packageName string

		if pkg == "cli" {
			packageName = "github.com/urfave/cli/v2"
		} else {
			packageName = fmt.Sprintf("github.com/urfave/cli/v2/%s", pkg)
		}

		coverProfile := fmt.Sprintf("--coverprofile=%s.coverprofile", pkg)

		err := runCmd("go", "test", "-v", coverProfile, packageName)
		if err != nil {
			return err
		}
	}

	return testCleanup()
}

func testCleanup() error {
	var out bytes.Buffer

	for _, pkg := range packages {
		file, err := os.Open(fmt.Sprintf("%s.coverprofile", pkg))
		if err != nil {
			return err
		}

		b, err := ioutil.ReadAll(file)
		if err != nil {
			return err
		}

		out.Write(b)
		err = file.Close()
		if err != nil {
			return err
		}

		err = os.Remove(fmt.Sprintf("%s.coverprofile", pkg))
		if err != nil {
			return err
		}
	}

	outFile, err := os.Create("coverage.txt")
	if err != nil {
		return err
	}

	_, err = out.WriteTo(outFile)
	if err != nil {
		return err
	}

	err = outFile.Close()
	if err != nil {
		return err
	}

	return nil
}

func GfmrunActionFunc(c *cli.Context) error {
	filename := c.Args().Get(0)
	if filename == "" {
		filename = "README.md"
	}

	file, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer file.Close()

	var counter int
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		if strings.Contains(scanner.Text(), "package main") {
			counter++
		}
	}

	err = file.Close()
	if err != nil {
		return err
	}

	err = scanner.Err()
	if err != nil {
		return err
	}

	return runCmd("gfmrun", "-c", fmt.Sprint(counter), "-s", filename)
}

func TocActionFunc(c *cli.Context) error {
	filename := c.Args().Get(0)
	if filename == "" {
		filename = "README.md"
	}

	err := runCmd("markdown-toc", "-i", filename)
	if err != nil {
		return err
	}

	err = runCmd("git", "diff", "--exit-code")
	if err != nil {
		return err
	}

	return nil
}

// checkBinarySizeActionFunc checks the size of an example binary to ensure that we are keeping size down
// this was originally inspired by https://github.com/urfave/cli/issues/1055, and followed up on as a part
// of https://github.com/urfave/cli/issues/1057
func checkBinarySizeActionFunc(c *cli.Context) (err error) {
	const (
		sourceFilePath       = "./internal/example/example.go"
		builtFilePath        = "./internal/example/built-example"
		desiredMinBinarySize = 4.8
		desiredMaxBinarySize = 5.0
		badNewsEmoji         = "🚨"
		goodNewsEmoji        = "✨"
		checksPassedEmoji    = "✅"
		mbStringFormatter    = "%.1fMB"
	)

	// build example binary
	err = runCmd("go", "build", "-o", builtFilePath, sourceFilePath)
	if err != nil {
		return err
	}

	// get file into
	fileInfo, err := os.Stat(builtFilePath)
	if err != nil {
		return err
	}

	// get human readable size, in MB with one decimal place.
	// example output is: 35.2MB. (note: this simply an example)
	// that output is much easier to reason about than the `35223432`
	// that you would see output without the rounding
	fileSize := fileInfo.Size()
	roundedFileSize := math.Round(float64(fileSize)/float64(1000000)*10) / 10
	roundedFileSizeString := fmt.Sprintf(mbStringFormatter, roundedFileSize)

	// check against bounds
	isLessThanDesiredMin := roundedFileSize < desiredMinBinarySize
	isMoreThanDesiredMax := roundedFileSize > desiredMaxBinarySize
	desiredMinSizeString := fmt.Sprintf(mbStringFormatter, desiredMinBinarySize)
	desiredMaxSizeString := fmt.Sprintf(mbStringFormatter, desiredMaxBinarySize)

	// show guidance
	fmt.Println(fmt.Sprintf("\n%s is the current binary size", roundedFileSizeString))
	// show guidance for min size
	if isLessThanDesiredMin {
		fmt.Println(fmt.Sprintf("  %s %s is the target min size", goodNewsEmoji, desiredMinSizeString))
		fmt.Println("") // visual spacing
		fmt.Println("     The binary is smaller than the target min size, which is great news!")
		fmt.Println("     That means that whatever you've done is shrinking the binary size.")
		fmt.Println("     You'll want to go into ./internal/build/build.go and decrease")
		fmt.Println("     the desiredMinBinarySize, and also probably decrease the ")
		fmt.Println("     desiredMaxBinarySize by the same amount. That will ensure that")
		fmt.Println("     future PRs will enforce the newly shrunk binary sizes.")
		fmt.Println("") // visual spacing
		os.Exit(1)
	} else {
		fmt.Println(fmt.Sprintf("  %s %s is the target min size", checksPassedEmoji, desiredMinSizeString))
	}
	// show guidance for max size
	if isMoreThanDesiredMax {
		fmt.Println(fmt.Sprintf("  %s %s is the target max size", badNewsEmoji, desiredMaxSizeString))
		fmt.Println("") // visual spacing
		os.Exit(1)
	} else {
		fmt.Println(fmt.Sprintf("  %s %s is the target max size", checksPassedEmoji, desiredMaxSizeString))
	}

	return nil
}
<commit_msg>decrease min<commit_after>// local build script file, similar to a makefile or collection of bash scripts in other projects

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"os"
	"os/exec"
	"strings"

	"github.com/urfave/cli/v2"
)

var packages = []string{"cli", "altsrc"}

func main() {
	app := cli.NewApp()

	app.Name = "builder"
	app.Usage = "Generates a new urfave/cli build!"

	app.Commands = cli.Commands{
		{
			Name:   "vet",
			Action: VetActionFunc,
		},
		{
			Name:   "test",
			Action: TestActionFunc,
		},
		{
			Name:   "gfmrun",
			Action: GfmrunActionFunc,
		},
		{
			Name:   "toc",
			Action: TocActionFunc,
		},
		{
			Name:   "check-binary-size",
			Action: checkBinarySizeActionFunc,
		},
	}

	err := app.Run(os.Args)
	if err != nil {
		log.Fatal(err)
	}
}

func runCmd(arg string, args ...string) error {
	cmd := exec.Command(arg, args...)

	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	return cmd.Run()
}

func VetActionFunc(_ *cli.Context) error {
	return runCmd("go", "vet")
}

func TestActionFunc(c *cli.Context) error {
	for _, pkg := range packages {
		var packageName string

		if pkg == "cli" {
			packageName = "github.com/urfave/cli/v2"
		} else {
			packageName = fmt.Sprintf("github.com/urfave/cli/v2/%s", pkg)
		}

		coverProfile := fmt.Sprintf("--coverprofile=%s.coverprofile", pkg)

		err := runCmd("go", "test", "-v", coverProfile, packageName)
		if err != nil {
			return err
		}
	}

	return testCleanup()
}

func testCleanup() error {
	var out bytes.Buffer

	for _, pkg := range packages {
		file, err := os.Open(fmt.Sprintf("%s.coverprofile", pkg))
		if err != nil {
			return err
		}

		b, err := ioutil.ReadAll(file)
		if err != nil {
			return err
		}

		out.Write(b)
		err = file.Close()
		if err != nil {
			return err
		}

		err = os.Remove(fmt.Sprintf("%s.coverprofile", pkg))
		if err != nil {
			return err
		}
	}

	outFile, err := os.Create("coverage.txt")
	if err != nil {
		return err
	}

	_, err = out.WriteTo(outFile)
	if err != nil {
		return err
	}

	err = outFile.Close()
	if err != nil {
		return err
	}

	return nil
}

func GfmrunActionFunc(c *cli.Context) error {
	filename := c.Args().Get(0)
	if filename == "" {
		filename = "README.md"
	}

	file, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer file.Close()

	var counter int
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		if strings.Contains(scanner.Text(), "package main") {
			counter++
		}
	}

	err = file.Close()
	if err != nil {
		return err
	}

	err = scanner.Err()
	if err != nil {
		return err
	}

	return runCmd("gfmrun", "-c", fmt.Sprint(counter), "-s", filename)
}

func TocActionFunc(c *cli.Context) error {
	filename := c.Args().Get(0)
	if filename == "" {
		filename = "README.md"
	}

	err := runCmd("markdown-toc", "-i", filename)
	if err != nil {
		return err
	}

	err = runCmd("git", "diff", "--exit-code")
	if err != nil {
		return err
	}

	return nil
}

// checkBinarySizeActionFunc checks the size of an example binary to ensure that we are keeping size down
// this was originally inspired by https://github.com/urfave/cli/issues/1055, and followed up on as a part
// of https://github.com/urfave/cli/issues/1057
func checkBinarySizeActionFunc(c *cli.Context) (err error) {
	const (
		sourceFilePath       = "./internal/example/example.go"
		builtFilePath        = "./internal/example/built-example"
		desiredMinBinarySize = 4.7
		desiredMaxBinarySize = 5.0
		badNewsEmoji         = "🚨"
		goodNewsEmoji        = "✨"
		checksPassedEmoji    = "✅"
		mbStringFormatter    = "%.1fMB"
	)

	// build example binary
	err = runCmd("go", "build", "-o", builtFilePath, sourceFilePath)
	if err != nil {
		return err
	}

	// get file into
	fileInfo, err := os.Stat(builtFilePath)
	if err != nil {
		return err
	}

	// get human readable size, in MB with one decimal place.
	// example output is: 35.2MB. (note: this simply an example)
	// that output is much easier to reason about than the `35223432`
	// that you would see output without the rounding
	fileSize := fileInfo.Size()
	roundedFileSize := math.Round(float64(fileSize)/float64(1000000)*10) / 10
	roundedFileSizeString := fmt.Sprintf(mbStringFormatter, roundedFileSize)

	// check against bounds
	isLessThanDesiredMin := roundedFileSize < desiredMinBinarySize
	isMoreThanDesiredMax := roundedFileSize > desiredMaxBinarySize
	desiredMinSizeString := fmt.Sprintf(mbStringFormatter, desiredMinBinarySize)
	desiredMaxSizeString := fmt.Sprintf(mbStringFormatter, desiredMaxBinarySize)

	// show guidance
	fmt.Println(fmt.Sprintf("\n%s is the current binary size", roundedFileSizeString))
	// show guidance for min size
	if isLessThanDesiredMin {
		fmt.Println(fmt.Sprintf("  %s %s is the target min size", goodNewsEmoji, desiredMinSizeString))
		fmt.Println("") // visual spacing
		fmt.Println("     The binary is smaller than the target min size, which is great news!")
		fmt.Println("     That means that whatever you've done is shrinking the binary size.")
		fmt.Println("     You'll want to go into ./internal/build/build.go and decrease")
		fmt.Println("     the desiredMinBinarySize, and also probably decrease the ")
		fmt.Println("     desiredMaxBinarySize by the same amount. That will ensure that")
		fmt.Println("     future PRs will enforce the newly shrunk binary sizes.")
		fmt.Println("") // visual spacing
		os.Exit(1)
	} else {
		fmt.Println(fmt.Sprintf("  %s %s is the target min size", checksPassedEmoji, desiredMinSizeString))
	}
	// show guidance for max size
	if isMoreThanDesiredMax {
		fmt.Println(fmt.Sprintf("  %s %s is the target max size", badNewsEmoji, desiredMaxSizeString))
		fmt.Println("") // visual spacing
		os.Exit(1)
	} else {
		fmt.Println(fmt.Sprintf("  %s %s is the target max size", checksPassedEmoji, desiredMaxSizeString))
	}

	return nil
}
<|endoftext|>