<commit_before>package union

import (
	"fmt"
	"io"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/ncw/rclone/fs"
	"github.com/ncw/rclone/fs/config/configmap"
	"github.com/ncw/rclone/fs/config/configstruct"
	"github.com/ncw/rclone/fs/hash"
	"github.com/pkg/errors"
)

// Register with Fs
func init() {
	fsi := &fs.RegInfo{
		Name:        "union",
		Description: "A stackable unification remote, which can appear to merge the contents of several remotes",
		NewFs:       NewFs,
		Options: []fs.Option{{
			Name:     "remotes",
			Help:     "List of space separated remotes.\nCan be 'remotea:test/dir remoteb:', '\"remotea:test/space dir\" remoteb:', etc.\nThe last remote is used to write to.",
			Required: true,
		}},
	}
	fs.Register(fsi)
}

// Options defines the configuration for this backend
type Options struct {
	Remotes fs.SpaceSepList `config:"remotes"`
}

// Fs represents a remote acd server
type Fs struct {
	name     string       // name of this remote
	features *fs.Features // optional features
	opt      Options      // options for this Fs
	root     string       // the path we are working on
	remotes  []fs.Fs      // slice of remotes
}

// Name of the remote (as passed into NewFs)
func (f *Fs) Name() string {
	return f.name
}

// Root of the remote (as passed into NewFs)
func (f *Fs) Root() string {
	return f.root
}

// String converts this Fs to a string
func (f *Fs) String() string {
	return fmt.Sprintf("union root '%s'", f.root)
}

// Features returns the optional features of this Fs
func (f *Fs) Features() *fs.Features {
	return f.features
}

// Rmdir removes the root directory of the Fs object
func (f *Fs) Rmdir(dir string) error {
	return f.remotes[len(f.remotes)-1].Rmdir(dir)
}

// Hashes returns hash.HashNone to indicate remote hashing is unavailable
func (f *Fs) Hashes() hash.Set {
	// This could probably be set if all remotes share the same hashing algorithm
	return hash.Set(hash.None)
}

// Mkdir makes the root directory of the Fs object
func (f *Fs) Mkdir(dir string) error {
	return f.remotes[len(f.remotes)-1].Mkdir(dir)
}

// Put in to the remote path with the modTime given of the given size
//
// May create the object even if it returns an error - if so
// will return the object and the error, otherwise will return
// nil and the error
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
	return f.remotes[len(f.remotes)-1].Put(in, src, options...)
}

// List the objects and directories in dir into entries.  The
// entries can be returned in any order but should be for a
// complete directory.
//
// dir should be "" to list the root, and should not have
// trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
	set := make(map[string]fs.DirEntry)
	found := false
	for _, remote := range f.remotes {
		var remoteEntries, err = remote.List(dir)
		if err == fs.ErrorDirNotFound {
			continue
		}
		if err != nil {
			return nil, errors.Wrapf(err, "List failed on %v", remote)
		}
		found = true
		for _, remoteEntry := range remoteEntries {
			set[remoteEntry.Remote()] = remoteEntry
		}
	}
	if !found {
		return nil, fs.ErrorDirNotFound
	}
	for key := range set {
		entries = append(entries, set[key])
	}
	return entries, nil
}

// NewObject creates a new remote union file object based on the first Object it finds (reverse remote order)
func (f *Fs) NewObject(path string) (fs.Object, error) {
	for i := range f.remotes {
		var remote = f.remotes[len(f.remotes)-i-1]
		var obj, err = remote.NewObject(path)
		if err == fs.ErrorObjectNotFound {
			continue
		}
		if err != nil {
			return nil, errors.Wrapf(err, "NewObject failed on %v", remote)
		}
		return obj, nil
	}
	return nil, fs.ErrorObjectNotFound
}

// Precision is the greatest Precision of all remotes
func (f *Fs) Precision() time.Duration {
	var greatestPrecision time.Duration
	for _, remote := range f.remotes {
		if remote.Precision() > greatestPrecision {
			greatestPrecision = remote.Precision()
		}
	}
	return greatestPrecision
}

// NewFs constructs an Fs from the path.
//
// The returned Fs is the actual Fs, referenced by remote in the config
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
	// Parse config into Options struct
	opt := new(Options)
	err := configstruct.Set(m, opt)
	if err != nil {
		return nil, err
	}
	if len(opt.Remotes) == 0 {
		return nil, errors.New("union can't point to an empty remote - check the value of the remotes setting")
	}
	if len(opt.Remotes) == 1 {
		return nil, errors.New("union can't point to a single remote - check the value of the remotes setting")
	}
	for _, remote := range opt.Remotes {
		if strings.HasPrefix(remote, name+":") {
			return nil, errors.New("can't point union remote at itself - check the value of the remote setting")
		}
	}

	var remotes []fs.Fs
	for i := range opt.Remotes {
		// Last remote first so we return the correct (last) matching fs in case of fs.ErrorIsFile
		var remote = opt.Remotes[len(opt.Remotes)-i-1]
		_, configName, fsPath, err := fs.ParseRemote(remote)
		if err != nil {
			return nil, err
		}
		var rootString = path.Join(fsPath, filepath.ToSlash(root))
		if configName != "local" {
			rootString = configName + ":" + rootString
		}
		myFs, err := fs.NewFs(rootString)
		if err != nil {
			if err == fs.ErrorIsFile {
				return myFs, err
			}
			return nil, err
		}
		remotes = append(remotes, myFs)
	}

	// Reverse the remotes again so they are in the order as before
	for i, j := 0, len(remotes)-1; i < j; i, j = i+1, j-1 {
		remotes[i], remotes[j] = remotes[j], remotes[i]
	}

	f := &Fs{
		name:    name,
		root:    root,
		opt:     *opt,
		remotes: remotes,
	}
	var features = (&fs.Features{
		CaseInsensitive:         true,
		DuplicateFiles:          false,
		ReadMimeType:            true,
		WriteMimeType:           true,
		CanHaveEmptyDirectories: true,
		BucketBased:             true,
	}).Fill(f)
	for _, remote := range f.remotes {
		features = features.Mask(remote)
	}
	f.features = features

	return f, nil
}

// Check the interfaces are satisfied
var (
	_ fs.Fs = &Fs{}
)
<commit_msg>union: implement optional interfaces (Move, DirMove, Copy etc) - fixes #2619<commit_after>package union

import (
	"fmt"
	"io"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/ncw/rclone/fs"
	"github.com/ncw/rclone/fs/config/configmap"
	"github.com/ncw/rclone/fs/config/configstruct"
	"github.com/ncw/rclone/fs/hash"
	"github.com/pkg/errors"
)

// Register with Fs
func init() {
	fsi := &fs.RegInfo{
		Name:        "union",
		Description: "A stackable unification remote, which can appear to merge the contents of several remotes",
		NewFs:       NewFs,
		Options: []fs.Option{{
			Name:     "remotes",
			Help:     "List of space separated remotes.\nCan be 'remotea:test/dir remoteb:', '\"remotea:test/space dir\" remoteb:', etc.\nThe last remote is used to write to.",
			Required: true,
		}},
	}
	fs.Register(fsi)
}

// Options defines the configuration for this backend
type Options struct {
	Remotes fs.SpaceSepList `config:"remotes"`
}

// Fs represents a remote acd server
type Fs struct {
	name     string       // name of this remote
	features *fs.Features // optional features
	opt      Options      // options for this Fs
	root     string       // the path we are working on
	remotes  []fs.Fs      // slice of remotes
	wr       fs.Fs        // writable remote
	hashSet  hash.Set     // intersection of hash types
}

// Name of the remote (as passed into NewFs)
func (f *Fs) Name() string {
	return f.name
}

// Root of the remote (as passed into NewFs)
func (f *Fs) Root() string {
	return f.root
}

// String converts this Fs to a string
func (f *Fs) String() string {
	return fmt.Sprintf("union root '%s'", f.root)
}

// Features returns the optional features of this Fs
func (f *Fs) Features() *fs.Features {
	return f.features
}

// Rmdir removes the root directory of the Fs object
func (f *Fs) Rmdir(dir string) error {
	return f.wr.Rmdir(dir)
}

// Hashes returns hash.HashNone to indicate remote hashing is unavailable
func (f *Fs) Hashes() hash.Set {
	return f.hashSet
}

// Mkdir makes the root directory of the Fs object
func (f *Fs) Mkdir(dir string) error {
	return f.wr.Mkdir(dir)
}

// Purge all files in the root and the root directory
//
// Implement this if you have a way of deleting all the files
// quicker than just running Remove() on the result of List()
//
// Return an error if it doesn't exist
func (f *Fs) Purge() error {
	return f.wr.Features().Purge()
}

// Copy src to this remote using server side copy operations.
//
// This is stored with the remote path given
//
// It returns the destination Object and a possible error
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantCopy
func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
	if src.Fs() != f.wr {
		fs.Debugf(src, "Can't copy - not same remote type")
		return nil, fs.ErrorCantCopy
	}
	return f.wr.Features().Copy(src, remote)
}

// Move src to this remote using server side move operations.
//
// This is stored with the remote path given
//
// It returns the destination Object and a possible error
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantMove
func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
	if src.Fs() != f.wr {
		fs.Debugf(src, "Can't move - not same remote type")
		return nil, fs.ErrorCantMove
	}
	return f.wr.Features().Move(src, remote)
}

// DirMove moves src, srcRemote to this remote at dstRemote
// using server side move operations.
//
// Will only be called if src.Fs().Name() == f.Name()
//
// If it isn't possible then return fs.ErrorCantDirMove
//
// If destination exists then return fs.ErrorDirExists
func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
	srcFs, ok := src.(*Fs)
	if !ok {
		fs.Debugf(srcFs, "Can't move directory - not same remote type")
		return fs.ErrorCantDirMove
	}
	return f.wr.Features().DirMove(srcFs.wr, srcRemote, dstRemote)
}

// ChangeNotify calls the passed function with a path
// that has had changes. If the implementation
// uses polling, it should adhere to the given interval.
// At least one value will be written to the channel,
// specifying the initial value and updated values might
// follow. A 0 Duration should pause the polling.
// The ChangeNotify implemantion must empty the channel
// regulary. When the channel gets closed, the implemantion
// should stop polling and release resources.
func (f *Fs) ChangeNotify(fn func(string, fs.EntryType), ch <-chan time.Duration) {
	for _, remote := range f.remotes {
		if ChangeNotify := remote.Features().ChangeNotify; ChangeNotify != nil {
			ChangeNotify(fn, ch)
		}
	}
}

// DirCacheFlush resets the directory cache - used in testing
// as an optional interface
func (f *Fs) DirCacheFlush() {
	for _, remote := range f.remotes {
		if DirCacheFlush := remote.Features().DirCacheFlush; DirCacheFlush != nil {
			DirCacheFlush()
		}
	}
}

// PutStream uploads to the remote path with the modTime given of indeterminate size
//
// May create the object even if it returns an error - if so
// will return the object and the error, otherwise will return
// nil and the error
func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
	return f.wr.Features().PutStream(in, src, options...)
}

// About gets quota information from the Fs
func (f *Fs) About() (*fs.Usage, error) {
	return f.wr.Features().About()
}

// Put in to the remote path with the modTime given of the given size
//
// May create the object even if it returns an error - if so
// will return the object and the error, otherwise will return
// nil and the error
func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
	return f.wr.Put(in, src, options...)
}

// List the objects and directories in dir into entries.  The
// entries can be returned in any order but should be for a
// complete directory.
//
// dir should be "" to list the root, and should not have
// trailing slashes.
//
// This should return ErrDirNotFound if the directory isn't
// found.
func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
	set := make(map[string]fs.DirEntry)
	found := false
	for _, remote := range f.remotes {
		var remoteEntries, err = remote.List(dir)
		if err == fs.ErrorDirNotFound {
			continue
		}
		if err != nil {
			return nil, errors.Wrapf(err, "List failed on %v", remote)
		}
		found = true
		for _, remoteEntry := range remoteEntries {
			set[remoteEntry.Remote()] = remoteEntry
		}
	}
	if !found {
		return nil, fs.ErrorDirNotFound
	}
	for key := range set {
		entries = append(entries, set[key])
	}
	return entries, nil
}

// NewObject creates a new remote union file object based on the first Object it finds (reverse remote order)
func (f *Fs) NewObject(path string) (fs.Object, error) {
	for i := range f.remotes {
		var remote = f.remotes[len(f.remotes)-i-1]
		var obj, err = remote.NewObject(path)
		if err == fs.ErrorObjectNotFound {
			continue
		}
		if err != nil {
			return nil, errors.Wrapf(err, "NewObject failed on %v", remote)
		}
		return obj, nil
	}
	return nil, fs.ErrorObjectNotFound
}

// Precision is the greatest Precision of all remotes
func (f *Fs) Precision() time.Duration {
	var greatestPrecision time.Duration
	for _, remote := range f.remotes {
		if remote.Precision() > greatestPrecision {
			greatestPrecision = remote.Precision()
		}
	}
	return greatestPrecision
}

// NewFs constructs an Fs from the path.
//
// The returned Fs is the actual Fs, referenced by remote in the config
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
	// Parse config into Options struct
	opt := new(Options)
	err := configstruct.Set(m, opt)
	if err != nil {
		return nil, err
	}
	if len(opt.Remotes) == 0 {
		return nil, errors.New("union can't point to an empty remote - check the value of the remotes setting")
	}
	if len(opt.Remotes) == 1 {
		return nil, errors.New("union can't point to a single remote - check the value of the remotes setting")
	}
	for _, remote := range opt.Remotes {
		if strings.HasPrefix(remote, name+":") {
			return nil, errors.New("can't point union remote at itself - check the value of the remote setting")
		}
	}

	var remotes []fs.Fs
	for i := range opt.Remotes {
		// Last remote first so we return the correct (last) matching fs in case of fs.ErrorIsFile
		var remote = opt.Remotes[len(opt.Remotes)-i-1]
		_, configName, fsPath, err := fs.ParseRemote(remote)
		if err != nil {
			return nil, err
		}
		var rootString = path.Join(fsPath, filepath.ToSlash(root))
		if configName != "local" {
			rootString = configName + ":" + rootString
		}
		myFs, err := fs.NewFs(rootString)
		if err != nil {
			if err == fs.ErrorIsFile {
				return myFs, err
			}
			return nil, err
		}
		remotes = append(remotes, myFs)
	}

	// Reverse the remotes again so they are in the order as before
	for i, j := 0, len(remotes)-1; i < j; i, j = i+1, j-1 {
		remotes[i], remotes[j] = remotes[j], remotes[i]
	}

	f := &Fs{
		name:    name,
		root:    root,
		opt:     *opt,
		remotes: remotes,
		wr:      remotes[len(remotes)-1],
	}
	var features = (&fs.Features{
		CaseInsensitive:         true,
		DuplicateFiles:          false,
		ReadMimeType:            true,
		WriteMimeType:           true,
		CanHaveEmptyDirectories: true,
		BucketBased:             true,
		SetTier:                 true,
		GetTier:                 true,
	}).Fill(f)
	features = features.Mask(f.wr) // mask the features just on the writable fs

	// FIXME maybe should be masking the bools here?

	// Clear ChangeNotify and DirCacheFlush if all are nil
	clearChangeNotify := true
	clearDirCacheFlush := true
	for _, remote := range f.remotes {
		remoteFeatures := remote.Features()
		if remoteFeatures.ChangeNotify != nil {
			clearChangeNotify = false
		}
		if remoteFeatures.DirCacheFlush != nil {
			clearDirCacheFlush = false
		}
	}
	if clearChangeNotify {
		features.ChangeNotify = nil
	}
	if clearDirCacheFlush {
		features.DirCacheFlush = nil
	}

	f.features = features

	// Get common intersection of hashes
	hashSet := f.remotes[0].Hashes()
	for _, remote := range f.remotes[1:] {
		hashSet = hashSet.Overlap(remote.Hashes())
	}
	f.hashSet = hashSet

	return f, nil
}

// Check the interfaces are satisfied
var (
	_ fs.Fs              = (*Fs)(nil)
	_ fs.Purger          = (*Fs)(nil)
	_ fs.PutStreamer     = (*Fs)(nil)
	_ fs.Copier          = (*Fs)(nil)
	_ fs.Mover           = (*Fs)(nil)
	_ fs.DirMover        = (*Fs)(nil)
	_ fs.DirCacheFlusher = (*Fs)(nil)
	_ fs.ChangeNotifier  = (*Fs)(nil)
	_ fs.Abouter         = (*Fs)(nil)
)
<|endoftext|>


<commit_before>package influxdb

import (
	"regexp"
	"sort"
	"strings"
	"sync"
)

// Index is the in memory index structure for answering queries about measurements, tags
// and series within a database.
type Index struct {
	mu                  sync.RWMutex
	measurementIndex    map[string]*measurementIndex // map measurement name to its tag index
	seriesToMeasurement map[uint32]*Measurement      // map series id to its measurement
	series              map[uint32]*Series           // map series id to the Series object
	names               []string                     // sorted list of the measurement names
}

func NewIndex() *Index {
	return &Index{
		measurementIndex:    make(map[string]*measurementIndex),
		seriesToMeasurement: make(map[uint32]*Measurement),
		series:              make(map[uint32]*Series),
		names:               make([]string, 0),
	}
}

// Filter represents a tag filter when looking up other tags or measurements.
type Filter struct {
	Not   bool
	Key   string
	Value string
	Regex *regexp.Regexp
}

type Filters []*Filter

// SeriesIDs is a convenience type for sorting, checking equality, and doing union and
// intersection of collections of series ids.
type SeriesIDs []uint32

func (p SeriesIDs) Len() int           { return len(p) }
func (p SeriesIDs) Less(i, j int) bool { return p[i] < p[j] }
func (p SeriesIDs) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// Equals assumes that both are sorted. This is by design, no touchy!
func (a SeriesIDs) Equals(seriesIDs SeriesIDs) bool {
	if len(a) != len(seriesIDs) {
		return false
	}
	for i, s := range seriesIDs {
		if a[i] != s {
			return false
		}
	}
	return true
}

// Intersect returns a new collection of series ids in sorted order that is the intersection of the two.
// The two collections must already be sorted.
func (s SeriesIDs) Intersect(a SeriesIDs) SeriesIDs {
	l := s
	r := a

	// we want to iterate through the shortest one and stop
	if len(a) < len(s) {
		l = a
		r = s
	}

	// they're in sorted order so advance the counter as needed.
	// That is, don't run comparisons against lower values that we've already passed
	var i, j int

	ids := make([]uint32, 0, len(l))
	for i < len(l) {
		if l[i] == r[j] {
			ids = append(ids, l[i])
			i += 1
			j += 1
		} else if l[i] < r[j] {
			i += 1
		} else {
			j += 1
		}
	}

	return SeriesIDs(ids)
}

// Union returns a new collection of series ids in sorted order that is the union of the two.
// The two collections must already be sorted.
func (s SeriesIDs) Union(a SeriesIDs) SeriesIDs {
	l := s
	r := a

	ids := make([]uint32, 0, len(l)+len(r))
	var i, j int
	for i < len(l) && j < len(r) {
		if l[i] == r[j] {
			ids = append(ids, l[i])
			i += 1
			j += 1
		} else if l[i] < r[j] {
			ids = append(ids, l[i])
			i += 1
		} else {
			ids = append(ids, r[j])
			j += 1
		}
	}

	// now append the remainder
	if i < len(l) {
		ids = append(ids, l[i:]...)
	} else if j < len(r) {
		ids = append(ids, r[j:]...)
	}

	return ids
}

// Reject returns a new collection of series ids in sorted order with the passed in set removed from the original. This is useful for the NOT operator.
// The two collections must already be sorted.
func (l SeriesIDs) Reject(r SeriesIDs) SeriesIDs {
	var i, j int

	ids := make([]uint32, 0, len(l))
	for i < len(l) && j < len(r) {
		if l[i] == r[j] {
			i += 1
			j += 1
		} else if l[i] < r[j] {
			ids = append(ids, l[i])
			i += 1
		} else {
			j += 1
		}
	}

	// append the remainder
	if i < len(l) {
		ids = append(ids, l[i:]...)
	}

	return SeriesIDs(ids)
}

// Keeps a mapping of the series in a measurement
type measurementIndex struct {
	series         map[string]*Series // sorted tag string to the series object
	measurement    *Measurement
	seriesByTagset map[string]map[string]SeriesIDs // map from tag key to value to sorted set of series ids
	ids            SeriesIDs                       // sorted list of series IDs in this measurement
}

// addSeries will add a series to the measurementIndex. Returns false if already present
func (m *measurementIndex) addSeries(s *Series) bool {
	tagset := string(marshalTags(s.Tags))
	if _, ok := m.series[tagset]; ok {
		return false
	}
	m.series[tagset] = s
	m.ids = append(m.ids, s.ID)
	// the series ID should always be higher than all others because it's a new
	// series. So don't do the sort if we don't have to.
	if len(m.ids) > 1 && m.ids[len(m.ids)-1] < m.ids[len(m.ids)-2] {
		sort.Sort(m.ids)
	}

	// add this series id to the tag index on the measurement
	for k, v := range s.Tags {
		valueMap := m.seriesByTagset[k]
		if valueMap == nil {
			valueMap = make(map[string]SeriesIDs)
			m.seriesByTagset[k] = valueMap
		}
		ids := valueMap[v]
		ids = append(ids, s.ID)

		// most of the time the series ID will be higher than all others because it's a new
		// series. So don't do the sort if we don't have to.
		if len(ids) > 1 && ids[len(ids)-1] < ids[len(ids)-2] {
			sort.Sort(ids)
		}
		valueMap[v] = ids
	}

	return true
}

// seriesByTags returns the Series that matches the given tagset.
func (m *measurementIndex) seriesByTags(tags map[string]string) *Series {
	return m.series[string(marshalTags(tags))]
}

// sereisIDs returns the series ids for a given filter
func (m measurementIndex) seriesIDs(filter *Filter) (ids SeriesIDs) {
	values := m.seriesByTagset[filter.Key]
	if values == nil {
		return
	}

	// hanlde regex filters
	if filter.Regex != nil {
		for k, v := range values {
			if filter.Regex.MatchString(k) {
				if ids == nil {
					ids = v
				} else {
					ids = ids.Union(v)
				}
			}
		}
		if filter.Not {
			ids = m.ids.Reject(ids)
		}
		return
	}

	// this is for the value is not null query
	if filter.Not && filter.Value == "" {
		for _, v := range values {
			if ids == nil {
				ids = v
			} else {
				ids.Intersect(v)
			}
		}
		return
	}

	// get the ids that have the given key/value tag pair
	ids = SeriesIDs(values[filter.Value])

	// filter out these ids from the entire set if it's a not query
	if filter.Not {
		ids = m.ids.Reject(ids)
	}

	return
}

// AddSeries adds the series for the given measurement to the index. Returns false if already present
func (t *Index) AddSeries(name string, s *Series) bool {
	t.mu.Lock()
	defer t.mu.Unlock()

	// if there is a measurement for this id, it's already been added
	if t.seriesToMeasurement[s.ID] != nil {
		return false
	}

	// get or create the measurement index
	idx := t.measurementIndex[name]
	if idx == nil {
		idx = &measurementIndex{
			series:         make(map[string]*Series),
			measurement:    NewMeasurement(name),
			seriesByTagset: make(map[string]map[string]SeriesIDs),
			ids:            SeriesIDs(make([]uint32, 0)),
		}
		t.measurementIndex[name] = idx
		t.names = append(t.names, name)
		sort.Strings(t.names)
	}
	idx.measurement.Series = append(idx.measurement.Series, s)
	t.seriesToMeasurement[s.ID] = idx.measurement
	t.series[s.ID] = s

	// TODO: add this series to the global tag index

	b := idx.addSeries(s)
	return b
}

// AddField adds a field to the measurement name. Returns false if already present
func (t *Index) AddField(name string, f *Field) bool {
	panic("not implemented")
	return false
}

// SeriesIDs returns an array of series ids for the given measurements and filters to be applied to all.
// Filters are equivalent to and AND operation. If you want to do an OR, get the series IDs for one set,
// then get the series IDs for another set and use the SeriesIDs.Union to combine the two.
func (t *Index) SeriesIDs(names []string, filters Filters) SeriesIDs {
	t.mu.RLock()
	defer t.mu.RUnlock()

	// they want all ids if no filters are specified
	if len(filters) == 0 {
		ids := SeriesIDs(make([]uint32, 0))
		for _, idx := range t.measurementIndex {
			ids = ids.Union(idx.ids)
		}
		return ids
	}

	ids := SeriesIDs(make([]uint32, 0))
	for _, n := range names {
		ids = ids.Union(t.seriesIDsForName(n, filters))
	}

	return ids
}

// TagKeys returns a sorted array of unique tag keys for the given measurements.
func (t *Index) TagKeys(names []string) []string {
	t.mu.RLock()
	defer t.mu.RUnlock()

	if len(names) == 0 {
		names = t.names
	}

	keys := make(map[string]bool)
	for _, n := range names {
		idx := t.measurementIndex[n]
		if idx != nil {
			for k, _ := range idx.seriesByTagset {
				keys[k] = true
			}
		}
	}

	sortedKeys := make([]string, 0, len(keys))
	for k, _ := range keys {
		sortedKeys = append(sortedKeys, k)
	}
	sort.Strings(sortedKeys)

	return sortedKeys
}

//seriesIDsForName is the same as SeriesIDs, but for a specific measurement.
func (t *Index) seriesIDsForName(name string, filters Filters) SeriesIDs {
	idx := t.measurementIndex[name]
	if idx == nil {
		return nil
	}

	// process the filters one at a time to get the list of ids they return
	idsPerFilter := make([]SeriesIDs, len(filters), len(filters))
	for i, filter := range filters {
		idsPerFilter[i] = idx.seriesIDs(filter)
	}

	// collapse the set of ids
	allIDs := idsPerFilter[0]
	for i := 1; i < len(filters); i++ {
		allIDs = allIDs.Intersect(idsPerFilter[i])
	}

	return allIDs
}

// MeasurementBySeriesID returns the Measurement that is the parent of the given series id.
func (t *Index) MeasurementBySeriesID(id uint32) *Measurement {
	t.mu.RLock()
	defer t.mu.RUnlock()
	return t.seriesToMeasurement[id]
}

// MeasurementAndSeries returns the Measurement and the Series for a given measurement name and tag set.
func (t *Index) MeasurementAndSeries(name string, tags map[string]string) (*Measurement, *Series) {
	t.mu.RLock()
	defer t.mu.RUnlock()
	idx := t.measurementIndex[name]
	if idx == nil {
		return nil, nil
	}
	return idx.measurement, idx.seriesByTags(tags)
}

// SereiesByID returns the Series that has the given id.
func (t *Index) SeriesByID(id uint32) *Series {
	return t.series[id]
}

// Measurements returns all measurements that match the given filters.
func (t *Index) Measurements(filters []*Filter) []*Measurement {
	measurements := make([]*Measurement, 0, len(t.measurementIndex))
	for _, idx := range t.measurementIndex {
		measurements = append(measurements, idx.measurement)
	}
	return measurements
}

// Names returns all measuremet names in sorted order.
func (t *Index) Names() []string {
	t.mu.Lock()
	defer t.mu.Unlock()
	return t.names
}

// DropSeries will clear the index of all references to a series.
func (t *Index) DropSeries(id uint32) {
	panic("not implemented")
}

// DropMeasurement will clear the index of all references to a measurement and its child series.
func (t *Index) DropMeasurement(name string) {
	panic("not implemented")
}

// used to convert the tag set to bytes for use as a lookup key
func marshalTags(tags map[string]string) []byte {
	s := make([]string, 0, len(tags))
	// pull out keys to sort
	for k := range tags {
		s = append(s, k)
	}
	sort.Strings(s)

	// now append on the key values in key sorted order
	for _, k := range s {
		s = append(s, tags[k])
	}
	return []byte(strings.Join(s, "|"))
}
<commit_msg>Small stype changes based on feedback<commit_after>package influxdb

import (
	"regexp"
	"sort"
	"strings"
	"sync"
)

// Index is the in memory index structure for answering queries about measurements, tags
// and series within a database.
type Index struct {
	mu                  sync.RWMutex
	measurementIndex    map[string]*measurementIndex // map measurement name to its tag index
	seriesToMeasurement map[uint32]*Measurement      // map series id to its measurement
	series              map[uint32]*Series           // map series id to the Series object
	names               []string                     // sorted list of the measurement names
}

func NewIndex() *Index {
	return &Index{
		measurementIndex:    make(map[string]*measurementIndex),
		seriesToMeasurement: make(map[uint32]*Measurement),
		series:              make(map[uint32]*Series),
		names:               make([]string, 0),
	}
}

// Filter represents a tag filter when looking up other tags or measurements.
type Filter struct {
	Not   bool
	Key   string
	Value string
	Regex *regexp.Regexp
}

type Filters []*Filter

// SeriesIDs is a convenience type for sorting, checking equality, and doing union and
// intersection of collections of series ids.
type SeriesIDs []uint32

func (p SeriesIDs) Len() int           { return len(p) }
func (p SeriesIDs) Less(i, j int) bool { return p[i] < p[j] }
func (p SeriesIDs) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// Equals assumes that both are sorted. This is by design, no touchy!
func (a SeriesIDs) Equals(seriesIDs SeriesIDs) bool {
	if len(a) != len(seriesIDs) {
		return false
	}
	for i, s := range seriesIDs {
		if a[i] != s {
			return false
		}
	}
	return true
}

// Intersect returns a new collection of series ids in sorted order that is the intersection of the two.
// The two collections must already be sorted.
func (a SeriesIDs) Intersect(seriesIDs SeriesIDs) SeriesIDs {
	l := a
	r := seriesIDs

	// we want to iterate through the shortest one and stop
	if len(seriesIDs) < len(a) {
		l = seriesIDs
		r = a
	}

	// they're in sorted order so advance the counter as needed.
	// That is, don't run comparisons against lower values that we've already passed
	var i, j int

	ids := make([]uint32, 0, len(l))
	for i < len(l) {
		if l[i] == r[j] {
			ids = append(ids, l[i])
			i += 1
			j += 1
		} else if l[i] < r[j] {
			i += 1
		} else {
			j += 1
		}
	}

	return SeriesIDs(ids)
}

// Union returns a new collection of series ids in sorted order that is the union of the two.
// The two collections must already be sorted.
func (l SeriesIDs) Union(r SeriesIDs) SeriesIDs {
	ids := make([]uint32, 0, len(l)+len(r))
	var i, j int
	for i < len(l) && j < len(r) {
		if l[i] == r[j] {
			ids = append(ids, l[i])
			i += 1
			j += 1
		} else if l[i] < r[j] {
			ids = append(ids, l[i])
			i += 1
		} else {
			ids = append(ids, r[j])
			j += 1
		}
	}

	// now append the remainder
	if i < len(l) {
		ids = append(ids, l[i:]...)
	} else if j < len(r) {
		ids = append(ids, r[j:]...)
	}

	return ids
}

// Reject returns a new collection of series ids in sorted order with the passed in set removed from the original. This is useful for the NOT operator.
// The two collections must already be sorted.
func (l SeriesIDs) Reject(r SeriesIDs) SeriesIDs {
	var i, j int

	ids := make([]uint32, 0, len(l))
	for i < len(l) && j < len(r) {
		if l[i] == r[j] {
			i += 1
			j += 1
		} else if l[i] < r[j] {
			ids = append(ids, l[i])
			i += 1
		} else {
			j += 1
		}
	}

	// append the remainder
	if i < len(l) {
		ids = append(ids, l[i:]...)
	}

	return SeriesIDs(ids)
}

// Keeps a mapping of the series in a measurement
type measurementIndex struct {
	series         map[string]*Series // sorted tag string to the series object
	measurement    *Measurement
	seriesByTagset map[string]map[string]SeriesIDs // map from tag key to value to sorted set of series ids
	ids            SeriesIDs                       // sorted list of series IDs in this measurement
}

// addSeries will add a series to the measurementIndex. Returns false if already present
func (m *measurementIndex) addSeries(s *Series) bool {
	tagset := string(marshalTags(s.Tags))
	if _, ok := m.series[tagset]; ok {
		return false
	}
	m.series[tagset] = s
	m.ids = append(m.ids, s.ID)
	// the series ID should always be higher than all others because it's a new
	// series. So don't do the sort if we don't have to.
	if len(m.ids) > 1 && m.ids[len(m.ids)-1] < m.ids[len(m.ids)-2] {
		sort.Sort(m.ids)
	}

	// add this series id to the tag index on the measurement
	for k, v := range s.Tags {
		valueMap := m.seriesByTagset[k]
		if valueMap == nil {
			valueMap = make(map[string]SeriesIDs)
			m.seriesByTagset[k] = valueMap
		}
		ids := valueMap[v]
		ids = append(ids, s.ID)

		// most of the time the series ID will be higher than all others because it's a new
		// series. So don't do the sort if we don't have to.
		if len(ids) > 1 && ids[len(ids)-1] < ids[len(ids)-2] {
			sort.Sort(ids)
		}
		valueMap[v] = ids
	}

	return true
}

// seriesByTags returns the Series that matches the given tagset.
func (m *measurementIndex) seriesByTags(tags map[string]string) *Series {
	return m.series[string(marshalTags(tags))]
}

// sereisIDs returns the series ids for a given filter
func (m measurementIndex) seriesIDs(filter *Filter) (ids SeriesIDs) {
	values := m.seriesByTagset[filter.Key]
	if values == nil {
		return
	}

	// hanlde regex filters
	if filter.Regex != nil {
		for k, v := range values {
			if filter.Regex.MatchString(k) {
				if ids == nil {
					ids = v
				} else {
					ids = ids.Union(v)
				}
			}
		}
		if filter.Not {
			ids = m.ids.Reject(ids)
		}
		return
	}

	// this is for the value is not null query
	if filter.Not && filter.Value == "" {
		for _, v := range values {
			if ids == nil {
				ids = v
			} else {
				ids.Intersect(v)
			}
		}
		return
	}

	// get the ids that have the given key/value tag pair
	ids = SeriesIDs(values[filter.Value])

	// filter out these ids from the entire set if it's a not query
	if filter.Not {
		ids = m.ids.Reject(ids)
	}

	return
}

// AddSeries adds the series for the given measurement to the index. Returns false if already present
func (t *Index) AddSeries(name string, s *Series) bool {
	t.mu.Lock()
	defer t.mu.Unlock()

	// if there is a measurement for this id, it's already been added
	if t.seriesToMeasurement[s.ID] != nil {
		return false
	}

	// get or create the measurement index
	idx := t.measurementIndex[name]
	if idx == nil {
		idx = &measurementIndex{
			series:         make(map[string]*Series),
			measurement:    NewMeasurement(name),
			seriesByTagset: make(map[string]map[string]SeriesIDs),
			ids:            SeriesIDs(make([]uint32, 0)),
		}
		t.measurementIndex[name] = idx
		t.names = append(t.names, name)
		sort.Strings(t.names)
	}
	idx.measurement.Series = append(idx.measurement.Series, s)
	t.seriesToMeasurement[s.ID] = idx.measurement
	t.series[s.ID] = s

	// TODO: add this series to the global tag index

	b := idx.addSeries(s)
	return b
}

// AddField adds a field to the measurement name. Returns false if already present
func (t *Index) AddField(name string, f *Field) bool {
	panic("not implemented")
	return false
}

// SeriesIDs returns an array of series ids for the given measurements and filters to be applied to all.
// Filters are equivalent to and AND operation. If you want to do an OR, get the series IDs for one set,
// then get the series IDs for another set and use the SeriesIDs.Union to combine the two.
func (t *Index) SeriesIDs(names []string, filters Filters) SeriesIDs {
	t.mu.RLock()
	defer t.mu.RUnlock()

	// they want all ids if no filters are specified
	if len(filters) == 0 {
		ids := SeriesIDs(make([]uint32, 0))
		for _, idx := range t.measurementIndex {
			ids = ids.Union(idx.ids)
		}
		return ids
	}

	ids := SeriesIDs(make([]uint32, 0))
	for _, n := range names {
		ids = ids.Union(t.seriesIDsForName(n, filters))
	}

	return ids
}

// TagKeys returns a sorted array of unique tag keys for the given measurements.
func (t *Index) TagKeys(names []string) []string {
	t.mu.RLock()
	defer t.mu.RUnlock()

	if len(names) == 0 {
		names = t.names
	}

	keys := make(map[string]bool)
	for _, n := range names {
		idx := t.measurementIndex[n]
		if idx != nil {
			for k, _ := range idx.seriesByTagset {
				keys[k] = true
			}
		}
	}

	sortedKeys := make([]string, 0, len(keys))
	for k, _ := range keys {
		sortedKeys = append(sortedKeys, k)
	}
	sort.Strings(sortedKeys)

	return sortedKeys
}

//seriesIDsForName is the same as SeriesIDs, but for a specific measurement.
func (t *Index) seriesIDsForName(name string, filters Filters) SeriesIDs {
	idx := t.measurementIndex[name]
	if idx == nil {
		return nil
	}

	// process the filters one at a time to get the list of ids they return
	idsPerFilter := make([]SeriesIDs, len(filters), len(filters))
	for i, filter := range filters {
		idsPerFilter[i] = idx.seriesIDs(filter)
	}

	// collapse the set of ids
	allIDs := idsPerFilter[0]
	for i := 1; i < len(filters); i++ {
		allIDs = allIDs.Intersect(idsPerFilter[i])
	}

	return allIDs
}

// MeasurementBySeriesID returns the Measurement that is the parent of the given series id.
func (t *Index) MeasurementBySeriesID(id uint32) *Measurement {
	t.mu.RLock()
	defer t.mu.RUnlock()
	return t.seriesToMeasurement[id]
}

// MeasurementAndSeries returns the Measurement and the Series for a given measurement name and tag set.
func (t *Index) MeasurementAndSeries(name string, tags map[string]string) (*Measurement, *Series) {
	t.mu.RLock()
	defer t.mu.RUnlock()
	idx := t.measurementIndex[name]
	if idx == nil {
		return nil, nil
	}
	return idx.measurement, idx.seriesByTags(tags)
}

// SereiesByID returns the Series that has the given id.
func (t *Index) SeriesByID(id uint32) *Series {
	return t.series[id]
}

// Measurements returns all measurements that match the given filters.
func (t *Index) Measurements(filters []*Filter) []*Measurement {
	measurements := make([]*Measurement, 0, len(t.measurementIndex))
	for _, idx := range t.measurementIndex {
		measurements = append(measurements, idx.measurement)
	}
	return measurements
}

// Names returns all measuremet names in sorted order.
func (t *Index) Names() []string {
	t.mu.Lock()
	defer t.mu.Unlock()
	return t.names
}

// DropSeries will clear the index of all references to a series.
func (t *Index) DropSeries(id uint32) {
	panic("not implemented")
}

// DropMeasurement will clear the index of all references to a measurement and its child series.
func (t *Index) DropMeasurement(name string) {
	panic("not implemented")
}

// used to convert the tag set to bytes for use as a lookup key
func marshalTags(tags map[string]string) []byte {
	s := make([]string, 0, len(tags))
	// pull out keys to sort
	for k := range tags {
		s = append(s, k)
	}
	sort.Strings(s)

	// now append on the key values in key sorted order
	for _, k := range s {
		s = append(s, tags[k])
	}
	return []byte(strings.Join(s, "|"))
}
<|endoftext|>


<commit_before>// Copyright ©2014 The bíogo.bam Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package bam

import (
	"code.google.com/p/biogo.bam/bgzf"

	"errors"
	"sort"
)

var baiMagic = [4]byte{'B', 'A', 'I', 0x1}

const (
	tileWidth     = 0x4000
	statsDummyBin = 0x924a
)

func makeOffset(vOff uint64) bgzf.Offset {
	return bgzf.Offset{
		File:  int64(vOff >> 16),
		Block: uint16(vOff),
	}
}

func isZero(o bgzf.Offset) bool {
	return o == bgzf.Offset{}
}

func vOffset(o bgzf.Offset) int64 {
	return o.File<<16 | int64(o.Block)
}

func isPlaced(r *Record) bool {
	return r.Ref != nil && r.Pos != -1
}

type byBinNumber []Bin

func (b byBinNumber) Len() int           { return len(b) }
func (b byBinNumber) Less(i, j int) bool { return b[i].Bin < b[j].Bin }
func (b byBinNumber) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }

type byBeginOffset []Chunk

func (c byBeginOffset) Len() int           { return len(c) }
func (c byBeginOffset) Less(i, j int) bool { return vOffset(c[i].Begin) < vOffset(c[j].Begin) }
func (c byBeginOffset) Swap(i, j int)      { c[i], c[j] = c[j], c[i] }

type byVirtOffset []bgzf.Offset

func (o byVirtOffset) Len() int           { return len(o) }
func (o byVirtOffset) Less(i, j int) bool { return vOffset(o[i]) < vOffset(o[j]) }
func (o byVirtOffset) Swap(i, j int)      { o[i], o[j] = o[j], o[i] }

type Index struct {
	References []RefIndex
	Unmapped   *uint64
	isSorted   bool
	lastRecord int
}

type RefIndex struct {
	Bins      []Bin
	Stats     *IndexStats
	Intervals []bgzf.Offset
}

type Bin struct {
	Bin    uint32
	Chunks []Chunk
}

type IndexStats struct {
	Chunk    Chunk
	Mapped   uint64
	Unmapped uint64
}

type Chunk struct {
	Begin bgzf.Offset
	End   bgzf.Offset
}

func (i *Index) Sort() {
	if !i.isSorted {
		for _, ref := range i.References {
			sort.Sort(byBinNumber(ref.Bins))
			for _, bin := range ref.Bins {
				sort.Sort(byBeginOffset(bin.Chunks))
			}
			sort.Sort(byVirtOffset(ref.Intervals))
		}
		i.isSorted = true
	}
}

func (i *Index) Add(r *Record, c Chunk) error {
	if i.Unmapped == nil {
		i.Unmapped = new(uint64)
	}
	if !isPlaced(r) {
		*i.Unmapped++
		return nil
	}

	rid := r.Reference().ID()
	if rid < len(i.References)-1 {
		return errors.New("bam: attempt to add record out of reference ID sort order")
	}
	if rid == len(i.References) {
		i.References = append(i.References, RefIndex{})
	} else {
		refs := make([]RefIndex, rid+1)
		copy(refs, i.References)
		i.References = refs
	}
	ref := &i.References[rid]

	// Record bin information.
	b := uint32(r.Bin())
	for i, bin := range ref.Bins {
		if bin.Bin == b {
			for j, chunk := range ref.Bins[i].Chunks {
				if vOffset(chunk.End) > vOffset(c.Begin) {
					ref.Bins[i].Chunks[j].End = c.End
					goto found
				}
			}
			ref.Bins[i].Chunks = append(ref.Bins[i].Chunks, c)
			goto found
		}
	}
	i.isSorted = false // TODO(kortschak) Consider making use of this more effectively for bin search.
	ref.Bins = append(ref.Bins, Bin{
		Bin:    b,
		Chunks: []Chunk{c},
	})
found:

	// Record interval tile information.
	biv := r.Start() / tileWidth
	if r.Start() < i.lastRecord {
		return errors.New("bam: attempt to add record out of position sort order")
	}
	i.lastRecord = r.Start()
	eiv := r.End() / tileWidth
	if eiv == len(ref.Intervals) {
		if eiv > biv {
			panic("bam: unexpected alignment length")
		}
		ref.Intervals = append(ref.Intervals, c.Begin)
	} else if eiv > len(ref.Intervals) {
		intvs := make([]bgzf.Offset, eiv)
		if len(ref.Intervals) > biv {
			biv = len(ref.Intervals)
		}
		for iv, offset := range intvs[biv:eiv] {
			if !isZero(offset) {
				panic("bam: unexpected non-zero offset")
			}
			intvs[iv+biv] = c.Begin
		}
		copy(intvs, ref.Intervals)
		ref.Intervals = intvs
	}

	// Record index stats.
	if ref.Stats == nil {
		ref.Stats = &IndexStats{
			Chunk: c,
		}
	} else {
		ref.Stats.Chunk.End = c.End
	}
	if r.Flags&Unmapped == 0 {
		ref.Stats.Mapped++
	} else {
		ref.Stats.Unmapped++
	}

	return nil
}

func (i *Index) Chunks(rid, beg, end int) []Chunk {
	if rid >= len(i.References) {
		return nil
	}
	i.Sort()
	ref := i.References[rid]

	iv := beg / tileWidth
	if iv >= len(ref.Intervals) {
		return nil
	}

	// Collect candidate chunks according to the scheme described in
	// the SAM spec under section 5 Indexing BAM.
	var chunks []Chunk
	for _, bin := range reg2bins(beg, end) {
		b := uint32(bin)
		c := sort.Search(len(ref.Bins), func(i int) bool { return ref.Bins[i].Bin >= b })
		if c < len(ref.Bins) && ref.Bins[c].Bin == b {
			for _, chunk := range ref.Bins[c].Chunks {
				// Here we check all tiles starting from the left end of the
				// query region until we get a non-zero offset. The spec states
				// that we only need to check tiles that contain beg. That is
				// not correct since we may have no alignments at the left end
				// of the query region.
				for j, tile := range ref.Intervals[iv:] {
					if isZero(tile) {
						continue
					}
					tbeg := (j + iv) * tileWidth
					tend := tbeg + tileWidth
					// We allow adjacent alignment since samtools behaviour here
					// has always irritated me and it is cheap to discard these
					// later if they are not wanted.
					if tend >= beg && tbeg <= end && vOffset(chunk.End) > vOffset(tile) {
						chunks = append(chunks, chunk)
						break
					}
				}
			}
		}
	}

	// Sort and merge overlaps.
	if !sort.IsSorted(byBeginOffset(chunks)) {
		sort.Sort(byBeginOffset(chunks))
	}
	for c := 1; c < len(chunks); c++ {
		leftChunk := &chunks[c-1]
		rightChunk := &chunks[c]
		leftEndOffset := vOffset(leftChunk.End)
		if leftEndOffset >= vOffset(rightChunk.Begin) {
			rightChunk.Begin = leftChunk.Begin
			if leftEndOffset > vOffset(rightChunk.End) {
				rightChunk.End = leftChunk.End
			}
			chunks = append(chunks[:c-1], chunks[c:]...)
			c--
		}
	}

	return chunks
}
<commit_msg>Implement chunk merging<commit_after>// Copyright ©2014 The bíogo.bam Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package bam

import (
	"code.google.com/p/biogo.bam/bgzf"

	"errors"
	"sort"
)

var baiMagic = [4]byte{'B', 'A', 'I', 0x1}

const (
	tileWidth     = 0x4000
	statsDummyBin = 0x924a
)

func makeOffset(vOff uint64) bgzf.Offset {
	return bgzf.Offset{
		File:  int64(vOff >> 16),
		Block: uint16(vOff),
	}
}

func isZero(o bgzf.Offset) bool {
	return o == bgzf.Offset{}
}

func vOffset(o bgzf.Offset) int64 {
	return o.File<<16 | int64(o.Block)
}

func isPlaced(r *Record) bool {
	return r.Ref != nil && r.Pos != -1
}

type byBinNumber []Bin

func (b byBinNumber) Len() int           { return len(b) }
func (b byBinNumber) Less(i, j int) bool { return b[i].Bin < b[j].Bin }
func (b byBinNumber) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }

type byBeginOffset []Chunk

func (c byBeginOffset) Len() int           { return len(c) }
func (c byBeginOffset) Less(i, j int) bool { return vOffset(c[i].Begin) < vOffset(c[j].Begin) }
func (c byBeginOffset) Swap(i, j int)      { c[i], c[j] = c[j], c[i] }

type byVirtOffset []bgzf.Offset

func (o byVirtOffset) Len() int           { return len(o) }
func (o byVirtOffset) Less(i, j int) bool { return vOffset(o[i]) < vOffset(o[j]) }
func (o byVirtOffset) Swap(i, j int)      { o[i], o[j] = o[j], o[i] }

type Index struct {
	References []RefIndex
	Unmapped   *uint64
	isSorted   bool
	lastRecord int
}

type RefIndex struct {
	Bins      []Bin
	Stats     *IndexStats
	Intervals []bgzf.Offset
}

type Bin struct {
	Bin    uint32
	Chunks []Chunk
}

type IndexStats struct {
	Chunk    Chunk
	Mapped   uint64
	Unmapped uint64
}

type Chunk struct {
	Begin bgzf.Offset
	End   bgzf.Offset
}

func (i *Index) Sort() {
	if !i.isSorted {
		for _, ref := range i.References {
			sort.Sort(byBinNumber(ref.Bins))
			for _, bin := range ref.Bins {
				sort.Sort(byBeginOffset(bin.Chunks))
			}
			sort.Sort(byVirtOffset(ref.Intervals))
		}
		i.isSorted = true
	}
}

func (i *Index) Add(r *Record, c Chunk) error {
	if i.Unmapped == nil {
		i.Unmapped = new(uint64)
	}
	if !isPlaced(r) {
		*i.Unmapped++
		return nil
	}

	rid := r.Reference().ID()
	if rid < len(i.References)-1 {
		return errors.New("bam: attempt to add record out of reference ID sort order")
	}
	if rid == len(i.References) {
		i.References = append(i.References, RefIndex{})
	} else {
		refs := make([]RefIndex, rid+1)
		copy(refs, i.References)
		i.References = refs
	}
	ref := &i.References[rid]

	// Record bin information.
	b := uint32(r.Bin())
	for i, bin := range ref.Bins {
		if bin.Bin == b {
			for j, chunk := range ref.Bins[i].Chunks {
				if vOffset(chunk.End) > vOffset(c.Begin) {
					ref.Bins[i].Chunks[j].End = c.End
					goto found
				}
			}
			ref.Bins[i].Chunks = append(ref.Bins[i].Chunks, c)
			goto found
		}
	}
	i.isSorted = false // TODO(kortschak) Consider making use of this more effectively for bin search.
	ref.Bins = append(ref.Bins, Bin{
		Bin:    b,
		Chunks: []Chunk{c},
	})
found:

	// Record interval tile information.
	biv := r.Start() / tileWidth
	if r.Start() < i.lastRecord {
		return errors.New("bam: attempt to add record out of position sort order")
	}
	i.lastRecord = r.Start()
	eiv := r.End() / tileWidth
	if eiv == len(ref.Intervals) {
		if eiv > biv {
			panic("bam: unexpected alignment length")
		}
		ref.Intervals = append(ref.Intervals, c.Begin)
	} else if eiv > len(ref.Intervals) {
		intvs := make([]bgzf.Offset, eiv)
		if len(ref.Intervals) > biv {
			biv = len(ref.Intervals)
		}
		for iv, offset := range intvs[biv:eiv] {
			if !isZero(offset) {
				panic("bam: unexpected non-zero offset")
			}
			intvs[iv+biv] = c.Begin
		}
		copy(intvs, ref.Intervals)
		ref.Intervals = intvs
	}

	// Record index stats.
	if ref.Stats == nil {
		ref.Stats = &IndexStats{
			Chunk: c,
		}
	} else {
		ref.Stats.Chunk.End = c.End
	}
	if r.Flags&Unmapped == 0 {
		ref.Stats.Mapped++
	} else {
		ref.Stats.Unmapped++
	}

	return nil
}

func (i *Index) Chunks(rid, beg, end int) []Chunk {
	if rid >= len(i.References) {
		return nil
	}
	i.Sort()
	ref := i.References[rid]

	iv := beg / tileWidth
	if iv >= len(ref.Intervals) {
		return nil
	}

	// Collect candidate chunks according to the scheme described in
	// the SAM spec under section 5 Indexing BAM.
	var chunks []Chunk
	for _, bin := range reg2bins(beg, end) {
		b := uint32(bin)
		c := sort.Search(len(ref.Bins), func(i int) bool { return ref.Bins[i].Bin >= b })
		if c < len(ref.Bins) && ref.Bins[c].Bin == b {
			for _, chunk := range ref.Bins[c].Chunks {
				// Here we check all tiles starting from the left end of the
				// query region until we get a non-zero offset. The spec states
				// that we only need to check tiles that contain beg. That is
				// not correct since we may have no alignments at the left end
				// of the query region.
				for j, tile := range ref.Intervals[iv:] {
					if isZero(tile) {
						continue
					}
					tbeg := (j + iv) * tileWidth
					tend := tbeg + tileWidth
					// We allow adjacent alignment since samtools behaviour here
					// has always irritated me and it is cheap to discard these
					// later if they are not wanted.
					if tend >= beg && tbeg <= end && vOffset(chunk.End) > vOffset(tile) {
						chunks = append(chunks, chunk)
						break
					}
				}
			}
		}
	}

	// Sort and merge overlaps.
	if !sort.IsSorted(byBeginOffset(chunks)) {
		sort.Sort(byBeginOffset(chunks))
	}

	return adjacent(chunks)
}

type Strategy func([]Chunk) []Chunk

var (
	Identity Strategy = identity
	Adjacent Strategy = adjacent
	Squash   Strategy = squash
)

func CompressorStrategy(near int64) Strategy {
	const blockMask = 1<<16 - 1
	return func(chunks []Chunk) []Chunk {
		if len(chunks) == 0 {
			return nil
		}
		for c := 1; c < len(chunks); c++ {
			leftChunk := chunks[c-1]
			leftChunk.End.File += near
			rightChunk := &chunks[c]
			leftEndOffset := vOffset(leftChunk.End)
			if leftEndOffset&^blockMask >= vOffset(rightChunk.Begin)&^blockMask {
				rightChunk.Begin = leftChunk.Begin
				if leftEndOffset > vOffset(rightChunk.End) {
					rightChunk.End = leftChunk.End
				}
				chunks = append(chunks[:c-1], chunks[c:]...)
				c--
			}
		}
		return chunks
	}
}

func identity(chunks []Chunk) []Chunk { return chunks }

func adjacent(chunks []Chunk) []Chunk {
	if len(chunks) == 0 {
		return nil
	}
	for c := 1; c < len(chunks); c++ {
		leftChunk := chunks[c-1]
		rightChunk := &chunks[c]
		leftEndOffset := vOffset(leftChunk.End)
		if leftEndOffset >= vOffset(rightChunk.Begin) {
			rightChunk.Begin = leftChunk.Begin
			if leftEndOffset > vOffset(rightChunk.End) {
				rightChunk.End = leftChunk.End
			}
			chunks = append(chunks[:c-1], chunks[c:]...)
			c--
		}
	}
	return chunks
}

func squash(chunks []Chunk) []Chunk {
	if len(chunks) == 0 {
		return nil
	}
	left := chunks[0].Begin
	right := chunks[0].End
	for _, c := range chunks[1:] {
		if vOffset(c.End) > vOffset(right) {
			right = c.End
		}
	}
	return []Chunk{{Begin: left, End: right}}
}

func (i *Index) MergeChunks(s Strategy) {
	if s == nil {
		return
	}
	for r, ref := range i.References {
		for b, bin := range ref.Bins {
			if !sort.IsSorted(byBeginOffset(bin.Chunks)) {
				sort.Sort(byBeginOffset(bin.Chunks))
			}
			i.References[r].Bins[b].Chunks = s(bin.Chunks)
			if !sort.IsSorted(byBeginOffset(bin.Chunks)) {
				sort.Sort(byBeginOffset(bin.Chunks))
			}
		}
	}
}
<|endoftext|>


<commit_before>package rpm

import (
	"time"
)

// Header index value data types.
const (
	IndexDataTypeNull int = iota
	IndexDataTypeChar
	IndexDataTypeInt8
	IndexDataTypeInt16
	IndexDataTypeInt32
	IndexDataTypeInt64
	IndexDataTypeString
	IndexDataTypeBinary
	IndexDataTypeStringArray
	IndexDataTypeI8NString
)

// An IndexEntry is a rpm key/value tag stored in the package header.
type IndexEntry struct {
	Tag       int
	Type      int
	Offset    int
	ItemCount int
	Value     interface{}
}

// IndexEntries is an array of IndexEntry structs.
type IndexEntries []IndexEntry

// IndexByTag returns a pointer to an IndexEntry with the given tag ID or nil if
// the tag is not found.
func (c IndexEntries) IndexByTag(tag int) *IndexEntry {
	for _, e := range c {
		if e.Tag == tag {
			return &e
		}
	}

	return nil
}

// StringByTag returns the string value of an IndexEntry or an empty string if
// the tag is not found or has no value.
func (c IndexEntries) StringByTag(tag int) string {
	i := c.IndexByTag(tag)
	if i == nil || i.Value == nil {
		return ""
	}

	s := i.Value.([]string)

	return s[0]
}

// StringsByTag returns the slice of string values of an IndexEntry or nil if
// the tag is not found or has no value.
func (c IndexEntries) StringsByTag(tag int) []string {
	i := c.IndexByTag(tag)
	if i == nil || i.Value == nil {
		return nil
	}

	return i.Value.([]string)
}

// IntsByTag returns the int64 values of an IndexEntry or nil if the tag is not
// found or has no value. Values with a lower range (E.g. int8) are cast as an
// int64.
func (c IndexEntries) IntsByTag(tag int) []int64 {
	ix := c.IndexByTag(tag)
	if ix != nil && ix.Value != nil {
		vals := make([]int64, ix.ItemCount)

		for i := 0; i < int(ix.ItemCount); i++ {
			switch ix.Type {
			case IndexDataTypeChar, IndexDataTypeInt8:
				vals[i] = int64(ix.Value.([]int8)[i])

			case IndexDataTypeInt16:
				vals[i] = int64(ix.Value.([]int16)[i])

			case IndexDataTypeInt32:
				vals[i] = int64(ix.Value.([]int32)[i])

			case IndexDataTypeInt64:
				vals[i] = ix.Value.([]int64)[i]
			}
		}

		return vals
	}

	return nil
}

// IntByTag returns the int64 value of an IndexEntry or 0 if the tag is not found
// or has no value. Values with a lower range (E.g. int8) are cast as an int64.
func (c IndexEntries) IntByTag(tag int) int64 {
	i := c.IndexByTag(tag)
	if i != nil && i.Value != nil {
		switch i.Type {
		case IndexDataTypeChar, IndexDataTypeInt8:
			return int64(i.Value.([]int8)[0])

		case IndexDataTypeInt16:
			return int64(i.Value.([]int16)[0])

		case IndexDataTypeInt32:
			return int64(i.Value.([]int32)[0])

		case IndexDataTypeInt64:
			return int64(i.Value.([]int64)[0])
		}
	}

	return 0
}

// BytesByTag returns the raw value of an IndexEntry or nil if the tag is not
// found or has no value.
func (c IndexEntries) BytesByTag(tag int) []byte {
	i := c.IndexByTag(tag)
	if i == nil || i.Value == nil {
		return nil
	}

	return i.Value.([]byte)
}

// TimesByTag returns the value of an IndexEntry as a slice of Go native
// timestamps or nil if the tag is not found or has no value.
func (c IndexEntries) TimesByTag(tag int) []time.Time {
	ix := c.IndexByTag(tag)

	if ix == nil || ix.Value == nil {
		return nil
	}

	vals := make([]time.Time, ix.ItemCount)
	for i := 0; i < ix.ItemCount; i++ {
		vals[i] = time.Unix(int64(ix.Value.([]int32)[i]), 0)
	}

	return vals
}

// TimeByTag returns the value of an IndexEntry as a Go native timestamp or
// zero-time if the tag is not found or has no value.
func (c IndexEntries) TimeByTag(tag int) time.Time {
	vals := c.TimesByTag(tag)
	if vals == nil || len(vals) == 0 {
		return time.Time{}
	}

	return vals[0]
}
<commit_msg>Improved index getters to prevent panics on failed type assertions<commit_after>package rpm

import (
	"time"
)

// Header index value data types.
const (
	IndexDataTypeNull int = iota
	IndexDataTypeChar
	IndexDataTypeInt8
	IndexDataTypeInt16
	IndexDataTypeInt32
	IndexDataTypeInt64
	IndexDataTypeString
	IndexDataTypeBinary
	IndexDataTypeStringArray
	IndexDataTypeI8NString
)

// An IndexEntry is a rpm key/value tag stored in the package header.
type IndexEntry struct {
	Tag       int
	Type      int
	Offset    int
	ItemCount int
	Value     interface{}
}

// IndexEntries is an array of IndexEntry structs.
type IndexEntries []IndexEntry

// IndexByTag returns a pointer to an IndexEntry with the given tag ID or nil if
// the tag is not found.
func (c IndexEntries) IndexByTag(tag int) *IndexEntry {
	for _, e := range c {
		if e.Tag == tag {
			return &e
		}
	}

	return nil
}

// StringsByTag returns the slice of string values of an IndexEntry or nil if
// the tag is not found or has no value.
func (c IndexEntries) StringsByTag(tag int) []string {
	i := c.IndexByTag(tag)
	if i == nil || i.Value == nil {
		return nil
	}
	s, ok := i.Value.([]string)
	if !ok {
		return nil
	}
	return s
}

// StringByTag returns the string value of an IndexEntry or an empty string if
// the tag is not found or has no value.
func (c IndexEntries) StringByTag(tag int) string {
	s := c.StringsByTag(tag)
	if s == nil || len(s) == 0 {
		return ""
	}
	return s[0]
}

// IntsByTag returns the int64 values of an IndexEntry or nil if the tag is not
// found or has no value. Values with a lower range (E.g. int8) are cast as an
// int64.
func (c IndexEntries) IntsByTag(tag int) []int64 {
	ix := c.IndexByTag(tag)
	if ix != nil && ix.Value != nil {
		vals := make([]int64, ix.ItemCount)

		for i := 0; i < int(ix.ItemCount); i++ {
			switch ix.Type {
			case IndexDataTypeChar, IndexDataTypeInt8:
				vals[i] = int64(ix.Value.([]int8)[i])

			case IndexDataTypeInt16:
				vals[i] = int64(ix.Value.([]int16)[i])

			case IndexDataTypeInt32:
				vals[i] = int64(ix.Value.([]int32)[i])

			case IndexDataTypeInt64:
				vals[i] = ix.Value.([]int64)[i]
			}
		}

		return vals
	}

	return nil
}

// IntByTag returns the int64 value of an IndexEntry or 0 if the tag is not found
// or has no value. Values with a lower range (E.g. int8) are cast as an int64.
func (c IndexEntries) IntByTag(tag int) int64 {
	i := c.IndexByTag(tag)
	if i != nil && i.Value != nil {
		switch i.Type {
		case IndexDataTypeChar, IndexDataTypeInt8:
			v, ok := i.Value.([]int8)
			if !ok {
				return 0
			}
			return int64(v[0])

		case IndexDataTypeInt16:
			v, ok := i.Value.([]int16)
			if !ok {
				return 0
			}
			return int64(v[0])

		case IndexDataTypeInt32:
			v, ok := i.Value.([]int32)
			if !ok {
				return 0
			}
			return int64(v[0])

		case IndexDataTypeInt64:
			v, ok := i.Value.([]int64)
			if !ok {
				return 0
			}
			return v[0]
		}
	}

	return 0
}

// BytesByTag returns the raw value of an IndexEntry or nil if the tag is not
// found or has no value.
func (c IndexEntries) BytesByTag(tag int) []byte {
	i := c.IndexByTag(tag)
	if i == nil || i.Value == nil {
		return nil
	}
	b, ok := i.Value.([]byte)
	if !ok {
		return nil
	}
	return b
}

// TimesByTag returns the value of an IndexEntry as a slice of Go native
// timestamps or nil if the tag is not found or has no value.
func (c IndexEntries) TimesByTag(tag int) []time.Time {
	ix := c.IndexByTag(tag)
	if ix == nil || ix.Value == nil {
		return nil
	}
	v, ok := ix.Value.([]int32)
	if !ok {
		return nil
	}
	vals := make([]time.Time, ix.ItemCount)
	for i := 0; i < ix.ItemCount; i++ {
		vals[i] = time.Unix(int64(v[i]), 0)
	}
	return vals
}

// TimeByTag returns the value of an IndexEntry as a Go native timestamp or
// zero-time if the tag is not found or has no value.
func (c IndexEntries) TimeByTag(tag int) time.Time {
	vals := c.TimesByTag(tag)
	if vals == nil || len(vals) == 0 {
		return time.Time{}
	}
	return vals[0]
}
<|endoftext|>


<commit_before>// Copyright (c) 2012-2015 Ugorji Nwoke. All rights reserved.
// Use of this source code is governed by a MIT license found in the LICENSE file.

package codec

// All non-std package dependencies live in this file,
// so porting to different environment is easy (just update functions).

func pruneSignExt(v []byte, pos bool) (n int) {
	if len(v) < 2 {
	} else if pos && v[0] == 0 {
		for ; v[n] == 0 && n+1 < len(v) && (v[n+1]&(1<<7) == 0); n++ {
		}
	} else if !pos && v[0] == 0xff {
		for ; v[n] == 0xff && n+1 < len(v) && (v[n+1]&(1<<7) != 0); n++ {
		}
	}
	return
}

// validate that this function is correct ...
// culled from OGRE (Object-Oriented Graphics Rendering Engine)
// function: halfToFloatI (http://stderr.org/doc/ogre-doc/api/OgreBitwise_8h-source.html)
func halfFloatToFloatBits(yy uint16) (d uint32) {
	y := uint32(yy)
	s := (y >> 15) & 0x01
	e := (y >> 10) & 0x1f
	m := y & 0x03ff

	if e == 0 {
		if m == 0 { // plu or minus 0
			return s << 31
		}
		// Denormalized number -- renormalize it
		for (m & 0x00000400) == 0 {
			m <<= 1
			e -= 1
		}
		e += 1
		const zz uint32 = 0x0400
		m &= ^zz
	} else if e == 31 {
		if m == 0 { // Inf
			return (s << 31) | 0x7f800000
		}
		return (s << 31) | 0x7f800000 | (m << 13) // NaN
	}
	e = e + (127 - 15)
	m = m << 13
	return (s << 31) | (e << 23) | m
}

// GrowCap will return a new capacity for a slice, given the following:
//   - oldCap: current capacity
//   - unit: in-memory size of an element
//   - num: number of elements to add
func growCap(oldCap, unit, num int) (newCap int) {
	// appendslice logic (if cap < 1024, *2, else *1.25):
	//   leads to many copy calls, especially when copying bytes.
	//   bytes.Buffer model (2*cap + n): much better for bytes.
	// smarter way is to take the byte-size of the appended element(type) into account

	// maintain 2 thresholds:
	// t1: if cap <= t1, newcap = 2x
	// t2: if cap <= t2, newcap = 1.5x
	//     else          newcap = 1.25x
	//
	// t1, t2 >= 1024 always.
	// This means that, if unit size >= 16, then always do 2x or 1.25x (ie t1, t2, t3 are all same)
	//
	// With this, appending for bytes increase by:
	//    100% up to 4K
	//     75% up to 16K
	//     25% beyond that

	// unit can be 0 e.g. for struct{}{}; handle that appropriately
	if unit <= 0 {
		return 1<<63 - 1 // math.MaxInt64
	}

	// handle if num < 0, cap=0, etc.

	var t1, t2 int // thresholds
	if unit <= 4 {
		t1, t2 = 4*1024, 16*1024
	} else if unit <= 16 {
		t1, t2 = unit*1*1024, unit*4*1024
	} else {
		t1, t2 = 1024, 1024
	}

	if oldCap <= 0 {
		newCap = 2
	} else if oldCap <= t1 { // [0,t1]
		newCap = oldCap * 8 / 4
	} else if oldCap <= t2 { // (t1,t2]
		newCap = oldCap * 6 / 4
	} else { // (t2,infinity]
		newCap = oldCap * 5 / 4
	}

	if num > 0 && newCap < num+oldCap {
		newCap = num + oldCap
	}

	// ensure newCap takes multiples of a cache line (size is a multiple of 64)
	t1 = newCap * unit
	t2 = t1 % 64
	if t2 != 0 {
		t1 += 64 - t2
		newCap = t1 / unit
	}

	return
}
<commit_msg>codec: growCap: accomodate 32-bit OS<commit_after>// Copyright (c) 2012-2015 Ugorji Nwoke. All rights reserved.
// Use of this source code is governed by a MIT license found in the LICENSE file.

package codec

// All non-std package dependencies live in this file,
// so porting to different environment is easy (just update functions).

func pruneSignExt(v []byte, pos bool) (n int) {
	if len(v) < 2 {
	} else if pos && v[0] == 0 {
		for ; v[n] == 0 && n+1 < len(v) && (v[n+1]&(1<<7) == 0); n++ {
		}
	} else if !pos && v[0] == 0xff {
		for ; v[n] == 0xff && n+1 < len(v) && (v[n+1]&(1<<7) != 0); n++ {
		}
	}
	return
}

// validate that this function is correct ...
// culled from OGRE (Object-Oriented Graphics Rendering Engine)
// function: halfToFloatI (http://stderr.org/doc/ogre-doc/api/OgreBitwise_8h-source.html)
func halfFloatToFloatBits(yy uint16) (d uint32) {
	y := uint32(yy)
	s := (y >> 15) & 0x01
	e := (y >> 10) & 0x1f
	m := y & 0x03ff

	if e == 0 {
		if m == 0 { // plu or minus 0
			return s << 31
		}
		// Denormalized number -- renormalize it
		for (m & 0x00000400) == 0 {
			m <<= 1
			e -= 1
		}
		e += 1
		const zz uint32 = 0x0400
		m &= ^zz
	} else if e == 31 {
		if m == 0 { // Inf
			return (s << 31) | 0x7f800000
		}
		return (s << 31) | 0x7f800000 | (m << 13) // NaN
	}
	e = e + (127 - 15)
	m = m << 13
	return (s << 31) | (e << 23) | m
}

// GrowCap will return a new capacity for a slice, given the following:
//   - oldCap: current capacity
//   - unit: in-memory size of an element
//   - num: number of elements to add
func growCap(oldCap, unit, num int) (newCap int) {
	// appendslice logic (if cap < 1024, *2, else *1.25):
	//   leads to many copy calls, especially when copying bytes.
	//   bytes.Buffer model (2*cap + n): much better for bytes.
	// smarter way is to take the byte-size of the appended element(type) into account

	// maintain 2 thresholds:
	// t1: if cap <= t1, newcap = 2x
	// t2: if cap <= t2, newcap = 1.5x
	//     else          newcap = 1.25x
	//
	// t1, t2 >= 1024 always.
	// This means that, if unit size >= 16, then always do 2x or 1.25x (ie t1, t2, t3 are all same)
	//
	// With this, appending for bytes increase by:
	//    100% up to 4K
	//     75% up to 16K
	//     25% beyond that

	// unit can be 0 e.g. for struct{}{}; handle that appropriately
	if unit <= 0 {
		if uint64(^uint(0)) == ^uint64(0) { // 64-bit
			return int(1<<63 - 1) // math.MaxInt64
		}
		return int(1<<31 - 1) //  math.MaxInt32
	}

	// handle if num < 0, cap=0, etc.

	var t1, t2 int // thresholds
	if unit <= 4 {
		t1, t2 = 4*1024, 16*1024
	} else if unit <= 16 {
		t1, t2 = unit*1*1024, unit*4*1024
	} else {
		t1, t2 = 1024, 1024
	}

	if oldCap <= 0 {
		newCap = 2
	} else if oldCap <= t1 { // [0,t1]
		newCap = oldCap * 8 / 4
	} else if oldCap <= t2 { // (t1,t2]
		newCap = oldCap * 6 / 4
	} else { // (t2,infinity]
		newCap = oldCap * 5 / 4
	}

	if num > 0 && newCap < num+oldCap {
		newCap = num + oldCap
	}

	// ensure newCap takes multiples of a cache line (size is a multiple of 64)
	t1 = newCap * unit
	t2 = t1 % 64
	if t2 != 0 {
		t1 += 64 - t2
		newCap = t1 / unit
	}

	return
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"syscall"
	"time"

	"github.com/fsouza/go-dockerclient"
	"path/filepath"
)

type DockerDynoDriver struct {
	d *Docker
}

func (dd *DockerDynoDriver) Build(release *Release) error {
	if err := dd.connectDocker(); err != nil {
		return err
	}

	stack := "heroku/cedar:14"
	si, err := dd.d.StackStat(stack)
	if err != nil {
		return err
	}
	if si == nil {
		log.Fatalf("Stack not found = %v\n", stack)
	}

	imageName, err := dd.d.BuildSlugImage(si, release)
	if err != nil {
		log.Fatal(err)
	}
	log.Println("Built image successfully")

	release.imageName = imageName
	return nil
}

func (dd *DockerDynoDriver) Start(ex *Executor) error {
	// Fill environment vector from Heroku configuration.
	var env []string
	for k, v := range ex.release.config {
		env = append(env, k+"="+v)
	}
    if os.Getenv("HEROKU_ACCESS_TOKEN") != "" {
		env = append(env, "HEROKU_ACCESS_TOKEN="+os.Getenv("HEROKU_ACCESS_TOKEN"))
	}

	if os.Getenv("CONTROL_DIR") != "" {
		env = append(env, "CONTROL_DIR="+os.Getenv("CONTROL_DIR"))
	}
	// attach a timestamp as some extra entropy because container names must be
	// unique
	name := fmt.Sprintf("%v.%v", ex.Name(), time.Now().Unix())
	container, err := dd.d.c.CreateContainer(docker.CreateContainerOptions{
		Name: name,
		Config: &docker.Config{
			Cmd: append([]string{"setuidgid", "app",
				"/hsup", "-d", "abspath", "-a",
				ex.release.appName, "run", "--"},
				ex.args...),
			Env: append(env, "HSUP_SKIP_BUILD=TRUE"),
			Image:   ex.release.imageName,
			Volumes: map[string]struct{}{"/hsup": struct{}{}},
		},
	})
	if err != nil {
		log.Fatal(err)
	}
	ex.container = container

	where, err := filepath.Abs(linuxAmd64Path())
	if err != nil {
		return err
	}

	err = dd.d.c.StartContainer(ex.container.ID, &docker.HostConfig{
		Binds: []string{where + ":/hsup"},
	})
	if err != nil {
		log.Fatal(err)
	}

	go dd.d.c.Logs(docker.LogsOptions{
		Container:    container.ID,
		Stdout:       true,
		Stderr:       true,
		Follow:       true,
		OutputStream: os.Stdout,
	})

	return nil
}

func (dd *DockerDynoDriver) Wait(ex *Executor) (s *ExitStatus) {
	code, err := dd.d.c.WaitContainer(ex.container.ID)
	return &ExitStatus{code: code, err: err}
}

func (dd *DockerDynoDriver) Stop(ex *Executor) error {
	log.Println("Stopping container for", ex.Name())
	dd.d.c.KillContainer(docker.KillContainerOptions{
		ID:     ex.container.ID,
		Signal: docker.Signal(syscall.SIGTERM)})
	return dd.d.c.StopContainer(ex.container.ID, 10)
}

func (dd *DockerDynoDriver) connectDocker() error {
	if dd.d == nil {
		dd.d = &Docker{}
		if err := dd.d.Connect(); err != nil {
			dd.d = nil
			return err
		}
	}

	return nil
}
<commit_msg>Gofmt<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"syscall"
	"time"

	"github.com/fsouza/go-dockerclient"
	"path/filepath"
)

type DockerDynoDriver struct {
	d *Docker
}

func (dd *DockerDynoDriver) Build(release *Release) error {
	if err := dd.connectDocker(); err != nil {
		return err
	}

	stack := "heroku/cedar:14"
	si, err := dd.d.StackStat(stack)
	if err != nil {
		return err
	}
	if si == nil {
		log.Fatalf("Stack not found = %v\n", stack)
	}

	imageName, err := dd.d.BuildSlugImage(si, release)
	if err != nil {
		log.Fatal(err)
	}
	log.Println("Built image successfully")

	release.imageName = imageName
	return nil
}

func (dd *DockerDynoDriver) Start(ex *Executor) error {
	// Fill environment vector from Heroku configuration.
	var env []string
	for k, v := range ex.release.config {
		env = append(env, k+"="+v)
	}
	if os.Getenv("HEROKU_ACCESS_TOKEN") != "" {
		env = append(env, "HEROKU_ACCESS_TOKEN="+os.Getenv("HEROKU_ACCESS_TOKEN"))
	}

	if os.Getenv("CONTROL_DIR") != "" {
		env = append(env, "CONTROL_DIR="+os.Getenv("CONTROL_DIR"))
	}
	// attach a timestamp as some extra entropy because container names must be
	// unique
	name := fmt.Sprintf("%v.%v", ex.Name(), time.Now().Unix())
	container, err := dd.d.c.CreateContainer(docker.CreateContainerOptions{
		Name: name,
		Config: &docker.Config{
			Cmd: append([]string{"setuidgid", "app",
				"/hsup", "-d", "abspath", "-a",
				ex.release.appName, "run", "--"},
				ex.args...),
			Env:     append(env, "HSUP_SKIP_BUILD=TRUE"),
			Image:   ex.release.imageName,
			Volumes: map[string]struct{}{"/hsup": struct{}{}},
		},
	})
	if err != nil {
		log.Fatal(err)
	}
	ex.container = container

	where, err := filepath.Abs(linuxAmd64Path())
	if err != nil {
		return err
	}

	err = dd.d.c.StartContainer(ex.container.ID, &docker.HostConfig{
		Binds: []string{where + ":/hsup"},
	})
	if err != nil {
		log.Fatal(err)
	}

	go dd.d.c.Logs(docker.LogsOptions{
		Container:    container.ID,
		Stdout:       true,
		Stderr:       true,
		Follow:       true,
		OutputStream: os.Stdout,
	})

	return nil
}

func (dd *DockerDynoDriver) Wait(ex *Executor) (s *ExitStatus) {
	code, err := dd.d.c.WaitContainer(ex.container.ID)
	return &ExitStatus{code: code, err: err}
}

func (dd *DockerDynoDriver) Stop(ex *Executor) error {
	log.Println("Stopping container for", ex.Name())
	dd.d.c.KillContainer(docker.KillContainerOptions{
		ID:     ex.container.ID,
		Signal: docker.Signal(syscall.SIGTERM)})
	return dd.d.c.StopContainer(ex.container.ID, 10)
}

func (dd *DockerDynoDriver) connectDocker() error {
	if dd.d == nil {
		dd.d = &Docker{}
		if err := dd.d.Connect(); err != nil {
			dd.d = nil
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package build

import (
	"fmt"
	"github.com/mitchellh/packer/packer"
	"io/ioutil"
	"log"
	"sync"
)

type Command byte

func (Command) Run(env packer.Environment, args []string) int {
	if len(args) != 1 {
		env.Ui().Error("A single template argument is required.\n")
		return 1
	}

	// Read the file into a byte array so that we can parse the template
	log.Printf("Reading template: %s\n", args[0])
	tplData, err := ioutil.ReadFile(args[0])
	if err != nil {
		env.Ui().Error("Failed to read template file: %s\n", err.Error())
		return 1
	}

	// Parse the template into a machine-usable format
	log.Println("Parsing template...")
	tpl, err := packer.ParseTemplate(tplData)
	if err != nil {
		env.Ui().Error("Failed to parse template: %s\n", err.Error())
		return 1
	}

	// The component finder for our builds
	components := &packer.ComponentFinder{
		Builder: env.Builder,
		Hook:    env.Hook,
	}

	// Go through each builder and compile the builds that we care about
	buildNames := tpl.BuildNames()
	builds := make([]packer.Build, 0, len(buildNames))
	for _, buildName := range buildNames {
		log.Printf("Creating build: %s\n", buildName)
		build, err := tpl.Build(buildName, components)
		if err != nil {
			env.Ui().Error("Failed to create build '%s': \n\n%s\n", buildName, err.Error())
			return 1
		}

		builds = append(builds, build)
	}

	// Compile all the UIs for the builds
	buildUis := make(map[string]packer.Ui)
	for _, b := range builds {
		buildUis[b.Name()] = &packer.PrefixedUi{
			fmt.Sprintf("==> %s", b.Name()),
			env.Ui(),
		}
	}

	// Prepare all the builds
	for _, b := range builds {
		log.Printf("Preparing build: %s\n", b.Name())
		err := b.Prepare()
		if err != nil {
			env.Ui().Error("%s\n", err)
			return 1
		}
	}

	// Run all the builds in parallel and wait for them to complete
	var wg sync.WaitGroup
	for _, b := range builds {
		log.Printf("Starting build run: %s\n", b.Name())

		// Increment the waitgroup so we wait for this item to finish properly
		wg.Add(1)

		// Run the build in a goroutine
		go func() {
			defer wg.Done()
			b.Run(env.Ui())
		}()
	}

	wg.Wait()

	env.Ui().Say("YAY!\n")
	return 0
}

func (Command) Synopsis() string {
	return "build image(s) from template"
}
<commit_msg>command/build: properly prefix the build outputs<commit_after>package build

import (
	"fmt"
	"github.com/mitchellh/packer/packer"
	"io/ioutil"
	"log"
	"sync"
)

type Command byte

func (Command) Run(env packer.Environment, args []string) int {
	if len(args) != 1 {
		env.Ui().Error("A single template argument is required.\n")
		return 1
	}

	// Read the file into a byte array so that we can parse the template
	log.Printf("Reading template: %s\n", args[0])
	tplData, err := ioutil.ReadFile(args[0])
	if err != nil {
		env.Ui().Error("Failed to read template file: %s\n", err.Error())
		return 1
	}

	// Parse the template into a machine-usable format
	log.Println("Parsing template...")
	tpl, err := packer.ParseTemplate(tplData)
	if err != nil {
		env.Ui().Error("Failed to parse template: %s\n", err.Error())
		return 1
	}

	// The component finder for our builds
	components := &packer.ComponentFinder{
		Builder: env.Builder,
		Hook:    env.Hook,
	}

	// Go through each builder and compile the builds that we care about
	buildNames := tpl.BuildNames()
	builds := make([]packer.Build, 0, len(buildNames))
	for _, buildName := range buildNames {
		log.Printf("Creating build: %s\n", buildName)
		build, err := tpl.Build(buildName, components)
		if err != nil {
			env.Ui().Error("Failed to create build '%s': \n\n%s\n", buildName, err.Error())
			return 1
		}

		builds = append(builds, build)
	}

	// Compile all the UIs for the builds
	buildUis := make(map[string]packer.Ui)
	for _, b := range builds {
		buildUis[b.Name()] = &packer.PrefixedUi{
			fmt.Sprintf("==> %s", b.Name()),
			env.Ui(),
		}
	}

	// Prepare all the builds
	for _, b := range builds {
		log.Printf("Preparing build: %s\n", b.Name())
		err := b.Prepare()
		if err != nil {
			env.Ui().Error("%s\n", err)
			return 1
		}
	}

	// Run all the builds in parallel and wait for them to complete
	var wg sync.WaitGroup
	for _, b := range builds {
		log.Printf("Starting build run: %s\n", b.Name())

		// Increment the waitgroup so we wait for this item to finish properly
		wg.Add(1)

		// Run the build in a goroutine
		go func() {
			defer wg.Done()
			b.Run(buildUis[b.Name()])
		}()
	}

	wg.Wait()

	env.Ui().Say("YAY!\n")
	return 0
}

func (Command) Synopsis() string {
	return "build image(s) from template"
}
<|endoftext|>


<commit_before>package build

import (
	"fmt"
	"github.com/mitchellh/packer/packer"
	"io/ioutil"
	"log"
	"sync"
)

type Command byte

func (Command) Run(env packer.Environment, args []string) int {
	if len(args) != 1 {
		env.Ui().Error("A single template argument is required.\n")
		return 1
	}

	// Read the file into a byte array so that we can parse the template
	log.Printf("Reading template: %s\n", args[0])
	tplData, err := ioutil.ReadFile(args[0])
	if err != nil {
		env.Ui().Error("Failed to read template file: %s\n", err.Error())
		return 1
	}

	// Parse the template into a machine-usable format
	log.Println("Parsing template...")
	tpl, err := packer.ParseTemplate(tplData)
	if err != nil {
		env.Ui().Error("Failed to parse template: %s\n", err.Error())
		return 1
	}

	// The component finder for our builds
	components := &packer.ComponentFinder{
		Builder: env.Builder,
		Hook:    env.Hook,
	}

	// Go through each builder and compile the builds that we care about
	buildNames := tpl.BuildNames()
	builds := make([]packer.Build, 0, len(buildNames))
	for _, buildName := range buildNames {
		log.Printf("Creating build: %s\n", buildName)
		build, err := tpl.Build(buildName, components)
		if err != nil {
			env.Ui().Error("Failed to create build '%s': \n\n%s\n", buildName, err.Error())
			return 1
		}

		builds = append(builds, build)
	}

	// Compile all the UIs for the builds
	buildUis := make(map[string]packer.Ui)
	for _, b := range builds {
		buildUis[b.Name()] = &packer.PrefixedUi{
			fmt.Sprintf("==> %s", b.Name()),
			env.Ui(),
		}
	}

	// Prepare all the builds
	for _, b := range builds {
		log.Printf("Preparing build: %s\n", b.Name())
		err := b.Prepare()
		if err != nil {
			env.Ui().Error("%s\n", err)
			return 1
		}
	}

	// Run all the builds in parallel and wait for them to complete
	var wg sync.WaitGroup
	for _, b := range builds {
		log.Printf("Starting build run: %s\n", b.Name())

		// Increment the waitgroup so we wait for this item to finish properly
		wg.Add(1)

		// Run the build in a goroutine
		go func() {
			defer wg.Done()
			b.Run(buildUis[b.Name()])
		}()
	}

	wg.Wait()
	return 0
}

func (Command) Synopsis() string {
	return "build image(s) from template"
}
<commit_msg>command/build: Output artifact info<commit_after>package build

import (
	"fmt"
	"github.com/mitchellh/packer/packer"
	"io/ioutil"
	"log"
	"sync"
)

type Command byte

func (Command) Run(env packer.Environment, args []string) int {
	if len(args) != 1 {
		env.Ui().Error("A single template argument is required.\n")
		return 1
	}

	// Read the file into a byte array so that we can parse the template
	log.Printf("Reading template: %s\n", args[0])
	tplData, err := ioutil.ReadFile(args[0])
	if err != nil {
		env.Ui().Error("Failed to read template file: %s\n", err.Error())
		return 1
	}

	// Parse the template into a machine-usable format
	log.Println("Parsing template...")
	tpl, err := packer.ParseTemplate(tplData)
	if err != nil {
		env.Ui().Error("Failed to parse template: %s\n", err.Error())
		return 1
	}

	// The component finder for our builds
	components := &packer.ComponentFinder{
		Builder: env.Builder,
		Hook:    env.Hook,
	}

	// Go through each builder and compile the builds that we care about
	buildNames := tpl.BuildNames()
	builds := make([]packer.Build, 0, len(buildNames))
	for _, buildName := range buildNames {
		log.Printf("Creating build: %s\n", buildName)
		build, err := tpl.Build(buildName, components)
		if err != nil {
			env.Ui().Error("Failed to create build '%s': \n\n%s\n", buildName, err.Error())
			return 1
		}

		builds = append(builds, build)
	}

	// Compile all the UIs for the builds
	buildUis := make(map[string]packer.Ui)
	for _, b := range builds {
		buildUis[b.Name()] = &packer.PrefixedUi{
			fmt.Sprintf("==> %s", b.Name()),
			env.Ui(),
		}
	}

	// Prepare all the builds
	for _, b := range builds {
		log.Printf("Preparing build: %s\n", b.Name())
		err := b.Prepare()
		if err != nil {
			env.Ui().Error("%s\n", err)
			return 1
		}
	}

	// Run all the builds in parallel and wait for them to complete
	var wg sync.WaitGroup
	artifacts := make(map[string]packer.Artifact)
	for _, b := range builds {
		log.Printf("Starting build run: %s\n", b.Name())

		// Increment the waitgroup so we wait for this item to finish properly
		wg.Add(1)

		// Run the build in a goroutine
		go func() {
			defer wg.Done()
			artifacts[b.Name()] = b.Run(buildUis[b.Name()])
		}()
	}

	wg.Wait()

	// Output all the artifacts
	env.Ui().Say("==> The build completed! The artifacts created were:")
	for name, artifact := range artifacts {
		env.Ui().Say("--> %s:", name)
		env.Ui().Say(artifact.String())
	}

	return 0
}

func (Command) Synopsis() string {
	return "build image(s) from template"
}
<|endoftext|>


<commit_before>package detailed

import (
	"fmt"
	"strings"

	"github.com/weaveworks/scope/probe/docker"
	"github.com/weaveworks/scope/probe/host"
	"github.com/weaveworks/scope/probe/kubernetes"
	"github.com/weaveworks/scope/probe/overlay"
	"github.com/weaveworks/scope/probe/process"
)

var labels = map[string]string{
	docker.CPUTotalUsage:     "CPU",
	docker.ContainerCommand:  "Command",
	docker.ContainerCreated:  "Created",
	docker.ContainerHostname: "Hostname",
	docker.ContainerID:       "ID",
	docker.ContainerIPs:      "IPs",
	docker.ContainerPorts:    "Ports",
	docker.ContainerState:    "State",
	docker.ImageID:           "Image ID",
	docker.MemoryUsage:       "Memory",
	host.CPUUsage:            "CPU",
	host.HostName:            "Hostname",
	host.KernelVersion:       "Kernel version",
	host.Load15:              "Load (15m)",
	host.Load1:               "Load (1m)",
	host.Load5:               "Load (5m)",
	host.LocalNetworks:       "Local Networks",
	host.MemoryUsage:         "Memory",
	host.OS:                  "Operating system",
	host.Uptime:              "Uptime",
	kubernetes.Namespace:     "Namespace",
	kubernetes.PodCreated:    "Created",
	kubernetes.PodID:         "ID",
	overlay.WeaveDNSHostname: "Weave DNS Hostname",
	overlay.WeaveMACAddress:  "Weave MAC",
	process.CPUUsage:         "CPU",
	process.Cmdline:          "Command",
	process.MemoryUsage:      "Memory",
	process.PID:              "PID",
	process.PPID:             "Parent PID",
	process.Threads:          "# Threads",
}

// Label maps from the internal keys to the human-readable label for a piece
// of metadata/set/etc. If none is found the raw key will be returned.
func Label(key string) string {
	if label, ok := labels[key]; ok {
		return label
	}
	if strings.HasPrefix(key, "label_") {
		return fmt.Sprintf("Label %q", strings.TrimPrefix(key, "label_"))
	}
	return key
}
<commit_msg>couple labels were added while waiting to merge<commit_after>package detailed

import (
	"fmt"
	"strings"

	"github.com/weaveworks/scope/probe/docker"
	"github.com/weaveworks/scope/probe/host"
	"github.com/weaveworks/scope/probe/kubernetes"
	"github.com/weaveworks/scope/probe/overlay"
	"github.com/weaveworks/scope/probe/process"
)

var labels = map[string]string{
	docker.CPUTotalUsage:         "CPU",
	docker.ContainerCommand:      "Command",
	docker.ContainerCreated:      "Created",
	docker.ContainerHostname:     "Hostname",
	docker.ContainerID:           "ID",
	docker.ContainerIPs:          "IPs",
	docker.ContainerPorts:        "Ports",
	docker.ContainerRestartCount: "Restart #",
	docker.ContainerState:        "State",
	docker.ContainerUptime:       "Uptime",
	docker.ImageID:               "Image ID",
	docker.MemoryUsage:           "Memory",
	host.CPUUsage:                "CPU",
	host.HostName:                "Hostname",
	host.KernelVersion:           "Kernel version",
	host.Load15:                  "Load (15m)",
	host.Load1:                   "Load (1m)",
	host.Load5:                   "Load (5m)",
	host.LocalNetworks:           "Local Networks",
	host.MemoryUsage:             "Memory",
	host.OS:                      "Operating system",
	host.Uptime:                  "Uptime",
	kubernetes.Namespace:         "Namespace",
	kubernetes.PodCreated:        "Created",
	kubernetes.PodID:             "ID",
	overlay.WeaveDNSHostname:     "Weave DNS Hostname",
	overlay.WeaveMACAddress:      "Weave MAC",
	process.CPUUsage:             "CPU",
	process.Cmdline:              "Command",
	process.MemoryUsage:          "Memory",
	process.PID:                  "PID",
	process.PPID:                 "Parent PID",
	process.Threads:              "# Threads",
}

// Label maps from the internal keys to the human-readable label for a piece
// of metadata/set/etc. If none is found the raw key will be returned.
func Label(key string) string {
	if label, ok := labels[key]; ok {
		return label
	}
	if strings.HasPrefix(key, "label_") {
		return fmt.Sprintf("Label %q", strings.TrimPrefix(key, "label_"))
	}
	return key
}
<|endoftext|>


<commit_before>package command

import (
	"strings"
	"testing"

	"fmt"

	"github.com/hashicorp/nomad/nomad/mock"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/nomad/testutil"
	"github.com/mitchellh/cli"
	"github.com/posener/complete"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestJobEvalCommand_Implements(t *testing.T) {
	t.Parallel()
	var _ cli.Command = &JobEvalCommand{}
}

func TestJobEvalCommand_Fails(t *testing.T) {
	t.Parallel()
	ui := new(cli.MockUi)
	cmd := &JobEvalCommand{Meta: Meta{Ui: ui}}

	// Fails on misuse
	if code := cmd.Run([]string{"some", "bad", "args"}); code != 1 {
		t.Fatalf("expected exit code 1, got: %d", code)
	}
	if out := ui.ErrorWriter.String(); !strings.Contains(out, commandErrorText(cmd)) {
		t.Fatalf("expected help output, got: %s", out)
	}
	ui.ErrorWriter.Reset()

	// Fails when job ID is not specified
	if code := cmd.Run([]string{}); code != 1 {
		t.Fatalf("expect exit 1, got: %d", code)
	}
	if out := ui.ErrorWriter.String(); !strings.Contains(out, "This command takes one argument") {
		t.Fatalf("unexpected error: %v", out)
	}
	ui.ErrorWriter.Reset()

}

func TestJobEvalCommand_Run(t *testing.T) {
	t.Parallel()
	srv, client, url := testServer(t, true, nil)
	defer srv.Shutdown()

	// Wait for a node to be ready
	testutil.WaitForResult(func() (bool, error) {
		nodes, _, err := client.Nodes().List(nil)
		if err != nil {
			return false, err
		}
		for _, node := range nodes {
			if node.Status == structs.NodeStatusReady {
				return true, nil
			}
		}
		return false, fmt.Errorf("no ready nodes")
	}, func(err error) {
		t.Fatalf("err: %v", err)
	})

	ui := new(cli.MockUi)
	cmd := &JobEvalCommand{Meta: Meta{Ui: ui}}
	require := require.New(t)

	state := srv.Agent.Server().State()

	// Create a job
	job := mock.Job()
	err := state.UpsertJob(11, job)
	require.Nil(err)

	job, err = state.JobByID(nil, structs.DefaultNamespace, job.ID)
	require.Nil(err)

	// Create a failed alloc for the job
	alloc := mock.Alloc()
	alloc.Job = job
	alloc.JobID = job.ID
	alloc.TaskGroup = job.TaskGroups[0].Name
	alloc.Namespace = job.Namespace
	alloc.ClientStatus = structs.AllocClientStatusFailed
	err = state.UpsertAllocs(12, []*structs.Allocation{alloc})
	require.Nil(err)

	if code := cmd.Run([]string{"-address=" + url, "-force-reschedule", job.ID}); code != 0 {
		t.Fatalf("expected exit 0, got: %d", code)
	}

	// Lookup alloc again
	alloc, err = state.AllocByID(nil, alloc.ID)
	require.NotNil(alloc)
	require.Nil(err)
	require.True(*alloc.DesiredTransition.ForceReschedule)

}

func TestJobEvalCommand_AutocompleteArgs(t *testing.T) {
	assert := assert.New(t)
	t.Parallel()

	srv, _, url := testServer(t, true, nil)
	defer srv.Shutdown()

	ui := new(cli.MockUi)
	cmd := &JobEvalCommand{Meta: Meta{Ui: ui, flagAddress: url}}

	// Create a fake job
	state := srv.Agent.Server().State()
	j := mock.Job()
	assert.Nil(state.UpsertJob(1000, j))

	prefix := j.ID[:len(j.ID)-5]
	args := complete.Args{Last: prefix}
	predictor := cmd.AutocompleteArgs()

	res := predictor.Predict(args)
	assert.Equal(1, len(res))
	assert.Equal(j.ID, res[0])
}
<commit_msg>unit test for job eval should detach<commit_after>package command

import (
	"strings"
	"testing"

	"fmt"

	"github.com/hashicorp/nomad/nomad/mock"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/nomad/testutil"
	"github.com/mitchellh/cli"
	"github.com/posener/complete"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestJobEvalCommand_Implements(t *testing.T) {
	t.Parallel()
	var _ cli.Command = &JobEvalCommand{}
}

func TestJobEvalCommand_Fails(t *testing.T) {
	t.Parallel()
	ui := new(cli.MockUi)
	cmd := &JobEvalCommand{Meta: Meta{Ui: ui}}

	// Fails on misuse
	if code := cmd.Run([]string{"some", "bad", "args"}); code != 1 {
		t.Fatalf("expected exit code 1, got: %d", code)
	}
	if out := ui.ErrorWriter.String(); !strings.Contains(out, commandErrorText(cmd)) {
		t.Fatalf("expected help output, got: %s", out)
	}
	ui.ErrorWriter.Reset()

	// Fails when job ID is not specified
	if code := cmd.Run([]string{}); code != 1 {
		t.Fatalf("expect exit 1, got: %d", code)
	}
	if out := ui.ErrorWriter.String(); !strings.Contains(out, "This command takes one argument") {
		t.Fatalf("unexpected error: %v", out)
	}
	ui.ErrorWriter.Reset()

}

func TestJobEvalCommand_Run(t *testing.T) {
	t.Parallel()
	srv, client, url := testServer(t, true, nil)
	defer srv.Shutdown()

	// Wait for a node to be ready
	testutil.WaitForResult(func() (bool, error) {
		nodes, _, err := client.Nodes().List(nil)
		if err != nil {
			return false, err
		}
		for _, node := range nodes {
			if node.Status == structs.NodeStatusReady {
				return true, nil
			}
		}
		return false, fmt.Errorf("no ready nodes")
	}, func(err error) {
		t.Fatalf("err: %v", err)
	})

	ui := new(cli.MockUi)
	cmd := &JobEvalCommand{Meta: Meta{Ui: ui}}
	require := require.New(t)

	state := srv.Agent.Server().State()

	// Create a job
	job := mock.Job()
	err := state.UpsertJob(11, job)
	require.Nil(err)

	job, err = state.JobByID(nil, structs.DefaultNamespace, job.ID)
	require.Nil(err)

	// Create a failed alloc for the job
	alloc := mock.Alloc()
	alloc.Job = job
	alloc.JobID = job.ID
	alloc.TaskGroup = job.TaskGroups[0].Name
	alloc.Namespace = job.Namespace
	alloc.ClientStatus = structs.AllocClientStatusFailed
	err = state.UpsertAllocs(12, []*structs.Allocation{alloc})
	require.Nil(err)

	if code := cmd.Run([]string{"-address=" + url, "-force-reschedule", "-detach", job.ID}); code != 0 {
		t.Fatalf("expected exit 0, got: %d", code)
	}

	// Lookup alloc again
	alloc, err = state.AllocByID(nil, alloc.ID)
	require.NotNil(alloc)
	require.Nil(err)
	require.True(*alloc.DesiredTransition.ForceReschedule)

}

func TestJobEvalCommand_AutocompleteArgs(t *testing.T) {
	assert := assert.New(t)
	t.Parallel()

	srv, _, url := testServer(t, true, nil)
	defer srv.Shutdown()

	ui := new(cli.MockUi)
	cmd := &JobEvalCommand{Meta: Meta{Ui: ui, flagAddress: url}}

	// Create a fake job
	state := srv.Agent.Server().State()
	j := mock.Job()
	assert.Nil(state.UpsertJob(1000, j))

	prefix := j.ID[:len(j.ID)-5]
	args := complete.Args{Last: prefix}
	predictor := cmd.AutocompleteArgs()

	res := predictor.Predict(args)
	assert.Equal(1, len(res))
	assert.Equal(j.ID, res[0])
}
<|endoftext|>


<commit_before>package heroku

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

type Build struct {
	Id         string      `json:"id"`
	SourceBlob *SourceBlob `json:"source_blob"`
	Status     string      `json:"status"`
}

type SourceBlob struct {
	Checksum string `json:"checksum,omitempty"`
	Url      string `json:"url"`
	Version  string `json:"url"`
}

func (c *Client) BuildCreate(appId string, sourceBlob *SourceBlob) (*Build, *http.Response, error) {
	body, err := json.Marshal(map[string]interface{}{
		"source_blob": sourceBlob,
	})
	fmt.Printf("BuildCreate body: %v\n", string(body))
	if err != nil {
		return nil, nil, err
	}

	url := c.BaseUrl() + fmt.Sprintf("/apps/%v/builds", appId)
	resp, err := c.MakeRequest("POST", url, &body)
	if err != nil {
		return nil, resp, err
	}

	var build Build
	err = c.readResponseInto(resp, &build)
	if err != nil {
		return nil, resp, err
	}

	return &build, resp, nil
}

func (c *Client) BuildInfo(appId string, id string) (*Build, *http.Response, error) {
	url := c.BaseUrl() + fmt.Sprintf("/apps/%v/builds/%v", appId, id)
	resp, err := c.MakeRequest("GET", url, nil)
	if err != nil {
		return nil, resp, err
	}

	var build Build
	err = c.readResponseInto(resp, &build)
	if err != nil {
		return nil, resp, err
	}

	return &build, resp, nil
}

func (c *Client) readResponse(resp *http.Response) ([]byte, error) {
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode >= 400 && c.Debug {
		fmt.Printf("[Heroku] Error: code=%v %v", resp.StatusCode, string(body))
	}

	return body, nil
}

func (c *Client) readResponseInto(resp *http.Response, val interface{}) error {
	body, err := c.readResponse(resp)
	if err != nil {
		return err
	}

	err = json.Unmarshal(body, val)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Try to fix building of JSON for Heroku build creation<commit_after>package heroku

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

type Build struct {
	Id         string      `json:"id"`
	SourceBlob *SourceBlob `json:"source_blob"`
	Status     string      `json:"status"`
}

type SourceBlob struct {
	Checksum string `json:"checksum,omitempty"`
	Url      string `json:"url"`
	Version  string `json:"url"`
}

type buildCreateRequest struct {
	SourceBlob *SourceBlob `json:"source_blob"`
}

func (c *Client) BuildCreate(appId string, sourceBlob *SourceBlob) (*Build, *http.Response, error) {
	body, err := json.Marshal(&buildCreateRequest{
		SourceBlob: sourceBlob,
	})
	fmt.Printf("BuildCreate body: %v\n", string(body))
	if err != nil {
		return nil, nil, err
	}

	url := c.BaseUrl() + fmt.Sprintf("/apps/%v/builds", appId)
	resp, err := c.MakeRequest("POST", url, &body)
	if err != nil {
		return nil, resp, err
	}

	var build Build
	err = c.readResponseInto(resp, &build)
	if err != nil {
		return nil, resp, err
	}

	return &build, resp, nil
}

func (c *Client) BuildInfo(appId string, id string) (*Build, *http.Response, error) {
	url := c.BaseUrl() + fmt.Sprintf("/apps/%v/builds/%v", appId, id)
	resp, err := c.MakeRequest("GET", url, nil)
	if err != nil {
		return nil, resp, err
	}

	var build Build
	err = c.readResponseInto(resp, &build)
	if err != nil {
		return nil, resp, err
	}

	return &build, resp, nil
}

func (c *Client) readResponse(resp *http.Response) ([]byte, error) {
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode >= 400 && c.Debug {
		fmt.Printf("[Heroku] Error: code=%v %v\n", resp.StatusCode, string(body))
	}

	return body, nil
}

func (c *Client) readResponseInto(resp *http.Response, val interface{}) error {
	body, err := c.readResponse(resp)
	if err != nil {
		return err
	}

	err = json.Unmarshal(body, val)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package sum

import "github.com/catorpilor/leetcode/utils"

func pathSum(root *utils.TreeNode) int {
	return 0
}
<commit_msg>solve 1022 use backtracking<commit_after>package sum

import "github.com/catorpilor/leetcode/utils"

func pathSum(root *utils.TreeNode) int {
	return useBackTracking(root)
}

// useBackTracking time compelxity O(N), space complexity O(log(N))
func useBackTracking(root *utils.TreeNode) int {
	var ans int
	helper(root, &ans, []int{})
	return ans
}

func helper(root *utils.TreeNode, ans *int, path []int) {
	if root == nil {
		return
	}
	np := len(path)
	path = append(path, root.Val)
	if root.Left == nil && root.Right == nil {
		// it's a leaf node
		*ans += genNum(path)
		path = path[:np]
		return
	}
	helper(root.Left, ans, path)
	helper(root.Right, ans, path)
}

func genNum(nums []int) int {
	var ans int
	n := len(nums)
	for i := 0; i < n; i++ {
		ans += nums[i] * (1 << (n - 1 - i))
	}
	return ans
}
<|endoftext|>