<commit_before>// Walking directories

package fs

import (
	"bytes"
	"fmt"
	"path"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/pkg/errors"
)

// ErrorSkipDir is used as a return value from Walk to indicate that the
// directory named in the call is to be skipped. It is not returned as
// an error by any function.
var ErrorSkipDir = errors.New("skip this directory")

// ErrorCantListR is returned by WalkR if the underlying Fs isn't
// capable of doing a recursive listing.
var ErrorCantListR = errors.New("recursive directory listing not available")

// WalkFunc is the type of the function called for directory
// visited by Walk. The path argument contains remote path to the directory.
//
// If there was a problem walking to directory named by path, the
// incoming error will describe the problem and the function can
// decide how to handle that error (and Walk will not descend into
// that directory). If an error is returned, processing stops. The
// sole exception is when the function returns the special value
// ErrorSkipDir. If the function returns ErrorSkipDir, Walk skips the
// directory's contents entirely.
type WalkFunc func(path string, entries DirEntries, err error) error

// Walk lists the directory.
//
// If includeAll is not set it will use the filters defined.
//
// If maxLevel is < 0 then it will recurse indefinitely, else it will
// only do maxLevel levels.
//
// It calls fn for each tranche of DirEntries read.
//
// Note that fn will not be called concurrently whereas the directory
// listing will proceed concurrently.
//
// Parent directories are always listed before their children
//
// This is implemented by WalkR if Config.UseRecursiveListing is true
// and f supports it and level > 1, or WalkN otherwise.
//
// NB (f, path) to be replaced by fs.Dir at some point
func Walk(f Fs, path string, includeAll bool, maxLevel int, fn WalkFunc) error {
	if (maxLevel < 0 || maxLevel > 1) && Config.UseListR && f.Features().ListR != nil {
		return WalkR(f, path, includeAll, maxLevel, fn)
	}
	return WalkN(f, path, includeAll, maxLevel, fn)
}

// WalkN lists the directory.
//
// It implements Walk using non recursive directory listing.
func WalkN(f Fs, path string, includeAll bool, maxLevel int, fn WalkFunc) error {
	return walk(f, path, includeAll, maxLevel, fn, ListDirSorted)
}

// WalkR lists the directory.
//
// It implements Walk using recursive directory listing if
// available, or returns ErrorCantListR if not.
func WalkR(f Fs, path string, includeAll bool, maxLevel int, fn WalkFunc) error {
	listR := f.Features().ListR
	if listR == nil {
		return ErrorCantListR
	}
	return walkR(f, path, includeAll, maxLevel, fn, listR)
}

type listDirFunc func(fs Fs, includeAll bool, dir string) (entries DirEntries, err error)

func walk(f Fs, path string, includeAll bool, maxLevel int, fn WalkFunc, listDir listDirFunc) error {
	var (
		wg         sync.WaitGroup // sync closing of go routines
		traversing sync.WaitGroup // running directory traversals
		doClose    sync.Once      // close the channel once
		mu         sync.Mutex     // stop fn being called concurrently
	)
	// listJob describe a directory listing that needs to be done
	type listJob struct {
		remote string
		depth  int
	}

	in := make(chan listJob, Config.Checkers)
	errs := make(chan error, 1)
	quit := make(chan struct{})
	closeQuit := func() {
		doClose.Do(func() {
			close(quit)
			go func() {
				for _ = range in {
					traversing.Done()
				}
			}()
		})
	}
	for i := 0; i < Config.Checkers; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for {
				select {
				case job, ok := <-in:
					if !ok {
						return
					}
					entries, err := listDir(f, includeAll, job.remote)
					var jobs []listJob
					if err == nil && job.depth != 0 {
						entries.ForDir(func(dir Directory) {
							// Recurse for the directory
							jobs = append(jobs, listJob{
								remote: dir.Remote(),
								depth:  job.depth - 1,
							})
						})
					}
					mu.Lock()
					err = fn(job.remote, entries, err)
					mu.Unlock()
					// NB once we have passed entries to fn we mustn't touch it again
					if err != nil && err != ErrorSkipDir {
						traversing.Done()
						Stats.Error(err)
						Errorf(job.remote, "error listing: %v", err)
						closeQuit()
						// Send error to error channel if space
						select {
						case errs <- err:
						default:
						}
						continue
					}
					if err == nil && len(jobs) > 0 {
						traversing.Add(len(jobs))
						go func() {
							// Now we have traversed this directory, send these
							// jobs off for traversal in the background
							for _, newJob := range jobs {
								in <- newJob
							}
						}()
					}
					traversing.Done()
				case <-quit:
					return
				}
			}
		}()
	}
	// Start the process
	traversing.Add(1)
	in <- listJob{
		remote: path,
		depth:  maxLevel - 1,
	}
	traversing.Wait()
	close(in)
	wg.Wait()
	close(errs)
	// return the first error returned or nil
	return <-errs
}

// DirTree is a map of directories to entries
type DirTree map[string]DirEntries

// parentDir finds the parent directory of path
func parentDir(entryPath string) string {
	dirPath := path.Dir(entryPath)
	if dirPath == "." {
		dirPath = ""
	}
	return dirPath
}

// add an entry to the tree
func (dt DirTree) add(entry DirEntry) {
	dirPath := parentDir(entry.Remote())
	dt[dirPath] = append(dt[dirPath], entry)
}

// add a directory entry to the tree
func (dt DirTree) addDir(entry DirEntry) {
	dt.add(entry)
	// create the directory itself if it doesn't exist already
	dirPath := entry.Remote()
	if _, ok := dt[dirPath]; !ok {
		dt[dirPath] = nil
	}
}

// Find returns the DirEntry for filePath or nil if not found
func (dt DirTree) Find(filePath string) (parentPath string, entry DirEntry) {
	parentPath = parentDir(filePath)
	for _, entry := range dt[parentPath] {
		if entry.Remote() == filePath {
			return parentPath, entry
		}
	}
	return parentPath, nil
}

// check that dirPath has a *Dir in its parent
func (dt DirTree) checkParent(root, dirPath string) {
	if dirPath == root {
		return
	}
	parentPath, entry := dt.Find(dirPath)
	if entry != nil {
		return
	}
	dt[parentPath] = append(dt[parentPath], NewDir(dirPath, time.Now()))
	dt.checkParent(root, parentPath)
}

// check every directory in the tree has *Dir in its parent
func (dt DirTree) checkParents(root string) {
	for dirPath := range dt {
		dt.checkParent(root, dirPath)
	}
}

// Sort sorts all the Entries
func (dt DirTree) Sort() {
	for _, entries := range dt {
		sort.Stable(entries)
	}
}

// Dirs returns the directories in sorted order
func (dt DirTree) Dirs() (dirNames []string) {
	for dirPath := range dt {
		dirNames = append(dirNames, dirPath)
	}
	sort.Strings(dirNames)
	return dirNames
}

// String emits a simple representation of the DirTree
func (dt DirTree) String() string {
	out := new(bytes.Buffer)
	for _, dir := range dt.Dirs() {
		fmt.Fprintf(out, "%s/\n", dir)
		for _, entry := range dt[dir] {
			flag := ""
			if _, ok := entry.(Directory); ok {
				flag = "/"
			}
			fmt.Fprintf(out, "  %s%s\n", path.Base(entry.Remote()), flag)
		}
	}
	return out.String()
}

func walkRDirTree(f Fs, path string, includeAll bool, maxLevel int, listR ListRFn) (DirTree, error) {
	dirs := make(DirTree)
	var mu sync.Mutex
	err := listR(path, func(entries DirEntries) error {
		mu.Lock()
		defer mu.Unlock()
		for _, entry := range entries {
			slashes := strings.Count(entry.Remote(), "/")
			switch x := entry.(type) {
			case Object:
				// Make sure we don't delete excluded files if not required
				if includeAll || Config.Filter.IncludeObject(x) {
					if maxLevel < 0 || slashes <= maxLevel-1 {
						dirs.add(x)
					} else {
						// Make sure we include any parent directories of excluded objects
						dirPath := x.Remote()
						for ; slashes > maxLevel-1; slashes-- {
							dirPath = parentDir(dirPath)
						}
						dirs.checkParent(path, dirPath)
					}
				} else {
					Debugf(x, "Excluded from sync (and deletion)")
				}
			case Directory:
				if includeAll || Config.Filter.IncludeDirectory(x.Remote()) {
					if maxLevel < 0 || slashes <= maxLevel-1 {
						if slashes == maxLevel-1 {
							// Just add the object if at maxLevel
							dirs.add(x)
						} else {
							dirs.addDir(x)
						}
					}
				} else {
					Debugf(x, "Excluded from sync (and deletion)")
				}
			}
		}
		return nil
	})
	if err != nil {
		return nil, err
	}
	dirs.checkParents(path)
	if len(dirs) == 0 {
		dirs[path] = nil
	}
	dirs.Sort()
	return dirs, nil
}

// Create a DirTree using List
func walkNDirTree(f Fs, path string, includeAll bool, maxLevel int, listDir listDirFunc) (DirTree, error) {
	dirs := make(DirTree)
	fn := func(dirPath string, entries DirEntries, err error) error {
		if err == nil {
			dirs[dirPath] = entries
		}
		return err
	}
	err := walk(f, path, includeAll, maxLevel, fn, listDir)
	if err != nil {
		return nil, err
	}
	return dirs, nil
}

// NewDirTree returns a DirTree filled with the directory listing
// using the parameters supplied.
//
// If includeAll is not set it will use the filters defined.
//
// If maxLevel is < 0 then it will recurse indefinitely, else it will
// only do maxLevel levels.
//
// This is implemented by WalkR if Config.UseRecursiveListing is true
// and f supports it and level > 1, or WalkN otherwise.
//
// NB (f, path) to be replaced by fs.Dir at some point
func NewDirTree(f Fs, path string, includeAll bool, maxLevel int) (DirTree, error) {
	if ListR := f.Features().ListR; (maxLevel < 0 || maxLevel > 1) && Config.UseListR && ListR != nil {
		return walkRDirTree(f, path, includeAll, maxLevel, ListR)
	}
	return walkNDirTree(f, path, includeAll, maxLevel, ListDirSorted)
}

func walkR(f Fs, path string, includeAll bool, maxLevel int, fn WalkFunc, listR ListRFn) error {
	dirs, err := walkRDirTree(f, path, includeAll, maxLevel, listR)
	if err != nil {
		return err
	}
	skipping := false
	skipPrefix := ""
	emptyDir := DirEntries{}
	for _, dirPath := range dirs.Dirs() {
		if skipping {
			// Skip over directories as required
			if strings.HasPrefix(dirPath, skipPrefix) {
				continue
			}
			skipping = false
		}
		entries := dirs[dirPath]
		if entries == nil {
			entries = emptyDir
		}
		err = fn(dirPath, entries, nil)
		if err == ErrorSkipDir {
			skipping = true
			skipPrefix = dirPath
			if skipPrefix != "" {
				skipPrefix += "/"
			}
		} else if err != nil {
			return err
		}
	}
	return nil
}

// WalkGetAll runs Walk getting all the results
func WalkGetAll(f Fs, path string, includeAll bool, maxLevel int) (objs []Object, dirs []Directory, err error) {
	err = Walk(f, path, includeAll, maxLevel, func(dirPath string, entries DirEntries, err error) error {
		if err != nil {
			return err
		}
		for _, entry := range entries {
			switch x := entry.(type) {
			case Object:
				objs = append(objs, x)
			case Directory:
				dirs = append(dirs, x)
			}
		}
		return nil
	})
	return
}

// ListRHelper is used in the implementation of ListR to accumulate DirEntries
type ListRHelper struct {
	callback ListRCallback
	entries  DirEntries
}

// NewListRHelper should be called from ListR with the callback passed in
func NewListRHelper(callback ListRCallback) *ListRHelper {
	return &ListRHelper{
		callback: callback,
	}
}

// send sends the stored entries to the callback if there are >= max
// entries.
func (lh *ListRHelper) send(max int) (err error) {
	if len(lh.entries) >= max {
		err = lh.callback(lh.entries)
		lh.entries = lh.entries[:0]
	}
	return err
}

// Add an entry to the stored entries and send them if there are more
// than a certain amount
func (lh *ListRHelper) Add(entry DirEntry) error {
	if entry == nil {
		return nil
	}
	lh.entries = append(lh.entries, entry)
	return lh.send(100)
}

// Flush the stored entries (if any) sending them to the callback
func (lh *ListRHelper) Flush() error {
	return lh.send(1)
}
<commit_msg>DirTree.Prune: deletes several directories<commit_after>// Walking directories

package fs

import (
	"bytes"
	"fmt"
	"path"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/pkg/errors"
)

// ErrorSkipDir is used as a return value from Walk to indicate that the
// directory named in the call is to be skipped. It is not returned as
// an error by any function.
var ErrorSkipDir = errors.New("skip this directory")

// ErrorCantListR is returned by WalkR if the underlying Fs isn't
// capable of doing a recursive listing.
var ErrorCantListR = errors.New("recursive directory listing not available")

// WalkFunc is the type of the function called for directory
// visited by Walk. The path argument contains remote path to the directory.
//
// If there was a problem walking to directory named by path, the
// incoming error will describe the problem and the function can
// decide how to handle that error (and Walk will not descend into
// that directory). If an error is returned, processing stops. The
// sole exception is when the function returns the special value
// ErrorSkipDir. If the function returns ErrorSkipDir, Walk skips the
// directory's contents entirely.
type WalkFunc func(path string, entries DirEntries, err error) error

// Walk lists the directory.
//
// If includeAll is not set it will use the filters defined.
//
// If maxLevel is < 0 then it will recurse indefinitely, else it will
// only do maxLevel levels.
//
// It calls fn for each tranche of DirEntries read.
//
// Note that fn will not be called concurrently whereas the directory
// listing will proceed concurrently.
//
// Parent directories are always listed before their children
//
// This is implemented by WalkR if Config.UseRecursiveListing is true
// and f supports it and level > 1, or WalkN otherwise.
//
// NB (f, path) to be replaced by fs.Dir at some point
func Walk(f Fs, path string, includeAll bool, maxLevel int, fn WalkFunc) error {
	if (maxLevel < 0 || maxLevel > 1) && Config.UseListR && f.Features().ListR != nil {
		return WalkR(f, path, includeAll, maxLevel, fn)
	}
	return WalkN(f, path, includeAll, maxLevel, fn)
}

// WalkN lists the directory.
//
// It implements Walk using non recursive directory listing.
func WalkN(f Fs, path string, includeAll bool, maxLevel int, fn WalkFunc) error {
	return walk(f, path, includeAll, maxLevel, fn, ListDirSorted)
}

// WalkR lists the directory.
//
// It implements Walk using recursive directory listing if
// available, or returns ErrorCantListR if not.
func WalkR(f Fs, path string, includeAll bool, maxLevel int, fn WalkFunc) error {
	listR := f.Features().ListR
	if listR == nil {
		return ErrorCantListR
	}
	return walkR(f, path, includeAll, maxLevel, fn, listR)
}

type listDirFunc func(fs Fs, includeAll bool, dir string) (entries DirEntries, err error)

func walk(f Fs, path string, includeAll bool, maxLevel int, fn WalkFunc, listDir listDirFunc) error {
	var (
		wg         sync.WaitGroup // sync closing of go routines
		traversing sync.WaitGroup // running directory traversals
		doClose    sync.Once      // close the channel once
		mu         sync.Mutex     // stop fn being called concurrently
	)
	// listJob describe a directory listing that needs to be done
	type listJob struct {
		remote string
		depth  int
	}

	in := make(chan listJob, Config.Checkers)
	errs := make(chan error, 1)
	quit := make(chan struct{})
	closeQuit := func() {
		doClose.Do(func() {
			close(quit)
			go func() {
				for _ = range in {
					traversing.Done()
				}
			}()
		})
	}
	for i := 0; i < Config.Checkers; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for {
				select {
				case job, ok := <-in:
					if !ok {
						return
					}
					entries, err := listDir(f, includeAll, job.remote)
					var jobs []listJob
					if err == nil && job.depth != 0 {
						entries.ForDir(func(dir Directory) {
							// Recurse for the directory
							jobs = append(jobs, listJob{
								remote: dir.Remote(),
								depth:  job.depth - 1,
							})
						})
					}
					mu.Lock()
					err = fn(job.remote, entries, err)
					mu.Unlock()
					// NB once we have passed entries to fn we mustn't touch it again
					if err != nil && err != ErrorSkipDir {
						traversing.Done()
						Stats.Error(err)
						Errorf(job.remote, "error listing: %v", err)
						closeQuit()
						// Send error to error channel if space
						select {
						case errs <- err:
						default:
						}
						continue
					}
					if err == nil && len(jobs) > 0 {
						traversing.Add(len(jobs))
						go func() {
							// Now we have traversed this directory, send these
							// jobs off for traversal in the background
							for _, newJob := range jobs {
								in <- newJob
							}
						}()
					}
					traversing.Done()
				case <-quit:
					return
				}
			}
		}()
	}
	// Start the process
	traversing.Add(1)
	in <- listJob{
		remote: path,
		depth:  maxLevel - 1,
	}
	traversing.Wait()
	close(in)
	wg.Wait()
	close(errs)
	// return the first error returned or nil
	return <-errs
}

// DirTree is a map of directories to entries
type DirTree map[string]DirEntries

// parentDir finds the parent directory of path
func parentDir(entryPath string) string {
	dirPath := path.Dir(entryPath)
	if dirPath == "." {
		dirPath = ""
	}
	return dirPath
}

// add an entry to the tree
func (dt DirTree) add(entry DirEntry) {
	dirPath := parentDir(entry.Remote())
	dt[dirPath] = append(dt[dirPath], entry)
}

// add a directory entry to the tree
func (dt DirTree) addDir(entry DirEntry) {
	dt.add(entry)
	// create the directory itself if it doesn't exist already
	dirPath := entry.Remote()
	if _, ok := dt[dirPath]; !ok {
		dt[dirPath] = nil
	}
}

// Find returns the DirEntry for filePath or nil if not found
func (dt DirTree) Find(filePath string) (parentPath string, entry DirEntry) {
	parentPath = parentDir(filePath)
	for _, entry := range dt[parentPath] {
		if entry.Remote() == filePath {
			return parentPath, entry
		}
	}
	return parentPath, nil
}

// check that dirPath has a *Dir in its parent
func (dt DirTree) checkParent(root, dirPath string) {
	if dirPath == root {
		return
	}
	parentPath, entry := dt.Find(dirPath)
	if entry != nil {
		return
	}
	dt[parentPath] = append(dt[parentPath], NewDir(dirPath, time.Now()))
	dt.checkParent(root, parentPath)
}

// check every directory in the tree has *Dir in its parent
func (dt DirTree) checkParents(root string) {
	for dirPath := range dt {
		dt.checkParent(root, dirPath)
	}
}

// Sort sorts all the Entries
func (dt DirTree) Sort() {
	for _, entries := range dt {
		sort.Stable(entries)
	}
}

// Dirs returns the directories in sorted order
func (dt DirTree) Dirs() (dirNames []string) {
	for dirPath := range dt {
		dirNames = append(dirNames, dirPath)
	}
	sort.Strings(dirNames)
	return dirNames
}

// Prune remove directories from a directory tree. dirNames contains
// all directories to remove as keys, with true as values. dirNames
// will be modified in the function.
func (dt DirTree) Prune(dirNames map[string]bool) error {
	// We use map[string]bool to avoid recursion (and potential
	// stack exhaustion).

	// First we need delete directories from their parents.
	for dName, remove := range dirNames {
		if !remove {
			// Currently all values should be
			// true, therefore this should not
			// happen. But this makes function
			// more predictable.
			Infof(dName, "Directory in the map for prune, but the value is false")
			continue
		}
		if dName == "" {
			// if dName is root, do nothing (no parent exist)
			continue
		}
		parent := parentDir(dName)
		// It may happen that dt does not have a dName key,
		// since directory was excluded based on a filter. In
		// such case the loop will be skipped.
		for i, entry := range dt[parent] {
			switch x := entry.(type) {
			case Directory:
				if x.Remote() == dName {
					// the slice is not sorted yet
					// to delete item
					// a) replace it with the last one
					dt[parent][i] = dt[parent][len(dt[parent])-1]
					// b) remove last
					dt[parent] = dt[parent][:len(dt[parent])-1]
					// we modify a slice within a loop, but we stop
					// iterating immediately
					break
				}
			case Object:
				// do nothing
			default:
				return errors.Errorf("unknown object type %T", entry)

			}
		}
	}

	for len(dirNames) > 0 {
		// According to golang specs, if new keys were added
		// during range iteration, they may be skipped.
		for dName, remove := range dirNames {
			if !remove {
				Infof(dName, "Directory in the map for prune, but the value is false")
				continue
			}
			// First, add all subdirectories to dirNames.

			// It may happen that dt[dName] does not exist.
			// If so, the loop will be skipped.
			for _, entry := range dt[dName] {
				switch x := entry.(type) {
				case Directory:
					excludeDir := x.Remote()
					dirNames[excludeDir] = true
				case Object:
					// do nothing
				default:
					return errors.Errorf("unknown object type %T", entry)

				}
			}
			// Then remove current directory from DirTree
			delete(dt, dName)
			// and from dirNames
			delete(dirNames, dName)
		}
	}
	return nil
}

// String emits a simple representation of the DirTree
func (dt DirTree) String() string {
	out := new(bytes.Buffer)
	for _, dir := range dt.Dirs() {
		fmt.Fprintf(out, "%s/\n", dir)
		for _, entry := range dt[dir] {
			flag := ""
			if _, ok := entry.(Directory); ok {
				flag = "/"
			}
			fmt.Fprintf(out, "  %s%s\n", path.Base(entry.Remote()), flag)
		}
	}
	return out.String()
}

func walkRDirTree(f Fs, path string, includeAll bool, maxLevel int, listR ListRFn) (DirTree, error) {
	dirs := make(DirTree)
	var mu sync.Mutex
	err := listR(path, func(entries DirEntries) error {
		mu.Lock()
		defer mu.Unlock()
		for _, entry := range entries {
			slashes := strings.Count(entry.Remote(), "/")
			switch x := entry.(type) {
			case Object:
				// Make sure we don't delete excluded files if not required
				if includeAll || Config.Filter.IncludeObject(x) {
					if maxLevel < 0 || slashes <= maxLevel-1 {
						dirs.add(x)
					} else {
						// Make sure we include any parent directories of excluded objects
						dirPath := x.Remote()
						for ; slashes > maxLevel-1; slashes-- {
							dirPath = parentDir(dirPath)
						}
						dirs.checkParent(path, dirPath)
					}
				} else {
					Debugf(x, "Excluded from sync (and deletion)")
				}
			case Directory:
				if includeAll || Config.Filter.IncludeDirectory(x.Remote()) {
					if maxLevel < 0 || slashes <= maxLevel-1 {
						if slashes == maxLevel-1 {
							// Just add the object if at maxLevel
							dirs.add(x)
						} else {
							dirs.addDir(x)
						}
					}
				} else {
					Debugf(x, "Excluded from sync (and deletion)")
				}
			}
		}
		return nil
	})
	if err != nil {
		return nil, err
	}
	dirs.checkParents(path)
	if len(dirs) == 0 {
		dirs[path] = nil
	}
	dirs.Sort()
	return dirs, nil
}

// Create a DirTree using List
func walkNDirTree(f Fs, path string, includeAll bool, maxLevel int, listDir listDirFunc) (DirTree, error) {
	dirs := make(DirTree)
	fn := func(dirPath string, entries DirEntries, err error) error {
		if err == nil {
			dirs[dirPath] = entries
		}
		return err
	}
	err := walk(f, path, includeAll, maxLevel, fn, listDir)
	if err != nil {
		return nil, err
	}
	return dirs, nil
}

// NewDirTree returns a DirTree filled with the directory listing
// using the parameters supplied.
//
// If includeAll is not set it will use the filters defined.
//
// If maxLevel is < 0 then it will recurse indefinitely, else it will
// only do maxLevel levels.
//
// This is implemented by WalkR if Config.UseRecursiveListing is true
// and f supports it and level > 1, or WalkN otherwise.
//
// NB (f, path) to be replaced by fs.Dir at some point
func NewDirTree(f Fs, path string, includeAll bool, maxLevel int) (DirTree, error) {
	if ListR := f.Features().ListR; (maxLevel < 0 || maxLevel > 1) && Config.UseListR && ListR != nil {
		return walkRDirTree(f, path, includeAll, maxLevel, ListR)
	}
	return walkNDirTree(f, path, includeAll, maxLevel, ListDirSorted)
}

func walkR(f Fs, path string, includeAll bool, maxLevel int, fn WalkFunc, listR ListRFn) error {
	dirs, err := walkRDirTree(f, path, includeAll, maxLevel, listR)
	if err != nil {
		return err
	}
	skipping := false
	skipPrefix := ""
	emptyDir := DirEntries{}
	for _, dirPath := range dirs.Dirs() {
		if skipping {
			// Skip over directories as required
			if strings.HasPrefix(dirPath, skipPrefix) {
				continue
			}
			skipping = false
		}
		entries := dirs[dirPath]
		if entries == nil {
			entries = emptyDir
		}
		err = fn(dirPath, entries, nil)
		if err == ErrorSkipDir {
			skipping = true
			skipPrefix = dirPath
			if skipPrefix != "" {
				skipPrefix += "/"
			}
		} else if err != nil {
			return err
		}
	}
	return nil
}

// WalkGetAll runs Walk getting all the results
func WalkGetAll(f Fs, path string, includeAll bool, maxLevel int) (objs []Object, dirs []Directory, err error) {
	err = Walk(f, path, includeAll, maxLevel, func(dirPath string, entries DirEntries, err error) error {
		if err != nil {
			return err
		}
		for _, entry := range entries {
			switch x := entry.(type) {
			case Object:
				objs = append(objs, x)
			case Directory:
				dirs = append(dirs, x)
			}
		}
		return nil
	})
	return
}

// ListRHelper is used in the implementation of ListR to accumulate DirEntries
type ListRHelper struct {
	callback ListRCallback
	entries  DirEntries
}

// NewListRHelper should be called from ListR with the callback passed in
func NewListRHelper(callback ListRCallback) *ListRHelper {
	return &ListRHelper{
		callback: callback,
	}
}

// send sends the stored entries to the callback if there are >= max
// entries.
func (lh *ListRHelper) send(max int) (err error) {
	if len(lh.entries) >= max {
		err = lh.callback(lh.entries)
		lh.entries = lh.entries[:0]
	}
	return err
}

// Add an entry to the stored entries and send them if there are more
// than a certain amount
func (lh *ListRHelper) Add(entry DirEntry) error {
	if entry == nil {
		return nil
	}
	lh.entries = append(lh.entries, entry)
	return lh.send(100)
}

// Flush the stored entries (if any) sending them to the callback
func (lh *ListRHelper) Flush() error {
	return lh.send(1)
}
<|endoftext|>


<commit_before>package gen

import (
	"bytes"
	"encoding/json"
	"errors"
	"os"
	"path/filepath"
	"sync"

	"github.com/gocraft/health"
	"github.com/robertkrimen/otto"
	"github.com/robertkrimen/otto/underscore"
	"github.com/spf13/afero"
)

//package specfic halth stream. This can be used to provide consistent
//application wise health check stream.
//
// It is safe for concurrent usage.
var stream *health.Stream

func init() {
	stream = health.NewStream()
}

const (
	scriptsDir = "scripts"
	initDir    = "init"
	planDir    = "plan"
	configFile = "config.json"
	indexFile  = "index.js"
)

//Initilize initializes the build process. Any stages after this will have the generator
//already bootstraped.
//
// It is possible to bootstrap the generator from the project( User's side) by
// providing an entry javascript file in the default path of
// scripts/init/index.js which will be executed and you can overide the default
// entry script which is evaluated internally
//
// Initialzation is offloaded to the javascript runtine of the generator..Any
// error returned is a build error.
func Initilize(ctx *Context) error {

	// global values
	_ = ctx.VM.Set("fileTree", fileTree(ctx.FS, ctx.WorkDir))
	_ = ctx.VM.Set("sys", ctx.Sys)
	_ = ctx.VM.Set("defaultPlan", defaultPlan())
	_ = ctx.VM.Set("emptyPlan", &Plan{})
	_ = ctx.VM.Set("addCustomPlan", func(call otto.FunctionCall) otto.Value {
		arg, err := call.Argument(0).ToString()
		if err != nil {
			panicOtto(err)
		}
		p := &Plan{}
		err = json.Unmarshal([]byte(arg), p)
		if err != nil {
			panicOtto(err)
		}
		v, err := call.Otto.Get("Wu")
		if err != nil {
			panicOtto(err)
		}
		wu := v.Object()
		err = wu.Set("customPlan", p)
		if err != nil {
			panicOtto(err)
		}
		ctx.Sys.Plan = p
		return otto.UndefinedValue()
	})

	_, err := ctx.VM.Eval(entryScript())
	if err != nil {
		return err
	}
	// evaluate project provided entry script if provided. We ignore if the file
	// is not provided but any errors arsing from evaluating a provided script is
	// a built error.
	entryFile := filepath.Join(scriptsDir, initDir, indexFile)
	err = EvaluateFile(ctx.FS, ctx.VM, entryFile)
	if err != nil {
		if !os.IsNotExist(err) {
			return err
		}
	}
	return nil
}

//fileTree provides an array of all files found in the root( which is suppose to
//be the working directory.
//
//The file index is built only once and evaluated once too, then it is cached.
//This operates in the otto runtime.
func fileTree(fs afero.Fs, root string) func(otto.FunctionCall) otto.Value {
	var tree []string
	var v otto.Value
	return func(call otto.FunctionCall) otto.Value {
		if tree != nil {
			return v
		}
		ferr := afero.Walk(fs, "/", func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			if !info.IsDir() {
				tree = append(tree, path)
			}
			return nil
		})
		if ferr != nil {
			tree = nil
			panicOtto(ferr)
		}
		v, _ = call.Otto.ToValue(tree)
		return v
	}
}

func defaultSystem() *System {
	return &System{
		Boot: &Boot{},
		Plan: defaultPlan(),
	}
}

//EvaluateFile  opens the file in the specified path and evaluates it withing the context of
// the javascript runtine.
//
// The evaluated javascript code can mutate the global state. Use execFile to
// execute the javascript without mutating the state of the generato'r
// javascript runtime.
func EvaluateFile(fs afero.Fs, vm *otto.Otto, path string) error {
	data, err := afero.ReadFile(fs, path)
	if err != nil {
		return err
	}
	_, err = vm.Eval(data)
	if err != nil {
		return err
	}
	return nil
}

func defaultVM(sys *System) *otto.Otto {
	return otto.New()
}

//Configure configures the ctx with default values. This make sure the ctx is in
//good shape to be passed around to any other stages.
//
// It is safe to pass an eunintialized *Context, when you want to expect default
// behaviour.
//
// passing unitialized *Context will make the current working directory as the
// context WOrkDir. This can have unintended effect so be warned and just make
// sure the context passed has WorkDir set if you inted to do something
// otherwise.
func Configure(ctx *Context) error {
	if ctx.WorkDir == "" {

		// we assing the current working directory as the context WOrkDir. This
		// is ugly, and may result into bugs. But it is important for a WorkDir
		// to be set that is why this is left here.
		wd, err := os.Getwd()
		if err != nil {
			return err
		}
		ctx.WorkDir = wd
	}
	if ctx.Out == nil {
		ctx.Out = os.Stdout
	}
	if ctx.Job == nil {
		if stream.Sinks == nil {
			stream.AddSink(&health.WriterSink{Writer: ctx.Out})
		}
		ctx.Job = stream.NewJob("generate:" + ctx.WorkDir)
	}
	if ctx.FS == nil {
		ctx.FS = afero.NewOsFs()
	}
	if ctx.Sys == nil {
		ctx.Sys = defaultSystem()
	}
	if ctx.VM == nil {
		ctx.VM = defaultVM(ctx.Sys)
	}

	// ensure everything is relative to the working directory
	// The working directory is where the  directory in which the project to be
	// built lives.
	//
	// All file operations, happens within the diretory. So to access the
	// configuration file for example it is located in /config.json.
	ctx.FS = afero.NewBasePathFs(ctx.FS, ctx.WorkDir)
	af := afero.Afero{Fs: ctx.FS}
	data, err := af.ReadFile(configFile)
	if err != nil {
		return err
	}
	cfg := &Config{}
	err = json.Unmarshal(data, cfg)
	if err != nil {
		return err
	}
	ctx.Sys.Config = cfg
	return AddRequire(ctx)
}

//AddRequire adds require function to the ctx VM. It also loads the modules that
//are shipped with the package( a.k.a builtin modules).
func AddRequire(ctx *Context) error {
	// Add reuire
	req := NewRequire(ctx.FS, scriptsDir)
	if ctx.Sys.Config != nil {
		req.Paths = append(req.Paths, ctx.Sys.Config.PluginDir)
	}
	err := RegisterBuiltins(ctx, req)
	if err != nil {
		return err
	}
	_ = ctx.VM.Set("require", req.Load)
	return nil
}

//PlanExecution is the planning phase of the generator. First a project specific plan
//file is evaluated, the file is loacetd in /scripts_dir/plan_fir/index.js for
//example /scripts/plan/index.js. Second the existing system plan is merged with
//default plan.
//
//TODO: Use user defined plan only when it is set, and use default plan only
//when there is no user defined plan.
//
// This is executed to prepare the Plan object, which is the blueprint on how
// the whole execution is going to take place.
func PlanExecution(ctx *Context) error {
	pFile := ctx.Sys.Boot.PlanFile
	if pFile == "" {
		pFile = filepath.Join(scriptsDir, planDir, indexFile)
	}
	err := EvaluateFile(ctx.FS, ctx.VM, pFile)
	if err != nil {
		if !os.IsNotExist(err) {
			return err
		}
	}
	o, err := ctx.VM.Call("Wu.prepare", nil, ctx.Sys.Plan)
	if err != nil {
		return err
	}
	ok, err := o.ToBoolean()
	if err != nil {
		return err
	}
	if !ok {
		return errors.New("trouble preparing plan")
	}
	return nil
}

//Execute  executes the plans that are outilend by the plans found in ctx.
//
// Execution is done in two phases. First the files are evaluated using the
// matching plans. Plans are per file. Secondly the rendering of the site is
// done.
//
// Evaluation of files is done concurrently, then the results are aggregated
// back together.
//
// Rendering is done synchronously. As there is a lot of cross refences and
// stuffs to be considered, the rendering function is provided by the plan.
func Execute(ctx *Context) error {
	o, err := ctx.VM.Call("fileTree", nil)
	if err != nil {
		return err
	}
	ov, err := o.Export()
	if err != nil {
		return err
	}
	if ov == nil {
		// No files to mess with
		return errors.New("no files to build")
	}
	files, ok := ov.([]string)
	if !ok {
		// Some fish
		return err
	}

	var wg sync.WaitGroup
	var list FileList
	var errlist []error
	p := ctx.Sys.Plan
	for i := range files {
		wg.Add(1)
		go func(fn string) {
			defer wg.Done()
			s, err := p.FindStrategy(fn)
			if err != nil {
				errlist = append(errlist, err)
				return
			}
			f, err := ExecStrategy(ctx, fn, s)
			if err != nil {
				errlist = append(errlist, err)
				return
			}
			list = append(list, f)
		}(files[i])

	}
	wg.Wait()
	return ExecPlan(ctx, list, p)
}

//ExecStrategy executes the stratedy using the given ctx. The endgame is a *File
//as we are only interested in processing the file  named filename.
//
// The strategy outlines how filename is processed.
func ExecStrategy(ctx *Context, filename string, s *Strategy) (*File, error) {
	b, err := afero.ReadFile(ctx.FS, filename)
	if err != nil {
		return nil, err
	}
	f := &File{
		Name:     filename,
		Contents: string(b),
	}
	vm := ctx.VM.Copy()
	d, _ := json.Marshal(f)
	//wu, err := vm.Get("Wu")
	//if err != nil {
	//return nil, err
	//}
	//if !wu.IsObject() {
	//return nil, errors.New("no Wu object")
	//}
	//wo := wu.Object()
	//process, err := wo.Get("process")
	//if err != nil {
	//return nil, err
	//}
	out := &bytes.Buffer{}
	//_, err = process.Call(wo.Value(), s, string(d), out)
	_, err = vm.Call("Wu.process", nil, s, string(d), out)
	if err != nil {
		return nil, err
	}
	fd := &File{}
	err = json.Unmarshal(out.Bytes(), fd)
	if err != nil {
		return nil, err
	}
	return fd, nil
}

//ExecPlan executes plan s for processing the FileList fi using the conetx ctx.
func ExecPlan(ctx *Context, fl FileList, s *Plan) error {
	return nil
}

//RegisterBuiltins adds important modules  like underscore and fs.
func RegisterBuiltins(ctx *Context, r *Require) error {
	f := &fileSys{}
	f.Fs = ctx.FS
	v := f.export().ToValue(ctx.VM)
	r.addToCache("fs", v)
	r.addToCache("markdown", Markdown().ToValue(ctx.VM))
	_, err := ctx.VM.Run(underscore.Source())
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>gen: Cleanup<commit_after>package gen

import (
	"bytes"
	"encoding/json"
	"errors"
	"os"
	"path/filepath"
	"sync"

	"github.com/gocraft/health"
	"github.com/robertkrimen/otto"
	"github.com/robertkrimen/otto/underscore"
	"github.com/spf13/afero"
)

//package specfic halth stream. This can be used to provide consistent
//application wise health check stream.
//
// It is safe for concurrent usage.
var stream *health.Stream

func init() {
	stream = health.NewStream()
}

const (
	scriptsDir = "scripts"
	initDir    = "init"
	planDir    = "plan"
	configFile = "config.json"
	indexFile  = "index.js"
)

//Initilize initializes the build process. Any stages after this will have the generator
//already bootstraped.
//
// It is possible to bootstrap the generator from the project( User's side) by
// providing an entry javascript file in the default path of
// scripts/init/index.js which will be executed and you can overide the default
// entry script which is evaluated internally
//
// Initialzation is offloaded to the javascript runtine of the generator..Any
// error returned is a build error.
func Initilize(ctx *Context) error {

	// global values
	_ = ctx.VM.Set("fileTree", fileTree(ctx.FS, ctx.WorkDir))
	_ = ctx.VM.Set("sys", ctx.Sys)
	_ = ctx.VM.Set("defaultPlan", defaultPlan())
	_ = ctx.VM.Set("emptyPlan", &Plan{})
	_ = ctx.VM.Set("addCustomPlan", func(call otto.FunctionCall) otto.Value {
		arg, err := call.Argument(0).ToString()
		if err != nil {
			panicOtto(err)
		}
		p := &Plan{}
		err = json.Unmarshal([]byte(arg), p)
		if err != nil {
			panicOtto(err)
		}
		v, err := call.Otto.Get("Wu")
		if err != nil {
			panicOtto(err)
		}
		wu := v.Object()
		err = wu.Set("customPlan", p)
		if err != nil {
			panicOtto(err)
		}
		ctx.Sys.Plan = p
		return otto.UndefinedValue()
	})

	_, err := ctx.VM.Eval(entryScript())
	if err != nil {
		return err
	}
	// evaluate project provided entry script if provided. We ignore if the file
	// is not provided but any errors arsing from evaluating a provided script is
	// a built error.
	entryFile := filepath.Join(scriptsDir, initDir, indexFile)
	err = EvaluateFile(ctx.FS, ctx.VM, entryFile)
	if err != nil {
		if !os.IsNotExist(err) {
			return err
		}
	}
	return nil
}

//fileTree provides an array of all files found in the root( which is suppose to
//be the working directory.
//
//The file index is built only once and evaluated once too, then it is cached.
//This operates in the otto runtime.
func fileTree(fs afero.Fs, root string) func(otto.FunctionCall) otto.Value {
	var tree []string
	var v otto.Value
	return func(call otto.FunctionCall) otto.Value {
		if tree != nil {
			return v
		}
		ferr := afero.Walk(fs, "/", func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			if !info.IsDir() {
				tree = append(tree, path)
			}
			return nil
		})
		if ferr != nil {
			tree = nil
			panicOtto(ferr)
		}
		v, _ = call.Otto.ToValue(tree)
		return v
	}
}

func defaultSystem() *System {
	return &System{
		Boot: &Boot{},
		Plan: defaultPlan(),
	}
}

//EvaluateFile  opens the file in the specified path and evaluates it withing the context of
// the javascript runtine.
//
// The evaluated javascript code can mutate the global state. Use execFile to
// execute the javascript without mutating the state of the generato'r
// javascript runtime.
func EvaluateFile(fs afero.Fs, vm *otto.Otto, path string) error {
	data, err := afero.ReadFile(fs, path)
	if err != nil {
		return err
	}
	_, err = vm.Eval(data)
	if err != nil {
		return err
	}
	return nil
}

func defaultVM(sys *System) *otto.Otto {
	return otto.New()
}

//Configure configures the ctx with default values. This make sure the ctx is in
//good shape to be passed around to any other stages.
//
// It is safe to pass an eunintialized *Context, when you want to expect default
// behaviour.
//
// passing unitialized *Context will make the current working directory as the
// context WOrkDir. This can have unintended effect so be warned and just make
// sure the context passed has WorkDir set if you inted to do something
// otherwise.
func Configure(ctx *Context) error {
	if ctx.WorkDir == "" {

		// we assing the current working directory as the context WOrkDir. This
		// is ugly, and may result into bugs. But it is important for a WorkDir
		// to be set that is why this is left here.
		wd, err := os.Getwd()
		if err != nil {
			return err
		}
		ctx.WorkDir = wd
	}
	if ctx.Out == nil {
		ctx.Out = os.Stdout
	}
	if ctx.Job == nil {
		if stream.Sinks == nil {
			stream.AddSink(&health.WriterSink{Writer: ctx.Out})
		}
		ctx.Job = stream.NewJob("generate:" + ctx.WorkDir)
	}
	if ctx.FS == nil {
		ctx.FS = afero.NewOsFs()
	}
	if ctx.Sys == nil {
		ctx.Sys = defaultSystem()
	}
	if ctx.VM == nil {
		ctx.VM = defaultVM(ctx.Sys)
	}

	// ensure everything is relative to the working directory
	// The working directory is where the  directory in which the project to be
	// built lives.
	//
	// All file operations, happens within the diretory. So to access the
	// configuration file for example it is located in /config.json.
	ctx.FS = afero.NewBasePathFs(ctx.FS, ctx.WorkDir)
	af := afero.Afero{Fs: ctx.FS}
	data, err := af.ReadFile(configFile)
	if err != nil {
		return err
	}
	cfg := &Config{}
	err = json.Unmarshal(data, cfg)
	if err != nil {
		return err
	}
	ctx.Sys.Config = cfg
	return AddRequire(ctx)
}

//AddRequire adds require function to the ctx VM. It also loads the modules that
//are shipped with the package( a.k.a builtin modules).
func AddRequire(ctx *Context) error {
	// Add reuire
	req := NewRequire(ctx.FS, scriptsDir)
	if ctx.Sys.Config != nil {
		req.Paths = append(req.Paths, ctx.Sys.Config.PluginDir)
	}
	err := RegisterBuiltins(ctx, req)
	if err != nil {
		return err
	}
	_ = ctx.VM.Set("require", req.Load)
	return nil
}

//PlanExecution is the planning phase of the generator. First a project specific plan
//file is evaluated, the file is loacetd in /scripts_dir/plan_fir/index.js for
//example /scripts/plan/index.js. Second the existing system plan is merged with
//default plan.
//
//TODO: Use user defined plan only when it is set, and use default plan only
//when there is no user defined plan.
//
// This is executed to prepare the Plan object, which is the blueprint on how
// the whole execution is going to take place.
func PlanExecution(ctx *Context) error {
	pFile := ctx.Sys.Boot.PlanFile
	if pFile == "" {
		pFile = filepath.Join(scriptsDir, planDir, indexFile)
	}
	err := EvaluateFile(ctx.FS, ctx.VM, pFile)
	if err != nil {
		if !os.IsNotExist(err) {
			return err
		}
	}
	o, err := ctx.VM.Call("Wu.prepare", nil, ctx.Sys.Plan)
	if err != nil {
		return err
	}
	ok, err := o.ToBoolean()
	if err != nil {
		return err
	}
	if !ok {
		return errors.New("trouble preparing plan")
	}
	return nil
}

//Execute  executes the plans that are outilend by the plans found in ctx.
//
// Execution is done in two phases. First the files are evaluated using the
// matching plans. Plans are per file. Secondly the rendering of the site is
// done.
//
// Evaluation of files is done concurrently, then the results are aggregated
// back together.
//
// Rendering is done synchronously. As there is a lot of cross refences and
// stuffs to be considered, the rendering function is provided by the plan.
func Execute(ctx *Context) error {
	o, err := ctx.VM.Call("fileTree", nil)
	if err != nil {
		return err
	}
	ov, err := o.Export()
	if err != nil {
		return err
	}
	if ov == nil {
		// No files to mess with
		return errors.New("no files to build")
	}
	files, ok := ov.([]string)
	if !ok {
		// Some fish
		return err
	}

	var wg sync.WaitGroup
	var list FileList
	var errlist []error
	p := ctx.Sys.Plan
	for i := range files {
		wg.Add(1)
		go func(fn string) {
			defer wg.Done()
			s, err := p.FindStrategy(fn)
			if err != nil {
				errlist = append(errlist, err)
				return
			}
			f, err := ExecStrategy(ctx, fn, s)
			if err != nil {
				errlist = append(errlist, err)
				return
			}
			list = append(list, f)
		}(files[i])

	}
	wg.Wait()
	return ExecPlan(ctx, list, p)
}

//ExecStrategy executes the stratedy using the given ctx. The endgame is a *File
//as we are only interested in processing the file  named filename.
//
// The strategy outlines how filename is processed.
func ExecStrategy(ctx *Context, filename string, s *Strategy) (*File, error) {
	b, err := afero.ReadFile(ctx.FS, filename)
	if err != nil {
		return nil, err
	}
	f := &File{
		Name:     filename,
		Contents: string(b),
	}
	vm := ctx.VM.Copy()
	d, _ := json.Marshal(f)
	out := &bytes.Buffer{}
	_, err = vm.Call("Wu.process", nil, s, string(d), out)
	if err != nil {
		return nil, err
	}
	fd := &File{}
	err = json.Unmarshal(out.Bytes(), fd)
	if err != nil {
		return nil, err
	}
	return fd, nil
}

//ExecPlan executes plan s for processing the FileList fi using the conetx ctx.
func ExecPlan(ctx *Context, fl FileList, s *Plan) error {
	return nil
}

//RegisterBuiltins adds important modules  like underscore and fs.
func RegisterBuiltins(ctx *Context, r *Require) error {
	f := &fileSys{}
	f.Fs = ctx.FS
	v := f.export().ToValue(ctx.VM)
	r.addToCache("fs", v)
	r.addToCache("markdown", Markdown().ToValue(ctx.VM))
	_, err := ctx.VM.Run(underscore.Source())
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package examples

import (
	"flag"
	"fmt"
	"go/build"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"testing"
)

var examplesTest = flag.Bool("examples", false, "run the examples tests")

var defaultURL = "https://github.com/git-fixtures/basic.git"

var args = map[string][]string{
	"checkout":    []string{defaultURL, tempFolder(), "35e85108805c84807bc66a02d91535e1e24b38b9"},
	"clone":       []string{defaultURL, tempFolder()},
	"custom_http": []string{defaultURL},
	"open":        []string{cloneRepository(defaultURL, tempFolder())},
	"progress":    []string{defaultURL, tempFolder()},
	"push":        []string{setEmptyRemote(cloneRepository(defaultURL, tempFolder()))},
	"showcase":    []string{defaultURL, tempFolder()},
}

var ignored = map[string]bool{}

var tempFolders = []string{}

func TestExamples(t *testing.T) {
	flag.Parse()
	if !*examplesTest && os.Getenv("CI") == "" {
		t.Skip("skipping examples tests, pass --examples to execute it")
		return
	}

	defer deleteTempFolders()

	examples, err := filepath.Glob(examplesFolder())
	if err != nil {
		t.Errorf("error finding tests: %s", err)
	}

	for _, example := range examples {
		_, name := filepath.Split(filepath.Dir(example))

		if ignored[name] {
			continue
		}

		t.Run(name, func(t *testing.T) {
			testExample(t, name, example)
		})
	}
}

func tempFolder() string {
	path, err := ioutil.TempDir("", "")
	CheckIfError(err)

	tempFolders = append(tempFolders, path)
	return path
}

func packageFolder() string {
	return filepath.Join(
		build.Default.GOPATH,
		"src", "gopkg.in/src-d/go-git.v4",
	)
}

func examplesFolder() string {
	return filepath.Join(
		packageFolder(),
		"_examples", "*", "main.go",
	)
}

func cloneRepository(url, folder string) string {
	cmd := exec.Command("git", "clone", url, folder)
	err := cmd.Run()
	CheckIfError(err)

	return folder
}

func createBareRepository(dir string) string {
	cmd := exec.Command("git", "init", "--bare", dir)
	err := cmd.Run()
	CheckIfError(err)

	return dir
}

func setEmptyRemote(dir string) string {
	remote := createBareRepository(tempFolder())
	setRemote(dir, remote)
	return dir
}

func setRemote(local, remote string) {
	cmd := exec.Command("git", "remote", "set-url", "origin", remote)
	cmd.Dir = local
	err := cmd.Run()
	CheckIfError(err)
}

func testExample(t *testing.T, name, example string) {
	cmd := exec.Command("go", append([]string{
		"run", filepath.Join(example),
	}, args[name]...)...)

	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	if err := cmd.Run(); err != nil {
		t.Errorf("error running cmd %q", err)
	}
}

func deleteTempFolders() {
	for _, folder := range tempFolders {
		err := os.RemoveAll(folder)
		CheckIfError(err)
	}
}
<commit_msg>examples: fix compilation error from #368<commit_after>package examples

import (
	"flag"
	"go/build"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"testing"
)

var examplesTest = flag.Bool("examples", false, "run the examples tests")

var defaultURL = "https://github.com/git-fixtures/basic.git"

var args = map[string][]string{
	"checkout":    []string{defaultURL, tempFolder(), "35e85108805c84807bc66a02d91535e1e24b38b9"},
	"clone":       []string{defaultURL, tempFolder()},
	"custom_http": []string{defaultURL},
	"open":        []string{cloneRepository(defaultURL, tempFolder())},
	"progress":    []string{defaultURL, tempFolder()},
	"push":        []string{setEmptyRemote(cloneRepository(defaultURL, tempFolder()))},
	"showcase":    []string{defaultURL, tempFolder()},
}

var ignored = map[string]bool{}

var tempFolders = []string{}

func TestExamples(t *testing.T) {
	flag.Parse()
	if !*examplesTest && os.Getenv("CI") == "" {
		t.Skip("skipping examples tests, pass --examples to execute it")
		return
	}

	defer deleteTempFolders()

	examples, err := filepath.Glob(examplesFolder())
	if err != nil {
		t.Errorf("error finding tests: %s", err)
	}

	for _, example := range examples {
		_, name := filepath.Split(filepath.Dir(example))

		if ignored[name] {
			continue
		}

		t.Run(name, func(t *testing.T) {
			testExample(t, name, example)
		})
	}
}

func tempFolder() string {
	path, err := ioutil.TempDir("", "")
	CheckIfError(err)

	tempFolders = append(tempFolders, path)
	return path
}

func packageFolder() string {
	return filepath.Join(
		build.Default.GOPATH,
		"src", "gopkg.in/src-d/go-git.v4",
	)
}

func examplesFolder() string {
	return filepath.Join(
		packageFolder(),
		"_examples", "*", "main.go",
	)
}

func cloneRepository(url, folder string) string {
	cmd := exec.Command("git", "clone", url, folder)
	err := cmd.Run()
	CheckIfError(err)

	return folder
}

func createBareRepository(dir string) string {
	cmd := exec.Command("git", "init", "--bare", dir)
	err := cmd.Run()
	CheckIfError(err)

	return dir
}

func setEmptyRemote(dir string) string {
	remote := createBareRepository(tempFolder())
	setRemote(dir, remote)
	return dir
}

func setRemote(local, remote string) {
	cmd := exec.Command("git", "remote", "set-url", "origin", remote)
	cmd.Dir = local
	err := cmd.Run()
	CheckIfError(err)
}

func testExample(t *testing.T, name, example string) {
	cmd := exec.Command("go", append([]string{
		"run", filepath.Join(example),
	}, args[name]...)...)

	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	if err := cmd.Run(); err != nil {
		t.Errorf("error running cmd %q", err)
	}
}

func deleteTempFolders() {
	for _, folder := range tempFolders {
		err := os.RemoveAll(folder)
		CheckIfError(err)
	}
}
<|endoftext|>


<commit_before>package lrp_bbs_test

import (
	"time"

	"github.com/cloudfoundry-incubator/runtime-schema/bbs/shared"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	"github.com/cloudfoundry/dropsonde/metric_sender/fake"
	"github.com/cloudfoundry/dropsonde/metrics"
	"github.com/cloudfoundry/storeadapter"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("LrpConvergence", func() {
	const pollingInterval = 5 * time.Second

	var (
		sender *fake.FakeMetricSender

		cellPresence models.CellPresence
	)

	processGuid := "process-guid"
	unclaimedProcessGuid := "unclaimed-process-guid"

	BeforeEach(func() {
		sender = fake.NewFakeMetricSender()
		metrics.Initialize(sender)

		cellPresence = models.CellPresence{
			CellID:     "the-cell-id",
			Stack:      "the-stack",
			RepAddress: "cell.example.com",
		}

		registerCell(cellPresence)

		a1 := models.NewActualLRP(processGuid, "instance-guid-1", cellPresence.CellID, "domain", 0, models.ActualLRPStateUnclaimed)
		createAndClaim(a1)

		a2 := models.NewActualLRP(processGuid, "instance-guid-2", cellPresence.CellID, "domain", 1, models.ActualLRPStateUnclaimed)
		createAndClaim(a2)

		_, err := bbs.CreateActualLRP(models.NewActualLRP(unclaimedProcessGuid, "instance-guid-3", "", "domain", 0, models.ActualLRPStateUnclaimed))
		Ω(err).ShouldNot(HaveOccurred())
	})

	It("logs the convergence", func() {
		bbs.ConvergeLRPs(pollingInterval)
		logMessages := logger.TestSink.LogMessages()
		Ω(logMessages[0]).Should(Equal(
			"test.converge-lrps.starting-convergence",
		))
		Ω(logMessages[len(logMessages)-1]).Should(Equal(
			"test.converge-lrps.finished-convergence",
		))
	})

	It("bumps the convergence counter", func() {
		Ω(sender.GetCounter("ConvergenceLRPRuns")).Should(Equal(uint64(0)))
		bbs.ConvergeLRPs(pollingInterval)
		Ω(sender.GetCounter("ConvergenceLRPRuns")).Should(Equal(uint64(1)))
		bbs.ConvergeLRPs(pollingInterval)
		Ω(sender.GetCounter("ConvergenceLRPRuns")).Should(Equal(uint64(2)))
	})

	It("reports the duration that it took to converge", func() {
		timeProvider.IntervalToAdvance = 500 * time.Nanosecond
		bbs.ConvergeLRPs(pollingInterval)

		reportedDuration := sender.GetValue("ConvergenceLRPDuration")
		Ω(reportedDuration.Unit).Should(Equal("nanos"))
		Ω(reportedDuration.Value).ShouldNot(BeZero())
	})

	Describe("pruning LRPs by cell", func() {
		JustBeforeEach(func() {
			bbs.ConvergeLRPs(pollingInterval)
		})

		Context("when no cell is missing", func() {
			It("should not prune any LRPs", func() {
				Ω(bbs.ActualLRPs()).Should(HaveLen(3))
			})
		})

		Context("when an cell is missing", func() {
			BeforeEach(func() {
				etcdClient.Delete(shared.CellSchemaPath(cellPresence.CellID))
			})

			It("should delete LRPs associated with said cell but not the unclaimed LRP", func() {
				lrps, err := bbs.ActualLRPs()
				Ω(err).ShouldNot(HaveOccurred())
				Ω(lrps).Should(HaveLen(1))
				Ω(lrps[0].ProcessGuid).Should(Equal(unclaimedProcessGuid))
			})

			It("should prune LRP directories for apps that are no longer running", func() {
				actual, err := etcdClient.ListRecursively(shared.ActualLRPSchemaRoot)
				Ω(err).ShouldNot(HaveOccurred())
				Ω(actual.ChildNodes).Should(HaveLen(1))
				Ω(actual.ChildNodes[0].Key).Should(Equal(shared.ActualLRPProcessDir(unclaimedProcessGuid)))
			})

			It("logs", func() {
				Ω(logger.TestSink.LogMessages()).Should(ContainElement("test.converge-lrps.detected-actual-with-missing-cell"))
			})
		})
	})

	Describe("when there is a desired LRP", func() {
		var desiredEvents <-chan models.DesiredLRPChange
		var desiredLRP models.DesiredLRP

		commenceWatching := func() {
			desiredEvents, _, _ = bbs.WatchForDesiredLRPChanges()
		}

		BeforeEach(func() {
			desiredLRP = models.DesiredLRP{
				Domain:      "tests",
				ProcessGuid: processGuid,
				Instances:   2,
				Stack:       "pancake",
				Action: &models.DownloadAction{
					From: "http://example.com",
					To:   "/tmp/internet",
				},
			}
		})

		Context("when the desired LRP has malformed JSON", func() {
			BeforeEach(func() {
				err := etcdClient.SetMulti([]storeadapter.StoreNode{
					{
						Key:   shared.DesiredLRPSchemaPathByProcessGuid("bogus-desired"),
						Value: []byte("ß"),
					},
				})

				Ω(err).ShouldNot(HaveOccurred())
			})

			It("should delete the bogus entry", func() {
				bbs.ConvergeLRPs(pollingInterval)
				_, err := etcdClient.Get(shared.DesiredLRPSchemaPathByProcessGuid("bogus-desired"))
				Ω(err).Should(MatchError(storeadapter.ErrorKeyNotFound))
			})

			It("bumps the deleted LRPs convergence counter", func() {
				Ω(sender.GetCounter("ConvergenceLRPsDeleted")).Should(Equal(uint64(0)))
				bbs.ConvergeLRPs(pollingInterval)
				Ω(sender.GetCounter("ConvergenceLRPsDeleted")).Should(Equal(uint64(1)))
			})

			It("logs", func() {
				bbs.ConvergeLRPs(pollingInterval)
				Ω(logger.TestSink.LogMessages()).Should(ContainElement("test.converge-lrps.pruning-invalid-desired-lrp-json"))
			})
		})

		Context("when the desired LRP has all its actual LRPs, and there are no extras", func() {
			BeforeEach(func() {
				bbs.DesireLRP(desiredLRP)
			})

			It("should not kick the desired LRP", func() {
				commenceWatching()
				bbs.ConvergeLRPs(pollingInterval)

				Consistently(desiredEvents).ShouldNot(Receive())
			})
		})

		Context("when the desired LRP is missing actuals", func() {
			BeforeEach(func() {
				desiredLRP.Instances = 3
				bbs.DesireLRP(desiredLRP)
			})

			It("should kick the desired LRP", func() {
				commenceWatching()
				bbs.ConvergeLRPs(pollingInterval)

				var noticedOnce models.DesiredLRPChange
				Eventually(desiredEvents).Should(Receive(&noticedOnce))
				Ω(*noticedOnce.After).Should(Equal(desiredLRP))
			})

			It("bumps the compare-and-swapped LRPs convergence counter", func() {
				Ω(sender.GetCounter("ConvergenceLRPsKicked")).Should(Equal(uint64(0)))
				bbs.ConvergeLRPs(pollingInterval)
				Ω(sender.GetCounter("ConvergenceLRPsKicked")).Should(Equal(uint64(1)))
			})

			It("logs", func() {
				bbs.ConvergeLRPs(pollingInterval)
				Ω(logger.TestSink.LogMessages()).Should(ContainElement("test.converge-lrps.detected-missing-instance"))
			})
		})

		Context("when the desired LRP has only unclaimed actuals", func() {
			BeforeEach(func() {
				desiredLRP.ProcessGuid = unclaimedProcessGuid
				desiredLRP.Instances = 1
				bbs.DesireLRP(desiredLRP)
			})

			It("does not kick the desired LRP", func() {
				commenceWatching()
				bbs.ConvergeLRPs(pollingInterval)

				Consistently(desiredEvents).ShouldNot(Receive())
			})

			It("bumps the compare-and-swapped LRPs convergence counter", func() {
				Ω(sender.GetCounter("ConvergenceLRPsKicked")).Should(Equal(uint64(0)))
				bbs.ConvergeLRPs(pollingInterval)
				Ω(sender.GetCounter("ConvergenceLRPsKicked")).Should(Equal(uint64(0)))
			})

			Context("and the unclaimed actual is stale", func() {
				BeforeEach(func() {
					timeProvider.Increment(pollingInterval + 1*time.Second)
				})

				It("resends a start auction for the actual", func() {
					Ω(startAuctionBBS.LRPStartAuctions()).Should(HaveLen(0))

					commenceWatching()
					bbs.ConvergeLRPs(pollingInterval)

					Ω(startAuctionBBS.LRPStartAuctions()).Should(HaveLen(1))
				})

				It("logs", func() {
					commenceWatching()
					bbs.ConvergeLRPs(pollingInterval)
					Ω(logger.TestSink.LogMessages()).Should(ContainElement("test.converge-lrps.resending-start-auction"))
				})
			})
		})

		Context("when there are extra actual LRPs", func() {
			BeforeEach(func() {
				desiredLRP.Instances = 1
				bbs.DesireLRP(desiredLRP)
			})

			It("should kick the desired LRP", func() {
				commenceWatching()
				bbs.ConvergeLRPs(pollingInterval)

				var noticedOnce models.DesiredLRPChange
				Eventually(desiredEvents).Should(Receive(&noticedOnce))
				Ω(*noticedOnce.After).Should(Equal(desiredLRP))
			})

			It("bumps the compare-and-swapped LRPs convergence counter", func() {
				Ω(sender.GetCounter("ConvergenceLRPsKicked")).Should(Equal(uint64(0)))
				bbs.ConvergeLRPs(pollingInterval)
				Ω(sender.GetCounter("ConvergenceLRPsKicked")).Should(Equal(uint64(1)))
			})

			It("logs", func() {
				bbs.ConvergeLRPs(pollingInterval)
				Ω(logger.TestSink.LogMessages()).Should(ContainElement("test.converge-lrps.detected-extra-instance"))
				Ω(logger.TestSink.LogMessages()).Should(ContainElement("test.converge-lrps.kicking-desired-lrp"))
			})
		})
	})

	Context("when there is an actual LRP with no matching desired LRP", func() {
		It("should emit a stop for the actual LRP", func() {
			bbs.ConvergeLRPs(pollingInterval)
			Ω(fakeCellClient.StopLRPInstanceCallCount()).Should(Equal(2))

			addr1, stop1 := fakeCellClient.StopLRPInstanceArgsForCall(0)
			addr2, stop2 := fakeCellClient.StopLRPInstanceArgsForCall(1)
			Ω(addr1).Should(Equal(cellPresence.RepAddress))
			Ω(addr2).Should(Equal(cellPresence.RepAddress))

			Ω([]string{stop1.InstanceGuid, stop2.InstanceGuid}).Should(ConsistOf([]string{
				"instance-guid-1",
				"instance-guid-2",
			}))
		})

		It("logs", func() {
			bbs.ConvergeLRPs(pollingInterval)
			Ω(logger.TestSink.LogMessages()).Should(ContainElement("test.converge-lrps.detected-undesired-instance"))
		})

		It("bumps the stopped LRPs convergence counter", func() {
			Ω(sender.GetCounter("ConvergenceLRPsStopped")).Should(Equal(uint64(0)))
			bbs.ConvergeLRPs(pollingInterval)
			Ω(sender.GetCounter("ConvergenceLRPsStopped")).Should(Equal(uint64(2)))
		})

		Context("and there is a stale unclaimed actual", func() {
			BeforeEach(func() {
				timeProvider.Increment(pollingInterval + 1*time.Second)
			})

			It("does not resends a start auction for the actual", func() {
				Ω(startAuctionBBS.LRPStartAuctions()).Should(HaveLen(0))

				bbs.ConvergeLRPs(pollingInterval)

				Ω(startAuctionBBS.LRPStartAuctions()).Should(HaveLen(0))
			})

			It("logs", func() {
				bbs.ConvergeLRPs(pollingInterval)
				Ω(logger.TestSink.LogMessages()).Should(ContainElement("test.converge-lrps.failed-to-find-desired-lrp-for-stale-unclaimed-actual-lrp"))
			})
		})
	})
})
<commit_msg>Remove redundant LRP convergent test<commit_after>package lrp_bbs_test

import (
	"time"

	"github.com/cloudfoundry-incubator/runtime-schema/bbs/shared"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	"github.com/cloudfoundry/dropsonde/metric_sender/fake"
	"github.com/cloudfoundry/dropsonde/metrics"
	"github.com/cloudfoundry/storeadapter"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("LrpConvergence", func() {
	const pollingInterval = 5 * time.Second

	var (
		sender *fake.FakeMetricSender

		cellPresence models.CellPresence
	)

	processGuid := "process-guid"
	unclaimedProcessGuid := "unclaimed-process-guid"

	BeforeEach(func() {
		sender = fake.NewFakeMetricSender()
		metrics.Initialize(sender)

		cellPresence = models.CellPresence{
			CellID:     "the-cell-id",
			Stack:      "the-stack",
			RepAddress: "cell.example.com",
		}

		registerCell(cellPresence)

		a1 := models.NewActualLRP(processGuid, "instance-guid-1", cellPresence.CellID, "domain", 0, models.ActualLRPStateUnclaimed)
		createAndClaim(a1)

		a2 := models.NewActualLRP(processGuid, "instance-guid-2", cellPresence.CellID, "domain", 1, models.ActualLRPStateUnclaimed)
		createAndClaim(a2)

		_, err := bbs.CreateActualLRP(models.NewActualLRP(unclaimedProcessGuid, "instance-guid-3", "", "domain", 0, models.ActualLRPStateUnclaimed))
		Ω(err).ShouldNot(HaveOccurred())
	})

	It("logs the convergence", func() {
		bbs.ConvergeLRPs(pollingInterval)
		logMessages := logger.TestSink.LogMessages()
		Ω(logMessages[0]).Should(Equal(
			"test.converge-lrps.starting-convergence",
		))
		Ω(logMessages[len(logMessages)-1]).Should(Equal(
			"test.converge-lrps.finished-convergence",
		))
	})

	It("bumps the convergence counter", func() {
		Ω(sender.GetCounter("ConvergenceLRPRuns")).Should(Equal(uint64(0)))
		bbs.ConvergeLRPs(pollingInterval)
		Ω(sender.GetCounter("ConvergenceLRPRuns")).Should(Equal(uint64(1)))
		bbs.ConvergeLRPs(pollingInterval)
		Ω(sender.GetCounter("ConvergenceLRPRuns")).Should(Equal(uint64(2)))
	})

	It("reports the duration that it took to converge", func() {
		timeProvider.IntervalToAdvance = 500 * time.Nanosecond
		bbs.ConvergeLRPs(pollingInterval)

		reportedDuration := sender.GetValue("ConvergenceLRPDuration")
		Ω(reportedDuration.Unit).Should(Equal("nanos"))
		Ω(reportedDuration.Value).ShouldNot(BeZero())
	})

	Describe("pruning LRPs by cell", func() {
		JustBeforeEach(func() {
			bbs.ConvergeLRPs(pollingInterval)
		})

		Context("when no cell is missing", func() {
			It("should not prune any LRPs", func() {
				Ω(bbs.ActualLRPs()).Should(HaveLen(3))
			})
		})

		Context("when an cell is missing", func() {
			BeforeEach(func() {
				etcdClient.Delete(shared.CellSchemaPath(cellPresence.CellID))
			})

			It("should delete LRPs associated with said cell but not the unclaimed LRP", func() {
				lrps, err := bbs.ActualLRPs()
				Ω(err).ShouldNot(HaveOccurred())
				Ω(lrps).Should(HaveLen(1))
				Ω(lrps[0].ProcessGuid).Should(Equal(unclaimedProcessGuid))
			})

			It("should prune LRP directories for apps that are no longer running", func() {
				actual, err := etcdClient.ListRecursively(shared.ActualLRPSchemaRoot)
				Ω(err).ShouldNot(HaveOccurred())
				Ω(actual.ChildNodes).Should(HaveLen(1))
				Ω(actual.ChildNodes[0].Key).Should(Equal(shared.ActualLRPProcessDir(unclaimedProcessGuid)))
			})

			It("logs", func() {
				Ω(logger.TestSink.LogMessages()).Should(ContainElement("test.converge-lrps.detected-actual-with-missing-cell"))
			})
		})
	})

	Describe("when there is a desired LRP", func() {
		var desiredEvents <-chan models.DesiredLRPChange
		var desiredLRP models.DesiredLRP

		commenceWatching := func() {
			desiredEvents, _, _ = bbs.WatchForDesiredLRPChanges()
		}

		BeforeEach(func() {
			desiredLRP = models.DesiredLRP{
				Domain:      "tests",
				ProcessGuid: processGuid,
				Instances:   2,
				Stack:       "pancake",
				Action: &models.DownloadAction{
					From: "http://example.com",
					To:   "/tmp/internet",
				},
			}
		})

		Context("when the desired LRP has malformed JSON", func() {
			BeforeEach(func() {
				err := etcdClient.SetMulti([]storeadapter.StoreNode{
					{
						Key:   shared.DesiredLRPSchemaPathByProcessGuid("bogus-desired"),
						Value: []byte("ß"),
					},
				})

				Ω(err).ShouldNot(HaveOccurred())
			})

			It("should delete the bogus entry", func() {
				bbs.ConvergeLRPs(pollingInterval)
				_, err := etcdClient.Get(shared.DesiredLRPSchemaPathByProcessGuid("bogus-desired"))
				Ω(err).Should(MatchError(storeadapter.ErrorKeyNotFound))
			})

			It("bumps the deleted LRPs convergence counter", func() {
				Ω(sender.GetCounter("ConvergenceLRPsDeleted")).Should(Equal(uint64(0)))
				bbs.ConvergeLRPs(pollingInterval)
				Ω(sender.GetCounter("ConvergenceLRPsDeleted")).Should(Equal(uint64(1)))
			})

			It("logs", func() {
				bbs.ConvergeLRPs(pollingInterval)
				Ω(logger.TestSink.LogMessages()).Should(ContainElement("test.converge-lrps.pruning-invalid-desired-lrp-json"))
			})
		})

		Context("when the desired LRP has all its actual LRPs, and there are no extras", func() {
			BeforeEach(func() {
				bbs.DesireLRP(desiredLRP)
			})

			It("should not kick the desired LRP", func() {
				commenceWatching()
				bbs.ConvergeLRPs(pollingInterval)

				Consistently(desiredEvents).ShouldNot(Receive())
			})
		})

		Context("when the desired LRP is missing actuals", func() {
			BeforeEach(func() {
				desiredLRP.Instances = 3
				bbs.DesireLRP(desiredLRP)
			})

			It("should kick the desired LRP", func() {
				commenceWatching()
				bbs.ConvergeLRPs(pollingInterval)

				var noticedOnce models.DesiredLRPChange
				Eventually(desiredEvents).Should(Receive(&noticedOnce))
				Ω(*noticedOnce.After).Should(Equal(desiredLRP))
			})

			It("bumps the compare-and-swapped LRPs convergence counter", func() {
				Ω(sender.GetCounter("ConvergenceLRPsKicked")).Should(Equal(uint64(0)))
				bbs.ConvergeLRPs(pollingInterval)
				Ω(sender.GetCounter("ConvergenceLRPsKicked")).Should(Equal(uint64(1)))
			})

			It("logs", func() {
				bbs.ConvergeLRPs(pollingInterval)
				Ω(logger.TestSink.LogMessages()).Should(ContainElement("test.converge-lrps.detected-missing-instance"))
			})
		})

		Context("when the desired LRP has stale unclaimed actuals", func() {
			BeforeEach(func() {
				desiredLRP.ProcessGuid = unclaimedProcessGuid
				desiredLRP.Instances = 1
				bbs.DesireLRP(desiredLRP)
				timeProvider.Increment(pollingInterval + 1*time.Second)
			})

			It("resends a start auction for the unclaimed actual", func() {
				Ω(startAuctionBBS.LRPStartAuctions()).Should(HaveLen(0))

				commenceWatching()
				bbs.ConvergeLRPs(pollingInterval)

				startAuctions, err := startAuctionBBS.LRPStartAuctions()
				Ω(err).ShouldNot(HaveOccurred())
				Ω(startAuctions).Should(HaveLen(1))
				Ω(startAuctions[0].DesiredLRP).Should(Equal(desiredLRP))
				Ω(startAuctions[0].InstanceGuid).Should(Equal("instance-guid-3"))
			})

			It("logs", func() {
				commenceWatching()
				bbs.ConvergeLRPs(pollingInterval)
				Ω(logger.TestSink.LogMessages()).Should(ContainElement("test.converge-lrps.resending-start-auction"))
			})
		})

		Context("when there are extra actual LRPs", func() {
			BeforeEach(func() {
				desiredLRP.Instances = 1
				bbs.DesireLRP(desiredLRP)
			})

			It("should kick the desired LRP", func() {
				commenceWatching()
				bbs.ConvergeLRPs(pollingInterval)

				var noticedOnce models.DesiredLRPChange
				Eventually(desiredEvents).Should(Receive(&noticedOnce))
				Ω(*noticedOnce.After).Should(Equal(desiredLRP))
			})

			It("bumps the compare-and-swapped LRPs convergence counter", func() {
				Ω(sender.GetCounter("ConvergenceLRPsKicked")).Should(Equal(uint64(0)))
				bbs.ConvergeLRPs(pollingInterval)
				Ω(sender.GetCounter("ConvergenceLRPsKicked")).Should(Equal(uint64(1)))
			})

			It("logs", func() {
				bbs.ConvergeLRPs(pollingInterval)
				Ω(logger.TestSink.LogMessages()).Should(ContainElement("test.converge-lrps.detected-extra-instance"))
				Ω(logger.TestSink.LogMessages()).Should(ContainElement("test.converge-lrps.kicking-desired-lrp"))
			})
		})
	})

	Context("when there is an actual LRP with no matching desired LRP", func() {
		It("should emit a stop for the actual LRP", func() {
			bbs.ConvergeLRPs(pollingInterval)
			Ω(fakeCellClient.StopLRPInstanceCallCount()).Should(Equal(2))

			addr1, stop1 := fakeCellClient.StopLRPInstanceArgsForCall(0)
			addr2, stop2 := fakeCellClient.StopLRPInstanceArgsForCall(1)
			Ω(addr1).Should(Equal(cellPresence.RepAddress))
			Ω(addr2).Should(Equal(cellPresence.RepAddress))

			Ω([]string{stop1.InstanceGuid, stop2.InstanceGuid}).Should(ConsistOf(
				"instance-guid-1",
				"instance-guid-2",
			))
		})

		It("logs", func() {
			bbs.ConvergeLRPs(pollingInterval)
			Ω(logger.TestSink.LogMessages()).Should(ContainElement("test.converge-lrps.detected-undesired-instance"))
		})

		It("bumps the stopped LRPs convergence counter", func() {
			Ω(sender.GetCounter("ConvergenceLRPsStopped")).Should(Equal(uint64(0)))
			bbs.ConvergeLRPs(pollingInterval)
			Ω(sender.GetCounter("ConvergenceLRPsStopped")).Should(Equal(uint64(2)))
		})

		Context("and there is a stale unclaimed actual", func() {
			BeforeEach(func() {
				timeProvider.Increment(pollingInterval + 1*time.Second)
			})

			It("does not resends a start auction for the actual", func() {
				Ω(startAuctionBBS.LRPStartAuctions()).Should(HaveLen(0))

				bbs.ConvergeLRPs(pollingInterval)

				Ω(startAuctionBBS.LRPStartAuctions()).Should(HaveLen(0))
			})

			It("logs", func() {
				bbs.ConvergeLRPs(pollingInterval)
				Ω(logger.TestSink.LogMessages()).Should(ContainElement("test.converge-lrps.failed-to-find-desired-lrp-for-stale-unclaimed-actual-lrp"))
			})
		})
	})
})
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"errors"
	"fmt"
	"log"
	"os"
	"os/exec"
	"strings"
)

type Option struct {
	Name     string
	Question string
	Default  string
}

var options = []Option{
	{
		Name:     "prod",
		Question: "What is your production environment branch?",
		Default:  "master",
	},
	{
		Name:     "other",
		Question: "What other environment branches do you have?",
		Default:  "stage dev",
	},
	{
		Name:     "types",
		Question: "What branch types do you have?",
		Default:  "feature hotfix",
	},
}

type Config struct {
	Prod  string
	Other []string
	Types []string
}

var config Config

func main() {
	if len(os.Args) < 2 {
		help()
	}

	switch os.Args[1] {
	case "init":
		cmdInit()
	case "branch":
		readConfig()
		cmdBranch(os.Args[2:])
	default:
		help()
	}
}

// Commands

func cmdInit() {
	values := map[string]string{}
	reader := bufio.NewReader(os.Stdin)

	for _, opt := range options {
		fmt.Printf("%s [%s] ", opt.Question, opt.Default)
		value, err := reader.ReadString('\n')
		if err != nil {
			panic(err)
		}
		if value == "\n" {
			values[opt.Name] = opt.Default
		} else {
			values[opt.Name] = value[:len(value)-1]
		}
	}

	for k, v := range values {
		err := exec.Command("git", "config", "--local", "--replace-all", "env-branch."+k, v).Run()
		if err != nil {
			panic(err)
		}
	}

	fmt.Println("You're ready to go.")
}

func cmdBranch(args []string) {
	if len(args) < 1 {
		help()
	}

	switch args[0] {
	case "start":
		if len(args) < 2 {
			help()
		}
		newBranch := args[1]

		gitCommand("checkout", config.Prod)
		gitCommand("pull", "--rebase", "origin", config.Prod)
		gitCommand("checkout", "-b", newBranch)

	case "deploy":

		if len(args) < 2 {
			help()
		}

		deployEnv := args[1]
		var feature string
		var err error

		if len(args) > 2 {
			feature = args[2]
		} else {
			feature, err = getCurrentBranch()
			if err != nil {
				panic(err)
			}
		}

		gitCommand("checkout", feature)
		gitCommand("pull", "--rebase", "origin", config.Prod)
		gitCommand("checkout", deployEnv)
		gitCommand("pull", "--rebase", "origin", deployEnv)
		gitCommand("merge", feature)
		gitCommand("push", "origin", deployEnv)

	default:
		help()
	}
}

// Everything else

func help() {
	fmt.Println("TODO: implement help")
	os.Exit(1)
}

func gitCommand(args ...string) {
	fmt.Printf("+ git %s\n", strings.Join(args, " "))
	cmd := exec.Command("git", args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin

	err := cmd.Run()

	if err != nil {
		log.Fatalf("Failed executing command: git %s\n", strings.Join(args, " "))
	}
}

func readConfig() {
	config = Config{}

	cfg := map[string]string{}

	for _, opt := range options {
		stdout, err := exec.Command("git", "config", "env-branch."+opt.Name).Output()
		if err != nil {
			log.Fatalf("This repo isn't git env enabled. Run 'git env init' first.")
		}
		cfg[opt.Name] = string(stdout)[:len(stdout)-1]
	}
	config.Prod = cfg["prod"]
	config.Other = strings.Split(cfg["other"], " ")
	config.Types = strings.Split(cfg["types"], " ")
}

func getCurrentBranch() (string, error) {
	stdout, err := exec.Command("git", "branch").Output()
	if err != nil {
		return "", err
	}

	lines := strings.Split(string(stdout), "\n")
	for _, line := range lines {
		if strings.HasPrefix(line, "* ") {
			items := strings.Split(line, " ")

			//TODO
			// return error if the branch is an environment

			return items[1], nil
		}
	}

	return "", errors.New("could not detect current branch")
}
<commit_msg>Disallow merging env branches into each other via branch deploy<commit_after>package main

import (
	"bufio"
	"errors"
	"fmt"
	"log"
	"os"
	"os/exec"
	"strings"
)

type Option struct {
	Name     string
	Question string
	Default  string
}

var options = []Option{
	{
		Name:     "prod",
		Question: "What is your production environment branch?",
		Default:  "master",
	},
	{
		Name:     "other",
		Question: "What other environment branches do you have?",
		Default:  "stage dev",
	},
	{
		Name:     "types",
		Question: "What branch types do you have?",
		Default:  "feature hotfix",
	},
}

type Config struct {
	Prod  string
	Other []string
	Types []string
}

func LoadConfig() Config {
	config = Config{}

	cfg := map[string]string{}

	for _, opt := range options {
		stdout, err := exec.Command("git", "config", "env-branch."+opt.Name).Output()
		if err != nil {
			log.Fatalf("This repo isn't git env enabled. Run 'git env init' first.")
		}
		cfg[opt.Name] = string(stdout)[:len(stdout)-1]
	}
	config.Prod = cfg["prod"]
	config.Other = strings.Split(cfg["other"], " ")
	config.Types = strings.Split(cfg["types"], " ")

	return config
}

func (c Config) IsEnv(branch string) bool {
	if branch == config.Prod {
		return true
	}
	for _, b := range config.Other {
		if branch == b {
			return true
		}
	}
	return false
}

var config Config

func main() {
	if len(os.Args) < 2 {
		help()
	}

	switch os.Args[1] {
	case "init":
		cmdInit()
	case "branch":
		config = LoadConfig()
		cmdBranch(os.Args[2:])
	default:
		help()
	}
}

// Commands

func cmdInit() {
	values := map[string]string{}
	reader := bufio.NewReader(os.Stdin)

	for _, opt := range options {
		fmt.Printf("%s [%s] ", opt.Question, opt.Default)
		value, err := reader.ReadString('\n')
		if err != nil {
			panic(err)
		}
		if value == "\n" {
			values[opt.Name] = opt.Default
		} else {
			values[opt.Name] = value[:len(value)-1]
		}
	}

	for k, v := range values {
		err := exec.Command("git", "config", "--local", "--replace-all", "env-branch."+k, v).Run()
		if err != nil {
			panic(err)
		}
	}

	fmt.Println("You're ready to go.")
}

func cmdBranch(args []string) {
	if len(args) < 1 {
		help()
	}

	switch args[0] {
	case "start":
		if len(args) < 2 {
			help()
		}
		newBranch := args[1]

		gitCommand("checkout", config.Prod)
		gitCommand("pull", "--rebase", "origin", config.Prod)
		gitCommand("checkout", "-b", newBranch)

	case "deploy":

		if len(args) < 2 {
			help()
		}

		deployEnv := args[1]
		var feature string
		var err error

		if len(args) > 2 {
			feature = args[2]
		} else {
			feature, err = getCurrentBranch()
			if err != nil {
				panic(err)
			}
		}

		if !config.IsEnv(deployEnv) {
			log.Fatalf("Branch %s is not an env branch. Can't merge a feature into it.\n", deployEnv)
		}

		if config.IsEnv(feature) {
			log.Fatalf("Branch %s is an env branch. Can't merge an env branch into another env branch.\n", deployEnv)
		}

		gitCommand("checkout", feature)
		gitCommand("pull", "--rebase", "origin", config.Prod)
		gitCommand("checkout", deployEnv)
		gitCommand("pull", "--rebase", "origin", deployEnv)
		gitCommand("merge", feature)
		gitCommand("push", "origin", deployEnv)

	default:
		help()
	}
}

// Everything else

func help() {
	fmt.Println("TODO: implement help")
	os.Exit(1)
}

func gitCommand(args ...string) {
	fmt.Printf("+ git %s\n", strings.Join(args, " "))
	cmd := exec.Command("git", args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin

	err := cmd.Run()

	if err != nil {
		log.Fatalf("Failed executing command: git %s\n", strings.Join(args, " "))
	}
}

func getCurrentBranch() (string, error) {
	stdout, err := exec.Command("git", "branch").Output()
	if err != nil {
		return "", err
	}

	lines := strings.Split(string(stdout), "\n")
	for _, line := range lines {
		if strings.HasPrefix(line, "* ") {
			items := strings.Split(line, " ")

			//TODO
			// return error if the branch is an environment

			return items[1], nil
		}
	}

	return "", errors.New("could not detect current branch")
}
<|endoftext|>


<commit_before>package bebber

import (
  "gopkg.in/mgo.v2"
  "github.com/gin-gonic/gin"
)

// Struct to store obj which are interesting for many Handler Functions
type Globals struct {
  Config Config
  DB MongoDBConn
}

type Config map[string]string

type MongoDBConn struct {
  Session mgo.Session
  DialInfo mgo.DialInfo
  DBName string
}

func MakeGlobalsHandler(fn func(*gin.Context, Globals), globals Globals) func(*gin.Context) {
  return func (c *gin.Context) {
           fn(c, globals)
         }
}
<commit_msg>change Globals struct and MonogDBConn struct use pointer<commit_after>package bebber

import (
  "gopkg.in/mgo.v2"
  "github.com/gin-gonic/gin"
)

// Struct to store obj which are interesting for many Handler Functions
type Globals struct {
  Config Config
  MongoDB MongoDBConn
}

type Config map[string]string

type MongoDBConn struct {
  Session *mgo.Session
  DialInfo *mgo.DialInfo
  DBName string
}

func MakeGlobalsHandler(fn func(*gin.Context, Globals), globals Globals) func(*gin.Context) {
  return func (c *gin.Context) {
           fn(c, globals)
         }
}
<|endoftext|>


<commit_before>package gls

import (
	"bytes"
	"errors"
	"fmt"
	"runtime"
	"strconv"
	"sync"
)

// Sourced https://github.com/bradfitz/http2/blob/dc0c5c000ec33e263612939744d51a3b68b9cece/gotrack.go
var goroutineSpace = []byte("goroutine ")
var littleBuf = sync.Pool{
	New: func() interface{} {
		buf := make([]byte, 64)
		return &buf
	},
}

func curGoroutineID() uint64 {
	bp := littleBuf.Get().(*[]byte)
	defer littleBuf.Put(bp)
	b := *bp
	b = b[:runtime.Stack(b, false)]
	// Parse the 4707 out of "goroutine 4707 ["
	b = bytes.TrimPrefix(b, goroutineSpace)
	i := bytes.IndexByte(b, ' ')
	if i < 0 {
		panic(fmt.Sprintf("No space found in %q", b))
	}
	b = b[:i]
	n, err := parseUintBytes(b, 10, 64)
	if err != nil {
		panic(fmt.Sprintf("Failed to parse goroutine ID out of %q: %v", b, err))
	}
	return n
}

// parseUintBytes is like strconv.ParseUint, but using a []byte.
func parseUintBytes(s []byte, base int, bitSize int) (n uint64, err error) {
	var cutoff, maxVal uint64

	if bitSize == 0 {
		bitSize = int(strconv.IntSize)
	}

	s0 := s
	switch {
	case len(s) < 1:
		err = strconv.ErrSyntax
		goto Error

	case 2 <= base && base <= 36:
		// valid base; nothing to do

	case base == 0:
		// Look for octal, hex prefix.
		switch {
		case s[0] == '0' && len(s) > 1 && (s[1] == 'x' || s[1] == 'X'):
			base = 16
			s = s[2:]
			if len(s) < 1 {
				err = strconv.ErrSyntax
				goto Error
			}
		case s[0] == '0':
			base = 8
		default:
			base = 10
		}

	default:
		err = errors.New("invalid base " + strconv.Itoa(base))
		goto Error
	}

	n = 0
	cutoff = cutoff64(base)
	maxVal = 1<<uint(bitSize) - 1

	for i := 0; i < len(s); i++ {
		var v byte
		d := s[i]
		switch {
		case '0' <= d && d <= '9':
			v = d - '0'
		case 'a' <= d && d <= 'z':
			v = d - 'a' + 10
		case 'A' <= d && d <= 'Z':
			v = d - 'A' + 10
		default:
			n = 0
			err = strconv.ErrSyntax
			goto Error
		}
		if int(v) >= base {
			n = 0
			err = strconv.ErrSyntax
			goto Error
		}

		if n >= cutoff {
			// n*base overflows
			n = 1<<64 - 1
			err = strconv.ErrRange
			goto Error
		}
		n *= uint64(base)

		n1 := n + uint64(v)
		if n1 < n || n1 > maxVal {
			// n+v overflows
			n = 1<<64 - 1
			err = strconv.ErrRange
			goto Error
		}
		n = n1
	}

	return n, nil

Error:
	return n, &strconv.NumError{Func: "ParseUint", Num: string(s0), Err: err}
}

// Return the first number n such that n*base >= 1<<64.
func cutoff64(base int) uint64 {
	if base < 2 {
		return 0
	}
	return (1<<64-1)/uint64(base) + 1
}
<commit_msg>rsc.io/grind .<commit_after>package gls

import (
	"bytes"
	"errors"
	"fmt"
	"runtime"
	"strconv"
	"sync"
)

// Sourced https://github.com/bradfitz/http2/blob/dc0c5c000ec33e263612939744d51a3b68b9cece/gotrack.go
var goroutineSpace = []byte("goroutine ")
var littleBuf = sync.Pool{
	New: func() interface{} {
		buf := make([]byte, 64)
		return &buf
	},
}

func curGoroutineID() uint64 {
	bp := littleBuf.Get().(*[]byte)
	defer littleBuf.Put(bp)
	b := *bp
	b = b[:runtime.Stack(b, false)]
	// Parse the 4707 out of "goroutine 4707 ["
	b = bytes.TrimPrefix(b, goroutineSpace)
	i := bytes.IndexByte(b, ' ')
	if i < 0 {
		panic(fmt.Sprintf("No space found in %q", b))
	}
	b = b[:i]
	n, err := parseUintBytes(b, 10, 64)
	if err != nil {
		panic(fmt.Sprintf("Failed to parse goroutine ID out of %q: %v", b, err))
	}
	return n
}

// parseUintBytes is like strconv.ParseUint, but using a []byte.
func parseUintBytes(s []byte, base int, bitSize int) (n uint64, err error) {
	var cutoff, maxVal uint64

	if bitSize == 0 {
		bitSize = int(strconv.IntSize)
	}

	s0 := s
	switch {
	case len(s) < 1:
		err = strconv.ErrSyntax
		return n, &strconv.NumError{Func: "ParseUint", Num: string(s0), Err: err}

	case 2 <= base && base <= 36:
		// valid base; nothing to do

	case base == 0:
		// Look for octal, hex prefix.
		switch {
		case s[0] == '0' && len(s) > 1 && (s[1] == 'x' || s[1] == 'X'):
			base = 16
			s = s[2:]
			if len(s) < 1 {
				err = strconv.ErrSyntax
				return n, &strconv.NumError{Func: "ParseUint", Num: string(s0), Err: err}
			}
		case s[0] == '0':
			base = 8
		default:
			base = 10
		}

	default:
		err = errors.New("invalid base " + strconv.Itoa(base))
		return n, &strconv.NumError{Func: "ParseUint", Num: string(s0), Err: err}
	}

	n = 0
	cutoff = cutoff64(base)
	maxVal = 1<<uint(bitSize) - 1

	for i := 0; i < len(s); i++ {
		var v byte
		d := s[i]
		switch {
		case '0' <= d && d <= '9':
			v = d - '0'
		case 'a' <= d && d <= 'z':
			v = d - 'a' + 10
		case 'A' <= d && d <= 'Z':
			v = d - 'A' + 10
		default:
			n = 0
			err = strconv.ErrSyntax
			return n, &strconv.NumError{Func: "ParseUint", Num: string(s0), Err: err}
		}
		if int(v) >= base {
			n = 0
			err = strconv.ErrSyntax
			return n, &strconv.NumError{Func: "ParseUint", Num: string(s0), Err: err}
		}

		if n >= cutoff {
			// n*base overflows
			n = 1<<64 - 1
			err = strconv.ErrRange
			return n, &strconv.NumError{Func: "ParseUint", Num: string(s0), Err: err}
		}
		n *= uint64(base)

		n1 := n + uint64(v)
		if n1 < n || n1 > maxVal {
			// n+v overflows
			n = 1<<64 - 1
			err = strconv.ErrRange
			return n, &strconv.NumError{Func: "ParseUint", Num: string(s0), Err: err}
		}
		n = n1
	}

	return n, nil
}

// Return the first number n such that n*base >= 1<<64.
func cutoff64(base int) uint64 {
	if base < 2 {
		return 0
	}
	return (1<<64-1)/uint64(base) + 1
}
<|endoftext|>


<commit_before>// Copyright 2013, 2014 Peter Vasil, Tomo Krajina. All
// rights reserved. Use of this source code is governed
// by a BSD-style license that can be found in the
// LICENSE file.

package gpx

import (
	"bytes"
	"encoding/xml"
	"errors"
	"io/ioutil"
	"os"
	"strings"
	"time"
)

// An array cannot be constant :( The first one if the default layout:
var TIMELAYOUTS = []string{
	"2006-01-02T15:04:05.000Z",
	"2006-01-02T15:04:05",
	"2006-01-02 15:04:05Z",
	"2006-01-02 15:04:05",
}

func init() {
	/*
		fmt.Println("----------------------------------------------------------------------------------------------------")
		fmt.Println("This API is experimental, it *will* change")
		fmt.Println("----------------------------------------------------------------------------------------------------")
	*/
}

type ToXmlParams struct {
	Version string
	Indent  bool
}

/*
 * Params are optional, you can set null to use GPXs Version and no indentation.
 */
func ToXml(g *GPX, params ToXmlParams) ([]byte, error) {
	version := g.Version
	if len(params.Version) > 0 {
		version = params.Version
	}
	indentation := params.Indent

	var gpxDoc interface{}
	if version == "1.0" {
		gpxDoc = convertToGpx10Models(g)
	} else if version == "1.1" {
		gpxDoc = convertToGpx11Models(g)
	} else {
		g.Version = "1.1"
		gpxDoc = convertToGpx11Models(g)
	}

	var buffer bytes.Buffer
	buffer.WriteString(xml.Header)
	if indentation {
		bytes, err := xml.MarshalIndent(gpxDoc, "", "	")
		if err != nil {
			return nil, err
		}
		buffer.Write(bytes)
	} else {
		bytes, err := xml.Marshal(gpxDoc)
		if err != nil {
			return nil, err
		}
		buffer.Write(bytes)
	}
	return buffer.Bytes(), nil
}

func guessGPXVersion(bytes []byte) (string, error) {
	bytesCount := 1000
	if len(bytes) < 1000 {
		bytesCount = len(bytes)
	}

	startOfDocument := string(bytes[:bytesCount])

	parts := strings.Split(startOfDocument, "<gpx")
	if len(parts) <= 1 {
		return "", errors.New("Invalid GPX file, cannot find version")
	}
	parts = strings.Split(parts[1], "version=")

	if len(parts) <= 1 {
		return "", errors.New("Invalid GPX file, cannot find version")
	}

	if len(parts[1]) < 10 {
		return "", errors.New("Invalid GPX file, cannot find version")
	}

	result := parts[1][1:4]

	return result, nil
}

func parseGPXTime(timestr string) (*time.Time, error) {
	if strings.Contains(timestr, ".") {
		// Probably seconds with milliseconds
		timestr = strings.Split(timestr, ".")[0]
	}
	timestr = strings.Trim(timestr, " \t\n\r")
	for _, timeLayout := range TIMELAYOUTS {
		t, err := time.Parse(timeLayout, timestr)

		if err == nil {
			return &t, nil
		}
	}

	return nil, errors.New("Cannot parse " + timestr)
}

func formatGPXTime(time *time.Time) string {
	if time == nil {
		return ""
	}
	if time.Year() <= 1 {
		// Invalid date:
		return ""
	}
	return time.Format(TIMELAYOUTS[0])
}

func ParseFile(fileName string) (*GPX, error) {
	f, err := os.Open(fileName)
	if err != nil {
		return nil, err
	}

	defer f.Close()

	bytes, err := ioutil.ReadAll(f)
	if err != nil {
		return nil, err
	}

	return ParseBytes(bytes)
}

func ParseBytes(bytes []byte) (*GPX, error) {
	version, err := guessGPXVersion(bytes)
	if err != nil {
		// Unknown version, try with 1.1
		version = "1.1"
	}

	if version == "1.0" {
		g := &gpx10Gpx{}
		err := xml.Unmarshal(bytes, &g)
		if err != nil {
			return nil, err
		}

		return convertFromGpx10Models(g), nil
	} else if version == "1.1" {
		g := &gpx11Gpx{}
		err := xml.Unmarshal(bytes, &g)
		if err != nil {
			return nil, err
		}

		return convertFromGpx11Models(g), nil
	} else {
		return nil, errors.New("Invalid version:" + version)
	}
}

func ParseString(str string) (*GPX, error) {
	return ParseBytes([]byte(str))
}
<commit_msg>Fix tests, default timelayout for formatting<commit_after>// Copyright 2013, 2014 Peter Vasil, Tomo Krajina. All
// rights reserved. Use of this source code is governed
// by a BSD-style license that can be found in the
// LICENSE file.

package gpx

import (
	"bytes"
	"encoding/xml"
	"errors"
	"io/ioutil"
	"os"
	"strings"
	"time"
)

const formattingTimelayout = "2006-01-02T15:04:05Z"

// An array cannot be constant :( The first one if the default layout:
var parsingTimelayouts = []string{
	"2006-01-02T15:04:05.000Z",
	formattingTimelayout,
	"2006-01-02T15:04:05",
	"2006-01-02 15:04:05Z",
	"2006-01-02 15:04:05",
}

func init() {
	/*
		fmt.Println("----------------------------------------------------------------------------------------------------")
		fmt.Println("This API is experimental, it *will* change")
		fmt.Println("----------------------------------------------------------------------------------------------------")
	*/
}

type ToXmlParams struct {
	Version string
	Indent  bool
}

/*
 * Params are optional, you can set null to use GPXs Version and no indentation.
 */
func ToXml(g *GPX, params ToXmlParams) ([]byte, error) {
	version := g.Version
	if len(params.Version) > 0 {
		version = params.Version
	}
	indentation := params.Indent

	var gpxDoc interface{}
	if version == "1.0" {
		gpxDoc = convertToGpx10Models(g)
	} else if version == "1.1" {
		gpxDoc = convertToGpx11Models(g)
	} else {
		g.Version = "1.1"
		gpxDoc = convertToGpx11Models(g)
	}

	var buffer bytes.Buffer
	buffer.WriteString(xml.Header)
	if indentation {
		bytes, err := xml.MarshalIndent(gpxDoc, "", "	")
		if err != nil {
			return nil, err
		}
		buffer.Write(bytes)
	} else {
		bytes, err := xml.Marshal(gpxDoc)
		if err != nil {
			return nil, err
		}
		buffer.Write(bytes)
	}
	return buffer.Bytes(), nil
}

func guessGPXVersion(bytes []byte) (string, error) {
	bytesCount := 1000
	if len(bytes) < 1000 {
		bytesCount = len(bytes)
	}

	startOfDocument := string(bytes[:bytesCount])

	parts := strings.Split(startOfDocument, "<gpx")
	if len(parts) <= 1 {
		return "", errors.New("Invalid GPX file, cannot find version")
	}
	parts = strings.Split(parts[1], "version=")

	if len(parts) <= 1 {
		return "", errors.New("Invalid GPX file, cannot find version")
	}

	if len(parts[1]) < 10 {
		return "", errors.New("Invalid GPX file, cannot find version")
	}

	result := parts[1][1:4]

	return result, nil
}

func parseGPXTime(timestr string) (*time.Time, error) {
	if strings.Contains(timestr, ".") {
		// Probably seconds with milliseconds
		timestr = strings.Split(timestr, ".")[0]
	}
	timestr = strings.Trim(timestr, " \t\n\r")
	for _, timeLayout := range parsingTimelayouts {
		t, err := time.Parse(timeLayout, timestr)

		if err == nil {
			return &t, nil
		}
	}

	return nil, errors.New("Cannot parse " + timestr)
}

func formatGPXTime(time *time.Time) string {
	if time == nil {
		return ""
	}
	if time.Year() <= 1 {
		// Invalid date:
		return ""
	}
	return time.Format(formattingTimelayout)
}

func ParseFile(fileName string) (*GPX, error) {
	f, err := os.Open(fileName)
	if err != nil {
		return nil, err
	}

	defer f.Close()

	bytes, err := ioutil.ReadAll(f)
	if err != nil {
		return nil, err
	}

	return ParseBytes(bytes)
}

func ParseBytes(bytes []byte) (*GPX, error) {
	version, err := guessGPXVersion(bytes)
	if err != nil {
		// Unknown version, try with 1.1
		version = "1.1"
	}

	if version == "1.0" {
		g := &gpx10Gpx{}
		err := xml.Unmarshal(bytes, &g)
		if err != nil {
			return nil, err
		}

		return convertFromGpx10Models(g), nil
	} else if version == "1.1" {
		g := &gpx11Gpx{}
		err := xml.Unmarshal(bytes, &g)
		if err != nil {
			return nil, err
		}

		return convertFromGpx11Models(g), nil
	} else {
		return nil, errors.New("Invalid version:" + version)
	}
}

func ParseString(str string) (*GPX, error) {
	return ParseBytes([]byte(str))
}
<|endoftext|>