<commit_before>package mstree

import (
	"bufio"
	"fmt"
	logging "github.com/op/go-logging"
	"io"
	"io/ioutil"
	"os"
	"strings"
	"time"
)

type MSTree struct {
	indexDir    string
	Root        *node
	syncChannel chan string
	fullReindex bool
}
type eventChan chan error
type TreeCreateError struct {
	msg string
}

func (tce *TreeCreateError) Error() string {
	return tce.msg
}

var (
	log *logging.Logger = logging.MustGetLogger("metricsearch")
)

func NewTree(indexDir string, syncBufferSize int) (*MSTree, error) {
	stat, err := os.Stat(indexDir)
	if err != nil {
		if os.IsNotExist(err) {
			err = os.MkdirAll(indexDir, os.FileMode(0755))
			if err != nil {
				log.Error(err.Error())
				return nil, err
			}
		} else {
			log.Error(err.Error())
			return nil, err
		}
	} else {
		if !stat.IsDir() {
			log.Error("'%s' exists and is not a directory", indexDir)
			return nil, &TreeCreateError{fmt.Sprintf("'%s' exists and is not a directory", indexDir)}
		}
	}
	syncChannel := make(chan string, syncBufferSize)
	root := newNode()
	tree := &MSTree{indexDir, root, syncChannel, false}
	log.Debug("Tree created. indexDir: %s syncBufferSize: %d", indexDir, syncBufferSize)
	go syncWorker(tree.indexDir, tree.syncChannel)
	log.Debug("Background index sync started")
	return tree, nil
}

func syncWorker(indexDir string, dataChannel chan string) {
	var err error
	fdCache := make(map[string]*os.File)
	defer func(fdCache map[string]*os.File) {
		for _, f := range fdCache {
			f.Close()
		}
	}(fdCache)
	for line := range dataChannel {
		tokens := strings.Split(line, ".")
		first := tokens[0]
		idxFilename := fmt.Sprintf("%s/%s.idx", indexDir, first)

		f, ok := fdCache[idxFilename]
		if !ok {
			f, err = os.OpenFile(idxFilename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.FileMode(0644))
			if err != nil {
				log.Error("Index update error: " + err.Error())
				continue
			}
			fdCache[idxFilename] = f
		}
		if len(tokens) > 1 {
			tail := strings.Join(tokens[1:], ".")
			_, err := io.WriteString(f, tail+"\n")
			if err != nil {
				log.Error("Index update error: " + err.Error())
				if f != nil {
					log.Debug(fmt.Sprintf("Closing file '%s'", f.Name()))
					f.Close()
				}
				delete(fdCache, idxFilename)
			} else {
				log.Debug("Metric '%s' synced to disk", line)
			}
		}
	}
}

func dumpWorker(idxFile string, idxNode *node, ev eventChan) {
	log.Debug("<%s> dumper started", idxFile)
	f, err := os.Create(idxFile)
	if err != nil {
		log.Debug("<%s> dumper finished with error: %s", idxFile, err.Error())
		ev <- err
		return
	}
	defer f.Close()
	idxNode.traverseDump("", f)
	log.Debug("<%s> dumper finished", idxFile)
	ev <- nil
}

func loadWorker(idxFile string, idxNode *node, ev eventChan) {
	log.Debug("<%s> loader started", idxFile)
	f, err := os.Open(idxFile)
	if err != nil {
		log.Error("<%s> loader finished with error: %s", idxFile, err.Error())
		ev <- err
		return
	}
	defer f.Close()
	inserted := true
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		line := strings.TrimRight(scanner.Text(), "\n")
		tokens := strings.Split(line, ".")
		idxNode.insert(tokens, &inserted)
	}
	log.Debug("<%s> loader finished", idxFile)
	ev <- nil
}

// TODO: channeled index writer
func (t *MSTree) Add(metric string) {
	if metric == "" {
		return
	}
	tokens := strings.Split(metric, ".")
	inserted := false
	t.Root.insert(tokens, &inserted)
	if !t.fullReindex && inserted && len(tokens) > 1 {
		t.syncChannel <- metric
	}
}

func (t *MSTree) LoadTxt(filename string, limit int) error {
	t.fullReindex = true
	f, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer f.Close()
	scanner := bufio.NewScanner(f)
	count := 0
	for scanner.Scan() {
		line := strings.TrimRight(scanner.Text(), "\n")
		t.Add(line)
		count++
		if count%1000000 == 0 {
			log.Info("Reindexed %d items", count)
		}
		if limit != -1 && count == limit {
			break
		}
	}
	log.Info("Reindexed %d items", count)
	err = t.DumpIndex()
	if err != nil {
		return err
	}
	t.fullReindex = false
	return nil
}

func (t *MSTree) DropIndex() error {
	files, err := ioutil.ReadDir(t.indexDir)
	if err != nil {
		log.Error("Error opening index: " + err.Error())
		return err
	}
	if len(files) > 0 {
		for _, file := range files {
			fName := fmt.Sprintf("%s/%s", t.indexDir, file.Name())
			if strings.HasSuffix(fName, ".idx") {
				err := os.Remove(fName)
				if err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func (t *MSTree) DumpIndex() error {
	log.Info("Syncinc the entire index")
	err := os.MkdirAll(t.indexDir, os.FileMode(0755))
	if err != nil {
		log.Error(err.Error())
		return err
	}
	procCount := 0
	ev := make(eventChan, len(t.Root.Children))
	for first, node := range t.Root.Children {
		idxFile := fmt.Sprintf("%s/%s.idx", t.indexDir, first)
		go dumpWorker(idxFile, node, ev)
		procCount++
	}
	var globalErr error = nil
	for e := range ev {
		procCount--
		if e != nil {
			globalErr = e
		}
		if procCount == 0 {
			break
		}
	}
	log.Info("Sync complete")
	return globalErr
}

func (t *MSTree) LoadIndex() error {
	var globalErr error = nil
	files, err := ioutil.ReadDir(t.indexDir)
	if err != nil {
		log.Error("Error loading index: " + err.Error())
		return err
	}
	if len(files) > 0 {
		ev := make(eventChan, len(files))
		procCount := 0
		for _, idxFile := range files {
			fName := idxFile.Name()
			if !strings.HasSuffix(fName, ".idx") {
				continue
			}
			pref := fName[:len(fName)-4]
			fName = fmt.Sprintf("%s/%s", t.indexDir, fName)
			idxNode := newNode()
			t.Root.Children[pref] = idxNode
			go loadWorker(fName, idxNode, ev)
			procCount++
		}
		tm := time.Now()

		for e := range ev {
			procCount--
			if e != nil {
				globalErr = e
			}
			if procCount == 0 {
				break
			}
		}
		log.Notice("Index load complete in %s", time.Now().Sub(tm).String())
	} else {
		log.Debug("Index is empty. Hope that's ok")
	}
	return globalErr
}

func (t *MSTree) Search(pattern string) []string {
	tokens := strings.Split(pattern, ".")
	nodesToSearch := make(map[string]*node)
	nodesToSearch[""] = t.Root
	for _, token := range tokens {
		prefRes := make(map[string]*node)
		for k, node := range nodesToSearch {
			sRes := node.search(token)
			if k == "" {
				// root node, no prefix
				for j, resNode := range sRes {
					prefRes[j] = resNode
				}
			} else {
				for j, resNode := range sRes {
					prefRes[k+"."+j] = resNode
				}
			}
		}
		nodesToSearch = prefRes
	}
	results := make([]string, len(nodesToSearch))
	i := 0
	for k, node := range nodesToSearch {
		if len(node.Children) == 0 {
			results[i] = k
		} else {
			results[i] = k + "."
		}
		i++
	}
	return results
}
<commit_msg>disable GC while reindexing<commit_after>package mstree

import (
	"bufio"
	"fmt"
	logging "github.com/op/go-logging"
	"io"
	"io/ioutil"
	"os"
	"runtime/debug"
	"strings"
	"time"
)

type MSTree struct {
	indexDir    string
	Root        *node
	syncChannel chan string
	fullReindex bool
}
type eventChan chan error
type TreeCreateError struct {
	msg string
}

func (tce *TreeCreateError) Error() string {
	return tce.msg
}

var (
	log *logging.Logger = logging.MustGetLogger("metricsearch")
)

func NewTree(indexDir string, syncBufferSize int) (*MSTree, error) {
	stat, err := os.Stat(indexDir)
	if err != nil {
		if os.IsNotExist(err) {
			err = os.MkdirAll(indexDir, os.FileMode(0755))
			if err != nil {
				log.Error(err.Error())
				return nil, err
			}
		} else {
			log.Error(err.Error())
			return nil, err
		}
	} else {
		if !stat.IsDir() {
			log.Error("'%s' exists and is not a directory", indexDir)
			return nil, &TreeCreateError{fmt.Sprintf("'%s' exists and is not a directory", indexDir)}
		}
	}
	syncChannel := make(chan string, syncBufferSize)
	root := newNode()
	tree := &MSTree{indexDir, root, syncChannel, false}
	log.Debug("Tree created. indexDir: %s syncBufferSize: %d", indexDir, syncBufferSize)
	go syncWorker(tree.indexDir, tree.syncChannel)
	log.Debug("Background index sync started")
	return tree, nil
}

func syncWorker(indexDir string, dataChannel chan string) {
	var err error
	fdCache := make(map[string]*os.File)
	defer func(fdCache map[string]*os.File) {
		for _, f := range fdCache {
			f.Close()
		}
	}(fdCache)
	for line := range dataChannel {
		tokens := strings.Split(line, ".")
		first := tokens[0]
		idxFilename := fmt.Sprintf("%s/%s.idx", indexDir, first)

		f, ok := fdCache[idxFilename]
		if !ok {
			f, err = os.OpenFile(idxFilename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.FileMode(0644))
			if err != nil {
				log.Error("Index update error: " + err.Error())
				continue
			}
			fdCache[idxFilename] = f
		}
		if len(tokens) > 1 {
			tail := strings.Join(tokens[1:], ".")
			_, err := io.WriteString(f, tail+"\n")
			if err != nil {
				log.Error("Index update error: " + err.Error())
				if f != nil {
					log.Debug(fmt.Sprintf("Closing file '%s'", f.Name()))
					f.Close()
				}
				delete(fdCache, idxFilename)
			} else {
				log.Debug("Metric '%s' synced to disk", line)
			}
		}
	}
}

func dumpWorker(idxFile string, idxNode *node, ev eventChan) {
	log.Debug("<%s> dumper started", idxFile)
	f, err := os.Create(idxFile)
	if err != nil {
		log.Debug("<%s> dumper finished with error: %s", idxFile, err.Error())
		ev <- err
		return
	}
	defer f.Close()
	idxNode.traverseDump("", f)
	log.Debug("<%s> dumper finished", idxFile)
	ev <- nil
}

func loadWorker(idxFile string, idxNode *node, ev eventChan) {
	log.Debug("<%s> loader started", idxFile)
	f, err := os.Open(idxFile)
	if err != nil {
		log.Error("<%s> loader finished with error: %s", idxFile, err.Error())
		ev <- err
		return
	}
	defer f.Close()
	inserted := true
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		line := strings.TrimRight(scanner.Text(), "\n")
		tokens := strings.Split(line, ".")
		idxNode.insert(tokens, &inserted)
	}
	log.Debug("<%s> loader finished", idxFile)
	ev <- nil
}

// TODO: channeled index writer
func (t *MSTree) Add(metric string) {
	if metric == "" {
		return
	}
	tokens := strings.Split(metric, ".")
	inserted := false
	t.Root.insert(tokens, &inserted)
	if !t.fullReindex && inserted && len(tokens) > 1 {
		t.syncChannel <- metric
	}
}

func (t *MSTree) LoadTxt(filename string, limit int) error {
	t.fullReindex = true
	f, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer f.Close()

	// Turn GC off
	prevGC := debug.SetGCPercent(-1)
	// Defer to turn GC back on
	defer debug.SetGCPercent(prevGC)

	scanner := bufio.NewScanner(f)
	count := 0
	for scanner.Scan() {
		line := strings.TrimRight(scanner.Text(), "\n")
		t.Add(line)
		count++
		if count%1000000 == 0 {
			log.Info("Reindexed %d items", count)
		}
		if limit != -1 && count == limit {
			break
		}
	}
	log.Info("Reindexed %d items", count)
	err = t.DumpIndex()
	if err != nil {
		return err
	}
	t.fullReindex = false
	return nil
}

func (t *MSTree) DropIndex() error {
	files, err := ioutil.ReadDir(t.indexDir)
	if err != nil {
		log.Error("Error opening index: " + err.Error())
		return err
	}
	if len(files) > 0 {
		for _, file := range files {
			fName := fmt.Sprintf("%s/%s", t.indexDir, file.Name())
			if strings.HasSuffix(fName, ".idx") {
				err := os.Remove(fName)
				if err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func (t *MSTree) DumpIndex() error {
	log.Info("Syncinc the entire index")
	err := os.MkdirAll(t.indexDir, os.FileMode(0755))
	if err != nil {
		log.Error(err.Error())
		return err
	}
	procCount := 0
	ev := make(eventChan, len(t.Root.Children))
	for first, node := range t.Root.Children {
		idxFile := fmt.Sprintf("%s/%s.idx", t.indexDir, first)
		go dumpWorker(idxFile, node, ev)
		procCount++
	}
	var globalErr error = nil
	for e := range ev {
		procCount--
		if e != nil {
			globalErr = e
		}
		if procCount == 0 {
			break
		}
	}
	log.Info("Sync complete")
	return globalErr
}

func (t *MSTree) LoadIndex() error {
	var globalErr error = nil
	files, err := ioutil.ReadDir(t.indexDir)
	if err != nil {
		log.Error("Error loading index: " + err.Error())
		return err
	}
	if len(files) > 0 {

		// Turn GC off
		prevGC := debug.SetGCPercent(-1)
		// Defer to turn GC back on
		defer debug.SetGCPercent(prevGC)

		ev := make(eventChan, len(files))
		procCount := 0
		for _, idxFile := range files {
			fName := idxFile.Name()
			if !strings.HasSuffix(fName, ".idx") {
				continue
			}
			pref := fName[:len(fName)-4]
			fName = fmt.Sprintf("%s/%s", t.indexDir, fName)
			idxNode := newNode()
			t.Root.Children[pref] = idxNode
			go loadWorker(fName, idxNode, ev)
			procCount++
		}
		tm := time.Now()

		for e := range ev {
			procCount--
			if e != nil {
				globalErr = e
			}
			if procCount == 0 {
				break
			}
		}
		log.Notice("Index load complete in %s", time.Now().Sub(tm).String())
	} else {
		log.Debug("Index is empty. Hope that's ok")
	}
	return globalErr
}

func (t *MSTree) Search(pattern string) []string {
	tokens := strings.Split(pattern, ".")
	nodesToSearch := make(map[string]*node)
	nodesToSearch[""] = t.Root
	for _, token := range tokens {
		prefRes := make(map[string]*node)
		for k, node := range nodesToSearch {
			sRes := node.search(token)
			if k == "" {
				// root node, no prefix
				for j, resNode := range sRes {
					prefRes[j] = resNode
				}
			} else {
				for j, resNode := range sRes {
					prefRes[k+"."+j] = resNode
				}
			}
		}
		nodesToSearch = prefRes
	}
	results := make([]string, len(nodesToSearch))
	i := 0
	for k, node := range nodesToSearch {
		if len(node.Children) == 0 {
			results[i] = k
		} else {
			results[i] = k + "."
		}
		i++
	}
	return results
}
<|endoftext|>


<commit_before>package channels

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"math"

	"github.com/couchbaselabs/sync_gateway/base"
)

// Encodes a ChangeLog into a simple appendable data format.
func (ch *ChangeLog) Encode(w io.Writer) {
	writeSequence(ch.Since, w)
	for _, entry := range ch.Entries {
		entry.Encode(w, "")
	}
}

// Encodes a LogEntry in a format that can be appended to an encoded ChangeLog.
func (entry *LogEntry) Encode(w io.Writer, parent string) {
	entry.assertValid()
	var flagBuf [1]byte
	flagBuf[0] = entry.Flags
	w.Write(flagBuf[0:1])
	writeSequence(entry.Sequence, w)
	writeString(entry.DocID, w)
	writeString(entry.RevID, w)
	writeString(parent, w)
}

// Decodes an encoded ChangeLog.
func DecodeChangeLog(r *bytes.Reader, afterSeq uint64) *ChangeLog {
	type docAndRev struct {
		docID, revID string
	}

	ch := ChangeLog{
		Since:   readSequence(r),
		Entries: make([]*LogEntry, 0, 500),
	}
	parents := map[docAndRev]*LogEntry{}
	cleanup := false
	skipping := (afterSeq > 0)
	var flagBuf [1]byte
	for {
		n, _ := r.Read(flagBuf[0:1])
		if n == 0 {
			break // eof
		}
		if flagBuf[0] > kMaxFlag {
			pos, _ := r.Seek(0, 1)
			base.Warn("DecodeChangeLog: bad flags 0x%x, entry %d, offset %d",
				flagBuf[0], len(ch.Entries), pos-1)
			return nil
		}
		seq := readSequence(r)
		if skipping {
			if seq >= afterSeq {
				skipping = false
			}
			if seq <= afterSeq {
				skipString(r)
				skipString(r)
				skipString(r)
				continue // ignore this sequence
			}
		}

		entry := &LogEntry{
			Flags:    flagBuf[0],
			Sequence: seq,
			DocID:    readString(r),
			RevID:    readString(r),
		}
		if !entry.checkValid() {
			return nil
		}

		if parentID := readString(r); parentID != "" {
			if parent := parents[docAndRev{entry.DocID, parentID}]; parent != nil {
				// Clear out the parent rev that was overwritten by this one
				parent.DocID = ""
				parent.RevID = ""
				cleanup = true
			}
		}
		parents[docAndRev{entry.DocID, entry.RevID}] = entry

		ch.Entries = append(ch.Entries, entry)
	}

	// Now remove any overwritten entries:
	if cleanup {
		iDst := 0
		for iSrc, entry := range ch.Entries {
			if entry.DocID != "" { // only copy non-cleared entries
				if iDst < iSrc {
					ch.Entries[iDst] = entry
				}
				iDst++
			}
		}
		ch.Entries = ch.Entries[0:iDst]
	}

	if afterSeq > ch.Since {
		ch.Since = afterSeq
	}
	return &ch
}

// Removes the oldest entries to limit the log's length to `maxLength`.
// This is the same as ChangeLog.Truncate except it works directly on the encoded form, which is
// much faster than decoding+truncating+encoding.
func TruncateEncodedChangeLog(r *bytes.Reader, maxLength, minLength int, w io.Writer) (removed int, newLength int) {
	since := readSequence(r)
	// Find the starting position and sequence of each entry:
	entryPos := make([]int64, 0, 1000)
	entrySeq := make([]uint64, 0, 1000)
	for {
		pos, _ := r.Seek(0, 1)
		flags, err := r.ReadByte()
		if err != nil {
			break // eof
		}
		seq := readSequence(r)
		skipString(r)
		skipString(r)
		skipString(r)
		if flags > 7 {
			panic(fmt.Sprintf("TruncateEncodedChangeLog: bad flags 0x%x, entry %d, offset %d",
				flags, len(entryPos), pos))
		}

		entryPos = append(entryPos, pos)
		entrySeq = append(entrySeq, seq)
	}

	// How many entries to remove?
	// * Leave no more than maxLength entries
	// * Every sequence value removed should be less than every sequence remaining.
	// * The new 'since' value should be the maximum sequence removed.
	oldLength := len(entryPos)
	removed = oldLength - maxLength
	if removed <= 0 {
		removed = 0
	} else {
		pivot, newSince := findPivot(entrySeq, removed-1)
		removed = pivot + 1
		if oldLength-removed >= minLength {
			since = newSince
		} else {
			removed = 0
			base.Warn("TruncateEncodedChangeLog: Couldn't find a safe place to truncate")
			//TODO: Possibly find a pivot earlier than desired?
		}
	}

	// Write the updated Since and the remaining entries:
	writeSequence(since, w)
	r.Seek(entryPos[removed], 0)
	io.Copy(w, r)
	return removed, oldLength - removed
}

//////// UTILITY FUNCTIONS:

func writeSequence(seq uint64, w io.Writer) {
	var buf [16]byte
	len := binary.PutUvarint(buf[0:16], seq)
	w.Write(buf[0:len])
}

func readSequence(r io.ByteReader) uint64 {
	seq, _ := binary.ReadUvarint(r)
	return seq
}

func writeString(s string, w io.Writer) {
	b := []byte(s)
	length := len(b)
	if length > 255 {
		panic("Doc/rev ID too long to encode: " + s)
	}
	if err := binary.Write(w, binary.BigEndian, uint8(length)); err != nil {
		panic("Write failed")
	}
	if _, err := w.Write(b); err != nil {
		panic("writeString failed")
	}
}

func readLength(r io.Reader) uint8 {
	var lengthBuf [1]byte
	if _, err := r.Read(lengthBuf[0:1]); err != nil {
		panic("readString length failed")
	}
	return lengthBuf[0]
}

func readString(r io.Reader) string {
	length := readLength(r)
	data := make([]byte, length)
	if _, err := io.ReadFull(r, data); err != nil {
		panic("readString bytes failed")
	}
	return string(data)
}

func skipString(r io.ReadSeeker) {
	length := readLength(r)
	r.Seek(int64(length), 1)
}

// Finds a 'pivot' index, at or after minIndex, such that all array values before and at the pivot
// are less than all array values after it.
func findPivot(values []uint64, minIndex int) (pivot int, maxBefore uint64) {
	// First construct a table where minRight[i] is the minimum value in [i..n)
	n := len(values)
	minRight := make([]uint64, n)
	var min uint64 = math.MaxUint64
	for i := n - 1; i >= 0; i-- {
		if values[i] < min {
			min = values[i]
		}
		minRight[i] = min
	}
	// Now scan left-to-right tracking the running max and looking for a pivot:
	maxBefore = 0
	for pivot = 0; pivot < n-1; pivot++ {
		if values[pivot] > maxBefore {
			maxBefore = values[pivot]
		}
		if pivot >= minIndex && maxBefore < minRight[pivot+1] {
			break
		}
	}
	//log.Printf("PIVOT: %v @%d -> %d", values, minIndex, pivot)
	return
}
<commit_msg>Diligently check all I/O errors, & catch all panics, in DecodeChangeLog.<commit_after>package channels

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"math"

	"github.com/couchbaselabs/sync_gateway/base"
)

// Encodes a ChangeLog into a simple appendable data format.
func (ch *ChangeLog) Encode(w io.Writer) {
	writeSequence(ch.Since, w)
	for _, entry := range ch.Entries {
		entry.Encode(w, "")
	}
}

// Encodes a LogEntry in a format that can be appended to an encoded ChangeLog.
func (entry *LogEntry) Encode(w io.Writer, parent string) {
	entry.assertValid()
	writeUInt8(entry.Flags, w)
	writeSequence(entry.Sequence, w)
	writeString(entry.DocID, w)
	writeString(entry.RevID, w)
	writeString(parent, w)
}

// Decodes an encoded ChangeLog.
func DecodeChangeLog(r *bytes.Reader, afterSeq uint64) (log *ChangeLog) {
	defer func() {
		if panicMsg := recover(); panicMsg != nil {
			// decodeChangeLog panicked.
			base.Warn("Panic from DecodeChangeLog: %v", panicMsg)
		}
	}()
	return decodeChangeLog(r, afterSeq)
}

func decodeChangeLog(r *bytes.Reader, afterSeq uint64) *ChangeLog {
	type docAndRev struct {
		docID, revID string
	}

	ch := ChangeLog{
		Since:   readSequence(r),
		Entries: make([]*LogEntry, 0, 500),
	}
	parents := map[docAndRev]*LogEntry{}
	cleanup := false
	skipping := (afterSeq > 0)
	var flagBuf [1]byte
	for {
		n, err := r.Read(flagBuf[0:1])
		if n == 0 {
			if err == io.EOF {
				break
			}
			panic("Error reading flags")
		}
		if flagBuf[0] > kMaxFlag {
			pos, _ := r.Seek(0, 1)
			base.Warn("DecodeChangeLog: bad flags 0x%x, entry %d, offset %d",
				flagBuf[0], len(ch.Entries), pos-1)
			return nil
		}
		seq := readSequence(r)
		if skipping {
			if seq >= afterSeq {
				skipping = false
			}
			if seq <= afterSeq {
				skipString(r)
				skipString(r)
				skipString(r)
				continue // ignore this sequence
			}
		}

		entry := &LogEntry{
			Flags:    flagBuf[0],
			Sequence: seq,
			DocID:    readString(r),
			RevID:    readString(r),
		}
		if !entry.checkValid() {
			return nil
		}

		if parentID := readString(r); parentID != "" {
			if parent := parents[docAndRev{entry.DocID, parentID}]; parent != nil {
				// Clear out the parent rev that was overwritten by this one
				parent.DocID = ""
				parent.RevID = ""
				cleanup = true
			}
		}
		parents[docAndRev{entry.DocID, entry.RevID}] = entry

		ch.Entries = append(ch.Entries, entry)
	}

	// Now remove any overwritten entries:
	if cleanup {
		iDst := 0
		for iSrc, entry := range ch.Entries {
			if entry.DocID != "" { // only copy non-cleared entries
				if iDst < iSrc {
					ch.Entries[iDst] = entry
				}
				iDst++
			}
		}
		ch.Entries = ch.Entries[0:iDst]
	}

	if afterSeq > ch.Since {
		ch.Since = afterSeq
	}
	return &ch
}

// Removes the oldest entries to limit the log's length to `maxLength`.
// This is the same as ChangeLog.Truncate except it works directly on the encoded form, which is
// much faster than decoding+truncating+encoding.
func TruncateEncodedChangeLog(r *bytes.Reader, maxLength, minLength int, w io.Writer) (removed int, newLength int) {
	since := readSequence(r)
	// Find the starting position and sequence of each entry:
	entryPos := make([]int64, 0, 1000)
	entrySeq := make([]uint64, 0, 1000)
	for {
		pos, err := r.Seek(0, 1)
		if err != nil {
			panic("Seek??")
		}
		flags, err := r.ReadByte()
		if err != nil {
			if err == io.EOF {
				break // eof
			}
			panic("ReadByte failed")
		}
		seq := readSequence(r)
		skipString(r)
		skipString(r)
		skipString(r)
		if flags > kMaxFlag {
			panic(fmt.Sprintf("TruncateEncodedChangeLog: bad flags 0x%x, entry %d, offset %d",
				flags, len(entryPos), pos))
		}

		entryPos = append(entryPos, pos)
		entrySeq = append(entrySeq, seq)
	}

	// How many entries to remove?
	// * Leave no more than maxLength entries
	// * Every sequence value removed should be less than every sequence remaining.
	// * The new 'since' value should be the maximum sequence removed.
	oldLength := len(entryPos)
	removed = oldLength - maxLength
	if removed <= 0 {
		removed = 0
	} else {
		pivot, newSince := findPivot(entrySeq, removed-1)
		removed = pivot + 1
		if oldLength-removed >= minLength {
			since = newSince
		} else {
			removed = 0
			base.Warn("TruncateEncodedChangeLog: Couldn't find a safe place to truncate")
			//TODO: Possibly find a pivot earlier than desired?
		}
	}

	// Write the updated Since and the remaining entries:
	writeSequence(since, w)
	if _, err := r.Seek(entryPos[removed], 0); err != nil {
		panic("Seek back???")
	}
	if _, err := io.Copy(w, r); err != nil {
		panic("Copy???")
	}
	return removed, oldLength - removed
}

//////// UTILITY FUNCTIONS:

func writeUInt8(u uint8, w io.Writer) {
	var buf [1]byte
	buf[0] = u
	if _, err := w.Write(buf[0:1]); err != nil {
		panic("writeUInt8 failed")
	}
}

func readUInt8(r io.Reader) uint8 {
	var buf [1]byte
	if _, err := io.ReadFull(r, buf[0:1]); err != nil {
		panic("readUInt8 failed")
	}
	return buf[0]
}

func writeSequence(seq uint64, w io.Writer) {
	var buf [16]byte
	len := binary.PutUvarint(buf[0:16], seq)
	if _, err := w.Write(buf[0:len]); err != nil {
		panic("writeSequence failed")
	}
}

func readSequence(r io.ByteReader) uint64 {
	seq, err := binary.ReadUvarint(r)
	if err != nil {
		panic("readSequence failed")
	}
	return seq
}

func writeString(s string, w io.Writer) {
	b := []byte(s)
	length := len(b)
	if length > 255 {
		panic("Doc/rev ID too long to encode: " + s)
	}
	writeUInt8(uint8(length), w)
	if _, err := w.Write(b); err != nil {
		panic("writeString failed")
	}
}

func readString(r io.Reader) string {
	length := readUInt8(r)
	data := make([]byte, length)
	if _, err := io.ReadFull(r, data); err != nil {
		panic("readString failed")
	}
	return string(data)
}

func skipString(r io.ReadSeeker) {
	length := readUInt8(r)
	if _, err := r.Seek(int64(length), 1); err != nil {
		panic("skipString failed")
	}
}

// Finds a 'pivot' index, at or after minIndex, such that all array values before and at the pivot
// are less than all array values after it.
func findPivot(values []uint64, minIndex int) (pivot int, maxBefore uint64) {
	// First construct a table where minRight[i] is the minimum value in [i..n)
	n := len(values)
	minRight := make([]uint64, n)
	var min uint64 = math.MaxUint64
	for i := n - 1; i >= 0; i-- {
		if values[i] < min {
			min = values[i]
		}
		minRight[i] = min
	}
	// Now scan left-to-right tracking the running max and looking for a pivot:
	maxBefore = 0
	for pivot = 0; pivot < n-1; pivot++ {
		if values[pivot] > maxBefore {
			maxBefore = values[pivot]
		}
		if pivot >= minIndex && maxBefore < minRight[pivot+1] {
			break
		}
	}
	//log.Printf("PIVOT: %v @%d -> %d", values, minIndex, pivot)
	return
}
<|endoftext|>


<commit_before>package interactive

import (
	"fmt"
	"sort"

	"github.com/viktomas/godu/core"
)

type byLength []string

func (l byLength) Len() int           { return len(l) }
func (l byLength) Swap(i, j int)      { l[i], l[j] = l[j], l[i] }
func (l byLength) Less(i, j int) bool { return len(l[i]) > len(l[j]) }

// QuoteMarkedFiles takes files from the map and returns slice of qoted file paths
func QuoteMarkedFiles(markedFiles map[*core.File]struct{}) []string {
	quotedFiles := make([]string, len(markedFiles))
	i := 0
	for file := range markedFiles {
		quotedFiles[i] = fmt.Sprintf("'%s'", file.Path())
		i++
	}
	// sorting lenght of the path (assuming that we want to deleate files in subdirs first)
	// alfabetical sorting added for deterinism (map keys doesn't guarantee order)
	sort.Sort(sort.StringSlice(quotedFiles))
	sort.Sort(byLength(quotedFiles))
	return quotedFiles
}
<commit_msg>Fixing 2 typos thanks to @leonklingele<commit_after>package interactive

import (
	"fmt"
	"sort"

	"github.com/viktomas/godu/core"
)

type byLength []string

func (l byLength) Len() int           { return len(l) }
func (l byLength) Swap(i, j int)      { l[i], l[j] = l[j], l[i] }
func (l byLength) Less(i, j int) bool { return len(l[i]) > len(l[j]) }

// QuoteMarkedFiles takes files from the map and returns slice of qoted file paths
func QuoteMarkedFiles(markedFiles map[*core.File]struct{}) []string {
	quotedFiles := make([]string, len(markedFiles))
	i := 0
	for file := range markedFiles {
		quotedFiles[i] = fmt.Sprintf("'%s'", file.Path())
		i++
	}
	// sorting lenght of the path (assuming that we want to deleate files in subdirs first)
	// alphabetical sorting added for determinism (map keys doesn't guarantee order)
	sort.Sort(sort.StringSlice(quotedFiles))
	sort.Sort(byLength(quotedFiles))
	return quotedFiles
}
<|endoftext|>


<commit_before>package build

import (
	"github.com/antchfx/gxpath/internal/query"
	"github.com/antchfx/gxpath/xpath"
)

func predicate(q query.Query) func(xpath.NodeNavigator) bool {
	type Predicater interface {
		Test(xpath.NodeNavigator) bool
	}
	if p, ok := q.(Predicater); ok {
		return p.Test
	}
	return func(xpath.NodeNavigator) bool { return true }
}

// positionFunc is a XPath Node Set functions postion().
var positionFunc = func(q query.Query, t query.Iterator) interface{} {
	count := 0
	node := t.Current()
	curr := node.Current()
	node.MoveToFirst()
	test := predicate(q)
	for {
		if test(node) {
			count++
			if node.Current() == curr {
				break
			}
		}
		if !node.MoveToNext() {
			break
		}
	}
	return float64(count)
}

// lastFunc is a XPath Node Set functions last().
var lastFunc = func(q query.Query, t query.Iterator) interface{} {
	count := 0
	node := t.Current()
	node.MoveToFirst()
	test := predicate(q)
	for {
		if test(node) {
			count++
		}
		if !node.MoveToNext() {
			break
		}
	}
	return float64(count)
}

// countFunc is a XPath Node Set functions count(node-set).
var countFunc = func(q query.Query, t query.Iterator) interface{} {
	count := 0
	node := t.Current()
	node.MoveToFirst()
	test := predicate(q)
	for {
		if test(node) {
			count++
		}
		if !node.MoveToNext() {
			break
		}
	}
	return float64(count)
}
<commit_msg>fix: last(),position() functions<commit_after>package build

import (
	"github.com/antchfx/gxpath/internal/query"
	"github.com/antchfx/gxpath/xpath"
)

func predicate(q query.Query) func(xpath.NodeNavigator) bool {
	type Predicater interface {
		Test(xpath.NodeNavigator) bool
	}
	if p, ok := q.(Predicater); ok {
		return p.Test
	}
	return func(xpath.NodeNavigator) bool { return true }
}

// positionFunc is a XPath Node Set functions postion().
var positionFunc = func(q query.Query, t query.Iterator) interface{} {
	var (
		count = 0
		node  = t.Current()
		val   = node
	)
	node.MoveToFirst()
	test := predicate(q)
	for {
		if test(node) {
			count++
			if node.Current() == val {
				break
			}
		}
		if !node.MoveToNext() {
			break
		}
	}
	return float64(count)
}

// lastFunc is a XPath Node Set functions last().
var lastFunc = func(q query.Query, t query.Iterator) interface{} {
	var (
		count = 0
		node  = t.Current()
	)
	node.MoveToFirst()
	test := predicate(q)
	for {
		if test(node) {
			count++
		}
		if !node.MoveToNext() {
			break
		}
	}
	return float64(count)
}

// countFunc is a XPath Node Set functions count(node-set).
var countFunc = func(q query.Query, t query.Iterator) interface{} {
	var (
		count = 0
		node  = t.Current()
	)
	node.MoveToFirst()
	test := predicate(q)
	for {
		if test(node) {
			count++
		}
		if !node.MoveToNext() {
			break
		}
	}
	return float64(count)
}
<|endoftext|>


<commit_before>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gps

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sync"

	"github.com/pkg/errors"
)

// A Solution is returned by a solver run. It is mostly just a Lock, with some
// additional methods that report information about the solve run.
type Solution interface {
	Lock
	// The name of the ProjectAnalyzer used in generating this solution.
	AnalyzerName() string
	// The version of the ProjectAnalyzer used in generating this solution.
	AnalyzerVersion() int
	// The name of the Solver used in generating this solution.
	SolverName() string
	// The version of the Solver used in generating this solution.
	SolverVersion() int
	Attempts() int
}

type solution struct {
	// A list of the projects selected by the solver.
	p []LockedProject

	// The number of solutions that were attempted
	att int

	// The hash digest of the input opts
	hd []byte

	// The analyzer info
	analyzerInfo ProjectAnalyzerInfo

	// The solver used in producing this solution
	solv Solver
}

// WriteDepTree takes a basedir and a Lock, and exports all the projects
// listed in the lock to the appropriate target location within the basedir.
//
// If the goal is to populate a vendor directory, basedir should be the absolute
// path to that vendor directory, not its parent (a project root, typically).
//
// It requires a SourceManager to do the work, and takes a flag indicating
// whether or not to strip vendor directories contained in the exported
// dependencies.
func WriteDepTree(basedir string, l Lock, sm SourceManager, sv bool, logger *log.Logger) error {
	if l == nil {
		return fmt.Errorf("must provide non-nil Lock to WriteDepTree")
	}

	err := os.MkdirAll(basedir, 0777)
	if err != nil {
		return err
	}

	var wg sync.WaitGroup
	errCh := make(chan error, len(l.Projects()))

	for _, p := range l.Projects() {
		wg.Add(1)
		go func(p LockedProject) {
			to := filepath.FromSlash(filepath.Join(basedir, string(p.Ident().ProjectRoot)))
			logger.Printf("Writing out %s@%s", p.Ident().errString(), p.Version())

			if err := sm.ExportProject(p.Ident(), p.Version(), to); err != nil {
				errCh <- errors.Wrapf(err, "failed to export %s", p.Ident().ProjectRoot)
			}

			if sv {
				err := filepath.Walk(to, stripVendor)
				if err != nil {
					errCh <- errors.Wrapf(err, "failed to strip vendor from %s", p.Ident().ProjectRoot)
				}
			}
			wg.Done()
		}(p)
	}

	wg.Wait()
	close(errCh)

	if len(errCh) > 0 {
		logger.Println("Failed to write dep tree. The following errors occurred:")
		for err := range errCh {
			logger.Println(" * ", err)
		}

		removeAll(basedir)

		return errors.New("failed to write dep tree")
	}
	return nil
}

func (r solution) Projects() []LockedProject {
	return r.p
}

func (r solution) Attempts() int {
	return r.att
}

func (r solution) InputHash() []byte {
	return r.hd
}

func (r solution) AnalyzerName() string {
	return r.analyzerInfo.Name
}

func (r solution) AnalyzerVersion() int {
	return r.analyzerInfo.Version
}

func (r solution) SolverName() string {
	return r.solv.Name()
}

func (r solution) SolverVersion() int {
	return r.solv.Version()
}
<commit_msg>Accommodate error channel size properly<commit_after>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gps

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sync"

	"github.com/pkg/errors"
)

// A Solution is returned by a solver run. It is mostly just a Lock, with some
// additional methods that report information about the solve run.
type Solution interface {
	Lock
	// The name of the ProjectAnalyzer used in generating this solution.
	AnalyzerName() string
	// The version of the ProjectAnalyzer used in generating this solution.
	AnalyzerVersion() int
	// The name of the Solver used in generating this solution.
	SolverName() string
	// The version of the Solver used in generating this solution.
	SolverVersion() int
	Attempts() int
}

type solution struct {
	// A list of the projects selected by the solver.
	p []LockedProject

	// The number of solutions that were attempted
	att int

	// The hash digest of the input opts
	hd []byte

	// The analyzer info
	analyzerInfo ProjectAnalyzerInfo

	// The solver used in producing this solution
	solv Solver
}

// WriteDepTree takes a basedir and a Lock, and exports all the projects
// listed in the lock to the appropriate target location within the basedir.
//
// If the goal is to populate a vendor directory, basedir should be the absolute
// path to that vendor directory, not its parent (a project root, typically).
//
// It requires a SourceManager to do the work, and takes a flag indicating
// whether or not to strip vendor directories contained in the exported
// dependencies.
func WriteDepTree(basedir string, l Lock, sm SourceManager, sv bool, logger *log.Logger) error {
	if l == nil {
		return fmt.Errorf("must provide non-nil Lock to WriteDepTree")
	}

	err := os.MkdirAll(basedir, 0777)
	if err != nil {
		return err
	}

	var wg sync.WaitGroup
	errCh := make(chan error, len(l.Projects()))

	for _, p := range l.Projects() {
		wg.Add(1)
		go func(p LockedProject) {
			defer wg.Done()
			to := filepath.FromSlash(filepath.Join(basedir, string(p.Ident().ProjectRoot)))
			logger.Printf("Writing out %s@%s", p.Ident().errString(), p.Version())

			if err := sm.ExportProject(p.Ident(), p.Version(), to); err != nil {
				errCh <- errors.Wrapf(err, "failed to export %s", p.Ident().ProjectRoot)
				return
			}

			if sv {
				err := filepath.Walk(to, stripVendor)
				if err != nil {
					errCh <- errors.Wrapf(err, "failed to strip vendor from %s", p.Ident().ProjectRoot)
				}
			}
			return
		}(p)
	}

	wg.Wait()
	close(errCh)

	if len(errCh) > 0 {
		logger.Println("Failed to write dep tree. The following errors occurred:")
		for err := range errCh {
			logger.Println(" * ", err)
		}

		removeAll(basedir)

		return errors.New("failed to write dep tree")
	}
	return nil
}

func (r solution) Projects() []LockedProject {
	return r.p
}

func (r solution) Attempts() int {
	return r.att
}

func (r solution) InputHash() []byte {
	return r.hd
}

func (r solution) AnalyzerName() string {
	return r.analyzerInfo.Name
}

func (r solution) AnalyzerVersion() int {
	return r.analyzerInfo.Version
}

func (r solution) SolverName() string {
	return r.solv.Name()
}

func (r solution) SolverVersion() int {
	return r.solv.Version()
}
<|endoftext|>


<commit_before>package slackevents

import (
	"encoding/json"

	"github.com/nlopes/slack"
)

type MessageActionResponse struct {
	ResponseType    string `json:"response_type"`
	ReplaceOriginal bool   `json:"replace_original"`
	Text            string `json:"text"`
}

type MessageActionEntity struct {
	ID     string `json:"id"`
	Domain string `json:"domain"`
}

type MessageAction struct {
	Type             string                   `json:"type"`
	Actions          []slack.AttachmentAction `json:"actions"`
	CallbackID       string                   `json:"callback_id"`
	Team             MessageActionEntity      `json:"team"`
	Channel          MessageActionEntity      `json:"channel"`
	User             MessageActionEntity      `json:"user"`
	ActionTimestamp  json.Number              `json:"action_ts"`
	MessageTimestamp json.Number              `json:"message_ts"`
	AttachmentID     json.Number              `json:"attachment_id"`
	Token            string                   `json:"token"`
	Message          slack.Message            `json:"message"`
	OriginalMessage  slack.Message            `json:"original_message"`
	ResponseURL      string                   `json:"response_url"`
	TriggerID        string                   `json:"trigger_id"`
}
<commit_msg>Added Name to MessageActionEntity<commit_after>package slackevents

import (
	"encoding/json"

	"github.com/nlopes/slack"
)

type MessageActionResponse struct {
	ResponseType    string `json:"response_type"`
	ReplaceOriginal bool   `json:"replace_original"`
	Text            string `json:"text"`
}

type MessageActionEntity struct {
	ID     string `json:"id"`
	Domain string `json:"domain"`
	Name   string `json:"name"`
}

type MessageAction struct {
	Type             string                   `json:"type"`
	Actions          []slack.AttachmentAction `json:"actions"`
	CallbackID       string                   `json:"callback_id"`
	Team             MessageActionEntity      `json:"team"`
	Channel          MessageActionEntity      `json:"channel"`
	User             MessageActionEntity      `json:"user"`
	ActionTimestamp  json.Number              `json:"action_ts"`
	MessageTimestamp json.Number              `json:"message_ts"`
	AttachmentID     json.Number              `json:"attachment_id"`
	Token            string                   `json:"token"`
	Message          slack.Message            `json:"message"`
	OriginalMessage  slack.Message            `json:"original_message"`
	ResponseURL      string                   `json:"response_url"`
	TriggerID        string                   `json:"trigger_id"`
}
<|endoftext|>


<commit_before>package checker

import (
	"io/ioutil"
	"path/filepath"
	"strings"
	"time"
)

import (
	"golog"
	"goyaml"
	"tritium/proto"
	"tritium/whale"
)

const TEST_FILE = "test/rewrite_test.yml"

type RewriteTestCase struct {
	Host     string
	Test     string
	Expected string
}

type ProjectTests struct {
	Host          []RewriteTestCase
	Link          []RewriteTestCase
	Cookie_Domain []RewriteTestCase
}

func read_test_cases(test_file string) (ProjectTests, error) {
	tests := ProjectTests{}
	fileinfo, err := ioutil.ReadFile(test_file)
	if err != nil {
		return tests, err
	}
	err = goyaml.Unmarshal([]byte(fileinfo), &tests)
	return tests, err
}

type create_http_cmd func(RewriteTestCase) string
type read_cmd_transform func(string) string

func create_host_http_req(test RewriteTestCase) string {
	request := "GET / HTTP/1.0\r\n"
	request += "Host: " + test.Test
	return request
}

func read_host_results(result string) string {
	lines := strings.Split(result, "\n")
	check := lines[1][6:]
	return strings.TrimSpace(check)
}

func create_link_http_res(test RewriteTestCase) string {
	response := "HTTP/1.0 200 OK\r\n"
	response += "Location: " + test.Test
	return response
}

func read_link_results(result string) string {
	lines := strings.Split(result, "\n")
	check := lines[1][10:]
	return strings.TrimSpace(check)
}

func create_cookie_http_res(test RewriteTestCase) string {
	response := "HTTP/1.0 200 OK\r\n"
	response += "Set-Cookie: Name=hi"
	if len(test.Test) != 0 {
		response += "; Domain=" + test.Test
	}
	return response
}

func read_cookie_results(result string) string {
	lines := strings.Split(result, "\n")
	for _, l := range lines {
		if strings.HasPrefix(l, "Set-Cookie:") {
			cooky := strings.Split(l[10:], ";")
			for _, c := range cooky {
				c = strings.TrimSpace(c)
				if strings.HasPrefix(strings.ToLower(c), "domain=") {
					dom := strings.Split(c, "=")
					return strings.TrimSpace(dom[1])
				}
			}
		}
	}
	return ""
}

//TODO Passing in request tsf every time because of the hack mentioned inside.
func (result *CheckResult) run_tests(test_type string, engine *whale.Whale, transformer *proto.Transform, req_transform *proto.Transform, rrules []*proto.RewriteRule, tests []RewriteTestCase, create_cmd create_http_cmd, read_result_cmd read_cmd_transform, logger *golog.Logger) bool {
	all_passed := true

	for i, current_test := range tests {
		env := map[string]string{}
		env["host"] = current_test.Host
		//TODO temporary hack to make cookie rewriters work.  The problem is that
		//the cookie rewriters hinge on environment variables that are set only
		//after rewriting request.ts, so we have to go through that transformation
		//first before we can do the response_post transformation for the cookies.

		// Temporarily just try to create a really large timeout.
		timeout := time.Now().Add(time.Duration(1) * time.Minute)
		if test_type == "Cookie" {
			req_cmd := "GET / HTTP/1.0\r\nHost: " + current_test.Host
			// run the engine to populate the environment
			_, exports, _ := engine.Run(req_transform, rrules, req_cmd, env, timeout)
			for _, arr := range exports {
				if len(arr) != 2 {
				} else if arr[0] == "set-cookie" {
					env[arr[0]] = env[arr[0]] + arr[1]
				} else {
					env[arr[0]] = arr[1]
				}
			}
			// env should now be populated with all the things we need, so we can run
			// the rest of the test.
		}

		http_cmd := create_cmd(current_test)
		test_result, _, _ := engine.Run(transformer, rrules, http_cmd, env, timeout)
		test_output := read_result_cmd(test_result)

		test_passed := test_output == current_test.Expected
		//TODO cookie hax, omnimobile and simple mobile translate cookie domains
		//differently, simply makes them more general, omni makes them more precise
		//in order to accomodate test passing on both, we compare that the result
		//is at least a subset of the domain that we expect.
		if test_type == "Cookie" {
			test_passed = whale.IsDomainCovered(test_output, current_test.Expected)
		}

		if test_passed {
			print(".")
		} else {
			result.AddRewriterWarning(test_type, i+1, "Expected: "+current_test.Expected+" Got: "+test_output)
			all_passed = false
		}
	}
	return all_passed
}

func (result *CheckResult) CheckRewriters(req_tsf *proto.Transform, post_tsf *proto.Transform, rrules []*proto.RewriteRule, projectPath string, logger *golog.Logger) bool {
	test_path := filepath.Join(projectPath, TEST_FILE)
	test_cases, err := read_test_cases(test_path)
	if err != nil {
		return false
	}

	engine := whale.NewEngine(logger)
	passed_host := result.run_tests("Host", engine, req_tsf, req_tsf, rrules, test_cases.Host, create_host_http_req, read_host_results, logger)
	passed_link := result.run_tests("Link", engine, post_tsf, req_tsf, rrules, test_cases.Link, create_link_http_res, read_link_results, logger)
	passed_cookie := result.run_tests("Cookie", engine, post_tsf, req_tsf, rrules, test_cases.Cookie_Domain, create_cookie_http_res, read_cookie_results, logger)

	return passed_host && passed_link && passed_cookie
}
<commit_msg>this function is not used at all. commented out.<commit_after>package checker

import (
	"io/ioutil"
	//"path/filepath"
	"strings"
	"time"
)

import (
	"golog"
	"goyaml"
	"tritium/proto"
	"tritium/whale"
)

const TEST_FILE = "test/rewrite_test.yml"

type RewriteTestCase struct {
	Host     string
	Test     string
	Expected string
}

type ProjectTests struct {
	Host          []RewriteTestCase
	Link          []RewriteTestCase
	Cookie_Domain []RewriteTestCase
}

func read_test_cases(test_file string) (ProjectTests, error) {
	tests := ProjectTests{}
	fileinfo, err := ioutil.ReadFile(test_file)
	if err != nil {
		return tests, err
	}
	err = goyaml.Unmarshal([]byte(fileinfo), &tests)
	return tests, err
}

type create_http_cmd func(RewriteTestCase) string
type read_cmd_transform func(string) string

func create_host_http_req(test RewriteTestCase) string {
	request := "GET / HTTP/1.0\r\n"
	request += "Host: " + test.Test
	return request
}

func read_host_results(result string) string {
	lines := strings.Split(result, "\n")
	check := lines[1][6:]
	return strings.TrimSpace(check)
}

func create_link_http_res(test RewriteTestCase) string {
	response := "HTTP/1.0 200 OK\r\n"
	response += "Location: " + test.Test
	return response
}

func read_link_results(result string) string {
	lines := strings.Split(result, "\n")
	check := lines[1][10:]
	return strings.TrimSpace(check)
}

func create_cookie_http_res(test RewriteTestCase) string {
	response := "HTTP/1.0 200 OK\r\n"
	response += "Set-Cookie: Name=hi"
	if len(test.Test) != 0 {
		response += "; Domain=" + test.Test
	}
	return response
}

func read_cookie_results(result string) string {
	lines := strings.Split(result, "\n")
	for _, l := range lines {
		if strings.HasPrefix(l, "Set-Cookie:") {
			cooky := strings.Split(l[10:], ";")
			for _, c := range cooky {
				c = strings.TrimSpace(c)
				if strings.HasPrefix(strings.ToLower(c), "domain=") {
					dom := strings.Split(c, "=")
					return strings.TrimSpace(dom[1])
				}
			}
		}
	}
	return ""
}

//TODO Passing in request tsf every time because of the hack mentioned inside.
func (result *CheckResult) run_tests(test_type string, engine *whale.Whale, transformer *proto.Transform, req_transform *proto.Transform, rrules []*proto.RewriteRule, tests []RewriteTestCase, create_cmd create_http_cmd, read_result_cmd read_cmd_transform, logger *golog.Logger) bool {
	all_passed := true

	for i, current_test := range tests {
		env := map[string]string{}
		env["host"] = current_test.Host
		//TODO temporary hack to make cookie rewriters work.  The problem is that
		//the cookie rewriters hinge on environment variables that are set only
		//after rewriting request.ts, so we have to go through that transformation
		//first before we can do the response_post transformation for the cookies.

		// Temporarily just try to create a really large timeout.
		timeout := time.Now().Add(time.Duration(1) * time.Minute)
		if test_type == "Cookie" {
			req_cmd := "GET / HTTP/1.0\r\nHost: " + current_test.Host
			// run the engine to populate the environment
			_, exports, _ := engine.Run(req_transform, rrules, req_cmd, env, timeout)
			for _, arr := range exports {
				if len(arr) != 2 {
				} else if arr[0] == "set-cookie" {
					env[arr[0]] = env[arr[0]] + arr[1]
				} else {
					env[arr[0]] = arr[1]
				}
			}
			// env should now be populated with all the things we need, so we can run
			// the rest of the test.
		}

		http_cmd := create_cmd(current_test)
		test_result, _, _ := engine.Run(transformer, rrules, http_cmd, env, timeout)
		test_output := read_result_cmd(test_result)

		test_passed := test_output == current_test.Expected
		//TODO cookie hax, omnimobile and simple mobile translate cookie domains
		//differently, simply makes them more general, omni makes them more precise
		//in order to accomodate test passing on both, we compare that the result
		//is at least a subset of the domain that we expect.
		if test_type == "Cookie" {
			test_passed = whale.IsDomainCovered(test_output, current_test.Expected)
		}

		if test_passed {
			print(".")
		} else {
			result.AddRewriterWarning(test_type, i+1, "Expected: "+current_test.Expected+" Got: "+test_output)
			all_passed = false
		}
	}
	return all_passed
}
/*
func (result *CheckResult) CheckRewriters(req_tsf *proto.Transform, post_tsf *proto.Transform, rrules []*proto.RewriteRule, projectPath string, logger *golog.Logger) bool {
	test_path := filepath.Join(projectPath, TEST_FILE)
	test_cases, err := read_test_cases(test_path)
	if err != nil {
		return false
	}

	engine := whale.NewEngine(logger)
	passed_host := result.run_tests("Host", engine, req_tsf, req_tsf, rrules, test_cases.Host, create_host_http_req, read_host_results, logger)
	passed_link := result.run_tests("Link", engine, post_tsf, req_tsf, rrules, test_cases.Link, create_link_http_res, read_link_results, logger)
	passed_cookie := result.run_tests("Cookie", engine, post_tsf, req_tsf, rrules, test_cases.Cookie_Domain, create_cookie_http_res, read_cookie_results, logger)

	return passed_host && passed_link && passed_cookie
}
*/<|endoftext|>


<commit_before>package parser

// #include "query_types.h"
// #include <stdlib.h>
import "C"

import (
	"common"
	"fmt"
	"time"
	"unsafe"
)

type From struct {
	TableName string
}

type Operation int

type Value struct {
	Name  string
	Elems []*Value
}

func (self *Value) IsFunctionCall() bool {
	return self.Elems != nil
}

type Expression struct {
	Left      interface{}
	Operation byte
	Right     *Expression
}

type BoolExpression struct {
	Left      *Expression
	Operation string
	Right     *Expression
}

type GroupByClause []*Value

func (self GroupByClause) GetGroupByTime() (*time.Duration, error) {
	for _, groupBy := range self {
		if groupBy.IsFunctionCall() {
			// TODO: check the number of arguments and return an error
			if len(groupBy.Elems) != 1 {
				return nil, common.NewQueryError(common.WrongNumberOfArguments, "time function only accepts one argument")
			}
			// TODO: check the function name
			// TODO: error checking
			arg := groupBy.Elems[0].Name
			duration, err := time.ParseDuration(arg)
			if err != nil {
				return nil, common.NewQueryError(common.InvalidArgument, fmt.Sprintf("invalid argument %s to the time function", arg))
			}
			return &duration, nil
		}
	}
	return nil, nil
}

type WhereCondition struct {
	isBooleanExpression bool
	Left                interface{}
	Operation           string
	Right               *WhereCondition
}

func (self *WhereCondition) GetBoolExpression() (*BoolExpression, bool) {
	if self.isBooleanExpression {
		return self.Left.(*BoolExpression), true
	}
	return nil, false
}

func (self *WhereCondition) GetLeftWhereCondition() (*WhereCondition, bool) {
	if !self.isBooleanExpression {
		return self.Left.(*WhereCondition), true
	}
	return nil, false
}

type Query struct {
	q             C.query
	closed        bool
	ColumnNames   []*Value
	Condition     *WhereCondition
	groupByClause GroupByClause
	Limit         int
	Ascending     bool
}

func (self *Query) GetColumnNames() []*Value {
	if self.ColumnNames != nil {
		return self.ColumnNames
	}

	self.ColumnNames = GetValueArray(self.q.c)
	return self.ColumnNames
}

func (self *Query) GetFromClause() *Value {
	return GetValue(self.q.f)
}

func (self *Expression) GetLeftValue() (*Value, bool) {
	if self.Operation == 0 {
		return self.Left.(*Value), true
	}
	return nil, false
}

func (self *Expression) GetLeftExpression() (*Expression, bool) {
	if self.Operation != 0 {
		return self.Left.(*Expression), true
	}
	return nil, false
}

func GetValueArray(array *C.value_array) []*Value {
	if array == nil {
		return nil
	}

	arr := uintptr(unsafe.Pointer(array.elems))
	elemSize := unsafe.Sizeof(*array.elems)
	size := uintptr(array.size)

	stringSlice := make([]*Value, 0, size)

	var i uintptr
	for i = 0; i < size; i++ {
		str := (**C.value)(unsafe.Pointer(arr + elemSize*i))
		stringSlice = append(stringSlice, GetValue(*str))
	}
	return stringSlice
}

func GetStringArray(array *C.array) []string {
	if array == nil {
		return nil
	}

	arr := uintptr(unsafe.Pointer(array.elems))
	elemSize := unsafe.Sizeof(*array.elems)
	size := uintptr(array.size)

	stringSlice := make([]string, 0, size)

	var i uintptr
	for i = 0; i < size; i++ {
		str := (**C.char)(unsafe.Pointer(arr + elemSize*i))
		stringSlice = append(stringSlice, C.GoString(*str))
	}
	return stringSlice
}

func GetValue(value *C.value) *Value {
	v := &Value{}
	v.Name = C.GoString(value.name)
	v.Elems = GetValueArray(value.args)

	return v
}

func GetExpression(expr *C.expression) *Expression {
	expression := &Expression{}
	if expr.op == 0 {
		expression.Left = GetValue((*C.value)(expr.left))
		expression.Operation = byte(expr.op)
		expression.Right = nil
	} else {
		expression.Left = GetExpression((*C.expression)(expr.left))
		expression.Operation = byte(expr.op)
		expression.Right = GetExpression((*C.expression)(unsafe.Pointer(expr.right)))
	}

	return expression
}

func GetBoolExpression(expr *C.bool_expression) *BoolExpression {
	boolExpression := &BoolExpression{}
	boolExpression.Left = GetExpression(expr.left)
	if expr.op != nil {
		boolExpression.Operation = C.GoString(expr.op)
		boolExpression.Right = GetExpression(expr.right)
	}

	return boolExpression
}

func GetWhereCondition(condition *C.condition) *WhereCondition {
	if condition.is_bool_expression != 0 {
		return &WhereCondition{
			isBooleanExpression: true,
			Left:                GetBoolExpression((*C.bool_expression)(condition.left)),
			Operation:           "",
			Right:               nil,
		}
	}

	c := &WhereCondition{}
	c.Left = GetWhereCondition((*C.condition)(condition.left))
	c.Operation = C.GoString(condition.op)
	c.Right = GetWhereCondition((*C.condition)(unsafe.Pointer(condition.right)))

	return c
}

func (self *Query) GetWhereCondition() *WhereCondition {
	if self.q.where_condition == nil {
		return nil
	}

	self.Condition = GetWhereCondition(self.q.where_condition)
	return self.Condition
}

func (self *Query) GetGroupByClause() GroupByClause {
	if self.groupByClause != nil {
		return self.groupByClause
	}

	if self.q.group_by == nil {
		self.groupByClause = GroupByClause{}
		return self.groupByClause
	}

	self.groupByClause = GetValueArray(self.q.group_by)
	return self.groupByClause
}

func (self *Query) Close() {
	if self.closed {
		return
	}

	C.close_query(&self.q)
	self.closed = true
}

func ParseQuery(query string) (*Query, error) {
	queryString := C.CString(query)
	defer C.free(unsafe.Pointer(queryString))
	q := C.parse_query(queryString)
	var err error
	if q.error != nil {
		str := C.GoString(q.error.err)
		err = fmt.Errorf("Error at %d:%d. %s", q.error.line, q.error.column, str)
		C.close_query(&q)
		return nil, err
	}
	return &Query{q, false, nil, nil, nil, int(q.limit), q.ascending != 0}, err
}
<commit_msg>convert C arrays to go slices.<commit_after>package parser

// #include "query_types.h"
// #include <stdlib.h>
import "C"

import (
	"common"
	"fmt"
	"reflect"
	"time"
	"unsafe"
)

type From struct {
	TableName string
}

type Operation int

type Value struct {
	Name  string
	Elems []*Value
}

func (self *Value) IsFunctionCall() bool {
	return self.Elems != nil
}

type Expression struct {
	Left      interface{}
	Operation byte
	Right     *Expression
}

type BoolExpression struct {
	Left      *Expression
	Operation string
	Right     *Expression
}

type GroupByClause []*Value

func (self GroupByClause) GetGroupByTime() (*time.Duration, error) {
	for _, groupBy := range self {
		if groupBy.IsFunctionCall() {
			// TODO: check the number of arguments and return an error
			if len(groupBy.Elems) != 1 {
				return nil, common.NewQueryError(common.WrongNumberOfArguments, "time function only accepts one argument")
			}
			// TODO: check the function name
			// TODO: error checking
			arg := groupBy.Elems[0].Name
			duration, err := time.ParseDuration(arg)
			if err != nil {
				return nil, common.NewQueryError(common.InvalidArgument, fmt.Sprintf("invalid argument %s to the time function", arg))
			}
			return &duration, nil
		}
	}
	return nil, nil
}

type WhereCondition struct {
	isBooleanExpression bool
	Left                interface{}
	Operation           string
	Right               *WhereCondition
}

func (self *WhereCondition) GetBoolExpression() (*BoolExpression, bool) {
	if self.isBooleanExpression {
		return self.Left.(*BoolExpression), true
	}
	return nil, false
}

func (self *WhereCondition) GetLeftWhereCondition() (*WhereCondition, bool) {
	if !self.isBooleanExpression {
		return self.Left.(*WhereCondition), true
	}
	return nil, false
}

type Query struct {
	q             C.query
	closed        bool
	ColumnNames   []*Value
	Condition     *WhereCondition
	groupByClause GroupByClause
	Limit         int
	Ascending     bool
}

func (self *Query) GetColumnNames() []*Value {
	if self.ColumnNames != nil {
		return self.ColumnNames
	}

	self.ColumnNames = GetValueArray(self.q.c)
	return self.ColumnNames
}

func (self *Query) GetFromClause() *Value {
	return GetValue(self.q.f)
}

func (self *Expression) GetLeftValue() (*Value, bool) {
	if self.Operation == 0 {
		return self.Left.(*Value), true
	}
	return nil, false
}

func (self *Expression) GetLeftExpression() (*Expression, bool) {
	if self.Operation != 0 {
		return self.Left.(*Expression), true
	}
	return nil, false
}

func setupSlice(hdr *reflect.SliceHeader, ptr unsafe.Pointer, size C.size_t) {
	hdr.Cap = int(size)
	hdr.Len = int(size)
	hdr.Data = uintptr(ptr)
}

func GetValueArray(array *C.value_array) []*Value {
	if array == nil {
		return nil
	}

	var values []*C.value
	setupSlice((*reflect.SliceHeader)((unsafe.Pointer(&values))), unsafe.Pointer(array.elems), array.size)

	valuesSlice := make([]*Value, 0, array.size)

	for _, value := range values {
		valuesSlice = append(valuesSlice, GetValue(value))
	}
	return valuesSlice
}

func GetStringArray(array *C.array) []string {
	if array == nil {
		return nil
	}

	var values []*C.char
	setupSlice((*reflect.SliceHeader)((unsafe.Pointer(&values))), unsafe.Pointer(array.elems), array.size)

	stringSlice := make([]string, 0, array.size)

	for _, value := range values {
		stringSlice = append(stringSlice, C.GoString(value))
	}
	return stringSlice
}

func GetValue(value *C.value) *Value {
	v := &Value{}
	v.Name = C.GoString(value.name)
	v.Elems = GetValueArray(value.args)

	return v
}

func GetExpression(expr *C.expression) *Expression {
	expression := &Expression{}
	if expr.op == 0 {
		expression.Left = GetValue((*C.value)(expr.left))
		expression.Operation = byte(expr.op)
		expression.Right = nil
	} else {
		expression.Left = GetExpression((*C.expression)(expr.left))
		expression.Operation = byte(expr.op)
		expression.Right = GetExpression((*C.expression)(unsafe.Pointer(expr.right)))
	}

	return expression
}

func GetBoolExpression(expr *C.bool_expression) *BoolExpression {
	boolExpression := &BoolExpression{}
	boolExpression.Left = GetExpression(expr.left)
	if expr.op != nil {
		boolExpression.Operation = C.GoString(expr.op)
		boolExpression.Right = GetExpression(expr.right)
	}

	return boolExpression
}

func GetWhereCondition(condition *C.condition) *WhereCondition {
	if condition.is_bool_expression != 0 {
		return &WhereCondition{
			isBooleanExpression: true,
			Left:                GetBoolExpression((*C.bool_expression)(condition.left)),
			Operation:           "",
			Right:               nil,
		}
	}

	c := &WhereCondition{}
	c.Left = GetWhereCondition((*C.condition)(condition.left))
	c.Operation = C.GoString(condition.op)
	c.Right = GetWhereCondition((*C.condition)(unsafe.Pointer(condition.right)))

	return c
}

func (self *Query) GetWhereCondition() *WhereCondition {
	if self.q.where_condition == nil {
		return nil
	}

	self.Condition = GetWhereCondition(self.q.where_condition)
	return self.Condition
}

func (self *Query) GetGroupByClause() GroupByClause {
	if self.groupByClause != nil {
		return self.groupByClause
	}

	if self.q.group_by == nil {
		self.groupByClause = GroupByClause{}
		return self.groupByClause
	}

	self.groupByClause = GetValueArray(self.q.group_by)
	return self.groupByClause
}

func (self *Query) Close() {
	if self.closed {
		return
	}

	C.close_query(&self.q)
	self.closed = true
}

func ParseQuery(query string) (*Query, error) {
	queryString := C.CString(query)
	defer C.free(unsafe.Pointer(queryString))
	q := C.parse_query(queryString)
	var err error
	if q.error != nil {
		str := C.GoString(q.error.err)
		err = fmt.Errorf("Error at %d:%d. %s", q.error.line, q.error.column, str)
		C.close_query(&q)
		return nil, err
	}
	return &Query{q, false, nil, nil, nil, int(q.limit), q.ascending != 0}, err
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package exec runs external commands. It wraps os.StartProcess to make it
// easier to remap stdin and stdout, connect I/O with pipes, and do other
// adjustments.
package exec

import (
	"bytes"
	"io"
	"os"
	"strconv"
)

// PathError records the name of a binary that was not
// found on the current $PATH.
type PathError struct {
	Name string
}

func (e *PathError) String() string {
	return "command " + strconv.Quote(e.Name) + " not found in $PATH"
}

// Cmd represents an external command being prepared or run.
type Cmd struct {
	// Path is the path of the command to run.
	//
	// This is the only field that must be set to a non-zero
	// value.
	Path string

	// Args is the command line arguments, including the command as Args[0].
	// If Args is empty, Run uses {Path}.
	//
	// In typical use, both Path and Args are set by calling Command.
	Args []string

	// Env specifies the environment of the process.
	// If Env is nil, Run uses the current process's environment.
	Env []string

	// Dir specifies the working directory of the command.
	// If Dir is the empty string, Run runs the command in the
	// process's current directory.
	Dir string

	// Stdin specifies the process's standard input.
	// If Stdin is nil, the process reads from DevNull.
	Stdin io.Reader

	// Stdout and Stderr specify the process's standard output and error.
	//
	// If either is nil, Run connects the
	// corresponding file descriptor to /dev/null.
	//
	// If Stdout and Stderr are are the same writer, at most one
	// goroutine at a time will call Write.
	Stdout io.Writer
	Stderr io.Writer

	err             os.Error // last error (from LookPath, stdin, stdout, stderr)
	process         *os.Process
	childFiles      []*os.File
	closeAfterStart []*os.File
	closeAfterWait  []*os.File
	goroutine       []func() os.Error
	errch           chan os.Error // one send per goroutine
}

// Command returns the Cmd struct to execute the named program with
// the given arguments.
//
// It sets Path and Args in the returned structure and zeroes the
// other fields.
//
// If name contains no path separators, Command uses LookPath to
// resolve the path to a complete name if possible. Otherwise it uses
// name directly.
//
// The returned Cmd's Args is constructed from the command name
// followed by the elements of arg, so arg should not include the
// command name itself. For example, Command("echo", "hello")
func Command(name string, arg ...string) *Cmd {
	aname, err := LookPath(name)
	if err != nil {
		aname = name
	}
	return &Cmd{
		Path: aname,
		Args: append([]string{name}, arg...),
		err:  err,
	}
}

// interfaceEqual protects against panics from doing equality tests on
// two interface with non-comparable underlying types
func interfaceEqual(a, b interface{}) bool {
	defer func() {
		recover()
	}()
	return a == b
}

func (c *Cmd) envv() []string {
	if c.Env != nil {
		return c.Env
	}
	return os.Environ()
}

func (c *Cmd) argv() []string {
	if len(c.Args) > 0 {
		return c.Args
	}
	return []string{c.Path}
}

func (c *Cmd) stdin() (f *os.File, err os.Error) {
	if c.Stdin == nil {
		f, err = os.Open(os.DevNull)
		c.closeAfterStart = append(c.closeAfterStart, f)
		return
	}

	if f, ok := c.Stdin.(*os.File); ok {
		return f, nil
	}

	pr, pw, err := os.Pipe()
	if err != nil {
		return
	}

	c.closeAfterStart = append(c.closeAfterStart, pr)
	c.closeAfterWait = append(c.closeAfterWait, pw)
	c.goroutine = append(c.goroutine, func() os.Error {
		_, err := io.Copy(pw, c.Stdin)
		if err1 := pw.Close(); err == nil {
			err = err1
		}
		return err
	})
	return pr, nil
}

func (c *Cmd) stdout() (f *os.File, err os.Error) {
	return c.writerDescriptor(c.Stdout)
}

func (c *Cmd) stderr() (f *os.File, err os.Error) {
	if c.Stderr != nil && interfaceEqual(c.Stderr, c.Stdout) {
		return c.childFiles[1], nil
	}
	return c.writerDescriptor(c.Stderr)
}

func (c *Cmd) writerDescriptor(w io.Writer) (f *os.File, err os.Error) {
	if w == nil {
		f, err = os.OpenFile(os.DevNull, os.O_WRONLY, 0)
		c.closeAfterStart = append(c.closeAfterStart, f)
		return
	}

	if f, ok := w.(*os.File); ok {
		return f, nil
	}

	pr, pw, err := os.Pipe()
	if err != nil {
		return
	}

	c.closeAfterStart = append(c.closeAfterStart, pw)
	c.closeAfterWait = append(c.closeAfterWait, pr)
	c.goroutine = append(c.goroutine, func() os.Error {
		_, err := io.Copy(w, pr)
		return err
	})
	return pw, nil
}

// Run runs the specified command and waits for it to complete.
//
// The returned error is nil if the command runs, has no problems
// copying stdin, stdout, and stderr, and exits with a zero exit
// status.
//
// If the command fails to run or doesn't complete successfully, the
// error is of type *os.Waitmsg. Other error types may be
// returned for I/O problems.
func (c *Cmd) Run() os.Error {
	if err := c.Start(); err != nil {
		return err
	}
	return c.Wait()
}

func (c *Cmd) Start() os.Error {
	if c.err != nil {
		return c.err
	}
	if c.process != nil {
		return os.NewError("exec: already started")
	}

	type F func(*Cmd) (*os.File, os.Error)
	for _, setupFd := range []F{(*Cmd).stdin, (*Cmd).stdout, (*Cmd).stderr} {
		fd, err := setupFd(c)
		if err != nil {
			return err
		}
		c.childFiles = append(c.childFiles, fd)
	}

	var err os.Error
	c.process, err = os.StartProcess(c.Path, c.argv(), &os.ProcAttr{
		Dir:   c.Dir,
		Files: c.childFiles,
		Env:   c.envv(),
	})
	if err != nil {
		return err
	}

	for _, fd := range c.closeAfterStart {
		fd.Close()
	}

	c.errch = make(chan os.Error, len(c.goroutine))
	for _, fn := range c.goroutine {
		go func(fn func() os.Error) {
			c.errch <- fn()
		}(fn)
	}

	return nil
}

func (c *Cmd) Wait() os.Error {
	if c.process == nil {
		return os.NewError("exec: not started")
	}
	msg, err := c.process.Wait(0)

	var copyError os.Error
	for _ = range c.goroutine {
		if err := <-c.errch; err != nil && copyError == nil {
			copyError = err
		}
	}

	for _, fd := range c.closeAfterWait {
		fd.Close()
	}

	if err != nil {
		return err
	} else if !msg.Exited() || msg.ExitStatus() != 0 {
		return msg
	}

	return copyError
}

// Output runs the command and returns its standard output.
func (c *Cmd) Output() ([]byte, os.Error) {
	var b bytes.Buffer
	c.Stdout = &b
	err := c.Run()
	return b.Bytes(), err
}

// CombinedOutput runs the command and returns its combined standard
// output and standard error.
func (c *Cmd) CombinedOutput() ([]byte, os.Error) {
	var b bytes.Buffer
	c.Stdout = &b
	c.Stderr = &b
	err := c.Run()
	return b.Bytes(), err
}
<commit_msg>exec: missing docs, errors<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package exec runs external commands. It wraps os.StartProcess to make it
// easier to remap stdin and stdout, connect I/O with pipes, and do other
// adjustments.
package exec

import (
	"bytes"
	"io"
	"os"
	"strconv"
)

// PathError records the name of a binary that was not
// found on the current $PATH.
type PathError struct {
	Name string
}

func (e *PathError) String() string {
	return "command " + strconv.Quote(e.Name) + " not found in $PATH"
}

// Cmd represents an external command being prepared or run.
type Cmd struct {
	// Path is the path of the command to run.
	//
	// This is the only field that must be set to a non-zero
	// value.
	Path string

	// Args is the command line arguments, including the command as Args[0].
	// If Args is empty, Run uses {Path}.
	//
	// In typical use, both Path and Args are set by calling Command.
	Args []string

	// Env specifies the environment of the process.
	// If Env is nil, Run uses the current process's environment.
	Env []string

	// Dir specifies the working directory of the command.
	// If Dir is the empty string, Run runs the command in the
	// process's current directory.
	Dir string

	// Stdin specifies the process's standard input.
	// If Stdin is nil, the process reads from DevNull.
	Stdin io.Reader

	// Stdout and Stderr specify the process's standard output and error.
	//
	// If either is nil, Run connects the
	// corresponding file descriptor to /dev/null.
	//
	// If Stdout and Stderr are are the same writer, at most one
	// goroutine at a time will call Write.
	Stdout io.Writer
	Stderr io.Writer

	err             os.Error // last error (from LookPath, stdin, stdout, stderr)
	process         *os.Process
	finished        bool // when Wait was called
	childFiles      []*os.File
	closeAfterStart []*os.File
	closeAfterWait  []*os.File
	goroutine       []func() os.Error
	errch           chan os.Error // one send per goroutine
}

// Command returns the Cmd struct to execute the named program with
// the given arguments.
//
// It sets Path and Args in the returned structure and zeroes the
// other fields.
//
// If name contains no path separators, Command uses LookPath to
// resolve the path to a complete name if possible. Otherwise it uses
// name directly.
//
// The returned Cmd's Args is constructed from the command name
// followed by the elements of arg, so arg should not include the
// command name itself. For example, Command("echo", "hello")
func Command(name string, arg ...string) *Cmd {
	aname, err := LookPath(name)
	if err != nil {
		aname = name
	}
	return &Cmd{
		Path: aname,
		Args: append([]string{name}, arg...),
		err:  err,
	}
}

// interfaceEqual protects against panics from doing equality tests on
// two interface with non-comparable underlying types
func interfaceEqual(a, b interface{}) bool {
	defer func() {
		recover()
	}()
	return a == b
}

func (c *Cmd) envv() []string {
	if c.Env != nil {
		return c.Env
	}
	return os.Environ()
}

func (c *Cmd) argv() []string {
	if len(c.Args) > 0 {
		return c.Args
	}
	return []string{c.Path}
}

func (c *Cmd) stdin() (f *os.File, err os.Error) {
	if c.Stdin == nil {
		f, err = os.Open(os.DevNull)
		c.closeAfterStart = append(c.closeAfterStart, f)
		return
	}

	if f, ok := c.Stdin.(*os.File); ok {
		return f, nil
	}

	pr, pw, err := os.Pipe()
	if err != nil {
		return
	}

	c.closeAfterStart = append(c.closeAfterStart, pr)
	c.closeAfterWait = append(c.closeAfterWait, pw)
	c.goroutine = append(c.goroutine, func() os.Error {
		_, err := io.Copy(pw, c.Stdin)
		if err1 := pw.Close(); err == nil {
			err = err1
		}
		return err
	})
	return pr, nil
}

func (c *Cmd) stdout() (f *os.File, err os.Error) {
	return c.writerDescriptor(c.Stdout)
}

func (c *Cmd) stderr() (f *os.File, err os.Error) {
	if c.Stderr != nil && interfaceEqual(c.Stderr, c.Stdout) {
		return c.childFiles[1], nil
	}
	return c.writerDescriptor(c.Stderr)
}

func (c *Cmd) writerDescriptor(w io.Writer) (f *os.File, err os.Error) {
	if w == nil {
		f, err = os.OpenFile(os.DevNull, os.O_WRONLY, 0)
		c.closeAfterStart = append(c.closeAfterStart, f)
		return
	}

	if f, ok := w.(*os.File); ok {
		return f, nil
	}

	pr, pw, err := os.Pipe()
	if err != nil {
		return
	}

	c.closeAfterStart = append(c.closeAfterStart, pw)
	c.closeAfterWait = append(c.closeAfterWait, pr)
	c.goroutine = append(c.goroutine, func() os.Error {
		_, err := io.Copy(w, pr)
		return err
	})
	return pw, nil
}

// Run starts the specified command and waits for it to complete.
//
// The returned error is nil if the command runs, has no problems
// copying stdin, stdout, and stderr, and exits with a zero exit
// status.
//
// If the command fails to run or doesn't complete successfully, the
// error is of type *os.Waitmsg. Other error types may be
// returned for I/O problems.
func (c *Cmd) Run() os.Error {
	if err := c.Start(); err != nil {
		return err
	}
	return c.Wait()
}

// Start starts the specified command but does not wait for it to complete.
func (c *Cmd) Start() os.Error {
	if c.err != nil {
		return c.err
	}
	if c.process != nil {
		return os.NewError("exec: already started")
	}

	type F func(*Cmd) (*os.File, os.Error)
	for _, setupFd := range []F{(*Cmd).stdin, (*Cmd).stdout, (*Cmd).stderr} {
		fd, err := setupFd(c)
		if err != nil {
			return err
		}
		c.childFiles = append(c.childFiles, fd)
	}

	var err os.Error
	c.process, err = os.StartProcess(c.Path, c.argv(), &os.ProcAttr{
		Dir:   c.Dir,
		Files: c.childFiles,
		Env:   c.envv(),
	})
	if err != nil {
		return err
	}

	for _, fd := range c.closeAfterStart {
		fd.Close()
	}

	c.errch = make(chan os.Error, len(c.goroutine))
	for _, fn := range c.goroutine {
		go func(fn func() os.Error) {
			c.errch <- fn()
		}(fn)
	}

	return nil
}

// Wait waits for the command to exit.
// It must have been started by Start.
//
// The returned error is nil if the command runs, has no problems
// copying stdin, stdout, and stderr, and exits with a zero exit
// status.
//
// If the command fails to run or doesn't complete successfully, the
// error is of type *os.Waitmsg. Other error types may be
// returned for I/O problems.
func (c *Cmd) Wait() os.Error {
	if c.process == nil {
		return os.NewError("exec: not started")
	}
	if c.finished {
		return os.NewError("exec: Wait was already called")
	}
	c.finished = true
	msg, err := c.process.Wait(0)

	var copyError os.Error
	for _ = range c.goroutine {
		if err := <-c.errch; err != nil && copyError == nil {
			copyError = err
		}
	}

	for _, fd := range c.closeAfterWait {
		fd.Close()
	}

	if err != nil {
		return err
	} else if !msg.Exited() || msg.ExitStatus() != 0 {
		return msg
	}

	return copyError
}

// Output runs the command and returns its standard output.
func (c *Cmd) Output() ([]byte, os.Error) {
	if c.Stdout != nil {
		return nil, os.NewError("exec: Stdout already set")
	}
	var b bytes.Buffer
	c.Stdout = &b
	err := c.Run()
	return b.Bytes(), err
}

// CombinedOutput runs the command and returns its combined standard
// output and standard error.
func (c *Cmd) CombinedOutput() ([]byte, os.Error) {
	if c.Stdout != nil {
		return nil, os.NewError("exec: Stdout already set")
	}
	if c.Stderr != nil {
		return nil, os.NewError("exec: Stderr already set")
	}
	var b bytes.Buffer
	c.Stdout = &b
	c.Stderr = &b
	err := c.Run()
	return b.Bytes(), err
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package sort provides primitives for sorting slices and user-defined
// collections.
package sort

import "math"

// A type, typically a collection, that satisfies sort.Interface can be
// sorted by the routines in this package.  The methods require that the
// elements of the collection be enumerated by an integer index.
type Interface interface {
	// Len is the number of elements in the collection.
	Len() int
	// Less returns whether the element with index i should sort
	// before the element with index j.
	Less(i, j int) bool
	// Swap swaps the elements with indexes i and j.
	Swap(i, j int)
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

// Insertion sort
func insertionSort(data Interface, a, b int) {
	for i := a + 1; i < b; i++ {
		for j := i; j > a && data.Less(j, j-1); j-- {
			data.Swap(j, j-1)
		}
	}
}

// siftDown implements the heap property on data[lo, hi).
// first is an offset into the array where the root of the heap lies.
func siftDown(data Interface, lo, hi, first int) {
	root := lo
	for {
		child := 2*root + 1
		if child >= hi {
			break
		}
		if child+1 < hi && data.Less(first+child, first+child+1) {
			child++
		}
		if !data.Less(first+root, first+child) {
			return
		}
		data.Swap(first+root, first+child)
		root = child
	}
}

func heapSort(data Interface, a, b int) {
	first := a
	lo := 0
	hi := b - a

	// Build heap with greatest element at top.
	for i := (hi - 1) / 2; i >= 0; i-- {
		siftDown(data, i, hi, first)
	}

	// Pop elements, largest first, into end of data.
	for i := hi - 1; i >= 0; i-- {
		data.Swap(first, first+i)
		siftDown(data, lo, i, first)
	}
}

// Quicksort, following Bentley and McIlroy,
// ``Engineering a Sort Function,'' SP&E November 1993.

// medianOfThree moves the median of the three values data[a], data[b], data[c] into data[a].
func medianOfThree(data Interface, a, b, c int) {
	m0 := b
	m1 := a
	m2 := c
	// bubble sort on 3 elements
	if data.Less(m1, m0) {
		data.Swap(m1, m0)
	}
	if data.Less(m2, m1) {
		data.Swap(m2, m1)
	}
	if data.Less(m1, m0) {
		data.Swap(m1, m0)
	}
	// now data[m0] <= data[m1] <= data[m2]
}

func swapRange(data Interface, a, b, n int) {
	for i := 0; i < n; i++ {
		data.Swap(a+i, b+i)
	}
}

func doPivot(data Interface, lo, hi int) (midlo, midhi int) {
	m := lo + (hi-lo)/2 // Written like this to avoid integer overflow.
	if hi-lo > 40 {
		// Tukey's ``Ninther,'' median of three medians of three.
		s := (hi - lo) / 8
		medianOfThree(data, lo, lo+s, lo+2*s)
		medianOfThree(data, m, m-s, m+s)
		medianOfThree(data, hi-1, hi-1-s, hi-1-2*s)
	}
	medianOfThree(data, lo, m, hi-1)

	// Invariants are:
	//	data[lo] = pivot (set up by ChoosePivot)
	//	data[lo <= i < a] = pivot
	//	data[a <= i < b] < pivot
	//	data[b <= i < c] is unexamined
	//	data[c <= i < d] > pivot
	//	data[d <= i < hi] = pivot
	//
	// Once b meets c, can swap the "= pivot" sections
	// into the middle of the slice.
	pivot := lo
	a, b, c, d := lo+1, lo+1, hi, hi
	for b < c {
		if data.Less(b, pivot) { // data[b] < pivot
			b++
			continue
		}
		if !data.Less(pivot, b) { // data[b] = pivot
			data.Swap(a, b)
			a++
			b++
			continue
		}
		if data.Less(pivot, c-1) { // data[c-1] > pivot
			c--
			continue
		}
		if !data.Less(c-1, pivot) { // data[c-1] = pivot
			data.Swap(c-1, d-1)
			c--
			d--
			continue
		}
		// data[b] > pivot; data[c-1] < pivot
		data.Swap(b, c-1)
		b++
		c--
	}

	n := min(b-a, a-lo)
	swapRange(data, lo, b-n, n)

	n = min(hi-d, d-c)
	swapRange(data, c, hi-n, n)

	return lo + b - a, hi - (d - c)
}

func quickSort(data Interface, a, b, maxDepth int) {
	for b-a > 7 {
		if maxDepth == 0 {
			heapSort(data, a, b)
			return
		}
		maxDepth--
		mlo, mhi := doPivot(data, a, b)
		// Avoiding recursion on the larger subproblem guarantees
		// a stack depth of at most lg(b-a).
		if mlo-a < b-mhi {
			quickSort(data, a, mlo, maxDepth)
			a = mhi // i.e., quickSort(data, mhi, b)
		} else {
			quickSort(data, mhi, b, maxDepth)
			b = mlo // i.e., quickSort(data, a, mlo)
		}
	}
	if b-a > 1 {
		insertionSort(data, a, b)
	}
}

// Sort sorts data.
// The algorithm used is not guaranteed to be a stable sort.
func Sort(data Interface) {
	// Switch to heapsort if depth of 2*ceil(lg(n)) is reached.
	n := data.Len()
	maxDepth := 0
	for 1<<uint(maxDepth) < n {
		maxDepth++
	}
	maxDepth *= 2
	quickSort(data, 0, n, maxDepth)
}

// IsSorted reports whether data is sorted.
func IsSorted(data Interface) bool {
	n := data.Len()
	for i := n - 1; i > 0; i-- {
		if data.Less(i, i-1) {
			return false
		}
	}
	return true
}

// Convenience types for common cases

// IntSlice attaches the methods of Interface to []int, sorting in increasing order.
type IntSlice []int

func (p IntSlice) Len() int           { return len(p) }
func (p IntSlice) Less(i, j int) bool { return p[i] < p[j] }
func (p IntSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// Sort is a convenience method.
func (p IntSlice) Sort() { Sort(p) }

// Float64Slice attaches the methods of Interface to []float64, sorting in increasing order.
type Float64Slice []float64

func (p Float64Slice) Len() int           { return len(p) }
func (p Float64Slice) Less(i, j int) bool { return p[i] < p[j] || math.IsNaN(p[i]) && !math.IsNaN(p[j]) }
func (p Float64Slice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// Sort is a convenience method.
func (p Float64Slice) Sort() { Sort(p) }

// StringSlice attaches the methods of Interface to []string, sorting in increasing order.
type StringSlice []string

func (p StringSlice) Len() int           { return len(p) }
func (p StringSlice) Less(i, j int) bool { return p[i] < p[j] }
func (p StringSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// Sort is a convenience method.
func (p StringSlice) Sort() { Sort(p) }

// Convenience wrappers for common cases

// Ints sorts a slice of ints in increasing order.
func Ints(a []int) { Sort(IntSlice(a)) }

// Float64s sorts a slice of float64s in increasing order.
func Float64s(a []float64) { Sort(Float64Slice(a)) }

// Strings sorts a slice of strings in increasing order.
func Strings(a []string) { Sort(StringSlice(a)) }

// IntsAreSorted tests whether a slice of ints is sorted in increasing order.
func IntsAreSorted(a []int) bool { return IsSorted(IntSlice(a)) }

// Float64sAreSorted tests whether a slice of float64s is sorted in increasing order.
func Float64sAreSorted(a []float64) bool { return IsSorted(Float64Slice(a)) }

// StringsAreSorted tests whether a slice of strings is sorted in increasing order.
func StringsAreSorted(a []string) bool { return IsSorted(StringSlice(a)) }
<commit_msg>sort: fix computation of maxDepth to avoid infinite loop<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package sort provides primitives for sorting slices and user-defined
// collections.
package sort

import "math"

// A type, typically a collection, that satisfies sort.Interface can be
// sorted by the routines in this package.  The methods require that the
// elements of the collection be enumerated by an integer index.
type Interface interface {
	// Len is the number of elements in the collection.
	Len() int
	// Less returns whether the element with index i should sort
	// before the element with index j.
	Less(i, j int) bool
	// Swap swaps the elements with indexes i and j.
	Swap(i, j int)
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

// Insertion sort
func insertionSort(data Interface, a, b int) {
	for i := a + 1; i < b; i++ {
		for j := i; j > a && data.Less(j, j-1); j-- {
			data.Swap(j, j-1)
		}
	}
}

// siftDown implements the heap property on data[lo, hi).
// first is an offset into the array where the root of the heap lies.
func siftDown(data Interface, lo, hi, first int) {
	root := lo
	for {
		child := 2*root + 1
		if child >= hi {
			break
		}
		if child+1 < hi && data.Less(first+child, first+child+1) {
			child++
		}
		if !data.Less(first+root, first+child) {
			return
		}
		data.Swap(first+root, first+child)
		root = child
	}
}

func heapSort(data Interface, a, b int) {
	first := a
	lo := 0
	hi := b - a

	// Build heap with greatest element at top.
	for i := (hi - 1) / 2; i >= 0; i-- {
		siftDown(data, i, hi, first)
	}

	// Pop elements, largest first, into end of data.
	for i := hi - 1; i >= 0; i-- {
		data.Swap(first, first+i)
		siftDown(data, lo, i, first)
	}
}

// Quicksort, following Bentley and McIlroy,
// ``Engineering a Sort Function,'' SP&E November 1993.

// medianOfThree moves the median of the three values data[a], data[b], data[c] into data[a].
func medianOfThree(data Interface, a, b, c int) {
	m0 := b
	m1 := a
	m2 := c
	// bubble sort on 3 elements
	if data.Less(m1, m0) {
		data.Swap(m1, m0)
	}
	if data.Less(m2, m1) {
		data.Swap(m2, m1)
	}
	if data.Less(m1, m0) {
		data.Swap(m1, m0)
	}
	// now data[m0] <= data[m1] <= data[m2]
}

func swapRange(data Interface, a, b, n int) {
	for i := 0; i < n; i++ {
		data.Swap(a+i, b+i)
	}
}

func doPivot(data Interface, lo, hi int) (midlo, midhi int) {
	m := lo + (hi-lo)/2 // Written like this to avoid integer overflow.
	if hi-lo > 40 {
		// Tukey's ``Ninther,'' median of three medians of three.
		s := (hi - lo) / 8
		medianOfThree(data, lo, lo+s, lo+2*s)
		medianOfThree(data, m, m-s, m+s)
		medianOfThree(data, hi-1, hi-1-s, hi-1-2*s)
	}
	medianOfThree(data, lo, m, hi-1)

	// Invariants are:
	//	data[lo] = pivot (set up by ChoosePivot)
	//	data[lo <= i < a] = pivot
	//	data[a <= i < b] < pivot
	//	data[b <= i < c] is unexamined
	//	data[c <= i < d] > pivot
	//	data[d <= i < hi] = pivot
	//
	// Once b meets c, can swap the "= pivot" sections
	// into the middle of the slice.
	pivot := lo
	a, b, c, d := lo+1, lo+1, hi, hi
	for b < c {
		if data.Less(b, pivot) { // data[b] < pivot
			b++
			continue
		}
		if !data.Less(pivot, b) { // data[b] = pivot
			data.Swap(a, b)
			a++
			b++
			continue
		}
		if data.Less(pivot, c-1) { // data[c-1] > pivot
			c--
			continue
		}
		if !data.Less(c-1, pivot) { // data[c-1] = pivot
			data.Swap(c-1, d-1)
			c--
			d--
			continue
		}
		// data[b] > pivot; data[c-1] < pivot
		data.Swap(b, c-1)
		b++
		c--
	}

	n := min(b-a, a-lo)
	swapRange(data, lo, b-n, n)

	n = min(hi-d, d-c)
	swapRange(data, c, hi-n, n)

	return lo + b - a, hi - (d - c)
}

func quickSort(data Interface, a, b, maxDepth int) {
	for b-a > 7 {
		if maxDepth == 0 {
			heapSort(data, a, b)
			return
		}
		maxDepth--
		mlo, mhi := doPivot(data, a, b)
		// Avoiding recursion on the larger subproblem guarantees
		// a stack depth of at most lg(b-a).
		if mlo-a < b-mhi {
			quickSort(data, a, mlo, maxDepth)
			a = mhi // i.e., quickSort(data, mhi, b)
		} else {
			quickSort(data, mhi, b, maxDepth)
			b = mlo // i.e., quickSort(data, a, mlo)
		}
	}
	if b-a > 1 {
		insertionSort(data, a, b)
	}
}

// Sort sorts data.
// The algorithm used is not guaranteed to be a stable sort.
func Sort(data Interface) {
	// Switch to heapsort if depth of 2*ceil(lg(n+1)) is reached.
	n := data.Len()
	maxDepth := 0
	for i := n; i > 0; i >>= 1 {
		maxDepth++
	}
	maxDepth *= 2
	quickSort(data, 0, n, maxDepth)
}

// IsSorted reports whether data is sorted.
func IsSorted(data Interface) bool {
	n := data.Len()
	for i := n - 1; i > 0; i-- {
		if data.Less(i, i-1) {
			return false
		}
	}
	return true
}

// Convenience types for common cases

// IntSlice attaches the methods of Interface to []int, sorting in increasing order.
type IntSlice []int

func (p IntSlice) Len() int           { return len(p) }
func (p IntSlice) Less(i, j int) bool { return p[i] < p[j] }
func (p IntSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// Sort is a convenience method.
func (p IntSlice) Sort() { Sort(p) }

// Float64Slice attaches the methods of Interface to []float64, sorting in increasing order.
type Float64Slice []float64

func (p Float64Slice) Len() int           { return len(p) }
func (p Float64Slice) Less(i, j int) bool { return p[i] < p[j] || math.IsNaN(p[i]) && !math.IsNaN(p[j]) }
func (p Float64Slice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// Sort is a convenience method.
func (p Float64Slice) Sort() { Sort(p) }

// StringSlice attaches the methods of Interface to []string, sorting in increasing order.
type StringSlice []string

func (p StringSlice) Len() int           { return len(p) }
func (p StringSlice) Less(i, j int) bool { return p[i] < p[j] }
func (p StringSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// Sort is a convenience method.
func (p StringSlice) Sort() { Sort(p) }

// Convenience wrappers for common cases

// Ints sorts a slice of ints in increasing order.
func Ints(a []int) { Sort(IntSlice(a)) }

// Float64s sorts a slice of float64s in increasing order.
func Float64s(a []float64) { Sort(Float64Slice(a)) }

// Strings sorts a slice of strings in increasing order.
func Strings(a []string) { Sort(StringSlice(a)) }

// IntsAreSorted tests whether a slice of ints is sorted in increasing order.
func IntsAreSorted(a []int) bool { return IsSorted(IntSlice(a)) }

// Float64sAreSorted tests whether a slice of float64s is sorted in increasing order.
func Float64sAreSorted(a []float64) bool { return IsSorted(Float64Slice(a)) }

// StringsAreSorted tests whether a slice of strings is sorted in increasing order.
func StringsAreSorted(a []string) bool { return IsSorted(StringSlice(a)) }
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package sync

import "runtime"

// Cond implements a condition variable, a rendezvous point
// for goroutines waiting for or announcing the occurrence
// of an event.
//
// Each Cond has an associated Locker L (often a *Mutex or *RWMutex),
// which must be held when changing the condition and
// when calling the Wait method.
type Cond struct {
	L       Locker // held while observing or changing the condition
	m       Mutex  // held to avoid internal races
	waiters int    // number of goroutines blocked on Wait
	sema    *uint32
}

// NewCond returns a new Cond with Locker l.
func NewCond(l Locker) *Cond {
	return &Cond{L: l}
}

// Wait atomically unlocks c.L and suspends execution
// of the calling goroutine.  After later resuming execution,
// Wait locks c.L before returning.
//
// Because L is not locked when Wait first resumes, the caller
// typically cannot assume that the condition is true when
// Wait returns.  Instead, the caller should Wait in a loop:
//
//    c.L.Lock()
//    for !condition() {
//        c.Wait()
//    }
//    ... make use of condition ...
//    c.L.Unlock()
//
func (c *Cond) Wait() {
	c.m.Lock()
	if c.sema == nil {
		c.sema = new(uint32)
	}
	s := c.sema
	c.waiters++
	c.m.Unlock()
	c.L.Unlock()
	runtime.Semacquire(s)
	c.L.Lock()
}

// Signal wakes one goroutine waiting on c, if there is any.
//
// It is allowed but not required for the caller to hold c.L
// during the call.
func (c *Cond) Signal() {
	c.m.Lock()
	if c.waiters > 0 {
		c.waiters--
		runtime.Semrelease(c.sema)
	}
	c.m.Unlock()
}

// Broadcast wakes all goroutines waiting on c.
//
// It is allowed but not required for the caller to hold c.L
// during the call.
func (c *Cond) Broadcast() {
	c.m.Lock()
	if c.waiters > 0 {
		s := c.sema
		n := c.waiters
		for i := 0; i < n; i++ {
			runtime.Semrelease(s)
		}
		// We just issued n wakeups via the semaphore s.
		// To ensure that they wake up the existing waiters
		// and not waiters that arrive after Broadcast returns,
		// clear c.sema.  The next operation will allocate
		// a new one.
		c.sema = nil
		c.waiters = 0
	}
	c.m.Unlock()
}
<commit_msg>sync: make package comment appear<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sync

import "runtime"

// Cond implements a condition variable, a rendezvous point
// for goroutines waiting for or announcing the occurrence
// of an event.
//
// Each Cond has an associated Locker L (often a *Mutex or *RWMutex),
// which must be held when changing the condition and
// when calling the Wait method.
type Cond struct {
	L       Locker // held while observing or changing the condition
	m       Mutex  // held to avoid internal races
	waiters int    // number of goroutines blocked on Wait
	sema    *uint32
}

// NewCond returns a new Cond with Locker l.
func NewCond(l Locker) *Cond {
	return &Cond{L: l}
}

// Wait atomically unlocks c.L and suspends execution
// of the calling goroutine.  After later resuming execution,
// Wait locks c.L before returning.
//
// Because L is not locked when Wait first resumes, the caller
// typically cannot assume that the condition is true when
// Wait returns.  Instead, the caller should Wait in a loop:
//
//    c.L.Lock()
//    for !condition() {
//        c.Wait()
//    }
//    ... make use of condition ...
//    c.L.Unlock()
//
func (c *Cond) Wait() {
	c.m.Lock()
	if c.sema == nil {
		c.sema = new(uint32)
	}
	s := c.sema
	c.waiters++
	c.m.Unlock()
	c.L.Unlock()
	runtime.Semacquire(s)
	c.L.Lock()
}

// Signal wakes one goroutine waiting on c, if there is any.
//
// It is allowed but not required for the caller to hold c.L
// during the call.
func (c *Cond) Signal() {
	c.m.Lock()
	if c.waiters > 0 {
		c.waiters--
		runtime.Semrelease(c.sema)
	}
	c.m.Unlock()
}

// Broadcast wakes all goroutines waiting on c.
//
// It is allowed but not required for the caller to hold c.L
// during the call.
func (c *Cond) Broadcast() {
	c.m.Lock()
	if c.waiters > 0 {
		s := c.sema
		n := c.waiters
		for i := 0; i < n; i++ {
			runtime.Semrelease(s)
		}
		// We just issued n wakeups via the semaphore s.
		// To ensure that they wake up the existing waiters
		// and not waiters that arrive after Broadcast returns,
		// clear c.sema.  The next operation will allocate
		// a new one.
		c.sema = nil
		c.waiters = 0
	}
	c.m.Unlock()
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sync

// Once is an object that will perform exactly one action.
type Once struct {
	m    Mutex
	done bool
}

// Do calls the function f if and only if the method is being called for the
// first time with this receiver.  In other words, given
// 	var once Once
// if once.Do(f) is called multiple times, only the first call will invoke f,
// even if f has a different value in each invocation.  A new instance of
// Once is required for each function to execute.
//
// Do is intended for initialization that must be run exactly once.  Since f
// is niladic, it may be necessary to use a function literal to capture the
// arguments to a function to be invoked by Do:
// 	config.once.Do(func() { config.init(filename) })
//
func (o *Once) Do(f func()) {
	o.m.Lock()
	defer o.m.Unlock()
	if !o.done {
		o.done = true
		f()
	}
}
<commit_msg>sync.once: document that Do cannot be invoked recursively. documentation change only.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sync

// Once is an object that will perform exactly one action.
type Once struct {
	m    Mutex
	done bool
}

// Do calls the function f if and only if the method is being called for the
// first time with this receiver.  In other words, given
// 	var once Once
// if Do(f) is called multiple times, only the first call will invoke f,
// even if f has a different value in each invocation.  A new instance of
// Once is required for each function to execute.
//
// Do is intended for initialization that must be run exactly once.  Since f
// is niladic, it may be necessary to use a function literal to capture the
// arguments to a function to be invoked by Do:
// 	config.once.Do(func() { config.init(filename) })
//
// Because no call to Do returns until the one call to f returns, if f causes
// Do to be called, it will deadlock.
//
func (o *Once) Do(f func()) {
	o.m.Lock()
	defer o.m.Unlock()
	if !o.done {
		o.done = true
		f()
	}
}
<|endoftext|>