<commit_before>package main

import (
	"os"
	"gopkg.in/alecthomas/kingpin.v2"
	"github.com/tears-of-noobs/bytefmt"
	"fmt"
	"github.com/sandreas/graft/pattern"
	"github.com/sandreas/graft/file"
	"strconv"
	"regexp"
	"path"
	"strings"
	"math"
	"time"
)

var (
	app = kingpin.New("graft", "A command-line tool to locate and transfer files")
	sourcePatternParameter = app.Arg("source-pattern", "source pattern - used to locate files (e.g. src/*)").Required().String()
	destinationPatternParameter = app.Arg("destination-pattern", "destination pattern for transfer (e.g. dst/$1)").Default("").String()

	exportTo = app.Flag("export-to", "export source listing to file, one line per found item").Default("").String()
	filesFrom = app.Flag("files-from", "import source listing from file, one line per item").Default("").String()

	minAge = app.Flag("min-age", " minimum age (e.g. -2 days, -8 weeks, 2015-10-10, etc.)").Default("").String()
	maxAge = app.Flag("max-age", "maximum age (e.g. 2 days, 8 weeks, 2015-10-10, etc.)").Default("").String()

	caseSensitive = app.Flag("case-sensitive", "be case sensitive when matching files and folders").Bool()
	dryRun = app.Flag("dry-run", "dry-run / simulation mode").Bool()
	hideMatches = app.Flag("hide-matches", "hide matches in search mode ($1: ...)").Bool()
	move = app.Flag("move", "move / rename files - do not make a copy").Bool()
	quiet = app.Flag("quiet", "quiet mode - do not show any output").Bool()
	regex = app.Flag("regex", "use a real regex instead of glob patterns (e.g. src/.*\\.jpg)").Bool()
	times = app.Flag("times", "transfer source modify times to destination").Bool()
)

var dirsToRemove = make([]string, 0)
func main() {
	kingpin.MustParse(app.Parse(os.Args[1:]))

	//firstBytes := int64(1003838300);
	//size := firstBytes * 5;
	//chunkSize:= int64(32*1024);
	//
	//handleProgress(firstBytes, size, chunkSize)
	//time.Sleep(3 * time.Second)
	//handleProgress(firstBytes*2, size, chunkSize)
	//time.Sleep(3 * time.Second)
	//handleProgress(firstBytes*3, size, chunkSize)
	//time.Sleep(3 * time.Second)
	//handleProgress(firstBytes*4, size, chunkSize)
	//time.Sleep(3 * time.Second)
	//handleProgress(firstBytes*5, size, chunkSize)
	//os.Exit(0)

	sourcePattern := *sourcePatternParameter
	destinationPattern := *destinationPatternParameter

	patternPath, pat := pattern.ParsePathPattern(sourcePattern)

	sourcePathStat, err := os.Stat(patternPath)

	if sourcePathStat.Mode().IsRegular() {
		if strings.HasSuffix(destinationPattern, "/") || strings.HasSuffix(destinationPattern, "\\") {
			destinationPattern += sourcePathStat.Name()
		}
		transferElementHandler(sourcePattern, destinationPattern)
		return
	}

	if destinationPattern == "" {
		searchIn := patternPath
		if patternPath == "" {
			searchIn = "./"
		}

		searchFor := ""
		if pat != "" {
			searchFor = pat
		}
		prntln("search in '" + searchIn + "': " + searchFor)

	} else if (*move) {
		prntln("move: " + sourcePattern + " => " + destinationPattern)
	} else {
		prntln("copy: " + sourcePattern + " => " + destinationPattern)
	}
	prntln("")

	if ! *regex {
		pat = pattern.GlobToRegex(pat)
	}

	caseInsensitiveQualifier := "(?i)"
	if *caseSensitive {
		caseInsensitiveQualifier = ""
	}

	compiledPattern, err := pattern.CompileNormalizedPathPattern(patternPath, caseInsensitiveQualifier + pat)
	if err == nil && compiledPattern.NumSubexp() == 0 && pat != "" {
		compiledPattern, err = pattern.CompileNormalizedPathPattern(patternPath, caseInsensitiveQualifier + "(" + pat + ")")
	}

	if err != nil {
		prntln("could not compile source pattern, please use slashes to qualify paths (recognized path: " + patternPath + ", pattern" + pat + ")")
		return
	}

	var matchingPaths []string

	if *filesFrom != "" {
		if ! file.Exists(*filesFrom) {
			prntln("Could not load files from " + *filesFrom)
			return
		}
		matchingPaths, err = file.ReadAllLinesFunc(*filesFrom, file.SkipEmptyLines)
	} else {
		 //matchingPaths, err = file.WalkPathByPattern(patternPath, compiledPattern, progressHandlerWalkPathByPattern)
		matchingFiles, _ := file.WalkPathFiltered(patternPath, func(f file.File, err error)(bool) {
			normalizedPath := pattern.NormalizeDirSep(f.Path)
			if ! compiledPattern.MatchString(normalizedPath) {
				return false
			}

			return minAgeFilter(f) && maxAgeFilter(f)
		}, progressHandlerWalkPathByPattern)

		for _, element := range matchingFiles {
			matchingPaths = append(matchingPaths, element.Path)
		}

		if *exportTo != "" {
			exportFile(*exportTo, matchingPaths)
		}
	}

	if err != nil {
		prntln("Could not load sources path " + patternPath + ":", err.Error())
		return
	}

	if destinationPattern == "" {
		for _, element := range matchingPaths {
			findElementHandler(element, compiledPattern)
		}
		return
	}

	dstPath, dstPatt := pattern.ParsePathPattern(destinationPattern)
	var dst string
	for _, element := range matchingPaths {
		if dstPatt == "" {
			dst = pattern.NormalizeDirSep(dstPath + element[len(patternPath)+1:])
		} else {
			dst = compiledPattern.ReplaceAllString(pattern.NormalizeDirSep(element), pattern.NormalizeDirSep(destinationPattern))
		}
		transferElementHandler(element, dst)
	}

	if *move {
		for _, dirToRemove := range dirsToRemove {
			os.Remove(dirToRemove)
		}
	}
	return
}

func minAgeFilter(f file.File)(bool) {
	if *minAge == "" {
		return true
	}

	minAgeTime, err := pattern.StrToAge(*minAge, time.Now())
	if err != nil {
		return false
	}
	return minAgeTime.UnixNano() > f.ModTime().UnixNano()
}

func maxAgeFilter(f file.File)(bool) {
	if *maxAge == "" {
		return true
	}

	maxAgeTime, err := pattern.StrToAge(*maxAge, time.Now())
	if err != nil {
		return false
	}
	return maxAgeTime.UnixNano() < f.ModTime().UnixNano()
}

func progressHandlerWalkPathByPattern(entriesWalked, entriesMatched int64, finished bool) (int64) {
	var progress string;
	if entriesMatched == 0 {
		progress = fmt.Sprintf("scanning - total: %d", entriesWalked)
	} else {
		progress = fmt.Sprintf("scanning - total: %d,  matches: %d", entriesWalked, entriesMatched)
	}
	// prnt("\x0c" + progressBar)
	prnt("\r" + progress)
	if finished {
		prntln("")
		prntln("")
	}
	if(entriesWalked > 1000) {
		return 500
	}
	return 100
}

func exportFile(file string, lines []string) {
	f, err := os.Create(*exportTo)
	if err != nil {
		prntln("could not create export file " + file + ": " + err.Error())
		return;
	}
	_, err = f.WriteString(strings.Join(lines, "\n"))
	defer f.Close()
	if err != nil {
		prntln("could not write export file " + file + ": " + err.Error())
	}

}

func appendRemoveDir(dir string) {
	if (*move) {
		dirsToRemove = append(dirsToRemove, dir)
	}
}

var timerLastUpdate time.Time
var reportInterval int64
var bytesLastUpdate int64

func startTimer(bytesTransferred, interval int64) {
	if bytesTransferred == 0 {
		timerLastUpdate = time.Now()
		reportInterval = interval
		bytesLastUpdate = 0
	}
}

func getReportStatus(bytesTransferred, size int64) (bool, float64, float64) {
	timeDiffNano := time.Now().UnixNano() - timerLastUpdate.UnixNano()
	 timeDiffSeconds := float64(timeDiffNano) / float64(time.Second)
	// if timeDiffSeconds >= float64(reportInterval) {
	if timeDiffNano >= reportInterval {
		bytesDiff := bytesTransferred - bytesLastUpdate
		bytesPerSecond := float64(float64(bytesDiff) / float64(timeDiffSeconds))
		percent := float64(bytesTransferred) / float64(size)

		bytesLastUpdate = bytesTransferred
		timerLastUpdate = time.Now()

		return true, bytesPerSecond, percent
	}

	return false, 0, 0
}

func handleProgress(bytesTransferred, size, chunkSize int64) (int64) {

	if size <= 0 {
		return chunkSize
	}

	startTimer(bytesTransferred, 1 * int64(time.Second))
	shouldReport, bytesPerSecond, percent := getReportStatus(bytesTransferred, size)
	if shouldReport {
		bandwidthOutput := " " + bytefmt.FormatBytes(bytesPerSecond, 2, true) + "/s"
		charCountWhenFullyTransmitted := 20
		progressChars := int(math.Floor(percent * float64(charCountWhenFullyTransmitted)))
		normalizedInt := percent * 100
		percentOutput := strconv.FormatFloat(normalizedInt, 'f', 2, 64)
		if bytesPerSecond == 0 {
			bandwidthOutput = ""
		}
		progressBar := fmt.Sprintf("[%-" + strconv.Itoa(charCountWhenFullyTransmitted + 1)+ "s] " +percentOutput +  "%%" + bandwidthOutput, strings.Repeat("=", progressChars) + ">")

		prnt("\r" + progressBar)
	}

	if bytesTransferred == size {
		prntln("")
	}

	return chunkSize
}

func prntln(a ...interface{}) (n int, err error) {
	if ! *quiet {
		return fmt.Println(a...)
	}
	return n, err
}

func prnt(a...interface{}) (n int, err error) {
	if ! *quiet {
		return fmt.Print(a...)
	}
	return n, err
}

func findElementHandler(element string, compiledPattern *regexp.Regexp) {
	prntln(element)
	if *hideMatches {
		return
	}
	elementMatches := pattern.BuildMatchList(compiledPattern, element)
	for i := 0; i < len(elementMatches); i++ {
		prntln("    $" + strconv.Itoa(i + 1) + ": " + elementMatches[i])
	}

}

func transferElementHandler(src, dst string) {

	prntln(src + " => " + dst)

	if *dryRun {
		return
	}

	srcStat, srcErr := os.Stat(src)

	if srcErr != nil {
		prntln("could not read source: ", srcErr)
		return
	}

	dstStat, _ := os.Stat(dst)
	dstExists := file.Exists(dst)
	if srcStat.IsDir() {
		if ! dstExists {
			if os.MkdirAll(dst, srcStat.Mode()) != nil {
				prntln("Could not create destination directory")
			}
			appendRemoveDir(dst)
			fixTimes(dst, srcStat)
			return
		}

		if dstStat.IsDir() {
			appendRemoveDir(dst)
			fixTimes(dst, srcStat)
			return
		}

		prntln("destination already exists as file, source is a directory")
		return
	}

	if dstExists && dstStat.IsDir() {
		prntln("destination already exists as directory, source is a file")
		return
	}

	srcDir := path.Dir(src)
	srcDirStat, _ := os.Stat(srcDir)

	dstDir := path.Dir(dst)
	if ! file.Exists(dstDir) {
		os.MkdirAll(dstDir, srcDirStat.Mode())
	}

	if *move {
		renameErr := os.Rename(src, dst)
		if renameErr == nil {
			appendRemoveDir(srcDir)
			fixTimes(dst, srcStat)
			return
		}
		prntln("Could not rename source")
		return
	}

	srcPointer, srcPointerErr := os.Open(src)
	if srcPointerErr != nil {
		prntln("Could not open source file")
		return
	}
	dstPointer, dstPointerErr := os.OpenFile(dst, os.O_WRONLY | os.O_CREATE, srcStat.Mode())

	if dstPointerErr != nil {
		prntln("Could not create destination file", dstPointerErr.Error())
		return
	}

	file.CopyResumed(srcPointer, dstPointer, handleProgress)
	fixTimes(dst, srcStat)
}

func fixTimes(dst string, inStats os.FileInfo) {
	if *times {
		os.Chtimes(dst, inStats.ModTime(), inStats.ModTime())
	}
}<commit_msg>Small fix for bandwidth output<commit_after>package main

import (
	"os"
	"gopkg.in/alecthomas/kingpin.v2"
	"github.com/tears-of-noobs/bytefmt"
	"fmt"
	"github.com/sandreas/graft/pattern"
	"github.com/sandreas/graft/file"
	"strconv"
	"regexp"
	"path"
	"strings"
	"math"
	"time"
)

var (
	app = kingpin.New("graft", "A command-line tool to locate and transfer files")
	sourcePatternParameter = app.Arg("source-pattern", "source pattern - used to locate files (e.g. src/*)").Required().String()
	destinationPatternParameter = app.Arg("destination-pattern", "destination pattern for transfer (e.g. dst/$1)").Default("").String()

	exportTo = app.Flag("export-to", "export source listing to file, one line per found item").Default("").String()
	filesFrom = app.Flag("files-from", "import source listing from file, one line per item").Default("").String()

	minAge = app.Flag("min-age", " minimum age (e.g. -2 days, -8 weeks, 2015-10-10, etc.)").Default("").String()
	maxAge = app.Flag("max-age", "maximum age (e.g. 2 days, 8 weeks, 2015-10-10, etc.)").Default("").String()

	caseSensitive = app.Flag("case-sensitive", "be case sensitive when matching files and folders").Bool()
	dryRun = app.Flag("dry-run", "dry-run / simulation mode").Bool()
	hideMatches = app.Flag("hide-matches", "hide matches in search mode ($1: ...)").Bool()
	move = app.Flag("move", "move / rename files - do not make a copy").Bool()
	quiet = app.Flag("quiet", "quiet mode - do not show any output").Bool()
	regex = app.Flag("regex", "use a real regex instead of glob patterns (e.g. src/.*\\.jpg)").Bool()
	times = app.Flag("times", "transfer source modify times to destination").Bool()
)

var dirsToRemove = make([]string, 0)
func main() {
	kingpin.MustParse(app.Parse(os.Args[1:]))

	//firstBytes := int64(1003838300);
	//size := firstBytes * 5;
	//chunkSize:= int64(32*1024);
	//
	//handleProgress(firstBytes, size, chunkSize)
	//time.Sleep(3 * time.Second)
	//handleProgress(firstBytes*2, size, chunkSize)
	//time.Sleep(3 * time.Second)
	//handleProgress(firstBytes*3, size, chunkSize)
	//time.Sleep(3 * time.Second)
	//handleProgress(firstBytes*4, size, chunkSize)
	//time.Sleep(3 * time.Second)
	//handleProgress(firstBytes*5, size, chunkSize)
	//os.Exit(0)

	sourcePattern := *sourcePatternParameter
	destinationPattern := *destinationPatternParameter

	patternPath, pat := pattern.ParsePathPattern(sourcePattern)

	sourcePathStat, err := os.Stat(patternPath)

	if sourcePathStat.Mode().IsRegular() {
		if strings.HasSuffix(destinationPattern, "/") || strings.HasSuffix(destinationPattern, "\\") {
			destinationPattern += sourcePathStat.Name()
		}
		transferElementHandler(sourcePattern, destinationPattern)
		return
	}

	if destinationPattern == "" {
		searchIn := patternPath
		if patternPath == "" {
			searchIn = "./"
		}

		searchFor := ""
		if pat != "" {
			searchFor = pat
		}
		prntln("search in '" + searchIn + "': " + searchFor)

	} else if (*move) {
		prntln("move: " + sourcePattern + " => " + destinationPattern)
	} else {
		prntln("copy: " + sourcePattern + " => " + destinationPattern)
	}
	prntln("")

	if ! *regex {
		pat = pattern.GlobToRegex(pat)
	}

	caseInsensitiveQualifier := "(?i)"
	if *caseSensitive {
		caseInsensitiveQualifier = ""
	}

	compiledPattern, err := pattern.CompileNormalizedPathPattern(patternPath, caseInsensitiveQualifier + pat)
	if err == nil && compiledPattern.NumSubexp() == 0 && pat != "" {
		compiledPattern, err = pattern.CompileNormalizedPathPattern(patternPath, caseInsensitiveQualifier + "(" + pat + ")")
	}

	if err != nil {
		prntln("could not compile source pattern, please use slashes to qualify paths (recognized path: " + patternPath + ", pattern" + pat + ")")
		return
	}

	var matchingPaths []string

	if *filesFrom != "" {
		if ! file.Exists(*filesFrom) {
			prntln("Could not load files from " + *filesFrom)
			return
		}
		matchingPaths, err = file.ReadAllLinesFunc(*filesFrom, file.SkipEmptyLines)
	} else {
		 //matchingPaths, err = file.WalkPathByPattern(patternPath, compiledPattern, progressHandlerWalkPathByPattern)
		matchingFiles, _ := file.WalkPathFiltered(patternPath, func(f file.File, err error)(bool) {
			normalizedPath := pattern.NormalizeDirSep(f.Path)
			if ! compiledPattern.MatchString(normalizedPath) {
				return false
			}

			return minAgeFilter(f) && maxAgeFilter(f)
		}, progressHandlerWalkPathByPattern)

		for _, element := range matchingFiles {
			matchingPaths = append(matchingPaths, element.Path)
		}

		if *exportTo != "" {
			exportFile(*exportTo, matchingPaths)
		}
	}

	if err != nil {
		prntln("Could not load sources path " + patternPath + ":", err.Error())
		return
	}

	if destinationPattern == "" {
		for _, element := range matchingPaths {
			findElementHandler(element, compiledPattern)
		}
		return
	}

	dstPath, dstPatt := pattern.ParsePathPattern(destinationPattern)
	var dst string
	for _, element := range matchingPaths {
		if dstPatt == "" {
			dst = pattern.NormalizeDirSep(dstPath + element[len(patternPath)+1:])
		} else {
			dst = compiledPattern.ReplaceAllString(pattern.NormalizeDirSep(element), pattern.NormalizeDirSep(destinationPattern))
		}
		transferElementHandler(element, dst)
	}

	if *move {
		for _, dirToRemove := range dirsToRemove {
			os.Remove(dirToRemove)
		}
	}
	return
}

func minAgeFilter(f file.File)(bool) {
	if *minAge == "" {
		return true
	}

	minAgeTime, err := pattern.StrToAge(*minAge, time.Now())
	if err != nil {
		return false
	}
	return minAgeTime.UnixNano() > f.ModTime().UnixNano()
}

func maxAgeFilter(f file.File)(bool) {
	if *maxAge == "" {
		return true
	}

	maxAgeTime, err := pattern.StrToAge(*maxAge, time.Now())
	if err != nil {
		return false
	}
	return maxAgeTime.UnixNano() < f.ModTime().UnixNano()
}

func progressHandlerWalkPathByPattern(entriesWalked, entriesMatched int64, finished bool) (int64) {
	var progress string;
	if entriesMatched == 0 {
		progress = fmt.Sprintf("scanning - total: %d", entriesWalked)
	} else {
		progress = fmt.Sprintf("scanning - total: %d,  matches: %d", entriesWalked, entriesMatched)
	}
	// prnt("\x0c" + progressBar)
	prnt("\r" + progress)
	if finished {
		prntln("")
		prntln("")
	}
	if(entriesWalked > 1000) {
		return 500
	}
	return 100
}

func exportFile(file string, lines []string) {
	f, err := os.Create(*exportTo)
	if err != nil {
		prntln("could not create export file " + file + ": " + err.Error())
		return;
	}
	_, err = f.WriteString(strings.Join(lines, "\n"))
	defer f.Close()
	if err != nil {
		prntln("could not write export file " + file + ": " + err.Error())
	}

}

func appendRemoveDir(dir string) {
	if (*move) {
		dirsToRemove = append(dirsToRemove, dir)
	}
}

var timerLastUpdate time.Time
var reportInterval int64
var bytesLastUpdate int64

func startTimer(bytesTransferred, interval int64) {
	if bytesTransferred == 0 {
		timerLastUpdate = time.Now()
		reportInterval = interval
		bytesLastUpdate = 0
	}
}

func getReportStatus(bytesTransferred, size int64) (bool, float64, float64) {
	timeDiffNano := time.Now().UnixNano() - timerLastUpdate.UnixNano()
	 timeDiffSeconds := float64(timeDiffNano) / float64(time.Second)
	// if timeDiffSeconds >= float64(reportInterval) {
	if timeDiffNano >= reportInterval {
		bytesDiff := bytesTransferred - bytesLastUpdate
		bytesPerSecond := float64(float64(bytesDiff) / float64(timeDiffSeconds))
		percent := float64(bytesTransferred) / float64(size)

		bytesLastUpdate = bytesTransferred
		timerLastUpdate = time.Now()

		return true, bytesPerSecond, percent
	}

	return false, 0, 0
}

func handleProgress(bytesTransferred, size, chunkSize int64) (int64) {

	if size <= 0 {
		return chunkSize
	}

	startTimer(bytesTransferred, 1 * int64(time.Second))
	shouldReport, bytesPerSecond, percent := getReportStatus(bytesTransferred, size)
	if shouldReport {
		bandwidthOutput := "   " + bytefmt.FormatBytes(bytesPerSecond, 2, true) + "/s"
		charCountWhenFullyTransmitted := 20
		progressChars := int(math.Floor(percent * float64(charCountWhenFullyTransmitted)))
		normalizedInt := percent * 100
		percentOutput := strconv.FormatFloat(normalizedInt, 'f', 2, 64)
		if bytesPerSecond == 0 {
			bandwidthOutput = ""
		}
		progressBar := fmt.Sprintf("[%-" + strconv.Itoa(charCountWhenFullyTransmitted + 1)+ "s] " +percentOutput +  "%%" + bandwidthOutput, strings.Repeat("=", progressChars) + ">")

		prnt("\r" + progressBar)
	}

	if bytesTransferred == size {
		prntln("")
	}

	return chunkSize
}

func prntln(a ...interface{}) (n int, err error) {
	if ! *quiet {
		return fmt.Println(a...)
	}
	return n, err
}

func prnt(a...interface{}) (n int, err error) {
	if ! *quiet {
		return fmt.Print(a...)
	}
	return n, err
}

func findElementHandler(element string, compiledPattern *regexp.Regexp) {
	prntln(element)
	if *hideMatches {
		return
	}
	elementMatches := pattern.BuildMatchList(compiledPattern, element)
	for i := 0; i < len(elementMatches); i++ {
		prntln("    $" + strconv.Itoa(i + 1) + ": " + elementMatches[i])
	}

}

func transferElementHandler(src, dst string) {

	prntln(src + " => " + dst)

	if *dryRun {
		return
	}

	srcStat, srcErr := os.Stat(src)

	if srcErr != nil {
		prntln("could not read source: ", srcErr)
		return
	}

	dstStat, _ := os.Stat(dst)
	dstExists := file.Exists(dst)
	if srcStat.IsDir() {
		if ! dstExists {
			if os.MkdirAll(dst, srcStat.Mode()) != nil {
				prntln("Could not create destination directory")
			}
			appendRemoveDir(dst)
			fixTimes(dst, srcStat)
			return
		}

		if dstStat.IsDir() {
			appendRemoveDir(dst)
			fixTimes(dst, srcStat)
			return
		}

		prntln("destination already exists as file, source is a directory")
		return
	}

	if dstExists && dstStat.IsDir() {
		prntln("destination already exists as directory, source is a file")
		return
	}

	srcDir := path.Dir(src)
	srcDirStat, _ := os.Stat(srcDir)

	dstDir := path.Dir(dst)
	if ! file.Exists(dstDir) {
		os.MkdirAll(dstDir, srcDirStat.Mode())
	}

	if *move {
		renameErr := os.Rename(src, dst)
		if renameErr == nil {
			appendRemoveDir(srcDir)
			fixTimes(dst, srcStat)
			return
		}
		prntln("Could not rename source")
		return
	}

	srcPointer, srcPointerErr := os.Open(src)
	if srcPointerErr != nil {
		prntln("Could not open source file")
		return
	}
	dstPointer, dstPointerErr := os.OpenFile(dst, os.O_WRONLY | os.O_CREATE, srcStat.Mode())

	if dstPointerErr != nil {
		prntln("Could not create destination file", dstPointerErr.Error())
		return
	}

	file.CopyResumed(srcPointer, dstPointer, handleProgress)
	fixTimes(dst, srcStat)
}

func fixTimes(dst string, inStats os.FileInfo) {
	if *times {
		os.Chtimes(dst, inStats.ModTime(), inStats.ModTime())
	}
}<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/gob"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"sync"
	"text/template"
	"time"
)

var lastSeenLimit = 30 * 24 * time.Hour
var lastSeenWriteThresh = time.Minute
var greeting *template.Template

// maps channel -> (nick -> last seen)
var lastSeen = struct {
	mtx sync.Mutex
	m   map[string]map[string]time.Time
}{}

func init() {
	readLastSeen()
	readGreeting()
}

func touchLastSeen(channel string, nick string) (absent time.Duration) {
	lastSeen.mtx.Lock()
	defer lastSeen.mtx.Unlock()

	c := lastSeen.m[channel]
	if c == nil {
		c = make(map[string]time.Time)
		lastSeen.m[channel] = c
	}
	t := c[nick]
	c[nick] = time.Now()
	return time.Now().Sub(t)
}

func runnerGreet(parsed Message) {
	botnick := *nick
	if botnick == Nick(parsed) {
		// we ignore ourselves
		return
	}
	nick := Nick(parsed)

	var channel string
	switch parsed.Command {
	case "JOIN":
		channel = parsed.Trailing
	case "PART":
		channel = Target(parsed)
	case "PRIVMSG":
		channel = Target(parsed)
	default:
		return
	}

	// TODO: Make channels configurable
	if channel != "#chaos-hd" {
		return
	}

	if channel == "" || channel[0] != '#' {
		log.Printf("Not greeting in non-channel %q", channel)
		return
	}

	// To handle renames of users correctly, we also save the hostmask. Only if
	// we've seen neither it's a genuinely new user.
	absentNick := touchLastSeen(channel, nick)
	absentHostmask := touchLastSeen(channel, Hostmask(parsed))
	seen := (absentNick <= lastSeenLimit) || (absentHostmask <= lastSeenLimit)
	if parsed.Command == "JOIN" && !seen {
		log.Printf("I have not seen %q in %q recently, so I'm greeting them", nick, channel)

		params := struct {
			Nick string
			Bot  string
		}{nick, botnick}

		var msg string

		msgBuf := new(bytes.Buffer)
		if greeting == nil {
			msg = fmt.Sprintf("Hey %s! o/", nick)
		} else if err := greeting.Execute(msgBuf, params); err != nil {
			log.Println("Could not render greeting:", err)
			msg = fmt.Sprintf("Hey %s! o/", nick)
		} else {
			msg = msgBuf.String()
		}
		Privmsg(channel, msg)
	}

	if absentNick > lastSeenWriteThresh || absentHostmask > lastSeenWriteThresh {
		writeLastSeen()
	}
}

func writeLastSeen() {
	lastSeen.mtx.Lock()
	defer lastSeen.mtx.Unlock()

	log.Println("Writing last-seen")

	// Take out the garbage
	for _, c := range lastSeen.m {
		for nick, last := range c {
			if time.Now().Sub(last) > lastSeenLimit {
				delete(c, nick)
			}
		}
	}

	// We write the file atomically to prevent corruption
	tmp, err := ioutil.TempFile(".", ".last-seen")
	if err != nil {
		log.Println("Could not create temporary file for last-seen:", err)
		return
	}
	defer os.Remove(tmp.Name())

	if err := gob.NewEncoder(tmp).Encode(lastSeen.m); err != nil {
		log.Println("Could not write last-seen:", err)
		return
	}

	if err := os.Rename(tmp.Name(), "last-seen"); err != nil {
		log.Println("Could not write last-seen:", err)
		return
	}
}

func readLastSeen() {
	lastSeen.mtx.Lock()
	defer lastSeen.mtx.Unlock()

	// Make sure, lastSeen is initialized to a sane, if empty, value.
	defer func() {
		if lastSeen.m == nil {
			lastSeen.m = make(map[string]map[string]time.Time)
		}
	}()

	f, err := os.Open("last-seen")
	if err != nil {
		log.Println("Could not read last-seen:", err)
		return
	}
	defer f.Close()

	var m map[string]map[string]time.Time
	if err := gob.NewDecoder(f).Decode(&m); err != nil {
		log.Println("Could not read last-seen:", err)
		return
	}
	lastSeen.m = m
}

func readGreeting() {
	t, err := template.ParseFiles("greeting.txt")
	if err != nil {
		log.Println("Could not parse greeting:", err)
		return
	}
	greeting = t
}
<commit_msg>greet: Strip trailing _ when remembering people<commit_after>package main

import (
	"bytes"
	"encoding/gob"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"sync"
	"text/template"
	"time"
)

var lastSeenLimit = 30 * 24 * time.Hour
var lastSeenWriteThresh = time.Minute
var greeting *template.Template

// maps channel -> (nick -> last seen)
var lastSeen = struct {
	mtx sync.Mutex
	m   map[string]map[string]time.Time
}{}

func init() {
	readLastSeen()
	readGreeting()
}

func touchLastSeen(channel string, nick string) (absent time.Duration) {
	lastSeen.mtx.Lock()
	defer lastSeen.mtx.Unlock()

	c := lastSeen.m[channel]
	if c == nil {
		c = make(map[string]time.Time)
		lastSeen.m[channel] = c
	}
	t := c[nick]
	c[nick] = time.Now()
	return time.Now().Sub(t)
}

func runnerGreet(parsed Message) {
	botnick := *nick
	if botnick == Nick(parsed) {
		// we ignore ourselves
		return
	}
	nick := Nick(parsed)

	var channel string
	switch parsed.Command {
	case "JOIN":
		channel = parsed.Trailing
	case "PART":
		channel = Target(parsed)
	case "PRIVMSG":
		channel = Target(parsed)
	default:
		return
	}

	// TODO: Make channels configurable
	if channel != "#chaos-hd" {
		return
	}

	if channel == "" || channel[0] != '#' {
		log.Printf("Not greeting in non-channel %q", channel)
		return
	}

	// To handle renames of users correctly, we also save the hostmask. Only if
	// we've seen neither it's a genuinely new user. We strip trailing _, they
	// usually appear for duplicate links when the original nick is taken by a
	// ghost.
	absentNick := touchLastSeen(channel, strings.TrimRight(nick, "_"))
	absentHostmask := touchLastSeen(channel, Hostmask(parsed))
	seen := (absentNick <= lastSeenLimit) || (absentHostmask <= lastSeenLimit)
	if parsed.Command == "JOIN" && !seen {
		log.Printf("I have not seen %q in %q recently, so I'm greeting them", nick, channel)

		params := struct {
			Nick string
			Bot  string
		}{nick, botnick}

		var msg string

		msgBuf := new(bytes.Buffer)
		if greeting == nil {
			msg = fmt.Sprintf("Hey %s! o/", nick)
		} else if err := greeting.Execute(msgBuf, params); err != nil {
			log.Println("Could not render greeting:", err)
			msg = fmt.Sprintf("Hey %s! o/", nick)
		} else {
			msg = msgBuf.String()
		}
		Privmsg(channel, msg)
	}

	if absentNick > lastSeenWriteThresh || absentHostmask > lastSeenWriteThresh {
		writeLastSeen()
	}
}

func writeLastSeen() {
	lastSeen.mtx.Lock()
	defer lastSeen.mtx.Unlock()

	log.Println("Writing last-seen")

	// Take out the garbage
	for _, c := range lastSeen.m {
		for nick, last := range c {
			if time.Now().Sub(last) > lastSeenLimit {
				delete(c, nick)
			}
		}
	}

	// We write the file atomically to prevent corruption
	tmp, err := ioutil.TempFile(".", ".last-seen")
	if err != nil {
		log.Println("Could not create temporary file for last-seen:", err)
		return
	}
	defer os.Remove(tmp.Name())

	if err := gob.NewEncoder(tmp).Encode(lastSeen.m); err != nil {
		log.Println("Could not write last-seen:", err)
		return
	}

	if err := os.Rename(tmp.Name(), "last-seen"); err != nil {
		log.Println("Could not write last-seen:", err)
		return
	}
}

func readLastSeen() {
	lastSeen.mtx.Lock()
	defer lastSeen.mtx.Unlock()

	// Make sure, lastSeen is initialized to a sane, if empty, value.
	defer func() {
		if lastSeen.m == nil {
			lastSeen.m = make(map[string]map[string]time.Time)
		}
	}()

	f, err := os.Open("last-seen")
	if err != nil {
		log.Println("Could not read last-seen:", err)
		return
	}
	defer f.Close()

	var m map[string]map[string]time.Time
	if err := gob.NewDecoder(f).Decode(&m); err != nil {
		log.Println("Could not read last-seen:", err)
		return
	}
	lastSeen.m = m
}

func readGreeting() {
	t, err := template.ParseFiles("greeting.txt")
	if err != nil {
		log.Println("Could not parse greeting:", err)
		return
	}
	greeting = t
}
<|endoftext|>


<commit_before>package pdb

import (
	"unicode"

	"github.com/TuftsBCB/seq"
)

func (m Model) seqAtomsGuess() []*Residue {
	seqres := m.Chain.Sequence
	mapping := make([]*Residue, len(seqres))
	if len(seqres) != len(m.Residues) {
		// This is a last ditch effort. Use the ATOM sequence number as an
		// index into the SEQRES residues.
		for _, r := range m.Residues {
			si := r.SequenceNum - 1
			if si >= 0 && si < len(seqres) && seqres[si] == r.Name {
				mapping[si] = r
			}
		}
		return mapping
	}

	for i, r := range m.Residues {
		if i < len(seqres) && seqres[i] == r.Name {
			mapping[i] = r
		}
	}
	return mapping
}

func (m Model) seqAtomsAlign() []*Residue {
	seqres := m.Chain.Sequence
	atomResidues := make([]seq.Residue, len(m.Residues))
	for i, r := range m.Residues {
		atomResidues[i] = r.Name
	}
	aligned := seq.NeedlemanWunsch(seqres, atomResidues, seq.MatBlosum62)

	mapped := make([]*Residue, len(seqres))
	atomi := 0
	for i, r := range aligned.B {
		if r == '-' {
			mapped[i] = nil
		} else {
			mapped[i] = m.Residues[atomi]
			atomi++
		}
	}
	return mapped
}

// Attempts to accomplish the same thing as seqAtomsWithMissing, but instead
// of mapping one residue at a time, we map *chunks* at a time. That is,
// a chunk is any group of contiguous residues.
func (m Model) seqAtomsChunksMerge() []*Residue {
	seqres := m.Chain.Sequence

	// Check to make sure that the total number of missing residues, plus the
	// total number of ATOM record residues equal the total number of residues
	// in the SEQRES records. Otherwise, the merge will fail.
	if len(seqres) != len(m.Chain.Missing)+len(m.Residues) {
		return m.seqAtomsAlign()
	}

	result := make([]*Residue, len(seqres))
	mchunks := chunk(m.Chain.Missing, m.Residues)
	fchunks := chunk(m.Residues, m.Chain.Missing)

	// If the PDB file is corrupted, a merge will fail.
	// So we fall back to alignment.
	if ok := merge(result, 0, seqres, nil, mchunks, fchunks); !ok {
		return m.seqAtomsAlign()
	}

	// X out any residues that don't have ATOM records.
	for i := range result {
		if result[i].Atoms == nil {
			result[i] = nil
		}
	}
	return result
}

func merge(result []*Residue, end int, seqres []seq.Residue,
	chunkToMerge []*Residue, mchunks, fchunks [][]*Residue) bool {

	if chunkToMerge != nil {
		i := 0
		for ; i < len(chunkToMerge); i++ {
			if chunkToMerge[i].Name != seqres[end+i] {
				return false
			}
			result[end+i] = chunkToMerge[i]
		}
		end += i
	}
	switch {
	case len(mchunks) == 0 && len(fchunks) == 0:
		return true
	case len(mchunks) == 0:
		return merge(result, end, seqres, fchunks[0], nil, fchunks[1:])
	case len(fchunks) == 0:
		return merge(result, end, seqres, mchunks[0], mchunks[1:], nil)
	}

	// This is a little weird. We really want to alternate chunks, since
	// a chunk is split presumably where there are holes.
	// But, we don't force it.
	// So we express a preference for it by trying the alternate first, and
	// fallback to similar chunk second.
	switch {
	case len(chunkToMerge) > 0 && chunkToMerge[0].Atoms == nil:
		// The current chunk is from missing, so prefer filled.
		return merge(result, end, seqres, fchunks[0], mchunks, fchunks[1:]) ||
			merge(result, end, seqres, mchunks[0], mchunks[1:], fchunks)
	case len(chunkToMerge) > 0 && chunkToMerge[0].Atoms != nil:
		// The current chunk is from filled, so prefer missing.
		return merge(result, end, seqres, mchunks[0], mchunks[1:], fchunks) ||
			merge(result, end, seqres, fchunks[0], mchunks, fchunks[1:])
	}

	// doesn't matter what we do here.
	// (this is the beginning)
	return merge(result, end, seqres, mchunks[0], mchunks[1:], fchunks) ||
		merge(result, end, seqres, fchunks[0], mchunks, fchunks[1:])
}

// chunk splits up residues in a list of contiguous segments.
func chunk(residues []*Residue, other []*Residue) [][]*Residue {
	if len(residues) == 0 {
		return nil
	}

	chunks := make([][]*Residue, 0)

	cur := make([]*Residue, 1)
	last := residues[0]
	cur[0] = last
	for i := 1; i < len(residues); i++ {
		r := residues[i]

		if last.isContiguous(r, other) {
			cur = append(cur, r)
			last = r
			continue
		}

		chunks = append(chunks, cur)
		cur = make([]*Residue, 1)
		cur[0] = r
		last = r
	}
	chunks = append(chunks, cur)

	return chunks
}

func (a Residue) less(b Residue) bool {
	return a.SequenceNum < b.SequenceNum ||
		(a.SequenceNum == b.SequenceNum && a.InsertionCode < b.InsertionCode)
}

func (a Residue) equals(b Residue) bool {
	return a.SequenceNum == b.SequenceNum && a.InsertionCode == b.InsertionCode
}

func (a *Residue) isContiguous(b *Residue, other []*Residue) bool {
	// If any residue in "other" is next to a or b, then we cannot claim
	// contiguity.
	for i := range other {
		if a.SequenceNum == other[i].SequenceNum {
			return false
		}
		if b.SequenceNum == other[i].SequenceNum {
			return false
		}
	}
	return a.isNext(b)
}

func (a *Residue) isNext(b *Residue) bool {
	asn, bsn := a.SequenceNum, b.SequenceNum

	// We make the assumption that any two residues with the same sequence
	// number are always contiguous.
	// Note that this has problems. Consider the case with two ATOM records
	// with sequence numbers 36 and 37, but a missing residue with sequence
	// number 36 and insertion code A. So it should go 36 -> 36A -> 37.
	// To remedy this, the caller must make sure that all missing (or filled)
	// residues don't have the same sequence number.
	if asn == bsn {
		return true
	}
	if asn+1 == bsn || asn-1 == bsn {
		return true
	}
	return false
}

type residues []*Residue

func (rs residues) String() string {
	bs := make([]byte, len(rs))
	for i, r := range rs {
		switch {
		case r == nil:
			bs[i] = '-'
		case r.Atoms == nil:
			bs[i] = byte(unicode.ToLower(rune(r.Name)))
		default:
			bs[i] = byte(r.Name)
		}
	}
	return string(bs)
}

type gappedResidues []*Residue

func (rs gappedResidues) String() string {
	bs := make([]byte, len(rs))
	for i, r := range rs {
		if r == nil || r.Atoms == nil {
			bs[i] = '-'
		} else {
			bs[i] = byte(r.Name)
		}
	}
	return string(bs)
}
<commit_msg>Update to new NeedlemanWunsch interface.<commit_after>package pdb

import (
	"unicode"

	"github.com/TuftsBCB/seq"
)

func (m Model) seqAtomsGuess() []*Residue {
	seqres := m.Chain.Sequence
	mapping := make([]*Residue, len(seqres))
	if len(seqres) != len(m.Residues) {
		// This is a last ditch effort. Use the ATOM sequence number as an
		// index into the SEQRES residues.
		for _, r := range m.Residues {
			si := r.SequenceNum - 1
			if si >= 0 && si < len(seqres) && seqres[si] == r.Name {
				mapping[si] = r
			}
		}
		return mapping
	}

	for i, r := range m.Residues {
		if i < len(seqres) && seqres[i] == r.Name {
			mapping[i] = r
		}
	}
	return mapping
}

func (m Model) seqAtomsAlign() []*Residue {
	seqres := m.Chain.Sequence
	atomResidues := make([]seq.Residue, len(m.Residues))
	for i, r := range m.Residues {
		atomResidues[i] = r.Name
	}
	aligned := seq.NeedlemanWunsch(seqres, atomResidues, seq.SubstBlosum62)

	mapped := make([]*Residue, len(seqres))
	atomi := 0
	for i, r := range aligned.B {
		if r == '-' {
			mapped[i] = nil
		} else {
			mapped[i] = m.Residues[atomi]
			atomi++
		}
	}
	return mapped
}

// Attempts to accomplish the same thing as seqAtomsWithMissing, but instead
// of mapping one residue at a time, we map *chunks* at a time. That is,
// a chunk is any group of contiguous residues.
func (m Model) seqAtomsChunksMerge() []*Residue {
	seqres := m.Chain.Sequence

	// Check to make sure that the total number of missing residues, plus the
	// total number of ATOM record residues equal the total number of residues
	// in the SEQRES records. Otherwise, the merge will fail.
	if len(seqres) != len(m.Chain.Missing)+len(m.Residues) {
		return m.seqAtomsAlign()
	}

	result := make([]*Residue, len(seqres))
	mchunks := chunk(m.Chain.Missing, m.Residues)
	fchunks := chunk(m.Residues, m.Chain.Missing)

	// If the PDB file is corrupted, a merge will fail.
	// So we fall back to alignment.
	if ok := merge(result, 0, seqres, nil, mchunks, fchunks); !ok {
		return m.seqAtomsAlign()
	}

	// X out any residues that don't have ATOM records.
	for i := range result {
		if result[i].Atoms == nil {
			result[i] = nil
		}
	}
	return result
}

func merge(result []*Residue, end int, seqres []seq.Residue,
	chunkToMerge []*Residue, mchunks, fchunks [][]*Residue) bool {

	if chunkToMerge != nil {
		i := 0
		for ; i < len(chunkToMerge); i++ {
			if chunkToMerge[i].Name != seqres[end+i] {
				return false
			}
			result[end+i] = chunkToMerge[i]
		}
		end += i
	}
	switch {
	case len(mchunks) == 0 && len(fchunks) == 0:
		return true
	case len(mchunks) == 0:
		return merge(result, end, seqres, fchunks[0], nil, fchunks[1:])
	case len(fchunks) == 0:
		return merge(result, end, seqres, mchunks[0], mchunks[1:], nil)
	}

	// This is a little weird. We really want to alternate chunks, since
	// a chunk is split presumably where there are holes.
	// But, we don't force it.
	// So we express a preference for it by trying the alternate first, and
	// fallback to similar chunk second.
	switch {
	case len(chunkToMerge) > 0 && chunkToMerge[0].Atoms == nil:
		// The current chunk is from missing, so prefer filled.
		return merge(result, end, seqres, fchunks[0], mchunks, fchunks[1:]) ||
			merge(result, end, seqres, mchunks[0], mchunks[1:], fchunks)
	case len(chunkToMerge) > 0 && chunkToMerge[0].Atoms != nil:
		// The current chunk is from filled, so prefer missing.
		return merge(result, end, seqres, mchunks[0], mchunks[1:], fchunks) ||
			merge(result, end, seqres, fchunks[0], mchunks, fchunks[1:])
	}

	// doesn't matter what we do here.
	// (this is the beginning)
	return merge(result, end, seqres, mchunks[0], mchunks[1:], fchunks) ||
		merge(result, end, seqres, fchunks[0], mchunks, fchunks[1:])
}

// chunk splits up residues in a list of contiguous segments.
func chunk(residues []*Residue, other []*Residue) [][]*Residue {
	if len(residues) == 0 {
		return nil
	}

	chunks := make([][]*Residue, 0)

	cur := make([]*Residue, 1)
	last := residues[0]
	cur[0] = last
	for i := 1; i < len(residues); i++ {
		r := residues[i]

		if last.isContiguous(r, other) {
			cur = append(cur, r)
			last = r
			continue
		}

		chunks = append(chunks, cur)
		cur = make([]*Residue, 1)
		cur[0] = r
		last = r
	}
	chunks = append(chunks, cur)

	return chunks
}

func (a Residue) less(b Residue) bool {
	return a.SequenceNum < b.SequenceNum ||
		(a.SequenceNum == b.SequenceNum && a.InsertionCode < b.InsertionCode)
}

func (a Residue) equals(b Residue) bool {
	return a.SequenceNum == b.SequenceNum && a.InsertionCode == b.InsertionCode
}

func (a *Residue) isContiguous(b *Residue, other []*Residue) bool {
	// If any residue in "other" is next to a or b, then we cannot claim
	// contiguity.
	for i := range other {
		if a.SequenceNum == other[i].SequenceNum {
			return false
		}
		if b.SequenceNum == other[i].SequenceNum {
			return false
		}
	}
	return a.isNext(b)
}

func (a *Residue) isNext(b *Residue) bool {
	asn, bsn := a.SequenceNum, b.SequenceNum

	// We make the assumption that any two residues with the same sequence
	// number are always contiguous.
	// Note that this has problems. Consider the case with two ATOM records
	// with sequence numbers 36 and 37, but a missing residue with sequence
	// number 36 and insertion code A. So it should go 36 -> 36A -> 37.
	// To remedy this, the caller must make sure that all missing (or filled)
	// residues don't have the same sequence number.
	if asn == bsn {
		return true
	}
	if asn+1 == bsn || asn-1 == bsn {
		return true
	}
	return false
}

type residues []*Residue

func (rs residues) String() string {
	bs := make([]byte, len(rs))
	for i, r := range rs {
		switch {
		case r == nil:
			bs[i] = '-'
		case r.Atoms == nil:
			bs[i] = byte(unicode.ToLower(rune(r.Name)))
		default:
			bs[i] = byte(r.Name)
		}
	}
	return string(bs)
}

type gappedResidues []*Residue

func (rs gappedResidues) String() string {
	bs := make([]byte, len(rs))
	for i, r := range rs {
		if r == nil || r.Atoms == nil {
			bs[i] = '-'
		} else {
			bs[i] = byte(r.Name)
		}
	}
	return string(bs)
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mako

import (
	"context"
	"log"
	"runtime"
	"strings"

	"cloud.google.com/go/compute/metadata"
	"knative.dev/pkg/injection/clients/kubeclient"

	"github.com/google/mako/go/quickstore"
	qpb "github.com/google/mako/proto/quickstore/quickstore_go_proto"
	"k8s.io/client-go/rest"
	"knative.dev/pkg/changeset"
	"knative.dev/pkg/controller"
	"knative.dev/pkg/injection"
)

const (
	// sidecarAddress is the address of the Mako sidecar to which we locally
	// write results, and it authenticates and publishes them to Mako after
	// assorted preprocessing.
	sidecarAddress = "localhost:9813"
)

// EscapeTag replaces characters that Mako doesn't accept with ones it does.
func EscapeTag(tag string) string {
	return strings.ReplaceAll(tag, ".", "_")
}

// Setup sets up the mako client for the provided benchmarkKey.
// It will add a few common tags and allow each benchmark to add custm tags as well.
// It returns the mako client handle to store metrics, a method to close the connection
// to mako server once done and error in case of failures.
func Setup(ctx context.Context, extraTags ...string) (context.Context, *quickstore.Quickstore, func(context.Context), error) {
	tags := append(MustGetTags(), extraTags...)
	// Get the commit of the benchmarks
	commitID, err := changeset.Get()
	if err != nil {
		return nil, nil, nil, err
	}

	// Setup a deployment informer, so that we can use the lister to track
	// desired and available pod counts.
	cfg, err := rest.InClusterConfig()
	if err != nil {
		return nil, nil, nil, err
	}
	ctx, informers := injection.Default.SetupInformers(ctx, cfg)
	if err := controller.StartInformers(ctx.Done(), informers...); err != nil {
		return nil, nil, nil, err
	}

	// Get the Kubernetes version from the API server.
	version, err := kubeclient.Get(ctx).Discovery().ServerVersion()
	if err != nil {
		return nil, nil, nil, err
	}

	// Decorate GCP metadata as tags (when we're running on GCP).
	if projectID, err := metadata.ProjectID(); err != nil {
		log.Printf("GCP project ID is not available: %v", err)
	} else {
		tags = append(tags, "project-id="+EscapeTag(projectID))
	}
	if zone, err := metadata.Zone(); err != nil {
		log.Printf("GCP zone is not available: %v", err)
	} else {
		tags = append(tags, "zone="+EscapeTag(zone))
	}
	if machineType, err := metadata.Get("instance/machine-type"); err != nil {
		log.Printf("GCP machine type is not available: %v", err)
	} else if parts := strings.Split(machineType, "/"); len(parts) != 4 {
		tags = append(tags, "instanceType="+EscapeTag(parts[3]))
	}

	qs, qclose, err := quickstore.NewAtAddress(ctx, &qpb.QuickstoreInput{
		BenchmarkKey: MustGetBenchmark(),
		Tags: append(tags,
			"commit="+commitID,
			"kubernetes="+EscapeTag(version.String()),
			EscapeTag(runtime.Version()),
		),
	}, sidecarAddress)
	if err != nil {
		return nil, nil, nil, err
	}
	return ctx, qs, qclose, nil
}
<commit_msg>Add the number of nodes as a Mako tag. (#611)<commit_after>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mako

import (
	"context"
	"fmt"
	"log"
	"runtime"
	"strings"

	"cloud.google.com/go/compute/metadata"
	"knative.dev/pkg/injection/clients/kubeclient"

	"github.com/google/mako/go/quickstore"
	qpb "github.com/google/mako/proto/quickstore/quickstore_go_proto"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/rest"
	"knative.dev/pkg/changeset"
	"knative.dev/pkg/controller"
	"knative.dev/pkg/injection"
)

const (
	// sidecarAddress is the address of the Mako sidecar to which we locally
	// write results, and it authenticates and publishes them to Mako after
	// assorted preprocessing.
	sidecarAddress = "localhost:9813"
)

// EscapeTag replaces characters that Mako doesn't accept with ones it does.
func EscapeTag(tag string) string {
	return strings.ReplaceAll(tag, ".", "_")
}

// Setup sets up the mako client for the provided benchmarkKey.
// It will add a few common tags and allow each benchmark to add custm tags as well.
// It returns the mako client handle to store metrics, a method to close the connection
// to mako server once done and error in case of failures.
func Setup(ctx context.Context, extraTags ...string) (context.Context, *quickstore.Quickstore, func(context.Context), error) {
	tags := append(MustGetTags(), extraTags...)
	// Get the commit of the benchmarks
	commitID, err := changeset.Get()
	if err != nil {
		return nil, nil, nil, err
	}

	// Setup a deployment informer, so that we can use the lister to track
	// desired and available pod counts.
	cfg, err := rest.InClusterConfig()
	if err != nil {
		return nil, nil, nil, err
	}
	ctx, informers := injection.Default.SetupInformers(ctx, cfg)
	if err := controller.StartInformers(ctx.Done(), informers...); err != nil {
		return nil, nil, nil, err
	}

	// Get the Kubernetes version from the API server.
	kc := kubeclient.Get(ctx)
	version, err := kc.Discovery().ServerVersion()
	if err != nil {
		return nil, nil, nil, err
	}

	// Determine the number of Kubernetes nodes through the kubernetes client.
	nodes, err := kc.CoreV1().Nodes().List(metav1.ListOptions{})
	if err != nil {
		return nil, nil, nil, err
	}
	tags = append(tags, "nodes="+fmt.Sprintf("%d", len(nodes.Items)))

	// Decorate GCP metadata as tags (when we're running on GCP).
	if projectID, err := metadata.ProjectID(); err != nil {
		log.Printf("GCP project ID is not available: %v", err)
	} else {
		tags = append(tags, "project-id="+EscapeTag(projectID))
	}
	if zone, err := metadata.Zone(); err != nil {
		log.Printf("GCP zone is not available: %v", err)
	} else {
		tags = append(tags, "zone="+EscapeTag(zone))
	}
	if machineType, err := metadata.Get("instance/machine-type"); err != nil {
		log.Printf("GCP machine type is not available: %v", err)
	} else if parts := strings.Split(machineType, "/"); len(parts) != 4 {
		tags = append(tags, "instanceType="+EscapeTag(parts[3]))
	}

	qs, qclose, err := quickstore.NewAtAddress(ctx, &qpb.QuickstoreInput{
		BenchmarkKey: MustGetBenchmark(),
		Tags: append(tags,
			"commit="+commitID,
			"kubernetes="+EscapeTag(version.String()),
			EscapeTag(runtime.Version()),
		),
	}, sidecarAddress)
	if err != nil {
		return nil, nil, nil, err
	}
	return ctx, qs, qclose, nil
}
<|endoftext|>


<commit_before>package gyazo

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"

	"github.com/google/go-querystring/query"
)

const (
	version         = "0.1"
	defaultEndpoint = "https://api.gyazo.com"
	uploadEndpoint  = "https://upload.gyazo.com"
)

// Client manages communication with the Gyazo API
type Client struct {
	// Gyazo API access token.
	token string

	// client provides request to API endpoints.
	client *http.Client

	// DefaultEndpint is Gyazo API endpoint.
	DefaultEndpoint string

	// UploadEndpint is Gyazo upload API endpoint.
	UploadEndpoint string
}

// Image represents a uploaded image.
//
// Gyazo API docs: https://gyazo.com/api/docs/image
type Image struct {
	ID           string `json:"image_id"`
	PermalinkURL string `json:"permalink_url"`
	ThumbURL     string `json:"thumb_url"`
	URL          string `json:"url"`
	Type         string `json:"type"`
	Star         string `json:"star"`
	CreatedAt    string `json:"created_at"`
}

// ListOptions specifies the optional parameters to the List.
type ListOptions struct {
	Page    int `url:"page,omitempty"`
	PerPage int `url:"per_page,omitempty"`
}

// NewClient returns a new Gyazo API client.
func NewClient(token string) (*Client, error) {
	if token == "" {
		return nil, errors.New("access token is empty")
	}

	c := &Client{token, http.DefaultClient, defaultEndpoint, uploadEndpoint}
	return c, nil
}

// List returns user images
func (c *Client) List(opts *ListOptions) (*[]Image, error) {
	url := c.DefaultEndpoint + "/api/images"
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", c.token))

	// Build and set query parameters
	if opts != nil {
		params, err := query.Values(opts)
		if err != nil {
			return nil, err
		}
		req.URL.RawQuery = params.Encode()
	}

	res, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	if res.StatusCode != http.StatusOK {
		return nil, errors.New(res.Status)
	}

	list := new([]Image)
	if err = json.NewDecoder(res.Body).Decode(&list); err != nil {
		return nil, err
	}
	return list, nil
}
<commit_msg>Use oauth2 package to OAuth2 authentication<commit_after>package gyazo

import (
	"encoding/json"
	"errors"
	"net/http"

	"github.com/google/go-querystring/query"
	"golang.org/x/oauth2"
)

const (
	version         = "0.1"
	defaultEndpoint = "https://api.gyazo.com"
	uploadEndpoint  = "https://upload.gyazo.com"
)

// Client manages communication with the Gyazo API
type Client struct {
	// Gyazo API access token.
	token string

	// client provides request to API endpoints.
	client *http.Client

	// DefaultEndpint is Gyazo API endpoint.
	DefaultEndpoint string

	// UploadEndpint is Gyazo upload API endpoint.
	UploadEndpoint string
}

// Image represents a uploaded image.
//
// Gyazo API docs: https://gyazo.com/api/docs/image
type Image struct {
	ID           string `json:"image_id"`
	PermalinkURL string `json:"permalink_url"`
	ThumbURL     string `json:"thumb_url"`
	URL          string `json:"url"`
	Type         string `json:"type"`
	Star         string `json:"star"`
	CreatedAt    string `json:"created_at"`
}

// ListOptions specifies the optional parameters to the List.
type ListOptions struct {
	Page    int `url:"page,omitempty"`
	PerPage int `url:"per_page,omitempty"`
}

// NewClient returns a new Gyazo API client.
func NewClient(token string) (*Client, error) {
	if token == "" {
		return nil, errors.New("access token is empty")
	}

	// Create an OAuth2 client to authentication
	oauthClient := oauth2.NewClient(
		oauth2.NoContext,
		oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}),
	)

	c := &Client{token, oauthClient, defaultEndpoint, uploadEndpoint}
	return c, nil
}

// List returns user images
func (c *Client) List(opts *ListOptions) (*[]Image, error) {
	url := c.DefaultEndpoint + "/api/images"
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}

	// Build and set query parameters
	if opts != nil {
		params, err := query.Values(opts)
		if err != nil {
			return nil, err
		}
		req.URL.RawQuery = params.Encode()
	}

	res, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	if res.StatusCode != http.StatusOK {
		return nil, errors.New(res.Status)
	}

	list := new([]Image)
	if err = json.NewDecoder(res.Body).Decode(&list); err != nil {
		return nil, err
	}
	return list, nil
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package cbft

import (
	"container/heap"
	"fmt"
	"sync"
)

type ConsistencyParams struct {
	// A Level value of "" means stale is ok; "at_plus" means we need
	// consistency at least at or beyond the consistency vector but
	// not before.
	Level string `json:"level"`

	// Keyed by indexName.
	Vectors map[string]ConsistencyVector `json:"vectors"`

	// TODO: Can user specify certain partition UUID (like vbucket UUID)?
}

// Key is partition, value is seq.
type ConsistencyVector map[string]uint64

type ConsistencyWaiter interface {
	ConsistencyWait(partition string,
		consistencyLevel string,
		consistencySeq uint64,
		cancelCh chan string) error
}

type ConsistencyWaitReq struct {
	ConsistencyLevel string
	ConsistencySeq   uint64
	CancelCh         chan string
	DoneCh           chan error
}

type ErrorConsistencyWait struct {
	Err    error  // The underlying, wrapped error.
	Status string // Short status reason, like "timeout", "cancelled", etc.

	// Keyed by partitionId, value is pair of start/end seq's.
	StartEndSeqs map[string][]uint64
}

func (e *ErrorConsistencyWait) Error() string {
	return fmt.Sprintf("ErrorConsistencyWait, startEndSeqs: %#v,"+
		" err: %v", e.StartEndSeqs, e.Err)
}

// ---------------------------------------------------------

func ConsistencyWaitDone(partition string, cancelCh chan string,
	doneCh chan error, currSeq func() uint64) error {
	seqStart := currSeq()

	select {
	case status := <-cancelCh:
		if status == "" { // For example, status might be "timeout".
			status = "cancelled"
		}

		rv := map[string][]uint64{}
		rv[partition] = []uint64{seqStart, currSeq()}

		err := fmt.Errorf("ConsistencyWaitDone cancelled, status: %s", status)

		return &ErrorConsistencyWait{ // TODO: track stats.
			Err:          err,
			Status:       status,
			StartEndSeqs: rv,
		}

	case err := <-doneCh:
		return err // TODO: track stats.
	}
}

func ConsistencyWaitPartitions(
	t ConsistencyWaiter,
	partitions []string,
	consistencyLevel string,
	consistencyVector map[string]uint64,
	cancelCh chan string) error {
	for _, partition := range partitions {
		consistencySeq := consistencyVector[partition]
		if consistencySeq > 0 {
			err := t.ConsistencyWait(partition,
				consistencyLevel, consistencySeq, cancelCh)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func ConsistencyWaitPIndex(pindex *PIndex, t ConsistencyWaiter,
	consistencyParams *ConsistencyParams, cancelCh chan string) error {
	if consistencyParams != nil &&
		consistencyParams.Level != "" &&
		consistencyParams.Vectors != nil {
		consistencyVector := consistencyParams.Vectors[pindex.IndexName]
		if consistencyVector != nil {
			err := ConsistencyWaitPartitions(t, pindex.sourcePartitionsArr,
				consistencyParams.Level, consistencyVector, cancelCh)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func ConsistencyWaitGroup(indexName string,
	consistencyParams *ConsistencyParams, cancelCh chan string,
	localPIndexes []*PIndex,
	addLocalPIndex func(*PIndex) error) error {
	var errConsistencyM sync.Mutex
	var errConsistency error

	var wg sync.WaitGroup

	for _, localPIndex := range localPIndexes {
		err := addLocalPIndex(localPIndex)
		if err != nil {
			return err
		}

		if consistencyParams != nil &&
			consistencyParams.Level != "" &&
			consistencyParams.Vectors != nil {
			consistencyVector := consistencyParams.Vectors[indexName]
			if consistencyVector != nil {
				wg.Add(1)
				go func(localPIndex *PIndex,
					consistencyVector map[string]uint64) {
					defer wg.Done()

					err := ConsistencyWaitPartitions(localPIndex.Dest,
						localPIndex.sourcePartitionsArr,
						consistencyParams.Level,
						consistencyVector,
						cancelCh)
					if err != nil {
						errConsistencyM.Lock()
						errConsistency = err
						errConsistencyM.Unlock()
					}
				}(localPIndex, consistencyVector)
			}
		}
	}

	wg.Wait()

	if errConsistency != nil {
		return errConsistency
	}

	if cancelCh != nil {
		select {
		case status := <-cancelCh:
			return fmt.Errorf("cancelled, status: %s", status)
		default:
		}
	}

	// TODO: There's likely a race here where at this point we've now
	// waited for all the (local) pindexes to reach the requested
	// consistency levels, but before we actually can use the
	// constructed alias and kick off a query, an adversary does a
	// rollback.  Using the alias to query after that might now be
	// incorrectly running against data some time back in the past.

	return nil
}

// ---------------------------------------------------------

// A cwrQueue is a consistency wait request queue, implementing the
// heap.Interface for consistencyWaitReq's.
type cwrQueue []*ConsistencyWaitReq

func (pq cwrQueue) Len() int { return len(pq) }

func (pq cwrQueue) Less(i, j int) bool {
	return pq[i].ConsistencySeq < pq[j].ConsistencySeq
}

func (pq cwrQueue) Swap(i, j int) {
	pq[i], pq[j] = pq[j], pq[i]
}

func (pq *cwrQueue) Push(x interface{}) {
	*pq = append(*pq, x.(*ConsistencyWaitReq))
}

func (pq *cwrQueue) Pop() interface{} {
	old := *pq
	n := len(old)
	item := old[n-1]
	*pq = old[0 : n-1]
	return item
}

// ---------------------------------------------------------

func RunConsistencyWaitQueue(
	cwrCh chan *ConsistencyWaitReq,
	m *sync.Mutex,
	cwrQueue *cwrQueue,
	currSeq func() uint64) {
	for cwr := range cwrCh {
		m.Lock()

		if cwr.ConsistencyLevel == "" {
			close(cwr.DoneCh) // We treat "" like stale=ok, so we're done.
		} else if cwr.ConsistencyLevel == "at_plus" {
			if cwr.ConsistencySeq > currSeq() {
				heap.Push(cwrQueue, cwr)
			} else {
				close(cwr.DoneCh)
			}
		} else {
			cwr.DoneCh <- fmt.Errorf("consistency wait unsupported level: %s,"+
				" cwr: %#v", cwr.ConsistencyLevel, cwr)
			close(cwr.DoneCh)
		}

		m.Unlock()
	}

	// If we reach here, then we're closing down so cancel/error any
	// callers waiting for consistency.
	m.Lock()
	defer m.Unlock()

	err := fmt.Errorf("consistency wait closed")

	for _, cwr := range *cwrQueue {
		cwr.DoneCh <- err
		close(cwr.DoneCh)
	}
}
<commit_msg>use goroutines to keep cwrQueue lock window short<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package cbft

import (
	"container/heap"
	"fmt"
	"sync"
)

type ConsistencyParams struct {
	// A Level value of "" means stale is ok; "at_plus" means we need
	// consistency at least at or beyond the consistency vector but
	// not before.
	Level string `json:"level"`

	// Keyed by indexName.
	Vectors map[string]ConsistencyVector `json:"vectors"`

	// TODO: Can user specify certain partition UUID (like vbucket UUID)?
}

// Key is partition, value is seq.
type ConsistencyVector map[string]uint64

type ConsistencyWaiter interface {
	ConsistencyWait(partition string,
		consistencyLevel string,
		consistencySeq uint64,
		cancelCh chan string) error
}

type ConsistencyWaitReq struct {
	ConsistencyLevel string
	ConsistencySeq   uint64
	CancelCh         chan string
	DoneCh           chan error
}

type ErrorConsistencyWait struct {
	Err    error  // The underlying, wrapped error.
	Status string // Short status reason, like "timeout", "cancelled", etc.

	// Keyed by partitionId, value is pair of start/end seq's.
	StartEndSeqs map[string][]uint64
}

func (e *ErrorConsistencyWait) Error() string {
	return fmt.Sprintf("ErrorConsistencyWait, startEndSeqs: %#v,"+
		" err: %v", e.StartEndSeqs, e.Err)
}

// ---------------------------------------------------------

func ConsistencyWaitDone(partition string, cancelCh chan string,
	doneCh chan error, currSeq func() uint64) error {
	seqStart := currSeq()

	select {
	case status := <-cancelCh:
		if status == "" { // For example, status might be "timeout".
			status = "cancelled"
		}

		rv := map[string][]uint64{}
		rv[partition] = []uint64{seqStart, currSeq()}

		err := fmt.Errorf("ConsistencyWaitDone cancelled, status: %s", status)

		return &ErrorConsistencyWait{ // TODO: track stats.
			Err:          err,
			Status:       status,
			StartEndSeqs: rv,
		}

	case err := <-doneCh:
		return err // TODO: track stats.
	}
}

func ConsistencyWaitPartitions(
	t ConsistencyWaiter,
	partitions []string,
	consistencyLevel string,
	consistencyVector map[string]uint64,
	cancelCh chan string) error {
	for _, partition := range partitions {
		consistencySeq := consistencyVector[partition]
		if consistencySeq > 0 {
			err := t.ConsistencyWait(partition,
				consistencyLevel, consistencySeq, cancelCh)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func ConsistencyWaitPIndex(pindex *PIndex, t ConsistencyWaiter,
	consistencyParams *ConsistencyParams, cancelCh chan string) error {
	if consistencyParams != nil &&
		consistencyParams.Level != "" &&
		consistencyParams.Vectors != nil {
		consistencyVector := consistencyParams.Vectors[pindex.IndexName]
		if consistencyVector != nil {
			err := ConsistencyWaitPartitions(t, pindex.sourcePartitionsArr,
				consistencyParams.Level, consistencyVector, cancelCh)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func ConsistencyWaitGroup(indexName string,
	consistencyParams *ConsistencyParams, cancelCh chan string,
	localPIndexes []*PIndex,
	addLocalPIndex func(*PIndex) error) error {
	var errConsistencyM sync.Mutex
	var errConsistency error

	var wg sync.WaitGroup

	for _, localPIndex := range localPIndexes {
		err := addLocalPIndex(localPIndex)
		if err != nil {
			return err
		}

		if consistencyParams != nil &&
			consistencyParams.Level != "" &&
			consistencyParams.Vectors != nil {
			consistencyVector := consistencyParams.Vectors[indexName]
			if consistencyVector != nil {
				wg.Add(1)
				go func(localPIndex *PIndex,
					consistencyVector map[string]uint64) {
					defer wg.Done()

					err := ConsistencyWaitPartitions(localPIndex.Dest,
						localPIndex.sourcePartitionsArr,
						consistencyParams.Level,
						consistencyVector,
						cancelCh)
					if err != nil {
						errConsistencyM.Lock()
						errConsistency = err
						errConsistencyM.Unlock()
					}
				}(localPIndex, consistencyVector)
			}
		}
	}

	wg.Wait()

	if errConsistency != nil {
		return errConsistency
	}

	if cancelCh != nil {
		select {
		case status := <-cancelCh:
			return fmt.Errorf("cancelled, status: %s", status)
		default:
		}
	}

	// TODO: There's likely a race here where at this point we've now
	// waited for all the (local) pindexes to reach the requested
	// consistency levels, but before we actually can use the
	// constructed alias and kick off a query, an adversary does a
	// rollback.  Using the alias to query after that might now be
	// incorrectly running against data some time back in the past.

	return nil
}

// ---------------------------------------------------------

// A cwrQueue is a consistency wait request queue, implementing the
// heap.Interface for consistencyWaitReq's.
type cwrQueue []*ConsistencyWaitReq

func (pq cwrQueue) Len() int { return len(pq) }

func (pq cwrQueue) Less(i, j int) bool {
	return pq[i].ConsistencySeq < pq[j].ConsistencySeq
}

func (pq cwrQueue) Swap(i, j int) {
	pq[i], pq[j] = pq[j], pq[i]
}

func (pq *cwrQueue) Push(x interface{}) {
	*pq = append(*pq, x.(*ConsistencyWaitReq))
}

func (pq *cwrQueue) Pop() interface{} {
	old := *pq
	n := len(old)
	item := old[n-1]
	*pq = old[0 : n-1]
	return item
}

// ---------------------------------------------------------

func RunConsistencyWaitQueue(
	cwrCh chan *ConsistencyWaitReq,
	m *sync.Mutex,
	cwrQueue *cwrQueue,
	currSeq func() uint64) {
	for cwr := range cwrCh {
		m.Lock()

		if cwr.ConsistencyLevel == "" {
			close(cwr.DoneCh) // We treat "" like stale=ok, so we're done.
		} else if cwr.ConsistencyLevel == "at_plus" {
			if cwr.ConsistencySeq > currSeq() {
				heap.Push(cwrQueue, cwr)
			} else {
				close(cwr.DoneCh)
			}
		} else {
			cwr.DoneCh <- fmt.Errorf("consistency wait unsupported level: %s,"+
				" cwr: %#v", cwr.ConsistencyLevel, cwr)
			close(cwr.DoneCh)
		}

		m.Unlock()
	}

	// If we reach here, then we're closing down so cancel/error any
	// callers waiting for consistency.
	m.Lock()
	defer m.Unlock()

	err := fmt.Errorf("consistency wait closed")

	for _, cwr := range *cwrQueue {
		// TODO: Perhaps extra goroutine here isn't necessary, but the
		// motivation is to keep cwrQueue's lock window short.
		go func(cwr *ConsistencyWaitReq) {
			cwr.DoneCh <- err
			close(cwr.DoneCh)
		}(cwr)
	}
}
<|endoftext|>


<commit_before>package haigo

import (
	"bytes"
	"encoding/json"
	"fmt"
	"text/template"

	"gopkg.in/mgo.v2"
	"gopkg.in/yaml.v2"
)

type param struct {
	Name string
	Type string
}

type HaigoQuery struct {
	Name        string  `yaml:"name"`
	Description string  `yaml:"description,omitempty"`
	QueryString string  `yaml:"query"`
	params      []param // TODO
}

type HaigoParams map[string]interface{}

// Execute - Returns configured mgo Query.
func (h *HaigoQuery) Execute(col *mgo.Collection, params HaigoParams) (*mgo.Query, error) {

	q, err := h.Query(params)
	if err != nil {
		return nil, err
	}

	return col.Find(q), nil
}

// HaigoFile - YAML formatted file with MongoDB Queries.
//
//	---
//    - name: basic-select
//      description: Basic MongoDB Select
//      query: '{"type": {{.type}} }'
//
//    - name: conditional
//      description: Conditional Query
//      query: '{
//         "type": "food",
//         "$or": [ { "qty": { "$gt": {{.qty}} } }, { "name": {{.name}} } ]
//      }'
type HaigoFile struct {
	Queries map[string]*HaigoQuery
}

func (m *HaigoFile) unmarshalYAML(data []byte) error {

	var hqs []HaigoQuery

	err := yaml.Unmarshal(data, &hqs)
	if err != nil {
		return err
	}

	qm := make(map[string]*HaigoQuery)

	for i := range hqs {
		qm[hqs[i].Name] = &hqs[i]
	}

	m.Queries = qm

	return nil
}

// sanitizeParams - Adds single quotes if param is a string (as needed by Mongo).
func sanitizeParams(params HaigoParams) HaigoParams {
	for k, v := range params {
		switch v.(type) {
		case string:
			params[k] = fmt.Sprintf("\"%s\"", v)
		}

	}
	return params
}

// Query - Accepts a params map and returns a map for use with the mgo `find()`
// function.
func (h *HaigoQuery) Query(params HaigoParams) (map[string]interface{}, error) {

	// Create the template
	t, err := template.New("haigo").Parse(h.QueryString)
	if err != nil {
		return nil, err
	}

	// Buffer to capture string
	buf := new(bytes.Buffer)

	// Execute template
	err = t.Execute(buf, sanitizeParams(params))
	if err != nil {
		return nil, err
	}

	// Unmarshal JSON into Map
	var m map[string]interface{}
	err = json.Unmarshal(buf.Bytes(), &m)
	if err != nil {
		return nil, err
	}

	return m, nil
}

// LoadQueryFile - Reads in Mongo Query File for use with Haigo.
func LoadQueryFile(file string) (*HaigoFile, error) {
	return parseMongoFile(file)
}
<commit_msg>include YAML sample<commit_after>package haigo

import (
	"bytes"
	"encoding/json"
	"fmt"
	"text/template"

	"gopkg.in/mgo.v2"
	"gopkg.in/yaml.v2"
)

type param struct {
	Name string
	Type string
}

type HaigoQuery struct {
	Name        string  `yaml:"name"`
	Description string  `yaml:"description,omitempty"`
	QueryString string  `yaml:"query"`
	params      []param // TODO
}

type HaigoParams map[string]interface{}

// Execute - Returns configured mgo Query.
func (h *HaigoQuery) Execute(col *mgo.Collection, params HaigoParams) (*mgo.Query, error) {

	q, err := h.Query(params)
	if err != nil {
		return nil, err
	}

	return col.Find(q), nil
}

// HaigoFile - YAML formatted file with MongoDB Queries.
//
// ---
//    - name: basic-select
//      description: Basic MongoDB Select
//      query: '{"type": {{.type}} }'
//
//    - name: conditional
//      description: Conditional Query
//      query: '{
//         "type": "food",
//         "$or": [ { "qty": { "$gt": {{.qty}} } }, { "name": {{.name}} } ]
//      }'
type HaigoFile struct {
	Queries map[string]*HaigoQuery
}

func (m *HaigoFile) unmarshalYAML(data []byte) error {

	var hqs []HaigoQuery

	err := yaml.Unmarshal(data, &hqs)
	if err != nil {
		return err
	}

	qm := make(map[string]*HaigoQuery)

	for i := range hqs {
		qm[hqs[i].Name] = &hqs[i]
	}

	m.Queries = qm

	return nil
}

// sanitizeParams - Adds single quotes if param is a string (as needed by Mongo).
func sanitizeParams(params HaigoParams) HaigoParams {
	for k, v := range params {
		switch v.(type) {
		case string:
			params[k] = fmt.Sprintf("\"%s\"", v)
		}

	}
	return params
}

// Query - Accepts a params map and returns a map for use with the mgo `find()`
// function.
func (h *HaigoQuery) Query(params HaigoParams) (map[string]interface{}, error) {

	// Create the template
	t, err := template.New("haigo").Parse(h.QueryString)
	if err != nil {
		return nil, err
	}

	// Buffer to capture string
	buf := new(bytes.Buffer)

	// Execute template
	err = t.Execute(buf, sanitizeParams(params))
	if err != nil {
		return nil, err
	}

	// Unmarshal JSON into Map
	var m map[string]interface{}
	err = json.Unmarshal(buf.Bytes(), &m)
	if err != nil {
		return nil, err
	}

	return m, nil
}

// LoadQueryFile - Reads in Mongo Query File for use with Haigo.
func LoadQueryFile(file string) (*HaigoFile, error) {
	return parseMongoFile(file)
}
<|endoftext|>


<commit_before>//hello<commit_msg>Add a hello world console output<commit_after>//hello

package main

import "fmt"

func main() {
	fmt.Println("Hello world")
}<|endoftext|>


<commit_before>package util

import (
	"crypto/rand"
	"math/big"
	"net/http"
	"time"

	"github.com/jetstack/cert-manager/pkg/logs"
)

// RetryBackoff is the ACME client RetryBackoff that filters rate limit errors to our retry loop
// inspired by acme/http.go
func RetryBackoff(n int, r *http.Request, res *http.Response) time.Duration {
	var jitter time.Duration
	if x, err := rand.Int(rand.Reader, big.NewInt(1000)); err == nil {
		// Set the minimum to 1ms to avoid a case where
		// an invalid Retry-After value is parsed into 0 below,
		// resulting in the 0 returned value which would unintentionally
		// stop the retries.
		jitter = (1 + time.Duration(x.Int64())) * time.Millisecond
	}
	if _, ok := res.Header["Retry-After"]; ok {
		// if Retry-After is set we should
		// error and let the cert-manager logic retry instead
		return -1
	}

	// don't retry more than 10 times
	if n > 10 {
		return -1
	}

	// classic backoff here in case we got no reply
	// eg. flakes
	if n < 1 {
		n = 1
	}

	logs.Log.V(logs.DebugLevel).WithValues("backoff", d).Info("Hit an error in golang.org/x/crypto/acme, retrying")

	d := time.Duration(1<<uint(n-1))*time.Second + jitter
	if d > 10*time.Second {
		return 10 * time.Second
	}
	return d
}
<commit_msg>Fix place of log<commit_after>package util

import (
	"crypto/rand"
	"math/big"
	"net/http"
	"time"

	"github.com/jetstack/cert-manager/pkg/logs"
)

// RetryBackoff is the ACME client RetryBackoff that filters rate limit errors to our retry loop
// inspired by acme/http.go
func RetryBackoff(n int, r *http.Request, res *http.Response) time.Duration {
	var jitter time.Duration
	if x, err := rand.Int(rand.Reader, big.NewInt(1000)); err == nil {
		// Set the minimum to 1ms to avoid a case where
		// an invalid Retry-After value is parsed into 0 below,
		// resulting in the 0 returned value which would unintentionally
		// stop the retries.
		jitter = (1 + time.Duration(x.Int64())) * time.Millisecond
	}
	if _, ok := res.Header["Retry-After"]; ok {
		// if Retry-After is set we should
		// error and let the cert-manager logic retry instead
		return -1
	}

	// don't retry more than 10 times
	if n > 10 {
		return -1
	}

	// classic backoff here in case we got no reply
	// eg. flakes
	if n < 1 {
		n = 1
	}

	d := time.Duration(1<<uint(n-1))*time.Second + jitter
	logs.Log.V(logs.DebugLevel).WithValues("backoff", d).Info("Hit an error in golang.org/x/crypto/acme, retrying")
	if d > 10*time.Second {
		return 10 * time.Second
	}
	return d
}
<|endoftext|>


<commit_before>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package builder

import (
	"os"
	"path/filepath"

	"mynewt.apache.org/newt/newt/pkg"
	"mynewt.apache.org/newt/newt/target"
	"mynewt.apache.org/newt/newt/toolchain"
	"mynewt.apache.org/newt/util"
)

type Builder struct {
	Packages map[*pkg.LocalPackage]*BuildPackage
	features map[string]bool
	apis     map[string]*BuildPackage

	Bsp          *pkg.BspPackage
	compilerPkg  *pkg.LocalPackage
	compilerInfo *toolchain.CompilerInfo

	target *target.Target
}

func NewBuilder(target *target.Target) (*Builder, error) {
	b := &Builder{}

	if err := b.Init(target); err != nil {
		return nil, err
	}

	return b, nil
}

func (b *Builder) Init(target *target.Target) error {
	b.target = target

	b.Packages = map[*pkg.LocalPackage]*BuildPackage{}
	b.features = map[string]bool{}
	b.apis = map[string]*BuildPackage{}

	return nil
}

func (b *Builder) Features() map[string]bool {
	return b.features
}

func (b *Builder) AddFeature(feature string) {
	b.features[feature] = true
}

func (b *Builder) AddPackage(npkg *pkg.LocalPackage) *BuildPackage {
	// Don't allow nil entries to the map
	if npkg == nil {
		panic("Cannot add nil package builder map")
	}

	bpkg := b.Packages[npkg]
	if bpkg == nil {
		bpkg = NewBuildPackage(npkg)
		b.Packages[npkg] = bpkg
	}

	return bpkg
}

// @return bool                 true if this is a new API.
func (b *Builder) AddApi(apiString string, bpkg *BuildPackage) bool {
	curBpkg := b.apis[apiString]
	if curBpkg == nil {
		b.apis[apiString] = bpkg
		return true
	} else {
		if curBpkg != bpkg {
			util.StatusMessage(util.VERBOSITY_QUIET,
				"Warning: API conflict: %s <-> %s\n", curBpkg.Name(),
				bpkg.Name())
		}
		return false
	}
}

func (b *Builder) loadDeps() error {
	// Circularly resolve dependencies, identities, APIs, and required APIs
	// until no new ones exist.
	for {
		reprocess := false
		for _, bpkg := range b.Packages {
			resolved, err := bpkg.Resolve(b)
			if err != nil {
				return err
			}

			if !resolved {
				reprocess = true
			}
		}

		if !reprocess {
			break
		}
	}

	util.StatusMessage(util.VERBOSITY_VERBOSE, "Building with the following "+
		"feature set: ["+b.FeatureString()+"]\n")

	return nil
}

// Recursively compiles all the .c and .s files in the specified directory.
// Architecture-specific files are also compiled.
func buildDir(srcDir string, c *toolchain.Compiler, arch string,
	ignDirs []string) error {

	// Quietly succeed if the source directory doesn't exist.
	if util.NodeNotExist(srcDir) {
		return nil
	}

	util.StatusMessage(util.VERBOSITY_VERBOSE,
		"compiling src in base directory: %s\n", srcDir)

	// Start from the source directory.
	if err := os.Chdir(srcDir); err != nil {
		return util.NewNewtError(err.Error())
	}

	// Don't recurse into destination directories.
	ignDirs = append(ignDirs, "obj")
	ignDirs = append(ignDirs, "bin")

	// Ignore architecture-specific source files for now.  Use a temporary
	// string slice here so that the "arch" directory is not ignored in the
	// subsequent architecture-specific compile phase.
	if err := c.RecursiveCompile(toolchain.COMPILER_TYPE_C,
		append(ignDirs, "arch")); err != nil {

		return err
	}

	archDir := srcDir + "/arch/" + arch + "/"
	util.StatusMessage(util.VERBOSITY_VERBOSE,
		"compiling architecture specific src pkgs in directory: %s\n",
		archDir)

	if util.NodeExist(archDir) {
		if err := os.Chdir(archDir); err != nil {
			return util.NewNewtError(err.Error())
		}
		if err := c.RecursiveCompile(toolchain.COMPILER_TYPE_C,
			ignDirs); err != nil {

			return err
		}

		// compile assembly sources in recursive compile as well
		if err := c.RecursiveCompile(toolchain.COMPILER_TYPE_ASM,
			ignDirs); err != nil {

			return err
		}
	}

	return nil
}

// Compiles and archives a package.
func (b *Builder) buildPackage(bpkg *BuildPackage) error {
	srcDir := bpkg.BasePath() + "/src"
	if util.NodeNotExist(srcDir) {
		// Nothing to compile.
		return nil
	}

	c, err := toolchain.NewCompiler(b.compilerPkg.BasePath(),
		b.PkgBinDir(bpkg.Name()), b.target.BuildProfile)
	if err != nil {
		return err
	}
	c.AddInfo(b.compilerInfo)

	ci, err := bpkg.CompilerInfo(b)
	if err != nil {
		return err
	}
	c.AddInfo(ci)

	// Build the package source in two phases:
	// 1. Non-test code.
	// 2. Test code (if the "test" feature is enabled).
	//
	// This is done in two passes because the structure of
	// architecture-specific directories is different for normal code and test
	// code, and not easy to generalize into a single operation:
	//     * src/arch/<target-arch>
	//     * src/test/arch/<target-arch>
	if err = buildDir(srcDir, c, b.target.Arch, []string{"test"}); err != nil {
		return err
	}
	if b.features["test"] {
		testSrcDir := srcDir + "/test"
		if err = buildDir(testSrcDir, c, b.target.Arch, nil); err != nil {
			return err
		}
	}

	// Create a static library ("archive").
	if err := os.Chdir(bpkg.BasePath() + "/"); err != nil {
		return util.NewNewtError(err.Error())
	}
	archiveFile := b.ArchivePath(bpkg.Name())
	if err = c.CompileArchive(archiveFile); err != nil {
		return err
	}

	return nil
}

func (b *Builder) link(elfName string) error {
	c, err := toolchain.NewCompiler(b.compilerPkg.BasePath(),
		b.PkgBinDir(elfName), b.target.BuildProfile)
	if err != nil {
		return err
	}

	pkgNames := []string{}
	for _, bpkg := range b.Packages {
		archivePath := b.ArchivePath(bpkg.Name())
		if util.NodeExist(archivePath) {
			pkgNames = append(pkgNames, archivePath)
		}
	}

	if b.Bsp.LinkerScript != "" {
		c.LinkerScript = b.Bsp.BasePath() + b.Bsp.LinkerScript
	}
	err = c.CompileElf(elfName, pkgNames)
	if err != nil {
		return err
	}

	return nil
}

// Populates the builder with all the packages that need to be built, and
// configures each package's build settings.  After this function executes,
// packages are ready to be built.
func (b *Builder) PrepBuild() error {
	if b.Bsp != nil {
		// Already prepped
		return nil
	}
	// Collect the seed packages.
	bspPkg := b.target.Bsp()
	if bspPkg == nil {
		if b.target.BspName == "" {
			return util.NewNewtError("BSP package not specified by target")
		} else {
			return util.NewNewtError("BSP package not found: " +
				b.target.BspName)
		}
	}

	b.Bsp = pkg.NewBspPackage(bspPkg)
	compilerPkg := b.resolveCompiler()
	if compilerPkg == nil {
		if b.Bsp.CompilerName == "" {
			return util.NewNewtError("Compiler package not specified by BSP")
		} else {
			return util.NewNewtError("Compiler package not found: " +
				b.Bsp.CompilerName)
		}
	}

	// An app package is not required (e.g., unit tests).
	appPkg := b.target.App()

	// Seed the builder with the app (if present), bsp, compiler, and target
	// packages.

	var appBpkg *BuildPackage
	if appPkg != nil {
		appBpkg = b.Packages[appPkg]
		if appBpkg == nil {
			appBpkg = b.AddPackage(appPkg)
		}
	}

	bspBpkg := b.Packages[bspPkg]
	if bspBpkg == nil {
		bspBpkg = b.AddPackage(bspPkg)
	}

	compilerBpkg := b.Packages[compilerPkg]
	if compilerBpkg == nil {
		compilerBpkg = b.AddPackage(compilerPkg)
	}

	targetBpkg := b.AddPackage(b.target.Package())

	// Populate the full set of packages to be built and resolve the feature
	// set.
	if err := b.loadDeps(); err != nil {
		return err
	}

	// Terminate if any package has an unmet API requirement.
	if err := b.verifyApisSatisfied(); err != nil {
		return err
	}

	// Populate the base set of compiler flags.  Flags from the following
	// packages get applied to every source file:
	//     * app (if present)
	//     * bsp
	//     * compiler
	//     * target

	baseCi := toolchain.NewCompilerInfo()

	// App flags.
	if appBpkg != nil {
		appCi, err := appBpkg.CompilerInfo(b)
		if err != nil {
			return err
		}
		baseCi.AddCompilerInfo(appCi)
	}

	// Bsp flags.
	bspCi, err := bspBpkg.CompilerInfo(b)
	if err != nil {
		return err
	}
	baseCi.AddCompilerInfo(bspCi)

	// Target flags.
	targetCi, err := targetBpkg.CompilerInfo(b)
	if err != nil {
		return err
	}
	baseCi.AddCompilerInfo(targetCi)

	// Note: Compiler flags get added when compiler is created.

	// Read the BSP configuration.  These settings are necessary for the link
	// step.
	b.Bsp.Reload(b.Features())

	b.compilerPkg = compilerPkg
	b.compilerInfo = baseCi

	return nil
}

func (b *Builder) Build() error {
	if b.target.App() == nil {
		if b.target.AppName == "" {
			return util.NewNewtError("App package not specified by target")
		} else {
			return util.NewNewtError("App package not found: " +
				b.target.AppName)
		}
	}

	// Populate the package and feature sets and calculate the base compiler
	// flags.
	err := b.PrepBuild()
	if err != nil {
		return err
	}

	// XXX: If any yml files have changed, a full rebuild is required.  We
	// don't currently check this.

	for _, bpkg := range b.Packages {
		err = b.buildPackage(bpkg)
		if err != nil {
			return err
		}
	}

	err = b.link(b.AppElfPath())
	if err != nil {
		return err
	}

	return nil
}

func (b *Builder) Test(p *pkg.LocalPackage) error {
	// Seed the builder with the package under test.
	testBpkg := b.AddPackage(p)

	// A few features are automatically supported when the test command is
	// used:
	//     * test:      ensures that the test code gets compiled.
	//     * selftest:  indicates that there is no app.
	b.AddFeature("test")
	b.AddFeature("selftest")

	// Populate the package and feature sets and calculate the base compiler
	// flags.
	err := b.PrepBuild()
	if err != nil {
		return err
	}

	// Define the PKG_TEST symbol while the package under test is being
	// compiled.  This symbol enables the appropriate main function that
	// usually comes from an app.
	testPkgCi, err := testBpkg.CompilerInfo(b)
	if err != nil {
		return err
	}
	testPkgCi.Cflags = append(testPkgCi.Cflags, "-DPKG_TEST")

	// XXX: If any yml files have changed, a full rebuild is required.  We
	// don't currently check this.

	for _, bpkg := range b.Packages {
		err = b.buildPackage(bpkg)
		if err != nil {
			return err
		}
	}

	testFilename := b.TestExePath(p.Name())
	err = b.link(testFilename)
	if err != nil {
		return err
	}

	// Run the tests.
	util.StatusMessage(util.VERBOSITY_DEFAULT, "Testing package %s\n", p.Name())

	if err := os.Chdir(filepath.Dir(testFilename)); err != nil {
		return err
	}

	o, err := util.ShellCommand(testFilename)
	if err != nil {
		util.StatusMessage(util.VERBOSITY_DEFAULT, "%s", string(o))

		return util.NewNewtError("Test crashed: " + testFilename)
	}

	util.StatusMessage(util.VERBOSITY_VERBOSE, "%s", string(o))
	util.StatusMessage(util.VERBOSITY_DEFAULT, "Test %s ok!\n", testFilename)

	return nil
}

func (b *Builder) Clean() error {
	path := b.BinDir()
	util.StatusMessage(util.VERBOSITY_VERBOSE, "Cleaning directory %s\n", path)
	err := os.RemoveAll(path)
	return err
}
<commit_msg>Specify cpp symbol indicating target architecture.<commit_after>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package builder

import (
	"os"
	"path/filepath"

	"mynewt.apache.org/newt/newt/pkg"
	"mynewt.apache.org/newt/newt/target"
	"mynewt.apache.org/newt/newt/toolchain"
	"mynewt.apache.org/newt/util"
)

type Builder struct {
	Packages map[*pkg.LocalPackage]*BuildPackage
	features map[string]bool
	apis     map[string]*BuildPackage

	Bsp          *pkg.BspPackage
	compilerPkg  *pkg.LocalPackage
	compilerInfo *toolchain.CompilerInfo

	target *target.Target
}

func NewBuilder(target *target.Target) (*Builder, error) {
	b := &Builder{}

	if err := b.Init(target); err != nil {
		return nil, err
	}

	return b, nil
}

func (b *Builder) Init(target *target.Target) error {
	b.target = target

	b.Packages = map[*pkg.LocalPackage]*BuildPackage{}
	b.features = map[string]bool{}
	b.apis = map[string]*BuildPackage{}

	return nil
}

func (b *Builder) Features() map[string]bool {
	return b.features
}

func (b *Builder) AddFeature(feature string) {
	b.features[feature] = true
}

func (b *Builder) AddPackage(npkg *pkg.LocalPackage) *BuildPackage {
	// Don't allow nil entries to the map
	if npkg == nil {
		panic("Cannot add nil package builder map")
	}

	bpkg := b.Packages[npkg]
	if bpkg == nil {
		bpkg = NewBuildPackage(npkg)
		b.Packages[npkg] = bpkg
	}

	return bpkg
}

// @return bool                 true if this is a new API.
func (b *Builder) AddApi(apiString string, bpkg *BuildPackage) bool {
	curBpkg := b.apis[apiString]
	if curBpkg == nil {
		b.apis[apiString] = bpkg
		return true
	} else {
		if curBpkg != bpkg {
			util.StatusMessage(util.VERBOSITY_QUIET,
				"Warning: API conflict: %s <-> %s\n", curBpkg.Name(),
				bpkg.Name())
		}
		return false
	}
}

func (b *Builder) loadDeps() error {
	// Circularly resolve dependencies, identities, APIs, and required APIs
	// until no new ones exist.
	for {
		reprocess := false
		for _, bpkg := range b.Packages {
			resolved, err := bpkg.Resolve(b)
			if err != nil {
				return err
			}

			if !resolved {
				reprocess = true
			}
		}

		if !reprocess {
			break
		}
	}

	util.StatusMessage(util.VERBOSITY_VERBOSE, "Building with the following "+
		"feature set: ["+b.FeatureString()+"]\n")

	return nil
}

// Recursively compiles all the .c and .s files in the specified directory.
// Architecture-specific files are also compiled.
func buildDir(srcDir string, c *toolchain.Compiler, arch string,
	ignDirs []string) error {

	// Quietly succeed if the source directory doesn't exist.
	if util.NodeNotExist(srcDir) {
		return nil
	}

	util.StatusMessage(util.VERBOSITY_VERBOSE,
		"compiling src in base directory: %s\n", srcDir)

	// Start from the source directory.
	if err := os.Chdir(srcDir); err != nil {
		return util.NewNewtError(err.Error())
	}

	// Don't recurse into destination directories.
	ignDirs = append(ignDirs, "obj")
	ignDirs = append(ignDirs, "bin")

	// Ignore architecture-specific source files for now.  Use a temporary
	// string slice here so that the "arch" directory is not ignored in the
	// subsequent architecture-specific compile phase.
	if err := c.RecursiveCompile(toolchain.COMPILER_TYPE_C,
		append(ignDirs, "arch")); err != nil {

		return err
	}

	archDir := srcDir + "/arch/" + arch + "/"
	util.StatusMessage(util.VERBOSITY_VERBOSE,
		"compiling architecture specific src pkgs in directory: %s\n",
		archDir)

	if util.NodeExist(archDir) {
		if err := os.Chdir(archDir); err != nil {
			return util.NewNewtError(err.Error())
		}
		if err := c.RecursiveCompile(toolchain.COMPILER_TYPE_C,
			ignDirs); err != nil {

			return err
		}

		// compile assembly sources in recursive compile as well
		if err := c.RecursiveCompile(toolchain.COMPILER_TYPE_ASM,
			ignDirs); err != nil {

			return err
		}
	}

	return nil
}

// Compiles and archives a package.
func (b *Builder) buildPackage(bpkg *BuildPackage) error {
	srcDir := bpkg.BasePath() + "/src"
	if util.NodeNotExist(srcDir) {
		// Nothing to compile.
		return nil
	}

	c, err := toolchain.NewCompiler(b.compilerPkg.BasePath(),
		b.PkgBinDir(bpkg.Name()), b.target.BuildProfile)
	if err != nil {
		return err
	}
	c.AddInfo(b.compilerInfo)

	ci, err := bpkg.CompilerInfo(b)
	if err != nil {
		return err
	}
	c.AddInfo(ci)

	// Build the package source in two phases:
	// 1. Non-test code.
	// 2. Test code (if the "test" feature is enabled).
	//
	// This is done in two passes because the structure of
	// architecture-specific directories is different for normal code and test
	// code, and not easy to generalize into a single operation:
	//     * src/arch/<target-arch>
	//     * src/test/arch/<target-arch>
	if err = buildDir(srcDir, c, b.target.Arch, []string{"test"}); err != nil {
		return err
	}
	if b.features["test"] {
		testSrcDir := srcDir + "/test"
		if err = buildDir(testSrcDir, c, b.target.Arch, nil); err != nil {
			return err
		}
	}

	// Create a static library ("archive").
	if err := os.Chdir(bpkg.BasePath() + "/"); err != nil {
		return util.NewNewtError(err.Error())
	}
	archiveFile := b.ArchivePath(bpkg.Name())
	if err = c.CompileArchive(archiveFile); err != nil {
		return err
	}

	return nil
}

func (b *Builder) link(elfName string) error {
	c, err := toolchain.NewCompiler(b.compilerPkg.BasePath(),
		b.PkgBinDir(elfName), b.target.BuildProfile)
	if err != nil {
		return err
	}

	pkgNames := []string{}
	for _, bpkg := range b.Packages {
		archivePath := b.ArchivePath(bpkg.Name())
		if util.NodeExist(archivePath) {
			pkgNames = append(pkgNames, archivePath)
		}
	}

	if b.Bsp.LinkerScript != "" {
		c.LinkerScript = b.Bsp.BasePath() + b.Bsp.LinkerScript
	}
	err = c.CompileElf(elfName, pkgNames)
	if err != nil {
		return err
	}

	return nil
}

// Populates the builder with all the packages that need to be built, and
// configures each package's build settings.  After this function executes,
// packages are ready to be built.
func (b *Builder) PrepBuild() error {
	if b.Bsp != nil {
		// Already prepped
		return nil
	}
	// Collect the seed packages.
	bspPkg := b.target.Bsp()
	if bspPkg == nil {
		if b.target.BspName == "" {
			return util.NewNewtError("BSP package not specified by target")
		} else {
			return util.NewNewtError("BSP package not found: " +
				b.target.BspName)
		}
	}

	b.Bsp = pkg.NewBspPackage(bspPkg)
	compilerPkg := b.resolveCompiler()
	if compilerPkg == nil {
		if b.Bsp.CompilerName == "" {
			return util.NewNewtError("Compiler package not specified by BSP")
		} else {
			return util.NewNewtError("Compiler package not found: " +
				b.Bsp.CompilerName)
		}
	}

	// An app package is not required (e.g., unit tests).
	appPkg := b.target.App()

	// Seed the builder with the app (if present), bsp, compiler, and target
	// packages.

	var appBpkg *BuildPackage
	if appPkg != nil {
		appBpkg = b.Packages[appPkg]
		if appBpkg == nil {
			appBpkg = b.AddPackage(appPkg)
		}
	}

	bspBpkg := b.Packages[bspPkg]
	if bspBpkg == nil {
		bspBpkg = b.AddPackage(bspPkg)
	}

	compilerBpkg := b.Packages[compilerPkg]
	if compilerBpkg == nil {
		compilerBpkg = b.AddPackage(compilerPkg)
	}

	targetBpkg := b.AddPackage(b.target.Package())

	// Populate the full set of packages to be built and resolve the feature
	// set.
	if err := b.loadDeps(); err != nil {
		return err
	}

	// Terminate if any package has an unmet API requirement.
	if err := b.verifyApisSatisfied(); err != nil {
		return err
	}

	// Populate the base set of compiler flags.  Flags from the following
	// packages get applied to every source file:
	//     * app (if present)
	//     * bsp
	//     * compiler (not added here)
	//     * target

	baseCi := toolchain.NewCompilerInfo()

	// App flags.
	if appBpkg != nil {
		appCi, err := appBpkg.CompilerInfo(b)
		if err != nil {
			return err
		}
		baseCi.AddCompilerInfo(appCi)
	}

	// Bsp flags.
	bspCi, err := bspBpkg.CompilerInfo(b)
	if err != nil {
		return err
	}
	baseCi.AddCompilerInfo(bspCi)

	// Target flags.
	targetCi, err := targetBpkg.CompilerInfo(b)
	if err != nil {
		return err
	}

	// Define a cpp symbol indicating the target architecture.
	// XXX: This should probably happen in the bsp after we move the arch field
	// from target to bsp.
	targetCi.Cflags = append(targetCi.Cflags, "-DARCH_"+b.target.Arch)

	baseCi.AddCompilerInfo(targetCi)

	// Note: Compiler flags get added when compiler is created.

	// Read the BSP configuration.  These settings are necessary for the link
	// step.
	b.Bsp.Reload(b.Features())

	b.compilerPkg = compilerPkg
	b.compilerInfo = baseCi

	return nil
}

func (b *Builder) Build() error {
	if b.target.App() == nil {
		if b.target.AppName == "" {
			return util.NewNewtError("App package not specified by target")
		} else {
			return util.NewNewtError("App package not found: " +
				b.target.AppName)
		}
	}

	// Populate the package and feature sets and calculate the base compiler
	// flags.
	err := b.PrepBuild()
	if err != nil {
		return err
	}

	// XXX: If any yml files have changed, a full rebuild is required.  We
	// don't currently check this.

	for _, bpkg := range b.Packages {
		err = b.buildPackage(bpkg)
		if err != nil {
			return err
		}
	}

	err = b.link(b.AppElfPath())
	if err != nil {
		return err
	}

	return nil
}

func (b *Builder) Test(p *pkg.LocalPackage) error {
	// Seed the builder with the package under test.
	testBpkg := b.AddPackage(p)

	// A few features are automatically supported when the test command is
	// used:
	//     * test:      ensures that the test code gets compiled.
	//     * selftest:  indicates that there is no app.
	b.AddFeature("test")
	b.AddFeature("selftest")

	// Populate the package and feature sets and calculate the base compiler
	// flags.
	err := b.PrepBuild()
	if err != nil {
		return err
	}

	// Define the PKG_TEST symbol while the package under test is being
	// compiled.  This symbol enables the appropriate main function that
	// usually comes from an app.
	testPkgCi, err := testBpkg.CompilerInfo(b)
	if err != nil {
		return err
	}
	testPkgCi.Cflags = append(testPkgCi.Cflags, "-DPKG_TEST")

	// XXX: If any yml files have changed, a full rebuild is required.  We
	// don't currently check this.

	for _, bpkg := range b.Packages {
		err = b.buildPackage(bpkg)
		if err != nil {
			return err
		}
	}

	testFilename := b.TestExePath(p.Name())
	err = b.link(testFilename)
	if err != nil {
		return err
	}

	// Run the tests.
	util.StatusMessage(util.VERBOSITY_DEFAULT, "Testing package %s\n", p.Name())

	if err := os.Chdir(filepath.Dir(testFilename)); err != nil {
		return err
	}

	o, err := util.ShellCommand(testFilename)
	if err != nil {
		util.StatusMessage(util.VERBOSITY_DEFAULT, "%s", string(o))

		return util.NewNewtError("Test crashed: " + testFilename)
	}

	util.StatusMessage(util.VERBOSITY_VERBOSE, "%s", string(o))
	util.StatusMessage(util.VERBOSITY_DEFAULT, "Test %s ok!\n", testFilename)

	return nil
}

func (b *Builder) Clean() error {
	path := b.BinDir()
	util.StatusMessage(util.VERBOSITY_VERBOSE, "Cleaning directory %s\n", path)
	err := os.RemoveAll(path)
	return err
}
<|endoftext|>