<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"

	"github.com/autogrow/go-jelly/ig"
)

type creds struct {
	User string `json:"user"`
	Pass string `json:"pass"`
}

func main() {
	var listDevices, listGrowrooms bool
	var id, gr string
	var printReadings, fmtJSON bool
	flag.BoolVar(&listDevices, "l", false, "list known devices")
	flag.BoolVar(&listGrowrooms, "g", false, "list growrooms")
	flag.StringVar(&id, "id", "", "serial number to work with")
	flag.StringVar(&gr, "gr", "", "growroom name to work with")
	flag.BoolVar(&printReadings, "r", false, "print readings")
	flag.BoolVar(&fmtJSON, "json", false, "format as JSON")
	flag.Parse()

	credsFile := os.Getenv("HOME") + "/.intelligrow/creds"
	creds, err := readCreds(credsFile)
	if err != nil {
		initCreds(credsFile)
		log.Fatalf("you need to enter your credentials in the file %s to continue", credsFile)
	}

	cl, err := ig.NewClient(creds.User, creds.Pass)
	if err != nil {
		log.Fatalf("failed to create IG client: %s", err)
	}

	switch {
	case listGrowrooms:
		fmt.Println("Growrooms:")
		for _, name := range cl.ListGrowrooms() {
			fmt.Printf("- %s", name)
		}

	case listDevices:
		fmt.Printf("%-12s %-18s %-10s %s", "Type", "ID", "Name", "Growroom")
		for _, d := range cl.Devices() {
			fmt.Printf("%-12s %-18s %-10s %s", d.Type, d.ID, d.DeviceName, d.Growroom)
		}

	case id != "" && printReadings:
		if doser, errd := cl.IntelliDoseBySerial(id); errd == nil {
			fmt.Printf("%-20s: %0.2f mS/cm²", "Nutrient", doser.Metrics.Ec)
			fmt.Printf("%-20s: %0.2f pH", "Acidity", doser.Metrics.PH)
			fmt.Printf("%-20s: %0.2f °C", "Water", doser.Metrics.NutTemp)
			return
		}

		if clim, errc := cl.IntelliClimateBySerial(id); errc == nil {
			fmt.Printf("%-20s: %0.2f °C", "Air Temp", clim.Metrics.AirTemp)
			fmt.Printf("%-20s: %0.2f %%H", "RH", clim.Metrics.Rh)
			fmt.Printf("%-20s: %0.2f kPa", "VPD", clim.Metrics.Vpd)
			fmt.Printf("%-20s: %0.2f ppm", "CO2", clim.Metrics.Co2)
			return
		}

		log.Fatalf("no device found with serial %s", id)

	case gr != "" && printReadings:
		room, ok := cl.Growroom(gr)
		if !ok {
			log.Fatalf("Growroom %s not found", gr)
		}

		if len(room.Devices.IntelliClimates) > 0 {
			fmt.Printf("%-20s: %0.2f °C", "Air Temp", room.Climate.AirTemp)
			fmt.Printf("%-20s: %0.2f %%H", "RH", room.Climate.RH)
			fmt.Printf("%-20s: %0.2f kPa", "VPD", room.Climate.VPD)
			fmt.Printf("%-20s: %0.2f ppm", "CO2", room.Climate.CO2)
		}

		if len(room.Devices.IntelliDoses) > 0 {
			fmt.Printf("%-20s: %0.2f mS/cm²", "Nutrient", room.Rootzone.EC)
			fmt.Printf("%-20s: %0.2f pH", "Acidity", room.Rootzone.PH)
			fmt.Printf("%-20s: %0.2f °C", "Water", room.Rootzone.Temp)
		}

	}
}

func initCreds(credsFile string) {
	data, err := json.Marshal(creds{})
	if err != nil {
		log.Fatalf("failed to encode empty creds file to put at %s", credsFile)
	}

	if err := os.Mkdir(filepath.Dir(credsFile), 0755); err != nil {
		log.Fatalf("failed to create creds file at %s: %s", credsFile, err)
	}

	if err := ioutil.WriteFile(credsFile, data, 0644); err != nil {
		log.Fatalf("failed to create creds file at %s: %s", credsFile, err)
	}
}

func readCreds(credsFile string) (creds, error) {
	creds := creds{}
	data, err := ioutil.ReadFile(credsFile)
	if err != nil {
		return creds, err
	}

	err = json.Unmarshal(data, &creds)
	if err != nil {
		log.Fatalf("failed to read creds file at %s: %s", credsFile, err)
	}

	if creds.Pass == "" || creds.User == "" {
		log.Fatalf("you need to enter your credentials in the file %s to continue", credsFile)
	}

	return creds, nil
}
<commit_msg>make sure to get data from API before printing readings, add JSON dump<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"

	"github.com/autogrow/go-jelly/ig"
)

type creds struct {
	User string `json:"user"`
	Pass string `json:"pass"`
}

func main() {
	var listDevices, listGrowrooms bool
	var id, gr string
	var printReadings, fmtJSON bool
	flag.BoolVar(&listDevices, "l", false, "list known devices")
	flag.BoolVar(&listGrowrooms, "g", false, "list growrooms")
	flag.StringVar(&id, "id", "", "serial number to work with")
	flag.StringVar(&gr, "gr", "", "growroom name to work with")
	flag.BoolVar(&printReadings, "r", false, "print readings")
	flag.BoolVar(&fmtJSON, "json", false, "format as JSON")
	flag.Parse()

	credsFile := os.Getenv("HOME") + "/.intelligrow/creds"
	creds, err := readCreds(credsFile)
	if err != nil {
		initCreds(credsFile)
		log.Fatalf("you need to enter your credentials in the file %s to continue", credsFile)
	}

	cl, err := ig.NewClient(creds.User, creds.Pass)
	if err != nil {
		log.Fatalf("failed to create IG client: %s", err)
	}

	switch {
	case listGrowrooms:
		fmt.Println("Growrooms:")
		for _, name := range cl.ListGrowrooms() {
			fmt.Printf("- %s\n", name)
		}

	case listDevices:
		fmt.Printf("%-12s %-18s %-10s %s\n", "Type", "ID", "Name", "Growroom")
		for _, d := range cl.Devices() {
			fmt.Printf("%-12s %-18s %-10s %s\n", d.Type, d.ID, d.DeviceName, d.Growroom)
		}

	case id != "":
		if doser, errd := cl.IntelliDoseBySerial(id); errd == nil {
			switch {
			case printReadings:
				_ = doser.GetMetrics()
				fmt.Printf("%-20s: %0.2f mS/cm²\n", "Nutrient", doser.Metrics.Ec)
				fmt.Printf("%-20s: %0.2f pH\n", "Acidity", doser.Metrics.PH)
				fmt.Printf("%-20s: %0.2f °C\n", "Water", doser.Metrics.NutTemp)
				return
			case fmtJSON:
				if err := doser.GetAll(); err != nil {
					log.Fatalf("failed to get the doser data: %s", err)
				}
				dumpJSON(doser)
			}
		}

		if clim, errc := cl.IntelliClimateBySerial(id); errc == nil {
			switch {
			case printReadings:
				_ = clim.GetMetrics()
				fmt.Printf("%-20s: %0.2f °C\n", "Air Temp", clim.Metrics.AirTemp)
				fmt.Printf("%-20s: %0.2f %%H\n", "RH", clim.Metrics.Rh)
				fmt.Printf("%-20s: %0.2f kPa\n", "VPD", clim.Metrics.Vpd)
				fmt.Printf("%-20s: %0.2f ppm\n", "CO2", clim.Metrics.Co2)
				return
			case fmtJSON:
				if err := clim.GetAll(); err != nil {
					log.Fatalf("failed to get the climate data: %s", err)
				}
				dumpJSON(clim)
			}
		}

		log.Fatalf("no device found with serial %s", id)

	case gr != "" && printReadings:
		room, ok := cl.Growroom(gr)

		if !ok {
			log.Fatalf("Growroom %s not found", gr)
		}

		if len(room.Devices.IntelliClimates) > 0 {
			fmt.Printf("%-20s: %0.2f °C\n", "Air Temp", room.Climate.AirTemp)
			fmt.Printf("%-20s: %0.2f %%H\n", "RH", room.Climate.RH)
			fmt.Printf("%-20s: %0.2f kPa\n", "VPD", room.Climate.VPD)
			fmt.Printf("%-20s: %0.2f ppm\n", "CO2", room.Climate.CO2)
		}

		if len(room.Devices.IntelliDoses) > 0 {
			fmt.Printf("%-20s: %0.2f mS/cm²\n", "Nutrient", room.Rootzone.EC)
			fmt.Printf("%-20s: %0.2f pH\n", "Acidity", room.Rootzone.PH)
			fmt.Printf("%-20s: %0.2f °C\n", "Water", room.Rootzone.Temp)
		}

	}
}

func initCreds(credsFile string) {
	data, err := json.Marshal(creds{})
	if err != nil {
		log.Fatalf("failed to encode empty creds file to put at %s", credsFile)
	}

	if err := os.Mkdir(filepath.Dir(credsFile), 0755); err != nil {
		log.Fatalf("failed to create creds file at %s: %s", credsFile, err)
	}

	if err := ioutil.WriteFile(credsFile, data, 0644); err != nil {
		log.Fatalf("failed to create creds file at %s: %s", credsFile, err)
	}
}

func readCreds(credsFile string) (creds, error) {
	creds := creds{}
	data, err := ioutil.ReadFile(credsFile)
	if err != nil {
		return creds, err
	}

	err = json.Unmarshal(data, &creds)
	if err != nil {
		log.Fatalf("failed to read creds file at %s: %s", credsFile, err)
	}

	if creds.Pass == "" || creds.User == "" {
		log.Fatalf("you need to enter your credentials in the file %s to continue", credsFile)
	}

	return creds, nil
}

func dumpJSON(v interface{}) {
	data, err := json.Marshal(v)
	if err != nil {
		log.Fatalf("failed to format JSON: %s", err)
	}

	fmt.Println(string(data))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"os/user"
	"strconv"
	"strings"

	"github.com/ianremmler/dgrl"
	"github.com/ianremmler/lit"
)

const usage = `usage:

lit [help | usage]              Show usage
lit init                        Initialize new issue tracker
lit new [<num>]                 Create num new issues (default: 1)
lit list [<sort>] <spec>        Show list of specified issues (default: open)
lit id [<sort>] <spec>          List ids of specified issues (default: open)
lit show [<sort>] <spec>        Show specified issues (default: open)
lit set <key> <val> <spec>      Set value for key in specified issues
lit tag (add|del) <tag> <spec>  Add or delete tag in specified issues
lit comment <id> [<text>]       Add issue comment (open editor if no text given)
lit edit <spec>                 Edit specified issues (default: open)
lit close <spec>                Close specified issues
lit reopen <spec>               Reopen specified issues

sort: (sortby|rsortby) <key>  Sort (reverse if rsortby) based on key

spec: all | <ids> | (with|without) <key> [<val>] | (less|greater) <key> <val>
	Specifies which issues to operate on
	Default open spec does not apply if input is piped
	Use 'comment' key to filter by comment contents and times`

const (
	// id, closed?, priority, assigned, tags, summary
	listFmt = "%-8.8s %-1.1s %-1.1s %-8.8s %-17.17s %s"
)

var (
	args        = os.Args[1:]
	it          = lit.New()
	listHdr     = fmt.Sprintf(listFmt, "id", "c", "p", "assigned", "tags", "summary")
	username    = "?"
	isStdinPipe = false
	cmd         string
)

func main() {
	log.SetFlags(0)
	log.SetPrefix("lit: ")

	if userEnv := os.Getenv("LIT_USER"); userEnv != "" {
		username = userEnv
	} else {
		if user, err := user.Current(); err == nil {
			if host, err := os.Hostname(); err == nil {
				username = fmt.Sprintf("%s@%s", user.Username, host)
			}
		}
	}

	// append args piped in from stdin
	if stat, err := os.Stdin.Stat(); err == nil && stat.Mode()&os.ModeNamedPipe != 0 {
		isStdinPipe = true
		if stdin, err := ioutil.ReadAll(os.Stdin); err == nil {
			stdinArgs := strings.Fields(string(stdin))
			args = append(args, stdinArgs...)
		}
	}

	if len(args) > 0 {
		cmd = args[0]
		args = args[1:]
	}
	switch cmd {
	case "", "-h", "-help", "--help", "help", "-u", "-usage", "--usage", "usage":
		usageCmd()
	case "list":
		listCmd()
	case "init":
		initCmd()
	case "new":
		newCmd()
	case "id":
		idCmd()
	case "show":
		showCmd()
	case "set":
		setCmd()
	case "tag":
		tagCmd()
	case "comment":
		commentCmd()
	case "edit":
		editCmd()
	case "close", "reopen":
		closeCmd()
	default:
		log.Fatalln(cmd + " is not a valid command")
	}
}

func usageCmd() {
	fmt.Println(usage)
}

func initCmd() {
	err := it.Init()
	checkErr(err)
}

func newCmd() {
	numIssues := 1
	if len(args) > 0 {
		num, err := strconv.ParseUint(args[0], 10, 16)
		checkErr(err)
		numIssues = int(num)
	}
	loadIssues()
	for i := 0; i < numIssues; i++ {
		id, err := it.NewIssue(username)
		checkErr(err)
		fmt.Println(id)
	}
	storeIssues()
}

func listCmd() {
	loadIssues()
	doSort, key, doAscend := dispOpts()
	ids := specIds(!isStdinPipe)
	if doSort {
		it.Sort(ids, key, doAscend)
	}
	fmt.Println(listHdr)
	for _, id := range ids {
		issue := it.Issue(id)
		if issue != nil {
			fmt.Println(listInfo(id, issue))
		}
	}
}

func idCmd() {
	loadIssues()
	doSort, key, doAscend := dispOpts()
	ids := specIds(!isStdinPipe)
	if doSort {
		it.Sort(ids, key, doAscend)
	}
	for _, id := range ids {
		if it.Issue(id) != nil {
			fmt.Println(id)
		}
	}
}

func showCmd() {
	loadIssues()
	doSort, key, doAscend := dispOpts()
	ids := specIds(!isStdinPipe)
	if doSort {
		it.Sort(ids, key, doAscend)
	}
	for _, id := range ids {
		fmt.Println(it.Issue(id))
	}
}

func setCmd() {
	if len(args) < 3 {
		log.Fatalln("set: you must specify a key, value, and spec")
	}
	key, val := args[0], args[1]
	args = args[2:]
	loadIssues()
	stamp := lit.Stamp(username)
	for _, id := range specIds(false) {
		issue := it.Issue(id)
		if issue == nil {
			log.Printf("set: error finding issue %s\n", id)
			continue
		}
		ok := lit.Set(issue, key, val)
		ok = ok && lit.Set(issue, "updated", stamp)
		if !ok {
			log.Printf("set: error updating fields in issue %s\n", id)
			continue
		}
	}
	storeIssues()
}

func tagCmd() {
	if len(args) < 3 {
		log.Fatalln("tag: you must specify an operation, tag, and spec")
	}
	op, tag := args[0], args[1]
	if op != "add" && op != "del" {
		log.Fatalf("tag: is not a valid operation\n")
	}
	doAdd := (op == "add")

	args = args[2:]
	loadIssues()
	stamp := lit.Stamp(username)
	for _, id := range specIds(false) {
		issue := it.Issue(id)
		if issue == nil {
			log.Printf("tag: error finding issue %s\n", id)
			continue
		}
		ok := lit.ModifyTag(issue, tag, doAdd)
		ok = ok && lit.Set(issue, "updated", stamp)
		if !ok {
			log.Printf("tag: error updating fields in issue %s\n", id)
			continue
		}
	}
	storeIssues()
}

func editCmd() {
	editor := getEditor()
	if editor == "" {
		log.Fatalln("edit: VISUAL or EDITOR environment variable must be set")
	}

	loadIssues()

	// create temp file
	tempFile, err := ioutil.TempFile("", "lit-")
	checkErr(err)
	filename := tempFile.Name()

	// load issue content into temp file
	ids := specIds(!isStdinPipe)
	toEdit := dgrl.NewRoot()
	for _, id := range ids {
		issue := it.Issue(id)
		if issue == nil {
			log.Printf("edit: error finding issue %s\n", id)
			continue
		}
		toEdit.Append(issue)
	}
	err = toEdit.Write(tempFile)
	checkErr(err)
	tempFile.Close()

	// get original file state
	origStat, err := os.Stat(filename)
	checkErr(err)

	// launch editor
	ed := exec.Command(editor, filename)
	ed.Stdin, ed.Stdout, ed.Stderr = os.Stdin, os.Stdout, os.Stderr
	err = ed.Run()
	checkErr(err)

	// get updated file state, compare to original
	newStat, err := os.Stat(filename)
	checkErr(err)
	if newStat.ModTime() == origStat.ModTime() {
		log.Fatalln("edit: file unchanged")
	}

	// parse issue from temp file
	tempFile, err = os.Open(filename)
	checkErr(err)
	edited := dgrl.NewParser().Parse(tempFile)
	tempFile.Close()
	if edited == nil {
		log.Fatalln("edit: error parsing file")
	}

	// update issues if we find a match
	didUpdate := false
	stamp := lit.Stamp(username)
	for _, id := range ids {
		issue := it.Issue(id)
		if issue == nil {
			// already printed error, so don't repeat here
			continue
		}
		for _, node := range edited.Kids() {
			if node.Type() == dgrl.BranchType && strings.HasPrefix(node.Key(), id) {
				if editedIssue, ok := node.(*dgrl.Branch); ok {
					*issue = *editedIssue
					if !lit.Set(issue, "updated", stamp) {
						log.Printf("edit: error setting update time for issue %s\n", id)
						continue
					}
					didUpdate = true
					break
				}
			}
		}
	}
	if !didUpdate {
		log.Fatalln("edit: did not update anything")
	}

	storeIssues()
}

func closeCmd() {
	loadIssues()
	stamp := lit.Stamp(username)
	for _, id := range specIds(false) {
		issue := it.Issue(id)
		if issue == nil {
			log.Printf("%s: error finding issue %s\n", cmd, id)
			continue
		}
		closedStamp := ""
		if cmd == "close" {
			closedStamp = stamp
		}
		ok := lit.Set(issue, "closed", closedStamp)
		ok = ok && lit.Set(issue, "updated", stamp)
		if !ok {
			log.Printf("%s: error updating fields for issue %s\n", cmd, id)
			continue
		}
	}
	storeIssues()
}

func commentCmd() {
	if len(args) < 1 {
		log.Fatalln("comment: you must specify an issue to comment on")
	}
	id := args[0]
	loadIssues()
	issue := it.Issue(id)
	if issue == nil {
		log.Fatalf("comment: error finding issue %s\n", id)
	}
	comment := ""
	if len(args) > 1 {
		comment = args[1]
	} else {
		comment = editComment()
	}
	stamp := lit.Stamp(username)
	commentBranch := dgrl.NewBranch(stamp)
	commentBranch.Append(dgrl.NewText(comment))
	issue.Append(commentBranch)
	if !lit.Set(issue, "updated", stamp) {
		log.Printf("comment: error setting update time for issue %s\n", id)
	}
	storeIssues()
}

func editComment() string {
	editor := getEditor()
	if editor == "" {
		log.Fatalln("comment: VISUAL or EDITOR environment variable must be set")
	}
	// create temp file
	tempFile, err := ioutil.TempFile("", "lit-")
	checkErr(err)
	filename := tempFile.Name()

	// get original file state
	origStat, err := os.Stat(filename)
	checkErr(err)

	// launch editor
	ed := exec.Command(editor, filename)
	ed.Stdin, ed.Stdout, ed.Stderr = os.Stdin, os.Stdout, os.Stderr
	err = ed.Run()
	checkErr(err)

	// get updated file state, compare to original
	newStat, err := os.Stat(filename)
	checkErr(err)
	if newStat.ModTime() == origStat.ModTime() {
		log.Fatalln("comment: file unchanged")
	}

	// read comment from file
	commentData, err := ioutil.ReadFile(filename)
	checkErr(err)
	return string(commentData)
}

func listInfo(id string, issue *dgrl.Branch) string {
	status := " "
	closed, _ := lit.Get(issue, "closed")
	if len(closed) > 0 {
		status = "*"
	}
	tags, _ := lit.Get(issue, "tags")
	priority, _ := lit.Get(issue, "priority")
	assigned, _ := lit.Get(issue, "assigned")
	summary, _ := lit.Get(issue, "summary")
	return fmt.Sprintf(listFmt, id, status, priority, assigned, tags, summary)
}

func keyval(kv []string) (string, string) {
	key, val := "", ""
	if len(kv) > 0 {
		key = kv[0]
	}
	if len(kv) > 1 {
		val = kv[1]
	}
	return key, val
}

func matchIds(kv []string, doesMatch bool) []string {
	key, val := keyval(kv)
	return it.Match(key, val, doesMatch)
}

func compareIds(kv []string, isLess bool) []string {
	key, val := keyval(kv)
	return it.Compare(key, val, isLess)
}

func dispOpts() (bool, string, bool) {
	switch {
	case len(args) == 0:
		return false, "", true
	case args[0] == "sortby" || args[0] == "rsortby":
		if len(args) < 2 {
			log.Fatalf("%s: sort requested, but no key given to sort by\n", cmd)
		}
		doSort := true
		doAscend := (args[0] == "sortby")
		key := args[1]
		args = args[2:]
		return doSort, key, doAscend
	}
	return false, "", true
}

func specIds(isDefaultOpen bool) []string {
	ids := []string{}
	switch {
	case len(args) == 0:
		if isDefaultOpen {
			ids = matchIds([]string{"closed", ""}, false)
		}
	case args[0] == "with":
		ids = matchIds(args[1:], true)
	case args[0] == "without":
		ids = matchIds(args[1:], false)
	case args[0] == "less":
		ids = compareIds(args[1:], true)
	case args[0] == "greater":
		ids = compareIds(args[1:], false)
	case args[0] == "all":
		ids = it.IssueIds()
	default:
		ids = args
	}
	return ids
}

func loadIssues() {
	err := it.Load()
	checkErr(err)
}

func storeIssues() {
	err := it.Store()
	checkErr(err)
}

func checkErr(err error) {
	if err != nil {
		log.Fatalf("%s: %s\n", cmd, err)
	}
}

func getEditor() string {
	editor := os.Getenv("VISUAL")
	if editor == "" {
		editor = os.Getenv("EDITOR")
	}
	return editor
}
<commit_msg>Don't complain on empty spec for set and tag cmds.<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"os/user"
	"strconv"
	"strings"

	"github.com/ianremmler/dgrl"
	"github.com/ianremmler/lit"
)

const usage = `usage:

lit [help | usage]              Show usage
lit init                        Initialize new issue tracker
lit new [<num>]                 Create num new issues (default: 1)
lit list [<sort>] <spec>        Show list of specified issues (default: open)
lit id [<sort>] <spec>          List ids of specified issues (default: open)
lit show [<sort>] <spec>        Show specified issues (default: open)
lit set <key> <val> <spec>      Set value for key in specified issues
lit tag (add|del) <tag> <spec>  Add or delete tag in specified issues
lit comment <id> [<text>]       Add issue comment (open editor if no text given)
lit edit <spec>                 Edit specified issues (default: open)
lit close <spec>                Close specified issues
lit reopen <spec>               Reopen specified issues

sort: (sortby|rsortby) <key>  Sort (reverse if rsortby) based on key

spec: all | <ids> | (with|without) <key> [<val>] | (less|greater) <key> <val>
	Specifies which issues to operate on
	Default open spec does not apply if input is piped
	Use 'comment' key to filter by comment contents and times`

const (
	// id, closed?, priority, assigned, tags, summary
	listFmt = "%-8.8s %-1.1s %-1.1s %-8.8s %-17.17s %s"
)

var (
	args        = os.Args[1:]
	it          = lit.New()
	listHdr     = fmt.Sprintf(listFmt, "id", "c", "p", "assigned", "tags", "summary")
	username    = "?"
	isStdinPipe = false
	cmd         string
)

func main() {
	log.SetFlags(0)
	log.SetPrefix("lit: ")

	if userEnv := os.Getenv("LIT_USER"); userEnv != "" {
		username = userEnv
	} else {
		if user, err := user.Current(); err == nil {
			if host, err := os.Hostname(); err == nil {
				username = fmt.Sprintf("%s@%s", user.Username, host)
			}
		}
	}

	// append args piped in from stdin
	if stat, err := os.Stdin.Stat(); err == nil && stat.Mode()&os.ModeNamedPipe != 0 {
		isStdinPipe = true
		if stdin, err := ioutil.ReadAll(os.Stdin); err == nil {
			stdinArgs := strings.Fields(string(stdin))
			args = append(args, stdinArgs...)
		}
	}

	if len(args) > 0 {
		cmd = args[0]
		args = args[1:]
	}
	switch cmd {
	case "", "-h", "-help", "--help", "help", "-u", "-usage", "--usage", "usage":
		usageCmd()
	case "list":
		listCmd()
	case "init":
		initCmd()
	case "new":
		newCmd()
	case "id":
		idCmd()
	case "show":
		showCmd()
	case "set":
		setCmd()
	case "tag":
		tagCmd()
	case "comment":
		commentCmd()
	case "edit":
		editCmd()
	case "close", "reopen":
		closeCmd()
	default:
		log.Fatalln(cmd + " is not a valid command")
	}
}

func usageCmd() {
	fmt.Println(usage)
}

func initCmd() {
	err := it.Init()
	checkErr(err)
}

func newCmd() {
	numIssues := 1
	if len(args) > 0 {
		num, err := strconv.ParseUint(args[0], 10, 16)
		checkErr(err)
		numIssues = int(num)
	}
	loadIssues()
	for i := 0; i < numIssues; i++ {
		id, err := it.NewIssue(username)
		checkErr(err)
		fmt.Println(id)
	}
	storeIssues()
}

func listCmd() {
	loadIssues()
	doSort, key, doAscend := dispOpts()
	ids := specIds(!isStdinPipe)
	if doSort {
		it.Sort(ids, key, doAscend)
	}
	fmt.Println(listHdr)
	for _, id := range ids {
		issue := it.Issue(id)
		if issue != nil {
			fmt.Println(listInfo(id, issue))
		}
	}
}

func idCmd() {
	loadIssues()
	doSort, key, doAscend := dispOpts()
	ids := specIds(!isStdinPipe)
	if doSort {
		it.Sort(ids, key, doAscend)
	}
	for _, id := range ids {
		if it.Issue(id) != nil {
			fmt.Println(id)
		}
	}
}

func showCmd() {
	loadIssues()
	doSort, key, doAscend := dispOpts()
	ids := specIds(!isStdinPipe)
	if doSort {
		it.Sort(ids, key, doAscend)
	}
	for _, id := range ids {
		fmt.Println(it.Issue(id))
	}
}

func setCmd() {
	if len(args) < 2 {
		log.Fatalln("set: you must specify a key and value")
	}
	key, val := args[0], args[1]
	args = args[2:]
	loadIssues()
	stamp := lit.Stamp(username)
	for _, id := range specIds(false) {
		issue := it.Issue(id)
		if issue == nil {
			log.Printf("set: error finding issue %s\n", id)
			continue
		}
		ok := lit.Set(issue, key, val)
		ok = ok && lit.Set(issue, "updated", stamp)
		if !ok {
			log.Printf("set: error updating fields in issue %s\n", id)
			continue
		}
	}
	storeIssues()
}

func tagCmd() {
	if len(args) < 2 {
		log.Fatalln("tag: you must specify an operation and tag")
	}
	op, tag := args[0], args[1]
	if op != "add" && op != "del" {
		log.Fatalf("tag: is not a valid operation\n")
	}
	doAdd := (op == "add")

	args = args[2:]
	loadIssues()
	stamp := lit.Stamp(username)
	for _, id := range specIds(false) {
		issue := it.Issue(id)
		if issue == nil {
			log.Printf("tag: error finding issue %s\n", id)
			continue
		}
		ok := lit.ModifyTag(issue, tag, doAdd)
		ok = ok && lit.Set(issue, "updated", stamp)
		if !ok {
			log.Printf("tag: error updating fields in issue %s\n", id)
			continue
		}
	}
	storeIssues()
}

func editCmd() {
	editor := getEditor()
	if editor == "" {
		log.Fatalln("edit: VISUAL or EDITOR environment variable must be set")
	}

	loadIssues()

	// create temp file
	tempFile, err := ioutil.TempFile("", "lit-")
	checkErr(err)
	filename := tempFile.Name()

	// load issue content into temp file
	ids := specIds(!isStdinPipe)
	toEdit := dgrl.NewRoot()
	for _, id := range ids {
		issue := it.Issue(id)
		if issue == nil {
			log.Printf("edit: error finding issue %s\n", id)
			continue
		}
		toEdit.Append(issue)
	}
	err = toEdit.Write(tempFile)
	checkErr(err)
	tempFile.Close()

	// get original file state
	origStat, err := os.Stat(filename)
	checkErr(err)

	// launch editor
	ed := exec.Command(editor, filename)
	ed.Stdin, ed.Stdout, ed.Stderr = os.Stdin, os.Stdout, os.Stderr
	err = ed.Run()
	checkErr(err)

	// get updated file state, compare to original
	newStat, err := os.Stat(filename)
	checkErr(err)
	if newStat.ModTime() == origStat.ModTime() {
		log.Fatalln("edit: file unchanged")
	}

	// parse issue from temp file
	tempFile, err = os.Open(filename)
	checkErr(err)
	edited := dgrl.NewParser().Parse(tempFile)
	tempFile.Close()
	if edited == nil {
		log.Fatalln("edit: error parsing file")
	}

	// update issues if we find a match
	didUpdate := false
	stamp := lit.Stamp(username)
	for _, id := range ids {
		issue := it.Issue(id)
		if issue == nil {
			// already printed error, so don't repeat here
			continue
		}
		for _, node := range edited.Kids() {
			if node.Type() == dgrl.BranchType && strings.HasPrefix(node.Key(), id) {
				if editedIssue, ok := node.(*dgrl.Branch); ok {
					*issue = *editedIssue
					if !lit.Set(issue, "updated", stamp) {
						log.Printf("edit: error setting update time for issue %s\n", id)
						continue
					}
					didUpdate = true
					break
				}
			}
		}
	}
	if !didUpdate {
		log.Fatalln("edit: did not update anything")
	}

	storeIssues()
}

func closeCmd() {
	loadIssues()
	stamp := lit.Stamp(username)
	for _, id := range specIds(false) {
		issue := it.Issue(id)
		if issue == nil {
			log.Printf("%s: error finding issue %s\n", cmd, id)
			continue
		}
		closedStamp := ""
		if cmd == "close" {
			closedStamp = stamp
		}
		ok := lit.Set(issue, "closed", closedStamp)
		ok = ok && lit.Set(issue, "updated", stamp)
		if !ok {
			log.Printf("%s: error updating fields for issue %s\n", cmd, id)
			continue
		}
	}
	storeIssues()
}

func commentCmd() {
	if len(args) < 1 {
		log.Fatalln("comment: you must specify an issue")
	}
	id := args[0]
	loadIssues()
	issue := it.Issue(id)
	if issue == nil {
		log.Fatalf("comment: error finding issue %s\n", id)
	}
	comment := ""
	if len(args) > 1 {
		comment = args[1]
	} else {
		comment = editComment()
	}
	stamp := lit.Stamp(username)
	commentBranch := dgrl.NewBranch(stamp)
	commentBranch.Append(dgrl.NewText(comment))
	issue.Append(commentBranch)
	if !lit.Set(issue, "updated", stamp) {
		log.Printf("comment: error setting update time for issue %s\n", id)
	}
	storeIssues()
}

func editComment() string {
	editor := getEditor()
	if editor == "" {
		log.Fatalln("comment: VISUAL or EDITOR environment variable must be set")
	}
	// create temp file
	tempFile, err := ioutil.TempFile("", "lit-")
	checkErr(err)
	filename := tempFile.Name()

	// get original file state
	origStat, err := os.Stat(filename)
	checkErr(err)

	// launch editor
	ed := exec.Command(editor, filename)
	ed.Stdin, ed.Stdout, ed.Stderr = os.Stdin, os.Stdout, os.Stderr
	err = ed.Run()
	checkErr(err)

	// get updated file state, compare to original
	newStat, err := os.Stat(filename)
	checkErr(err)
	if newStat.ModTime() == origStat.ModTime() {
		log.Fatalln("comment: file unchanged")
	}

	// read comment from file
	commentData, err := ioutil.ReadFile(filename)
	checkErr(err)
	return string(commentData)
}

func listInfo(id string, issue *dgrl.Branch) string {
	status := " "
	closed, _ := lit.Get(issue, "closed")
	if len(closed) > 0 {
		status = "*"
	}
	tags, _ := lit.Get(issue, "tags")
	priority, _ := lit.Get(issue, "priority")
	assigned, _ := lit.Get(issue, "assigned")
	summary, _ := lit.Get(issue, "summary")
	return fmt.Sprintf(listFmt, id, status, priority, assigned, tags, summary)
}

func keyval(kv []string) (string, string) {
	key, val := "", ""
	if len(kv) > 0 {
		key = kv[0]
	}
	if len(kv) > 1 {
		val = kv[1]
	}
	return key, val
}

func matchIds(kv []string, doesMatch bool) []string {
	key, val := keyval(kv)
	return it.Match(key, val, doesMatch)
}

func compareIds(kv []string, isLess bool) []string {
	key, val := keyval(kv)
	return it.Compare(key, val, isLess)
}

func dispOpts() (bool, string, bool) {
	switch {
	case len(args) == 0:
		return false, "", true
	case args[0] == "sortby" || args[0] == "rsortby":
		if len(args) < 2 {
			log.Fatalf("%s: sort requested, but no key given to sort by\n", cmd)
		}
		doSort := true
		doAscend := (args[0] == "sortby")
		key := args[1]
		args = args[2:]
		return doSort, key, doAscend
	}
	return false, "", true
}

func specIds(isDefaultOpen bool) []string {
	ids := []string{}
	switch {
	case len(args) == 0:
		if isDefaultOpen {
			ids = matchIds([]string{"closed", ""}, false)
		}
	case args[0] == "with":
		ids = matchIds(args[1:], true)
	case args[0] == "without":
		ids = matchIds(args[1:], false)
	case args[0] == "less":
		ids = compareIds(args[1:], true)
	case args[0] == "greater":
		ids = compareIds(args[1:], false)
	case args[0] == "all":
		ids = it.IssueIds()
	default:
		ids = args
	}
	return ids
}

func loadIssues() {
	err := it.Load()
	checkErr(err)
}

func storeIssues() {
	err := it.Store()
	checkErr(err)
}

func checkErr(err error) {
	if err != nil {
		log.Fatalf("%s: %s\n", cmd, err)
	}
}

func getEditor() string {
	editor := os.Getenv("VISUAL")
	if editor == "" {
		editor = os.Getenv("EDITOR")
	}
	return editor
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/nightlyone/lockfile"
	"io"
	"log"
	"log/syslog"
	"math/rand"
	"os"
	"os/exec"
	"path/filepath"
	"sync"
	"time"
)

// Avoid thundering herd problem on remote services used by this command.
// Interval will be 0, if this is not an issue.
func SpreadWait(interval time.Duration) {
	if interval > 0 {
		// Seed random generator with current process ID
		rand.Seed(int64(os.Getpid()))
		// Sleep for random amount of time within interval
		time.Sleep(time.Duration(rand.Int63n(int64(interval))))
	}
}

// Ok states that execution went well. Logs debug output and reports ok to
// monitoring.
func Ok() {
	log.Println("Ok")
	monitor(monitorOk, "")
}

// NotAvailable states that the command could not be started successfully. It
// might not be installed or has other problems.
func NotAvailable(err error) {
	s := fmt.Sprintln("Cannot start command: ", err)
	log.Println("FATAL:", s)
	monitor(monitorUnknown, s)
}

// TimedOut states that the command took too long and reports failure to the
// monitoring.
func TimedOut() {
	s := "execution took too long"
	log.Println("FATAL:", s)
	monitor(monitorCritical, s)
}

// Busy states that the command hangs and reports failure to the monitoring.
// Those tasks should be automatically killed, if it happens often.
func Busy() {
	s := "previous invocation of command still running"
	log.Println("FATAL:", s)
	monitor(monitorCritical, s)
}

// Failed states that the command didn't execute successfully and reports
// failure to the monitoring. Also Logs error output.
func Failed(err error) {
	s := fmt.Sprintln("Failed to execute: ", err)
	log.Println("FATAL:", s)
	monitor(monitorCritical, s)
}

var useSyslog bool

// derive logger
func getLogger() (logger io.Writer, err error) {
	if useSyslog {
		logger, err = syslog.New(syslog.LOG_NOTICE, monitoringEvent)
	} else {
		logger = os.Stderr
	}
	if err != nil {
		log.SetOutput(logger)
	}
	return logger, err
}

// pipe r to logger in the background
func logStream(r io.Reader, logger io.Writer, wg *sync.WaitGroup) {
	wg.Add(1)

	go func() {
		for {
			if _, err := io.Copy(logger, r); err != nil {
				break
			}
		}
		wg.Done()
	}()
}

func main() {
	var cmd *exec.Cmd
	var interval, timeout time.Duration
	var wg sync.WaitGroup

	// FIXME(mlafeldt) add command-line options for kill or wait on busy
	// state
	log.SetFlags(0)

	flag.DurationVar(&interval, "i", -1,
		"set execution interval for command, e.g. 45s, 2m, 1h30m, default: 1/10 of timeout")
	flag.DurationVar(&timeout, "t", 1*time.Minute,
		"set execution timeout for command, e.g. 45s, 2m, 1h30m, default: 1m")
	flag.BoolVar(&useSyslog, "l", false, "log via syslog")
	flag.Parse()

	if flag.NArg() < 1 {
		log.Fatal("FATAL: no command to execute")
		return
	}

	command := flag.Arg(0)
	monitoringEvent = filepath.Base(command)
	logger, err := getLogger()

	if interval >= timeout {
		log.Fatal("FATAL: interval >= timeout, no time left for actual command execution")
		return
	}

	if interval == -1 {
		interval = timeout / 10
	}

	loadMonitoringCommands()

	// FIXME(nightlyone) try two intervals instead of one?
	timer := time.AfterFunc(timeout, func() {
		TimedOut()
		if cmd != nil && cmd.Process != nil {
			cmd.Process.Kill()
			// FIXME(nightlyone) log the kill
		}
		os.Exit(0)
	})

	SpreadWait(interval)

	// Ensures that only one of these command runs concurrently on this
	// machine.  Also cleans up stale locks of dead instances.
	lock_dir := os.TempDir()
	os.Mkdir(filepath.Join(lock_dir, monitoringEvent), 0700)
	lock, _ := lockfile.New(filepath.Join(lock_dir, monitoringEvent, monitoringEvent+".lock"))
	if err := lock.TryLock(); err != nil {
		if err != lockfile.ErrBusy {
			log.Printf("ERROR: locking %s: reason: %v\n", lock, err)
		}
		timer.Stop()
		Busy()
		return
	}
	defer lock.Unlock()

	cmd = exec.Command(command, flag.Args()[1:]...)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}
	logStream(stdout, logger, &wg)

	stderr, err := cmd.StderrPipe()
	if err != nil {
		log.Fatal(err)
	}
	logStream(stderr, logger, &wg)

	if err := cmd.Start(); err != nil {
		timer.Stop()
		NotAvailable(err)
		return
	}

	if err := cmd.Wait(); err != nil {
		timer.Stop()
		Failed(err)
	} else {
		timer.Stop()
		Ok()
	}

	wg.Wait()
}
<commit_msg>allow user to disable logging of stderr/stdout<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/nightlyone/lockfile"
	"io"
	"log"
	"log/syslog"
	"math/rand"
	"os"
	"os/exec"
	"path/filepath"
	"sync"
	"time"
)

// Avoid thundering herd problem on remote services used by this command.
// Interval will be 0, if this is not an issue.
func SpreadWait(interval time.Duration) {
	if interval > 0 {
		// Seed random generator with current process ID
		rand.Seed(int64(os.Getpid()))
		// Sleep for random amount of time within interval
		time.Sleep(time.Duration(rand.Int63n(int64(interval))))
	}
}

// Ok states that execution went well. Logs debug output and reports ok to
// monitoring.
func Ok() {
	log.Println("Ok")
	monitor(monitorOk, "")
}

// NotAvailable states that the command could not be started successfully. It
// might not be installed or has other problems.
func NotAvailable(err error) {
	s := fmt.Sprintln("Cannot start command: ", err)
	log.Println("FATAL:", s)
	monitor(monitorUnknown, s)
}

// TimedOut states that the command took too long and reports failure to the
// monitoring.
func TimedOut() {
	s := "execution took too long"
	log.Println("FATAL:", s)
	monitor(monitorCritical, s)
}

// Busy states that the command hangs and reports failure to the monitoring.
// Those tasks should be automatically killed, if it happens often.
func Busy() {
	s := "previous invocation of command still running"
	log.Println("FATAL:", s)
	monitor(monitorCritical, s)
}

// Failed states that the command didn't execute successfully and reports
// failure to the monitoring. Also Logs error output.
func Failed(err error) {
	s := fmt.Sprintln("Failed to execute: ", err)
	log.Println("FATAL:", s)
	monitor(monitorCritical, s)
}

var useSyslog bool

// derive logger
func getLogger() (logger io.Writer, err error) {
	if useSyslog {
		logger, err = syslog.New(syslog.LOG_NOTICE, monitoringEvent)
	} else {
		logger = os.Stderr
	}
	if err != nil {
		log.SetOutput(logger)
	}
	return logger, err
}

// pipe r to logger in the background
func logStream(r io.Reader, logger io.Writer, wg *sync.WaitGroup) {
	wg.Add(1)

	go func() {
		for {
			if _, err := io.Copy(logger, r); err != nil {
				break
			}
		}
		wg.Done()
	}()
}

func main() {
	var cmd *exec.Cmd
	var interval, timeout time.Duration
	var wg sync.WaitGroup
	var pipeStderr, pipeStdout bool

	// FIXME(mlafeldt) add command-line options for kill or wait on busy
	// state
	log.SetFlags(0)

	flag.DurationVar(&interval, "i", -1,
		"set execution interval for command, e.g. 45s, 2m, 1h30m, default: 1/10 of timeout")
	flag.DurationVar(&timeout, "t", 1*time.Minute,
		"set execution timeout for command, e.g. 45s, 2m, 1h30m, default: 1m")
	flag.BoolVar(&useSyslog, "l", false, "log via syslog")
	flag.BoolVar(&pipeStderr, "e", true, "pipe stderr to log")
	flag.BoolVar(&pipeStdout, "o", true, "pipe stdout to log")
	flag.Parse()

	if flag.NArg() < 1 {
		log.Fatal("FATAL: no command to execute")
		return
	}

	command := flag.Arg(0)
	monitoringEvent = filepath.Base(command)
	logger, err := getLogger()
	if err != nil {
		log.Fatal("FATAL: cannot contact syslog")
	}

	if interval >= timeout {
		log.Fatal("FATAL: interval >= timeout, no time left for actual command execution")
		return
	}

	if interval == -1 {
		interval = timeout / 10
	}

	loadMonitoringCommands()

	// FIXME(nightlyone) try two intervals instead of one?
	timer := time.AfterFunc(timeout, func() {
		TimedOut()
		if cmd != nil && cmd.Process != nil {
			cmd.Process.Kill()
			// FIXME(nightlyone) log the kill
		}
		os.Exit(0)
	})

	SpreadWait(interval)

	// Ensures that only one of these command runs concurrently on this
	// machine.  Also cleans up stale locks of dead instances.
	lock_dir := os.TempDir()
	os.Mkdir(filepath.Join(lock_dir, monitoringEvent), 0700)
	lock, _ := lockfile.New(filepath.Join(lock_dir, monitoringEvent, monitoringEvent+".lock"))
	if err := lock.TryLock(); err != nil {
		if err != lockfile.ErrBusy {
			log.Printf("ERROR: locking %s: reason: %v\n", lock, err)
		}
		timer.Stop()
		Busy()
		return
	}
	defer lock.Unlock()

	cmd = exec.Command(command, flag.Args()[1:]...)

	if pipeStdout {
		stdout, err := cmd.StdoutPipe()
		if err != nil {
			log.Fatal(err)
		}
		logStream(stdout, logger, &wg)
	}

	if pipeStderr {
		stderr, err := cmd.StderrPipe()
		if err != nil {
			log.Fatal(err)
		}
		logStream(stderr, logger, &wg)
	}

	if err := cmd.Start(); err != nil {
		timer.Stop()
		NotAvailable(err)
		return
	}

	if err := cmd.Wait(); err != nil {
		timer.Stop()
		Failed(err)
	} else {
		timer.Stop()
		Ok()
	}

	wg.Wait()
}
<|endoftext|>


<commit_before>package cmd

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"

	"github.com/exercism/cli/api"
	"github.com/exercism/cli/config"
	"github.com/spf13/cobra"
)

// prepareCmd does necessary setup for Exercism and its tracks.
var prepareCmd = &cobra.Command{
	Use:     "prepare",
	Aliases: []string{"p"},
	Short:   "Prepare does setup for Exercism and its tracks.",
	Long: `Prepare downloads settings and dependencies for Exercism and the language tracks.

When called without any arguments, this downloads all the copy for the CLI so we
know what to say in all the various situations. It also provides an up-to-date list
of the API endpoints to use.

When called with a track ID, it will do specific setup for that track. This
might include downloading the files that the track maintainers have said are
necessary for the track in general. Any files that are only necessary for a specific
exercise will be downloaded along with the exercise.

To customize the CLI to suit your own preferences, use the configure command.
	`,
	RunE: func(cmd *cobra.Command, args []string) error {
		track, err := cmd.Flags().GetString("track")
		if err != nil {
			return err
		}

		if track == "" {
			fmt.Println("prepare called")
			return nil
		}
		cfg, err := config.NewUserConfig()
		if err != nil {
			return err
		}
		return prepareTrack(track, cfg)
	},
}

func prepareTrack(id string, cfg *config.UserConfig) error {
	client, err := api.NewClient(cfg.Token, cfg.APIBaseURL)
	if err != nil {
		return err
	}
	url := fmt.Sprintf("%s/tracks/%s", cfg.APIBaseURL, id)

	req, err := client.NewRequest("GET", url, nil)
	if err != nil {
		return err
	}

	res, err := client.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()

	var payload prepareTrackPayload

	if err := json.NewDecoder(res.Body).Decode(&payload); err != nil {
		return fmt.Errorf("unable to parse API response - %s", err)
	}

	if res.StatusCode != http.StatusOK {
		return errors.New(payload.Error.Message)
	}

	cliCfg, err := config.NewCLIConfig()
	if err != nil {
		return err
	}

	t, ok := cliCfg.Tracks[id]
	if !ok {
		t = config.NewTrack(id)
	}
	if payload.Track.TestPattern != "" {
		t.IgnorePatterns = append(t.IgnorePatterns, payload.Track.TestPattern)
	}
	cliCfg.Tracks[id] = t

	return cliCfg.Write()
}

type prepareTrackPayload struct {
	Track struct {
		ID          string `json:"id"`
		Language    string `json:"language"`
		TestPattern string `json:"test_pattern"`
	} `json:"track"`
	Error struct {
		Type    string `json:"type"`
		Message string `json:"message"`
	} `json:"error,omitempty"`
}

func initPrepareCmd() {
	prepareCmd.Flags().StringP("track", "t", "", "the track you want to prepare")
}

func init() {
	RootCmd.AddCommand(prepareCmd)
	initPrepareCmd()
}
<commit_msg>Normalize prepare command<commit_after>package cmd

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"

	"github.com/exercism/cli/api"
	"github.com/exercism/cli/config"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
)

// prepareCmd does necessary setup for Exercism and its tracks.
var prepareCmd = &cobra.Command{
	Use:     "prepare",
	Aliases: []string{"p"},
	Short:   "Prepare does setup for Exercism and its tracks.",
	Long: `Prepare downloads settings and dependencies for Exercism and the language tracks.

When called with a track ID, it will do specific setup for that track. This
might include downloading the files that the track maintainers have said are
necessary for the track in general. Any files that are only necessary for a specific
exercise will be downloaded along with the exercise.

To customize the CLI to suit your own preferences, use the configure command.
	`,
	RunE: func(cmd *cobra.Command, args []string) error {
		cfg := config.NewConfiguration()
		usrCfg, err := config.NewUserConfig()
		if err != nil {
			return err
		}
		cfg.UserConfig = usrCfg
		return runPrepare(cfg, cmd.Flags(), args)
	},
}

func runPrepare(cfg config.Configuration, flags *pflag.FlagSet, args []string) error {
	usrCfg := cfg.UserConfig

	track, err := flags.GetString("track")
	if err != nil {
		return err
	}

	if track == "" {
		return nil
	}
	client, err := api.NewClient(usrCfg.Token, usrCfg.APIBaseURL)
	if err != nil {
		return err
	}
	url := fmt.Sprintf("%s/tracks/%s", usrCfg.APIBaseURL, track)

	req, err := client.NewRequest("GET", url, nil)
	if err != nil {
		return err
	}

	res, err := client.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()

	var payload prepareTrackPayload

	if err := json.NewDecoder(res.Body).Decode(&payload); err != nil {
		return fmt.Errorf("unable to parse API response - %s", err)
	}

	if res.StatusCode != http.StatusOK {
		return errors.New(payload.Error.Message)
	}

	cliCfg, err := config.NewCLIConfig()
	if err != nil {
		return err
	}

	t, ok := cliCfg.Tracks[track]
	if !ok {
		t = config.NewTrack(track)
	}
	if payload.Track.TestPattern != "" {
		t.IgnorePatterns = append(t.IgnorePatterns, payload.Track.TestPattern)
	}
	cliCfg.Tracks[track] = t

	return cliCfg.Write()
}

type prepareTrackPayload struct {
	Track struct {
		ID          string `json:"id"`
		Language    string `json:"language"`
		TestPattern string `json:"test_pattern"`
	} `json:"track"`
	Error struct {
		Type    string `json:"type"`
		Message string `json:"message"`
	} `json:"error,omitempty"`
}

func initPrepareCmd() {
	prepareCmd.Flags().StringP("track", "t", "", "the track you want to prepare")
}

func init() {
	RootCmd.AddCommand(prepareCmd)
	initPrepareCmd()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"time"

	"github.com/girigiribauer/go-pwgen"
	"gopkg.in/urfave/cli.v1"
)

func main() {
	app := cli.NewApp()
	app.Name = "pw"
	app.Version = "0.1.0"
	app.Compiled = time.Now()
	app.Authors = []cli.Author{
		cli.Author{
			Name:  "girigiribauer",
			Email: "girigiribauer@gmail.com",
		},
	}
	app.Copyright = "(c) 2017 girigiribauer"
	app.Usage = "`pw` password generator by Golang"
	app.UsageText = "pw [options]"

	app.Flags = []cli.Flag{
		cli.IntFlag{
			Name:  "length, l",
			Usage: "password length (range is [8...64])",
			Value: 10,
		},
		cli.IntFlag{
			Name:  "count, c",
			Usage: "number of counts",
			Value: 10,
		},
		cli.BoolTFlag{
			Name:  "digit",
			Usage: "add password characters 0-9",
		},
		cli.BoolTFlag{
			Name:  "alphabetlarge",
			Usage: "add password characters A-Z",
		},
		cli.BoolTFlag{
			Name:  "alphabetsmall",
			Usage: "add password characters a-z",
		},
		cli.BoolFlag{
			Name:  "underscore, u",
			Usage: "add password character Underscore(_)",
		},
		cli.BoolFlag{
			Name:  "specialchars, s",
			Usage: "add password characters special characters, exclude from Space, Backslash, Underscore and Delete",
		},
	}

	app.Action = func(c *cli.Context) error {
		writer := os.Stdout
		err := pwgen.Pwgen(
			writer,
			c.Int("length"),
			c.Int("count"),
			c.Bool("digit"),
			c.Bool("alphabetlarge"),
			c.Bool("alphabetsmall"),
			c.Bool("underscore"),
			c.Bool("specialchars"),
		)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
		}

		return nil
	}

	app.Run(os.Args)
}
<commit_msg>fix version<commit_after>package main

import (
	"fmt"
	"os"
	"time"

	"github.com/girigiribauer/go-pwgen"
	"gopkg.in/urfave/cli.v1"
)

func main() {
	app := cli.NewApp()
	app.Name = "pw"
	app.Version = "0.2.1"
	app.Compiled = time.Now()
	app.Authors = []cli.Author{
		cli.Author{
			Name:  "girigiribauer",
			Email: "girigiribauer@gmail.com",
		},
	}
	app.Copyright = "(c) 2017 girigiribauer"
	app.Usage = "`pw` password generator by Golang"
	app.UsageText = "pw [options]"

	app.Flags = []cli.Flag{
		cli.IntFlag{
			Name:  "length, l",
			Usage: "password length (range is [8...64])",
			Value: 10,
		},
		cli.IntFlag{
			Name:  "count, c",
			Usage: "number of counts",
			Value: 10,
		},
		cli.BoolTFlag{
			Name:  "digit",
			Usage: "add password characters 0-9",
		},
		cli.BoolTFlag{
			Name:  "alphabetlarge",
			Usage: "add password characters A-Z",
		},
		cli.BoolTFlag{
			Name:  "alphabetsmall",
			Usage: "add password characters a-z",
		},
		cli.BoolFlag{
			Name:  "underscore, u",
			Usage: "add password character Underscore(_)",
		},
		cli.BoolFlag{
			Name:  "specialchars, s",
			Usage: "add password characters special characters, exclude from Space, Backslash, Underscore and Delete",
		},
	}

	app.Action = func(c *cli.Context) error {
		writer := os.Stdout
		err := pwgen.Pwgen(
			writer,
			c.Int("length"),
			c.Int("count"),
			c.Bool("digit"),
			c.Bool("alphabetlarge"),
			c.Bool("alphabetsmall"),
			c.Bool("underscore"),
			c.Bool("specialchars"),
		)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
		}

		return nil
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package osmpbf

import (
	"fmt"
	"io"
	"net/http"
	"os"
	"reflect"
	"runtime"
	"sync"
	"sync/atomic"
	"testing"
	"time"
)

const (
	// Originally downloaded from http://download.geofabrik.de/europe/great-britain/england/greater-london.html
	London    = "greater-london-140324.osm.pbf"
	LondonURL = "https://googledrive.com/host/0B8pisLiGtmqDR3dOR3hrWUpRTVE"
)

func parseTime(s string) time.Time {
	t, err := time.Parse(time.RFC3339, s)
	if err != nil {
		panic(err)
	}
	return t
}

var (
	IDsExpectedOrder = []string{
		// Start of dense nodes.
		"node/44", "node/47", "node/52", "node/58", "node/60",
		"node/79", // Just because way/79 is already there
		"node/2740703694", "node/2740703695", "node/2740703697",
		"node/2740703699", "node/2740703701",
		// End of dense nodes.

		// Start of ways.
		"way/73", "way/74", "way/75", "way/79", "way/482",
		"way/268745428", "way/268745431", "way/268745434", "way/268745436",
		"way/268745439",
		// End of ways.

		// Start of relations.
		"relation/69", "relation/94", "relation/152", "relation/245",
		"relation/332", "relation/3593436", "relation/3595575",
		"relation/3595798", "relation/3599126", "relation/3599127",
		// End of relations
	}

	IDs map[string]bool

	enc uint64 = 2729006
	ewc uint64 = 459055
	erc uint64 = 12833

	en = &Node{
		ID:  18088578,
		Lat: 51.5442632,
		Lon: -0.2010027,
		Tags: map[string]string{
			"alt_name":   "The King's Head",
			"amenity":    "pub",
			"created_by": "JOSM",
			"name":       "The Luminaire",
			"note":       "Live music venue too",
		},
		Info: Info{
			Version:   2,
			Timestamp: parseTime("2009-05-20T10:28:54Z"),
			Changeset: 1260468,
			Uid:       508,
			User:      "Welshie",
			Visible:   true,
		},
	}

	ew = &Way{
		ID: 4257116,
		NodeIDs: []int64{
			21544864, 333731851, 333731852, 333731850, 333731855,
			333731858, 333731854, 108047, 769984352, 21544864},
		Tags: map[string]string{
			"area":    "yes",
			"highway": "pedestrian",
			"name":    "Fitzroy Square",
		},
		Info: Info{
			Version:   7,
			Timestamp: parseTime("2013-08-07T12:08:39Z"),
			Changeset: 17253164,
			Uid:       1016290,
			User:      "Amaroussi",
			Visible:   true,
		},
	}

	er = &Relation{
		ID: 7677,
		Members: []Member{
			Member{ID: 4875932, Type: WayType, Role: "outer"},
			Member{ID: 4894305, Type: WayType, Role: "inner"},
		},
		Tags: map[string]string{
			"created_by": "Potlatch 0.9c",
			"type":       "multipolygon",
		},
		Info: Info{
			Version:   4,
			Timestamp: parseTime("2008-07-19T15:04:03Z"),
			Changeset: 540201,
			Uid:       3876,
			User:      "Edgemaster",
			Visible:   true,
		},
	}
)

func init() {
	IDs = make(map[string]bool)
	for _, id := range IDsExpectedOrder {
		IDs[id] = false
	}
}

func downloadTestOSMFile(t *testing.T) {
	if _, err := os.Stat(London); os.IsNotExist(err) {
		out, err := os.Create(London)
		if err != nil {
			t.Fatal(err)
		}
		defer out.Close()

		resp, err := http.Get(LondonURL)
		if err != nil {
			t.Fatal(err)
		}
		defer resp.Body.Close()

		if _, err := io.Copy(out, resp.Body); err != nil {
			t.Fatal(err)
		}
	} else if err != nil {
		t.Fatal(err)
	}
}

func TestDecode(t *testing.T) {
	downloadTestOSMFile(t)

	f, err := os.Open(London)
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()

	d := NewDecoder(f)
	err = d.Start(runtime.GOMAXPROCS(-1))
	if err != nil {
		t.Fatal(err)
	}

	var n *Node
	var w *Way
	var r *Relation
	var nc, wc, rc uint64
	var id string
	idsOrder := make([]string, 0, len(IDsExpectedOrder))
	for {
		if v, err := d.Decode(); err == io.EOF {
			break
		} else if err != nil {
			t.Fatal(err)
		} else {
			switch v := v.(type) {
			case *Node:
				nc++
				if v.ID == en.ID {
					n = v
				}
				id = fmt.Sprintf("node/%d", v.ID)
				if _, ok := IDs[id]; ok {
					idsOrder = append(idsOrder, id)
				}
			case *Way:
				wc++
				if v.ID == ew.ID {
					w = v
				}
				id = fmt.Sprintf("way/%d", v.ID)
				if _, ok := IDs[id]; ok {
					idsOrder = append(idsOrder, id)
				}
			case *Relation:
				rc++
				if v.ID == er.ID {
					r = v
				}
				id = fmt.Sprintf("relation/%d", v.ID)
				if _, ok := IDs[id]; ok {
					idsOrder = append(idsOrder, id)
				}
			default:
				t.Fatalf("unknown type %T", v)
			}
		}
	}

	if !reflect.DeepEqual(en, n) {
		t.Errorf("\nExpected: %#v\nActual:   %#v", en, n)
	}
	if !reflect.DeepEqual(ew, w) {
		t.Errorf("\nExpected: %#v\nActual:   %#v", ew, w)
	}
	if !reflect.DeepEqual(er, r) {
		t.Errorf("\nExpected: %#v\nActual:   %#v", er, r)
	}
	if enc != nc || ewc != wc || erc != rc {
		t.Errorf("\nExpected %7d nodes, %7d ways, %7d relations\nGot %7d nodes, %7d ways, %7d relations.",
			enc, ewc, erc, nc, wc, rc)
	}
	if !reflect.DeepEqual(IDsExpectedOrder, idsOrder) {
		t.Errorf("\nExpected: %v\nGot:      %v", IDsExpectedOrder, idsOrder)
	}
}

func TestDecodeConcurrent(t *testing.T) {
	downloadTestOSMFile(t)

	f, err := os.Open(London)
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()

	d := NewDecoder(f)
	err = d.Start(runtime.GOMAXPROCS(-1))
	if err != nil {
		t.Fatal(err)
	}

	var n *Node
	var w *Way
	var r *Relation
	var nc, wc, rc uint64
	var wg sync.WaitGroup
	for i := 0; i < 4; i++ {
		wg.Add(1)

		go func() {
			for {
				if v, err := d.Decode(); err == io.EOF {
					break
				} else if err != nil {
					t.Fatal(err)
				} else {
					switch v := v.(type) {
					case *Node:
						atomic.AddUint64(&nc, 1)
						if v.ID == en.ID {
							n = v
						}
					case *Way:
						atomic.AddUint64(&wc, 1)
						if v.ID == ew.ID {
							w = v
						}
					case *Relation:
						atomic.AddUint64(&rc, 1)
						if v.ID == er.ID {
							r = v
						}
					default:
						t.Fatalf("unknown type %T", v)
					}
				}
			}

			wg.Done()
		}()
	}
	wg.Wait()

	if !reflect.DeepEqual(en, n) {
		t.Errorf("\nExpected: %#v\nActual:   %#v", en, n)
	}
	if !reflect.DeepEqual(ew, w) {
		t.Errorf("\nExpected: %#v\nActual:   %#v", ew, w)
	}
	if !reflect.DeepEqual(er, r) {
		t.Errorf("\nExpected: %#v\nActual:   %#v", er, r)
	}
	if enc != nc || ewc != wc || erc != rc {
		t.Errorf("\nExpected %7d nodes, %7d ways, %7d relations\nGot %7d nodes, %7d ways, %7d relations",
			enc, ewc, erc, nc, wc, rc)
	}
}

func BenchmarkDecode(b *testing.B) {
	file := os.Getenv("OSMPBF_BENCHMARK_FILE")
	if file == "" {
		file = London
	}
	f, err := os.Open(file)
	if err != nil {
		b.Fatal(err)
	}
	defer f.Close()

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		f.Seek(0, 0)

		d := NewDecoder(f)
		err = d.Start(runtime.GOMAXPROCS(-1))
		if err != nil {
			b.Fatal(err)
		}

		var nc, wc, rc uint64
		start := time.Now()
		for {
			if v, err := d.Decode(); err == io.EOF {
				break
			} else if err != nil {
				b.Fatal(err)
			} else {
				switch v := v.(type) {
				case *Node:
					nc++
				case *Way:
					wc++
				case *Relation:
					rc++
				default:
					b.Fatalf("unknown type %T", v)
				}
			}
		}

		b.Logf("Done in %.3f seconds. Nodes: %d, Ways: %d, Relations: %d\n",
			time.Now().Sub(start).Seconds(), nc, wc, rc)
	}
}

func BenchmarkDecodeConcurrent(b *testing.B) {
	file := os.Getenv("OSMPBF_BENCHMARK_FILE")
	if file == "" {
		file = London
	}
	f, err := os.Open(file)
	if err != nil {
		b.Fatal(err)
	}
	defer f.Close()

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		f.Seek(0, 0)

		d := NewDecoder(f)
		err = d.Start(runtime.GOMAXPROCS(-1))
		if err != nil {
			b.Fatal(err)
		}

		var nc, wc, rc uint64
		start := time.Now()
		var wg sync.WaitGroup
		for i := 0; i < 4; i++ {
			wg.Add(1)

			go func() {
				for {
					if v, err := d.Decode(); err == io.EOF {
						break
					} else if err != nil {
						b.Fatal(err)
					} else {
						switch v := v.(type) {
						case *Node:
							atomic.AddUint64(&nc, 1)
						case *Way:
							atomic.AddUint64(&wc, 1)
						case *Relation:
							atomic.AddUint64(&rc, 1)
						default:
							b.Fatalf("unknown type %T", v)
						}
					}
				}

				wg.Done()
			}()
		}
		wg.Wait()

		b.Logf("Done in %.3f seconds. Nodes: %d, Ways: %d, Relations: %d\n",
			time.Now().Sub(start).Seconds(), nc, wc, rc)
	}
}
<commit_msg>Store test file in gist.<commit_after>package osmpbf

import (
	"fmt"
	"io"
	"net/http"
	"os"
	"reflect"
	"runtime"
	"sync"
	"sync/atomic"
	"testing"
	"time"
)

const (
	// Originally downloaded from http://download.geofabrik.de/europe/great-britain/england/greater-london.html
	// Stored at https://gist.github.com/AlekSi/d4369aa13cf1fc5ddfac3e91b67b2f7b
	London    = "greater-london-140324.osm.pbf"
	LondonURL = "https://gist.githubusercontent.com/AlekSi/d4369aa13cf1fc5ddfac3e91b67b2f7b/raw/f87959d6c9466547d9759971e071a15049b67ae2/greater-london-140324.osm.pbf"
)

func parseTime(s string) time.Time {
	t, err := time.Parse(time.RFC3339, s)
	if err != nil {
		panic(err)
	}
	return t
}

var (
	IDsExpectedOrder = []string{
		// Start of dense nodes.
		"node/44", "node/47", "node/52", "node/58", "node/60",
		"node/79", // Just because way/79 is already there
		"node/2740703694", "node/2740703695", "node/2740703697",
		"node/2740703699", "node/2740703701",
		// End of dense nodes.

		// Start of ways.
		"way/73", "way/74", "way/75", "way/79", "way/482",
		"way/268745428", "way/268745431", "way/268745434", "way/268745436",
		"way/268745439",
		// End of ways.

		// Start of relations.
		"relation/69", "relation/94", "relation/152", "relation/245",
		"relation/332", "relation/3593436", "relation/3595575",
		"relation/3595798", "relation/3599126", "relation/3599127",
		// End of relations
	}

	IDs map[string]bool

	enc uint64 = 2729006
	ewc uint64 = 459055
	erc uint64 = 12833

	en = &Node{
		ID:  18088578,
		Lat: 51.5442632,
		Lon: -0.2010027,
		Tags: map[string]string{
			"alt_name":   "The King's Head",
			"amenity":    "pub",
			"created_by": "JOSM",
			"name":       "The Luminaire",
			"note":       "Live music venue too",
		},
		Info: Info{
			Version:   2,
			Timestamp: parseTime("2009-05-20T10:28:54Z"),
			Changeset: 1260468,
			Uid:       508,
			User:      "Welshie",
			Visible:   true,
		},
	}

	ew = &Way{
		ID: 4257116,
		NodeIDs: []int64{
			21544864, 333731851, 333731852, 333731850, 333731855,
			333731858, 333731854, 108047, 769984352, 21544864},
		Tags: map[string]string{
			"area":    "yes",
			"highway": "pedestrian",
			"name":    "Fitzroy Square",
		},
		Info: Info{
			Version:   7,
			Timestamp: parseTime("2013-08-07T12:08:39Z"),
			Changeset: 17253164,
			Uid:       1016290,
			User:      "Amaroussi",
			Visible:   true,
		},
	}

	er = &Relation{
		ID: 7677,
		Members: []Member{
			Member{ID: 4875932, Type: WayType, Role: "outer"},
			Member{ID: 4894305, Type: WayType, Role: "inner"},
		},
		Tags: map[string]string{
			"created_by": "Potlatch 0.9c",
			"type":       "multipolygon",
		},
		Info: Info{
			Version:   4,
			Timestamp: parseTime("2008-07-19T15:04:03Z"),
			Changeset: 540201,
			Uid:       3876,
			User:      "Edgemaster",
			Visible:   true,
		},
	}
)

func init() {
	IDs = make(map[string]bool)
	for _, id := range IDsExpectedOrder {
		IDs[id] = false
	}
}

func downloadTestOSMFile(t *testing.T) {
	if _, err := os.Stat(London); os.IsNotExist(err) {
		resp, err := http.Get(LondonURL)
		if err != nil {
			t.Fatal(err)
		}
		defer resp.Body.Close()

		if resp.StatusCode != 200 {
			t.Fatalf("expected 200, got %d", resp.StatusCode)
		}

		out, err := os.Create(London)
		if err != nil {
			t.Fatal(err)
		}
		defer out.Close()

		if _, err = io.Copy(out, resp.Body); err != nil {
			t.Fatal(err)
		}
	} else if err != nil {
		t.Fatal(err)
	}
}

func TestDecode(t *testing.T) {
	downloadTestOSMFile(t)

	f, err := os.Open(London)
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()

	d := NewDecoder(f)
	err = d.Start(runtime.GOMAXPROCS(-1))
	if err != nil {
		t.Fatal(err)
	}

	var n *Node
	var w *Way
	var r *Relation
	var nc, wc, rc uint64
	var id string
	idsOrder := make([]string, 0, len(IDsExpectedOrder))
	for {
		if v, err := d.Decode(); err == io.EOF {
			break
		} else if err != nil {
			t.Fatal(err)
		} else {
			switch v := v.(type) {
			case *Node:
				nc++
				if v.ID == en.ID {
					n = v
				}
				id = fmt.Sprintf("node/%d", v.ID)
				if _, ok := IDs[id]; ok {
					idsOrder = append(idsOrder, id)
				}
			case *Way:
				wc++
				if v.ID == ew.ID {
					w = v
				}
				id = fmt.Sprintf("way/%d", v.ID)
				if _, ok := IDs[id]; ok {
					idsOrder = append(idsOrder, id)
				}
			case *Relation:
				rc++
				if v.ID == er.ID {
					r = v
				}
				id = fmt.Sprintf("relation/%d", v.ID)
				if _, ok := IDs[id]; ok {
					idsOrder = append(idsOrder, id)
				}
			default:
				t.Fatalf("unknown type %T", v)
			}
		}
	}

	if !reflect.DeepEqual(en, n) {
		t.Errorf("\nExpected: %#v\nActual:   %#v", en, n)
	}
	if !reflect.DeepEqual(ew, w) {
		t.Errorf("\nExpected: %#v\nActual:   %#v", ew, w)
	}
	if !reflect.DeepEqual(er, r) {
		t.Errorf("\nExpected: %#v\nActual:   %#v", er, r)
	}
	if enc != nc || ewc != wc || erc != rc {
		t.Errorf("\nExpected %7d nodes, %7d ways, %7d relations\nGot %7d nodes, %7d ways, %7d relations.",
			enc, ewc, erc, nc, wc, rc)
	}
	if !reflect.DeepEqual(IDsExpectedOrder, idsOrder) {
		t.Errorf("\nExpected: %v\nGot:      %v", IDsExpectedOrder, idsOrder)
	}
}

func TestDecodeConcurrent(t *testing.T) {
	downloadTestOSMFile(t)

	f, err := os.Open(London)
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()

	d := NewDecoder(f)
	err = d.Start(runtime.GOMAXPROCS(-1))
	if err != nil {
		t.Fatal(err)
	}

	var n *Node
	var w *Way
	var r *Relation
	var nc, wc, rc uint64
	var wg sync.WaitGroup
	for i := 0; i < 4; i++ {
		wg.Add(1)

		go func() {
			for {
				if v, err := d.Decode(); err == io.EOF {
					break
				} else if err != nil {
					t.Fatal(err)
				} else {
					switch v := v.(type) {
					case *Node:
						atomic.AddUint64(&nc, 1)
						if v.ID == en.ID {
							n = v
						}
					case *Way:
						atomic.AddUint64(&wc, 1)
						if v.ID == ew.ID {
							w = v
						}
					case *Relation:
						atomic.AddUint64(&rc, 1)
						if v.ID == er.ID {
							r = v
						}
					default:
						t.Fatalf("unknown type %T", v)
					}
				}
			}

			wg.Done()
		}()
	}
	wg.Wait()

	if !reflect.DeepEqual(en, n) {
		t.Errorf("\nExpected: %#v\nActual:   %#v", en, n)
	}
	if !reflect.DeepEqual(ew, w) {
		t.Errorf("\nExpected: %#v\nActual:   %#v", ew, w)
	}
	if !reflect.DeepEqual(er, r) {
		t.Errorf("\nExpected: %#v\nActual:   %#v", er, r)
	}
	if enc != nc || ewc != wc || erc != rc {
		t.Errorf("\nExpected %7d nodes, %7d ways, %7d relations\nGot %7d nodes, %7d ways, %7d relations",
			enc, ewc, erc, nc, wc, rc)
	}
}

func BenchmarkDecode(b *testing.B) {
	file := os.Getenv("OSMPBF_BENCHMARK_FILE")
	if file == "" {
		file = London
	}
	f, err := os.Open(file)
	if err != nil {
		b.Fatal(err)
	}
	defer f.Close()

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		f.Seek(0, 0)

		d := NewDecoder(f)
		err = d.Start(runtime.GOMAXPROCS(-1))
		if err != nil {
			b.Fatal(err)
		}

		var nc, wc, rc uint64
		start := time.Now()
		for {
			if v, err := d.Decode(); err == io.EOF {
				break
			} else if err != nil {
				b.Fatal(err)
			} else {
				switch v := v.(type) {
				case *Node:
					nc++
				case *Way:
					wc++
				case *Relation:
					rc++
				default:
					b.Fatalf("unknown type %T", v)
				}
			}
		}

		b.Logf("Done in %.3f seconds. Nodes: %d, Ways: %d, Relations: %d\n",
			time.Now().Sub(start).Seconds(), nc, wc, rc)
	}
}

func BenchmarkDecodeConcurrent(b *testing.B) {
	file := os.Getenv("OSMPBF_BENCHMARK_FILE")
	if file == "" {
		file = London
	}
	f, err := os.Open(file)
	if err != nil {
		b.Fatal(err)
	}
	defer f.Close()

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		f.Seek(0, 0)

		d := NewDecoder(f)
		err = d.Start(runtime.GOMAXPROCS(-1))
		if err != nil {
			b.Fatal(err)
		}

		var nc, wc, rc uint64
		start := time.Now()
		var wg sync.WaitGroup
		for i := 0; i < 4; i++ {
			wg.Add(1)

			go func() {
				for {
					if v, err := d.Decode(); err == io.EOF {
						break
					} else if err != nil {
						b.Fatal(err)
					} else {
						switch v := v.(type) {
						case *Node:
							atomic.AddUint64(&nc, 1)
						case *Way:
							atomic.AddUint64(&wc, 1)
						case *Relation:
							atomic.AddUint64(&rc, 1)
						default:
							b.Fatalf("unknown type %T", v)
						}
					}
				}

				wg.Done()
			}()
		}
		wg.Wait()

		b.Logf("Done in %.3f seconds. Nodes: %d, Ways: %d, Relations: %d\n",
			time.Now().Sub(start).Seconds(), nc, wc, rc)
	}
}
<|endoftext|>


<commit_before>package meta

import (
	"fmt"
	base "github.com/sjwhitworth/golearn/base"
	"math/rand"
	"runtime"
	"strings"
)

// BaggedModels train Classifiers on subsets of the original
// Instances and combine the results through voting
type BaggedModel struct {
	base.BaseClassifier
	Models             []base.Classifier
	selectedAttributes map[int][]base.Attribute
	RandomFeatures     int
}

func (b *BaggedModel) generateTrainingAttrs(model int, from *base.Instances) []base.Attribute {
	ret := make([]base.Attribute, 0)
	if b.RandomFeatures == 0 {
		for j := 0; j < from.Cols; j++ {
			attr := from.GetAttr(j)
			ret = append(ret, attr)
		}
	} else {
		for {
			if len(ret) >= b.RandomFeatures {
				break
			}
			attrIndex := rand.Intn(from.Cols)
			if attrIndex == from.ClassIndex {
				continue
			}
			attr := from.GetAttr(attrIndex)
			matched := false
			for _, a := range ret {
				if a.Equals(attr) {
					matched = true
					break
				}
			}
			if !matched {
				ret = append(ret, attr)
			}
		}
	}
	ret = append(ret, from.GetClassAttr())
	b.selectedAttributes[model] = ret
	return ret
}

func (b *BaggedModel) generatePredictionInstances(model int, from *base.Instances) *base.Instances {
	selected := b.selectedAttributes[model]
	return from.SelectAttributes(selected)
}

func (b *BaggedModel) generateTrainingInstances(model int, from *base.Instances) *base.Instances {
	insts := from.SampleWithReplacement(from.Rows)
	selected := b.generateTrainingAttrs(model, from)
	return insts.SelectAttributes(selected)
}

// AddModel adds a base.Classifier to the current model
func (b *BaggedModel) AddModel(m base.Classifier) {
	b.Models = append(b.Models, m)
}

// Train generates and trains each model on a randomised subset of
// Instances.
func (b *BaggedModel) Fit(from *base.Instances) {
	n := runtime.GOMAXPROCS(0)
	b.selectedAttributes = make(map[int][]base.Attribute)
	block := make(chan bool, n)
	for i, m := range b.Models {
		go func(c base.Classifier, f *base.Instances) {
			f = b.generateTrainingInstances(i, f)
			c.Fit(f)
			block <- true
		}(m, from)
	}
	for i := 0; i < len(b.Models); i++ {
		<-block
	}
}

// Predict gathers predictions from all the classifiers
// and outputs the most common (majority) class
//
// IMPORTANT: in the event of a tie, the first class which
// achieved the tie value is output.
func (b *BaggedModel) Predict(from *base.Instances) *base.Instances {
	n := runtime.GOMAXPROCS(0)
	// Channel to receive the results as they come in
	votes := make(chan *base.Instances, n)
	// Dispatch prediction generation
	for i, m := range b.Models {
		go func(c base.Classifier, f *base.Instances) {
			f = b.generatePredictionInstances(i, f)
			p := c.Predict(f)
			votes <- p
		}(m, from)
	}
	// Count the votes for each class
	voting := make(map[int](map[string]int))
	for _ = range b.Models { // Have to do this - deadlocks otherwise
		incoming := <-votes
		// Step through each prediction
		for j := 0; j < incoming.Rows; j++ {
			// Check if we've seen this class before...
			if _, ok := voting[j]; !ok {
				// If we haven't, create an entry
				voting[j] = make(map[string]int)
				// Continue on the current row
				j--
				continue
			}
			voting[j][incoming.GetClass(j)]++
		}
	}

	// Generate the overall consensus
	ret := from.GeneratePredictionVector()
	for i := range voting {
		maxClass := ""
		maxCount := 0
		// Find the most popular class
		for c := range voting[i] {
			votes := voting[i][c]
			if votes > maxCount {
				maxClass = c
				maxCount = votes
			}
		}
		ret.SetAttrStr(i, 0, maxClass)
	}
	return ret
}

// String returns a human-readable representation of the
// BaggedModel and everything it contains
func (b *BaggedModel) String() string {
	children := make([]string, 0)
	for i, m := range b.Models {
		children = append(children, fmt.Sprintf("%d: %s", i, m))
	}
	return fmt.Sprintf("BaggedModel(\n%s)", strings.Join(children, "\n\t"))
}
<commit_msg>Fixed race conditions<commit_after>package meta

import (
	"fmt"
	base "github.com/sjwhitworth/golearn/base"
	"math/rand"
	"runtime"
	"strings"
	"sync"
)

// BaggedModels train Classifiers on subsets of the original
// Instances and combine the results through voting
type BaggedModel struct {
	base.BaseClassifier
	Models             []base.Classifier
	RandomFeatures     int
	lock               sync.Mutex
	selectedAttributes map[int][]base.Attribute
}

func (b *BaggedModel) generateTrainingAttrs(model int, from *base.Instances) []base.Attribute {
	ret := make([]base.Attribute, 0)
	if b.RandomFeatures == 0 {
		for j := 0; j < from.Cols; j++ {
			attr := from.GetAttr(j)
			ret = append(ret, attr)
		}
	} else {
		for {
			if len(ret) >= b.RandomFeatures {
				break
			}
			attrIndex := rand.Intn(from.Cols)
			if attrIndex == from.ClassIndex {
				continue
			}
			attr := from.GetAttr(attrIndex)
			matched := false
			for _, a := range ret {
				if a.Equals(attr) {
					matched = true
					break
				}
			}
			if !matched {
				ret = append(ret, attr)
			}
		}
	}
	ret = append(ret, from.GetClassAttr())
	b.lock.Lock()
	b.selectedAttributes[model] = ret
	b.lock.Unlock()
	return ret
}

func (b *BaggedModel) generatePredictionInstances(model int, from *base.Instances) *base.Instances {
	selected := b.selectedAttributes[model]
	return from.SelectAttributes(selected)
}

func (b *BaggedModel) generateTrainingInstances(model int, from *base.Instances) *base.Instances {
	insts := from.SampleWithReplacement(from.Rows)
	selected := b.generateTrainingAttrs(model, from)
	return insts.SelectAttributes(selected)
}

// AddModel adds a base.Classifier to the current model
func (b *BaggedModel) AddModel(m base.Classifier) {
	b.Models = append(b.Models, m)
}

// Train generates and trains each model on a randomised subset of
// Instances.
func (b *BaggedModel) Fit(from *base.Instances) {
	n := runtime.GOMAXPROCS(0)
	b.selectedAttributes = make(map[int][]base.Attribute)
	block := make(chan bool, n)
	for i, m := range b.Models {
		go func(c base.Classifier, f *base.Instances, model int) {
			l := b.generateTrainingInstances(model, f)
			c.Fit(l)
			block <- true
		}(m, from, i)
	}
	for i := 0; i < len(b.Models); i++ {
		<-block
	}
}

// Predict gathers predictions from all the classifiers
// and outputs the most common (majority) class
//
// IMPORTANT: in the event of a tie, the first class which
// achieved the tie value is output.
func (b *BaggedModel) Predict(from *base.Instances) *base.Instances {
	n := runtime.GOMAXPROCS(0)
	// Channel to receive the results as they come in
	votes := make(chan *base.Instances, n)
	// Dispatch prediction generation
	for i, m := range b.Models {
		go func(c base.Classifier, f *base.Instances, model int) {
			l := b.generatePredictionInstances(model, f)
			p := c.Predict(l)
			votes <- p
		}(m, from, i)
	}
	// Count the votes for each class
	voting := make(map[int](map[string]int))
	for _ = range b.Models { // Have to do this - deadlocks otherwise
		incoming := <-votes
		// Step through each prediction
		for j := 0; j < incoming.Rows; j++ {
			// Check if we've seen this class before...
			if _, ok := voting[j]; !ok {
				// If we haven't, create an entry
				voting[j] = make(map[string]int)
				// Continue on the current row
				j--
				continue
			}
			voting[j][incoming.GetClass(j)]++
		}
	}

	// Generate the overall consensus
	ret := from.GeneratePredictionVector()
	for i := range voting {
		maxClass := ""
		maxCount := 0
		// Find the most popular class
		for c := range voting[i] {
			votes := voting[i][c]
			if votes > maxCount {
				maxClass = c
				maxCount = votes
			}
		}
		ret.SetAttrStr(i, 0, maxClass)
	}
	return ret
}

// String returns a human-readable representation of the
// BaggedModel and everything it contains
func (b *BaggedModel) String() string {
	children := make([]string, 0)
	for i, m := range b.Models {
		children = append(children, fmt.Sprintf("%d: %s", i, m))
	}
	return fmt.Sprintf("BaggedModel(\n%s)", strings.Join(children, "\n\t"))
}
<|endoftext|>


<commit_before>package db

import (
	"errors"
	"sync"

	"strings"

	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

var config *MongoConfig
var instance *mgo.Session
var instanceOnce sync.Once

type M bson.M

func (m M) Update(qs ...M) M {
	for _, q := range qs {
		for k, v := range q {
			m[k] = v
		}
	}
	return m
}

var ErrOperaBeforeInit = errors.New("please set db.SetOnFinishInit when needed operating db in init()")

// non-multhreads
var afterEvents []func()

func SetOnFinishInit(f func()) {
	if IsFinishInit() {
		f()
		return
	}
	afterEvents = append(afterEvents, f)
}

func IsFinishInit() bool {
	return instance != nil
}

func Setup(c *MongoConfig) {
	config = c
}

func ShareSession() *mgo.Session {
	instanceOnce.Do(func() {
		if instance == nil {
			if config == nil {
				panic(ErrOperaBeforeInit)
			}
			session, err := mgo.Dial(config.MongoDB)
			if err != nil {
				panic(err)
			}
			// Optional. Switch the session to a monotonic behavior.
			session.SetMode(mgo.Monotonic, true)
			instance = session
			for _, f := range afterEvents {
				f()
			}
		}
	})
	return instance
}

func InID(ids []string) (ret M) {
	return M{"_id": M{"$in": ObjectIds(ids)}}
}

func In(ids []string) M {
	return M{"$in": ObjectIds(ids)}
}

func ObjectIds(ids []string) (ret []bson.ObjectId) {
	ret = make([]bson.ObjectId, 0, len(ids))
	for _, id := range ids {
		id = strings.TrimSpace(id)
		if bson.IsObjectIdHex(id) {
			ret = append(ret, bson.ObjectIdHex(id))
		}
	}
	return
}

func NewSession() (session *mgo.Session) {
	return ShareSession().Copy()
}

func NewCollection(session *mgo.Session, dbName, name string) *mgo.Collection {
	if dbName == "" {
		return session.DB(config.DBName).C(name)
	}
	return session.DB(dbName).C(name)
}

func GetCol(dbName, col string) (session *mgo.Session, collection *mgo.Collection) {
	session = NewSession()
	collection = NewCollection(session, dbName, col)
	return
}

func NewObjectId() bson.ObjectId {
	return bson.NewObjectId()
}

func IsMgoNotFound(err error) bool {
	return err == mgo.ErrNotFound
}

func IsMgoDup(err error) bool {
	return mgo.IsDup(err)
}
<commit_msg>fix deadlock by moving afterEvents out of instanceOnce.Do<commit_after>package db

import (
	"errors"
	"sync"

	"strings"

	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

var config *MongoConfig
var instance *mgo.Session
var instanceOnce sync.Once

type M bson.M

func (m M) Update(qs ...M) M {
	for _, q := range qs {
		for k, v := range q {
			m[k] = v
		}
	}
	return m
}

var ErrOperaBeforeInit = errors.New("please set db.SetOnFinishInit when needed operating db in init()")

// non-multhreads
var afterEvents []func()

func SetOnFinishInit(f func()) {
	if IsFinishInit() {
		f()
		return
	}
	afterEvents = append(afterEvents, f)
}

func IsFinishInit() bool {
	return instance != nil
}

func Setup(c *MongoConfig) {
	config = c
}

func ShareSession() *mgo.Session {
	doInit := false
	instanceOnce.Do(func() {
		if instance == nil {
			if config == nil {
				panic(ErrOperaBeforeInit)
			}
			session, err := mgo.Dial(config.MongoDB)
			if err != nil {
				panic(err)
			}
			// Optional. Switch the session to a monotonic behavior.
			session.SetMode(mgo.Monotonic, true)
			instance = session
			doInit = true
		}
	})

	if doInit {
		for _, f := range afterEvents {
			f()
		}
	}
	return instance
}

func InID(ids []string) (ret M) {
	return M{"_id": M{"$in": ObjectIds(ids)}}
}

func In(ids []string) M {
	return M{"$in": ObjectIds(ids)}
}

func ObjectIds(ids []string) (ret []bson.ObjectId) {
	ret = make([]bson.ObjectId, 0, len(ids))
	for _, id := range ids {
		id = strings.TrimSpace(id)
		if bson.IsObjectIdHex(id) {
			ret = append(ret, bson.ObjectIdHex(id))
		}
	}
	return
}

func NewSession() (session *mgo.Session) {
	return ShareSession().Copy()
}

func NewCollection(session *mgo.Session, dbName, name string) *mgo.Collection {
	if dbName == "" {
		return session.DB(config.DBName).C(name)
	}
	return session.DB(dbName).C(name)
}

func GetCol(dbName, col string) (session *mgo.Session, collection *mgo.Collection) {
	session = NewSession()
	collection = NewCollection(session, dbName, col)
	return
}

func NewObjectId() bson.ObjectId {
	return bson.NewObjectId()
}

func IsMgoNotFound(err error) bool {
	return err == mgo.ErrNotFound
}

func IsMgoDup(err error) bool {
	return mgo.IsDup(err)
}
<|endoftext|>


<commit_before>package db

import (
	"fmt"
	"github.com/jinzhu/gorm"
	"github.com/tolexo/aero/conf"
	"math/rand"
	"net/url"
)

var engines map[string]gorm.DB
var ormInit []func(*gorm.DB)

func init() {
	engines = make(map[string]gorm.DB)
	initMySqlConn()
}
func initMySqlConn() {
	initMaster()
	initSlaves()
}
func initMaster() {
	lookup := "database.master"
	if conf.Exists(lookup) {
		connMySqlWrite = GetMySqlConnString(lookup)
	}
}
func initSlaves() {
	lookup := "database.slaves"
	if conf.Exists(lookup) {
		slaves := conf.StringSlice(lookup, []string{})
		connMySqlRead = make([]string, len(slaves))
		for i, container := range slaves {
			connMySqlRead[i] = GetMySqlConnString(container)
		}
	}
}
func GetMySqlConnString(container string) string {
	if !conf.Exists(container) {
		panic("Container for mysql configuration not found")
	}

	username := conf.String(container+".username", "")
	password := conf.String(container+".password", "")
	host := conf.String(container+".host", "")
	port := conf.String(container+".port", "")
	db := conf.String(container+".db", "")
	timezone := conf.String(container+".timezone", "")

	return fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?parseTime=true&loc=%s",
		username, password,
		host, port, db,
		url.QueryEscape(timezone),
	)
}

func GetDefaultConn(write bool) string {
	if write {
		return connMySqlWrite
	}

	if connMySqlRead == nil || len(connMySqlRead) == 0 {
		return connMySqlWrite
	}

	return connMySqlRead[rand.Intn(len(connMySqlRead))]
}

func GetMySqlConn(writable bool) (gorm.DB, error) {
	connStr := GetDefaultConn(writable)
	return GetOrm(connStr)
}

func GetOrm(connStr string) (ormObj gorm.DB, err error) {
	var ok bool

	if ormObj, ok = engines[connStr]; ok {
		return
	}
	// http://go-database-sql.org/accessing.html
	// the sql.DB object is designed to be long-lived
	if ormObj, err = gorm.Open("mysql", connStr); err == nil {
		if ormInit != nil {
			for _, fn := range ormInit {
				fn(&ormObj)
			}
		}
		engines[connStr] = ormObj
		return

	} else {
		return
	}
}

func DoOrmInit(fn func(*gorm.DB)) {
	// TODO: use mutex
	if ormInit == nil {
		ormInit = make([]func(*gorm.DB), 0)
	}
	ormInit = append(ormInit, fn)
}
<commit_msg>CLA-137 modified mysql adater<commit_after>package db

import (
	"fmt"
	"github.com/jinzhu/gorm"
	"github.com/tolexo/aero/conf"
	"math/rand"
	"net/url"
)

var engines map[string]gorm.DB
var ormInit []func(*gorm.DB)

func initMaster() {
	lookup := "database.master"
	if conf.Exists(lookup) {
		connMySqlWrite = getMySqlConnString(lookup)
	}
}
func initSlaves() {
	lookup := "database.slaves"
	if conf.Exists(lookup) {
		slaves := conf.StringSlice(lookup, []string{})
		connMySqlRead = make([]string, len(slaves))
		for i, container := range slaves {
			connMySqlRead[i] = getMySqlConnString(container)
		}
	}
}
func getMySqlConnString(container string) string {
	if !conf.Exists(container) {
		panic("Container for mysql configuration not found")
	}

	username := conf.String(container+".username", "")
	password := conf.String(container+".password", "")
	host := conf.String(container+".host", "")
	port := conf.String(container+".port", "")
	db := conf.String(container+".db", "")
	timezone := conf.String(container+".timezone", "")

	return fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?parseTime=true&loc=%s",
		username, password,
		host, port, db,
		url.QueryEscape(timezone),
	)
}

func getOrm(connStr string) (ormObj gorm.DB, err error) {
	var ok bool

	if ormObj, ok = engines[connStr]; ok {
		return
	}
	// http://go-database-sql.org/accessing.html
	// the sql.DB object is designed to be long-lived
	if ormObj, err = gorm.Open("mysql", connStr); err == nil {
		if ormInit != nil {
			for _, fn := range ormInit {
				fn(&ormObj)
			}
		}
		engines[connStr] = ormObj
		return

	} else {
		return
	}
}

func DoOrmInit(fn func(*gorm.DB)) {
	// TODO: use mutex
	if ormInit == nil {
		ormInit = make([]func(*gorm.DB), 0)
	}
	ormInit = append(ormInit, fn)
}

//Get MySql connection
func GetMySqlConn(writable bool) (gorm.DB, error) {
	engines = make(map[string]gorm.DB)
	if writable {
		initMaster()
		return getOrm(connMySqlWrite)
	} else {
		initSlaves()
		if connMySqlRead == nil || len(connMySqlRead) == 0 {
			return getOrm(connMySqlWrite)
		}
		return getOrm(connMySqlRead[rand.Intn(len(connMySqlRead))])

	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/DataDog/dd-trace-go/tracer"
	"github.com/Sirupsen/logrus"
	"github.com/flachnetz/dd-zipkin-proxy"
	"github.com/openzipkin/zipkin-go-opentracing/_thrift/gen-go/zipkincore"
	"regexp"
	"strconv"
	"strings"
)

func main() {
	converter := DefaultSpanConverter{}
	zipkinproxy.Main(converter.Convert)
}

var reHash = regexp.MustCompile("\\b(?:[a-f0-9]{32}|[a-f0-9]{24}|[a-f0-9-]{8}-[a-f0-9-]{4}-[a-f0-9-]{4}-[a-f0-9-]{4}-[a-f0-9-]{12})\\b")
var reNumber = regexp.MustCompile("\\b[0-9]{2,}\\b")
var reIwgHash = regexp.MustCompile("iwg\\.[A-Za-z0-9]{12}\\b")
var reEmail = regexp.MustCompile("emailAddress=[^%]+%40[^&]+")

func SimplifyResourceName(value string) string {
	// check if we need to apply the regexp by checking if a match is possible or not
	digitCount := 0
	hashCharCount := 0
	mightHaveEmailSign := false
	for _, char := range value {
		isDigit := char >= '0' && char <= '9'
		if isDigit {
			digitCount++
		}

		if isDigit || char >= 'a' && char <= 'f' {
			hashCharCount++
		}

		if char == '%' {
			mightHaveEmailSign = true
		}
	}

	// only search for hash, if we have enough chars for it
	if hashCharCount >= 24 {
		value = reHash.ReplaceAllString(value, "_HASH_")
	}

	// only replace numbers, if we have enough digits for a match
	if digitCount >= 2 {
		value = reNumber.ReplaceAllString(value, "_NUMBER_")
	}

	if strings.HasPrefix(value, "iwg.") {
		value = reIwgHash.ReplaceAllString(value, "iwg._HASH_")
	}

	if mightHaveEmailSign && digitCount >= 2 {
		value = reEmail.ReplaceAllString(value, "emailAddress=_EMAIL_")
	}

	return value
}

func RetractSensitiveData(value string) string {
	if strings.Contains(value, "emailAddress=") {
		value = reEmail.ReplaceAllString(value, "emailAddress=_EMAIL_")
	}

	return value
}

type DefaultSpanConverter struct {
	current  map[uint64]string
	previous map[uint64]string
}

func (converter *DefaultSpanConverter) Convert(span *zipkincore.Span) *tracer.Span {
	if span.Name == "watch-config-key-values" {
		return nil
	}

	name := SimplifyResourceName(span.Name)

	converted := &tracer.Span{
		SpanID:   uint64(span.ID),
		ParentID: uint64(span.GetParentID()),
		TraceID:  uint64(span.TraceID),
		Name:     name,
		Resource: name,
		Start:    1000 * span.GetTimestamp(),
		Duration: 1000 * span.GetDuration(),
		Sampled:  true,
	}

	// datadog traces use a trace of 0
	if converted.ParentID == converted.SpanID {
		converted.ParentID = 0
	}

	// split "http:/some/url" or "get:/some/url"
	for _, prefix := range []string{"http:/", "get:/", "post:/"} {
		if strings.HasPrefix(converted.Name, prefix) {
			converted.Resource = converted.Name[len(prefix)-1:]
			converted.Name = prefix[:len(prefix)-2]
			break
		}
	}

	// convert binary annotations (like tags)
	if len(span.BinaryAnnotations) > 0 {
		converted.Meta = make(map[string]string, len(span.BinaryAnnotations))
		for _, an := range span.BinaryAnnotations {
			if an.AnnotationType == zipkincore.AnnotationType_STRING {
				key := an.Key

				// rename keys to better match the datadog one.
				switch key {
				case "http.status":
					key = "http.status_code"

				case "client.url":
					key = "http.url"
				}

				converted.Meta[key] = string(an.Value)
			}

			if an.Host != nil && an.Host.ServiceName != "" {
				converted.Service = an.Host.ServiceName
			}
		}

		if url := converted.Meta["http.url"]; url != "" {
			converted.Resource = SimplifyResourceName(url)
			converted.Meta["http.url"] = RetractSensitiveData(url)
		}

		if status := converted.Meta["http.status_code"]; status != "" {
			if len(status) > 0 && '3' <= status[0] && status[0] <= '9' {
				if statusValue, err := strconv.Atoi(status); err == nil && statusValue >= 400 {
					converted.Error = int32(statusValue)
				}
			}
		}
	}

	extractInfoFromAnnotations(span, converted)

	// simplify some names
	if strings.HasPrefix(converted.Name, "http:") {
		converted.Service = converted.Name[5:]

	} else if converted.Name == "transaction" {
		converted.Service = "oracle"

	} else if sql := converted.Meta["sql"]; sql != "" {
		delete(converted.Meta, "sql")
		converted.Resource = sql
		converted.Service = "sql"

	} else if strings.HasPrefix(converted.Name, "redis:") {
		converted.Service = "redis"

		if key := converted.Meta["redis.key"]; key != "" {
			converted.Resource = SimplifyResourceName(key)

			// the hash is not really important later on. lets not spam datadog with it.
			delete(converted.Meta, "redis.key")
		}

	} else if converted.Service == "core-services" {
		lc := converted.Meta["lc"]

		switch {
		case strings.Contains(converted.Meta["http.url"], ":6080/"):
			converted.Service = "iwg-restrictor"
			converted.Name = iwgNameFromResource(converted.Resource, converted.Name)
			converted.Resource = dropDomainFromUrl(converted.Resource)

		case strings.Contains(converted.Meta["http.url"], ":2080/"):
			converted.Service = "instant-win-game"
			converted.Name = "iwg-game"
			converted.Resource = dropDomainFromUrl(converted.Resource)

		case lc != "" && lc != "servlet" && lc != "HttpClient":
			// TODO: find proper fix for this service name / name datadog magic
			//delete(converted.Meta, "lc")
			converted.Service = lc
		}
	}

	// If we could not get a service, we'll try to get it from the parent span.
	// Try first in the current map, then in the previous one.
	if converted.Service == "" {
		parentService := converter.current[converted.ParentID]
		if parentService != "" {
			converted.Service = parentService
		} else {
			parentService = converter.previous[converted.ParentID]
			if parentService != "" {
				converted.Service = parentService
			}
		}

		// if we did not get a service, use a fallback
		if converted.Service == "" {
			logrus.Warnf("Could not get a service for this span: %+v", span)
			converted.Service = "unknown"
		}
	}

	if lc := converted.Meta["lc"]; lc != "" {
		// TODO: here as well
		//delete(converted.Meta, "lc")

		if converted.Name == "" {
			converted.Name = lc
		}

		if lc == "consul" {
			converted.Service = "consul"
		}
	}

	if converted.Name == "transaction" || converted.Service == "redis" {
		converted.Type = "db"
	} else {
		converted.Type = "http"
	}

	if converted.Name == "hystrix" {
		converted.Resource = converted.Meta["thread"]
	} else if converted.Name == "" {
		logrus.Warnf("Could not get a name for this span: %+v converted: %+v", span, converted)
	}

	// if name and service differ than the overview page in datadog will only show the one with
	// most of the time spend. This is why we just rename it to the service here so that we can get a nice
	// overview of all resources belonging to the service. Can be removed in the future when datadog is changing things
	converted.Name = converted.Service

	// initialize history maps for span -> parent assignment
	const parentLookupMapSize = 40000
	if len(converter.current) >= parentLookupMapSize || converter.current == nil {
		converter.previous = converter.current
		converter.current = make(map[uint64]string, parentLookupMapSize)
	}

	// remember the service for a short while
	converter.current[converted.SpanID] = converted.Service

	return converted
}

func extractInfoFromAnnotations(span *zipkincore.Span, converted *tracer.Span) {
	// try to get the service from the cs/cr or sr/ss annotations
	var minTimestamp, maxTimestamp int64
	for _, an := range span.Annotations {
		if an.Value == "sr" && an.Host != nil && an.Host.ServiceName != "" {
			converted.Service = an.Host.ServiceName
		}

		if an.Timestamp < minTimestamp || minTimestamp == 0 {
			minTimestamp = an.Timestamp
		}

		if an.Timestamp > maxTimestamp {
			maxTimestamp = an.Timestamp
		}
	}

	if converted.Start == 0 {
		logrus.Warnf("Span had no start/duration, guessing from annotations: %s", identifySpan(span))
		converted.Start = 1000 * minTimestamp
		converted.Duration = 1000 * (maxTimestamp - minTimestamp)
	}
}

// Tries to get some identification for this span. The method tries
// to include the value of the local-component tag and the value of the tags name.
func identifySpan(span *zipkincore.Span) string {
	var name string
	for _, an := range span.BinaryAnnotations {
		if an.Key == "lc" {
			name = string(an.Value) + ":"
		}
	}

	return name + span.Name
}

func dropDomainFromUrl(url string) string {
	switch {
	case strings.HasPrefix(url, "http://"):
		index := strings.IndexRune(url[7:], '/')
		return url[7+index:]

	case strings.HasPrefix(url, "https://"):
		index := strings.IndexRune(url[8:], '/')
		return url[8+index:]
	}

	return url
}

func iwgNameFromResource(resource string, fallback string) string {
	splittedUrl := strings.Split(strings.Trim(resource, "/"), "/")
	if len(splittedUrl) == 1 && splittedUrl[0] != "" {
		return splittedUrl[0]
	} else if len(splittedUrl) > 1 {
		return splittedUrl[len(splittedUrl)-2] + "/" + splittedUrl[len(splittedUrl)-1]
	}
	return fallback
}
<commit_msg>remove querystring from urls<commit_after>package main

import (
	"github.com/DataDog/dd-trace-go/tracer"
	"github.com/Sirupsen/logrus"
	"github.com/flachnetz/dd-zipkin-proxy"
	"github.com/openzipkin/zipkin-go-opentracing/_thrift/gen-go/zipkincore"
	"regexp"
	"strconv"
	"strings"
)

func main() {
	converter := DefaultSpanConverter{}
	zipkinproxy.Main(converter.Convert)
}

var reHash = regexp.MustCompile("\\b(?:[a-f0-9]{32}|[a-f0-9]{24}|[a-f0-9-]{8}-[a-f0-9-]{4}-[a-f0-9-]{4}-[a-f0-9-]{4}-[a-f0-9-]{12})\\b")
var reNumber = regexp.MustCompile("\\b[0-9]{2,}\\b")
var reIwgHash = regexp.MustCompile("iwg\\.[A-Za-z0-9]{12}\\b")

func SimplifyResourceName(value string) string {
	// check if we need to apply the regexp by checking if a match is possible or not
	digitCount := 0
	hashCharCount := 0
	for idx, char := range value {
		isDigit := char >= '0' && char <= '9'
		if isDigit {
			digitCount++
		}

		if isDigit || char >= 'a' && char <= 'f' {
			hashCharCount++
		}
		if char == '?' {
			value = value[:idx]
			break
		}
	}

	// only search for hash, if we have enough chars for it
	if hashCharCount >= 24 {
		value = reHash.ReplaceAllString(value, "_HASH_")
	}

	// only replace numbers, if we have enough digits for a match
	if digitCount >= 2 {
		value = reNumber.ReplaceAllString(value, "_NUMBER_")
	}

	if strings.HasPrefix(value, "iwg.") {
		value = reIwgHash.ReplaceAllString(value, "iwg._HASH_")
	}

	return value
}

func RemoveQueryString(value string) string {
	idx := strings.IndexByte(value, '?')
	if idx >= 0 {
		return value[:idx]
	}
	return value
}

type DefaultSpanConverter struct {
	current  map[uint64]string
	previous map[uint64]string
}

func (converter *DefaultSpanConverter) Convert(span *zipkincore.Span) *tracer.Span {
	if span.Name == "watch-config-key-values" {
		return nil
	}

	name := SimplifyResourceName(span.Name)

	converted := &tracer.Span{
		SpanID:   uint64(span.ID),
		ParentID: uint64(span.GetParentID()),
		TraceID:  uint64(span.TraceID),
		Name:     name,
		Resource: name,
		Start:    1000 * span.GetTimestamp(),
		Duration: 1000 * span.GetDuration(),
		Sampled:  true,
	}

	// datadog traces use a trace of 0
	if converted.ParentID == converted.SpanID {
		converted.ParentID = 0
	}

	// split "http:/some/url" or "get:/some/url"
	for _, prefix := range []string{"http:/", "get:/", "post:/"} {
		if strings.HasPrefix(converted.Name, prefix) {
			converted.Resource = converted.Name[len(prefix)-1:]
			converted.Name = prefix[:len(prefix)-2]
			break
		}
	}

	// convert binary annotations (like tags)
	if len(span.BinaryAnnotations) > 0 {
		converted.Meta = make(map[string]string, len(span.BinaryAnnotations))
		for _, an := range span.BinaryAnnotations {
			if an.AnnotationType == zipkincore.AnnotationType_STRING {
				key := an.Key

				// rename keys to better match the datadog one.
				switch key {
				case "http.status":
					key = "http.status_code"

				case "client.url":
					key = "http.url"
				}

				converted.Meta[key] = string(an.Value)
			}

			if an.Host != nil && an.Host.ServiceName != "" {
				converted.Service = an.Host.ServiceName
			}
		}

		if url := converted.Meta["http.url"]; url != "" {
			converted.Resource = SimplifyResourceName(url)
			converted.Meta["http.url"] = RemoveQueryString(url)
		}

		if status := converted.Meta["http.status_code"]; status != "" {
			if len(status) > 0 && '3' <= status[0] && status[0] <= '9' {
				if statusValue, err := strconv.Atoi(status); err == nil && statusValue >= 400 {
					converted.Error = int32(statusValue)
				}
			}
		}
	}

	extractInfoFromAnnotations(span, converted)

	// simplify some names
	if strings.HasPrefix(converted.Name, "http:") {
		converted.Service = converted.Name[5:]

	} else if converted.Name == "transaction" {
		converted.Service = "oracle"

	} else if sql := converted.Meta["sql"]; sql != "" {
		delete(converted.Meta, "sql")
		converted.Resource = sql
		converted.Service = "sql"

	} else if strings.HasPrefix(converted.Name, "redis:") {
		converted.Service = "redis"

		if key := converted.Meta["redis.key"]; key != "" {
			converted.Resource = SimplifyResourceName(key)

			// the hash is not really important later on. lets not spam datadog with it.
			delete(converted.Meta, "redis.key")
		}

	} else if converted.Service == "core-services" {
		lc := converted.Meta["lc"]

		switch {
		case strings.Contains(converted.Meta["http.url"], ":6080/"):
			converted.Service = "iwg-restrictor"
			converted.Name = iwgNameFromResource(converted.Resource, converted.Name)
			converted.Resource = dropDomainFromUrl(converted.Resource)

		case strings.Contains(converted.Meta["http.url"], ":2080/"):
			converted.Service = "instant-win-game"
			converted.Name = "iwg-game"
			converted.Resource = dropDomainFromUrl(converted.Resource)

		case lc != "" && lc != "servlet" && lc != "HttpClient":
			// TODO: find proper fix for this service name / name datadog magic
			//delete(converted.Meta, "lc")
			converted.Service = lc
		}
	}

	// If we could not get a service, we'll try to get it from the parent span.
	// Try first in the current map, then in the previous one.
	if converted.Service == "" {
		parentService := converter.current[converted.ParentID]
		if parentService != "" {
			converted.Service = parentService
		} else {
			parentService = converter.previous[converted.ParentID]
			if parentService != "" {
				converted.Service = parentService
			}
		}

		// if we did not get a service, use a fallback
		if converted.Service == "" {
			logrus.Warnf("Could not get a service for this span: %+v", span)
			converted.Service = "unknown"
		}
	}

	if lc := converted.Meta["lc"]; lc != "" {
		// TODO: here as well
		//delete(converted.Meta, "lc")

		if converted.Name == "" {
			converted.Name = lc
		}

		if lc == "consul" {
			converted.Service = "consul"
		}
	}

	if converted.Name == "transaction" || converted.Service == "redis" {
		converted.Type = "db"
	} else {
		converted.Type = "http"
	}

	if converted.Name == "hystrix" {
		converted.Resource = converted.Meta["thread"]
	} else if converted.Name == "" {
		logrus.Warnf("Could not get a name for this span: %+v converted: %+v", span, converted)
	}

	// if name and service differ than the overview page in datadog will only show the one with
	// most of the time spend. This is why we just rename it to the service here so that we can get a nice
	// overview of all resources belonging to the service. Can be removed in the future when datadog is changing things
	converted.Name = converted.Service

	// initialize history maps for span -> parent assignment
	const parentLookupMapSize = 40000
	if len(converter.current) >= parentLookupMapSize || converter.current == nil {
		converter.previous = converter.current
		converter.current = make(map[uint64]string, parentLookupMapSize)
	}

	// remember the service for a short while
	converter.current[converted.SpanID] = converted.Service

	return converted
}

func extractInfoFromAnnotations(span *zipkincore.Span, converted *tracer.Span) {
	// try to get the service from the cs/cr or sr/ss annotations
	var minTimestamp, maxTimestamp int64
	for _, an := range span.Annotations {
		if an.Value == "sr" && an.Host != nil && an.Host.ServiceName != "" {
			converted.Service = an.Host.ServiceName
		}

		if an.Timestamp < minTimestamp || minTimestamp == 0 {
			minTimestamp = an.Timestamp
		}

		if an.Timestamp > maxTimestamp {
			maxTimestamp = an.Timestamp
		}
	}

	if converted.Start == 0 {
		logrus.Warnf("Span had no start/duration, guessing from annotations: %s", identifySpan(span))
		converted.Start = 1000 * minTimestamp
		converted.Duration = 1000 * (maxTimestamp - minTimestamp)
	}
}

// Tries to get some identification for this span. The method tries
// to include the value of the local-component tag and the value of the tags name.
func identifySpan(span *zipkincore.Span) string {
	var name string
	for _, an := range span.BinaryAnnotations {
		if an.Key == "lc" {
			name = string(an.Value) + ":"
		}
	}

	return name + span.Name
}

func dropDomainFromUrl(url string) string {
	switch {
	case strings.HasPrefix(url, "http://"):
		index := strings.IndexRune(url[7:], '/')
		return url[7+index:]

	case strings.HasPrefix(url, "https://"):
		index := strings.IndexRune(url[8:], '/')
		return url[8+index:]
	}

	return url
}

func iwgNameFromResource(resource string, fallback string) string {
	splittedUrl := strings.Split(strings.Trim(resource, "/"), "/")
	if len(splittedUrl) == 1 && splittedUrl[0] != "" {
		return splittedUrl[0]
	} else if len(splittedUrl) > 1 {
		return splittedUrl[len(splittedUrl)-2] + "/" + splittedUrl[len(splittedUrl)-1]
	}
	return fallback
}
<|endoftext|>