<commit_before>package command

import (
	"bufio"
	"flag"
	"fmt"
	"net"
	"sort"
	"strings"

	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
	"github.com/funkygao/golib/pipestream"
	consulapi "github.com/hashicorp/consul/api"
	"github.com/ryanuber/columnize"
)

// consul members will include:
// - zk cluster as server
// - agents
//   - brokers
//   - kateway
type Members struct {
	Ui  cli.Ui
	Cmd string

	brokerHosts, zkHosts, katewayHosts map[string]struct{}
	nodeHostMap                        map[string]string // consul members node->ip
}

func (this *Members) Run(args []string) (exitCode int) {
	var (
		zone        string
		showLoadAvg bool
		exec        string
		node        string
		role        string
	)
	cmdFlags := flag.NewFlagSet("members", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.BoolVar(&showLoadAvg, "l", false, "")
	cmdFlags.StringVar(&exec, "exec", "", "")
	cmdFlags.StringVar(&node, "n", "", "")
	cmdFlags.StringVar(&role, "r", "", "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if validateArgs(this, this.Ui).
		requireAdminRights("-exec").
		invalid(args) {
		return 2
	}

	zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
	this.fetchAllRunningHostsFromZk(zkzone)

	members := this.consulMembers()
	consulLiveNode := make([]string, 0, len(members))
	for _, member := range members {
		if member.Status == 1 {
			consulLiveNode = append(consulLiveNode, member.Addr)
		} else {
			this.Ui.Warn(fmt.Sprintf("%s %s status:%d", member.Name, member.Addr, member.Status))
		}
	}

	consulLiveMap := make(map[string]struct{})
	brokerN, zkN, katewayN, unknownN := 0, 0, 0, 0
	for _, node := range consulLiveNode {
		_, presentInBroker := this.brokerHosts[node]
		_, presentInZk := this.zkHosts[node]
		_, presentInKateway := this.katewayHosts[node]
		if presentInBroker {
			brokerN++
		}
		if presentInZk {
			zkN++
		}
		if presentInKateway {
			katewayN++
		}

		if !presentInBroker && !presentInZk && !presentInKateway {
			unknownN++
		}

		consulLiveMap[node] = struct{}{}
	}

	// all brokers should run consul
	for broker, _ := range this.brokerHosts {
		if _, present := consulLiveMap[broker]; !present {
			this.Ui.Warn(fmt.Sprintf("- %s", broker))
		}
	}

	switch {
	case showLoadAvg:
		this.displayLoadAvg(role)

	case exec != "":
		this.executeOnAll(exec, role, node)

	default:
		this.displayMembers(members, role)
	}

	// summary
	this.Ui.Output(fmt.Sprintf("Zk:%d Broker:%d Kateway:%d ?:%s => %d",
		zkN, brokerN, katewayN, color.Yellow("%d", unknownN), zkN+brokerN+katewayN+unknownN))

	return
}

func (this *Members) fetchAllRunningHostsFromZk(zkzone *zk.ZkZone) {
	this.brokerHosts = make(map[string]struct{})
	zkzone.ForSortedBrokers(func(cluster string, brokers map[string]*zk.BrokerZnode) {
		for _, brokerInfo := range brokers {
			this.brokerHosts[brokerInfo.Host] = struct{}{}
		}
	})

	this.zkHosts = make(map[string]struct{})
	for _, addr := range zkzone.ZkAddrList() {
		zkNode, _, err := net.SplitHostPort(addr)
		swallow(err)
		this.zkHosts[zkNode] = struct{}{}
	}

	this.katewayHosts = make(map[string]struct{})
	kws, err := zkzone.KatewayInfos()
	swallow(err)
	for _, kw := range kws {
		host, _, err := net.SplitHostPort(kw.PubAddr)
		swallow(err)
		this.katewayHosts[host] = struct{}{}
	}
}

// TODO role is ignored
func (this *Members) executeOnAll(execCmd string, role string, node string) {
	args := []string{"exec"}
	if node != "" {
		args = append(args, fmt.Sprintf("-node=%s", node))
	}
	args = append(args, strings.Split(execCmd, " ")...)
	cmd := pipestream.New("consul", args...)
	err := cmd.Open()
	swallow(err)
	defer cmd.Close()

	this.Ui.Info(fmt.Sprintf("%s ...", execCmd))

	lines := make([]string, 0)
	header := "Node|Host|Role|Result"
	lines = append(lines, header)

	scanner := bufio.NewScanner(cmd.Reader())
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		line := scanner.Text()
		if strings.Contains(line, "finished with exit code 0") ||
			strings.Contains(line, "completed / acknowledged") {
			continue
		}
		fields := strings.Fields(line)
		if len(fields) == 1 {
			continue
		}

		node := fields[0]
		if strings.HasSuffix(node, ":") {
			node = strings.TrimRight(node, ":")
		}

		host := this.nodeHostMap[node]
		lines = append(lines, fmt.Sprintf("%s|%s|%s|%s", node, host, this.roleOfHost(host),
			strings.Join(fields[1:], " ")))
	}

	if len(lines) > 1 {
		sort.Strings(lines[1:])
		this.Ui.Output(columnize.SimpleFormat(lines))
	}
}

func (this *Members) displayMembers(members []*consulapi.AgentMember, role string) {
	lines := make([]string, 0)
	header := "Node|Host|Role"
	lines = append(lines, header)
	for _, member := range members {
		hostRole := this.roleOfHost(member.Addr)
		if role != "" && role != hostRole {
			continue
		}

		lines = append(lines, fmt.Sprintf("%s|%s|%s", member.Name, member.Addr, hostRole))
	}

	if len(lines) > 1 {
		sort.Strings(lines[1:])
		this.Ui.Output(columnize.SimpleFormat(lines))
	}
}

func (this *Members) displayLoadAvg(role string) {
	cmd := pipestream.New("consul", "exec",
		"uptime", "|", "grep", "load")
	err := cmd.Open()
	swallow(err)
	defer cmd.Close()

	lines := make([]string, 0)
	header := "Node|Host|Role|Load Avg"
	lines = append(lines, header)

	scanner := bufio.NewScanner(cmd.Reader())
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		line := scanner.Text()
		fields := strings.Fields(line)
		node := fields[0]
		parts := strings.Split(line, "load average:")
		if len(parts) < 2 {
			continue
		}
		if strings.HasSuffix(node, ":") {
			node = strings.TrimRight(node, ":")
		}

		loadAvg := strings.TrimSpace(parts[1])
		if loadAvg[0] > '0' {
			loadAvg += " !"
			if loadAvg[0] > '2' {
				loadAvg += "!"
			}
			if loadAvg[0] > '4' {
				loadAvg += "!"
			}
		}

		host := this.nodeHostMap[node]
		hostRole := this.roleOfHost(host)
		if role != "" && hostRole != role {
			continue
		}

		lines = append(lines, fmt.Sprintf("%s|%s|%s|%s", node, host, hostRole, loadAvg))
	}

	if len(lines) > 1 {
		sort.Strings(lines[1:])
		this.Ui.Output(columnize.SimpleFormat(lines))
	}
}

func (this *Members) roleOfHost(host string) string {
	if _, present := this.brokerHosts[host]; present {
		return "B"
	}
	if _, present := this.zkHosts[host]; present {
		return "Z"
	}
	if _, present := this.katewayHosts[host]; present {
		return "K"
	}
	return "?"
}

func (this *Members) consulMembers() []*consulapi.AgentMember {
	cf := consulapi.DefaultConfig()
	client, err := consulapi.NewClient(cf)
	swallow(err)
	members, err := client.Agent().Members(false)
	swallow(err)

	m := make(map[string]*consulapi.AgentMember, len(members))
	sortedName := make([]string, 0, len(members))
	this.nodeHostMap = make(map[string]string, len(members))
	for _, member := range members {
		m[member.Name] = member
		this.nodeHostMap[member.Name] = member.Addr

		sortedName = append(sortedName, member.Name)
	}
	sort.Strings(sortedName)

	r := make([]*consulapi.AgentMember, 0, len(sortedName))
	for _, name := range sortedName {
		r = append(r, m[name])
	}

	return r
}

func (*Members) Synopsis() string {
	return "Verify consul members match kafka zone"
}

func (this *Members) Help() string {
	help := fmt.Sprintf(`
Usage: %s members [options]

    Verify consul members match kafka zone

    -z zone

    -r role
      Available roles: [Z|K|B]

    -l
      Display each member load average

    -exec <cmd>
      Execute cmd on all members and print the result by host
      e,g. gk members -exec "ifconfig bond0 | grep 'TX bytes'"

    -n node
      Execute cmd on a single node

`, this.Cmd)
	return strings.TrimSpace(help)
}
<commit_msg>gk exec on nodes with specified role<commit_after>package command

import (
	"bufio"
	"flag"
	"fmt"
	"net"
	"sort"
	"strings"
	"sync"

	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
	"github.com/funkygao/golib/pipestream"
	consulapi "github.com/hashicorp/consul/api"
	"github.com/ryanuber/columnize"
)

// consul members will include:
// - zk cluster as server
// - agents
//   - brokers
//   - kateway
type Members struct {
	Ui  cli.Ui
	Cmd string

	brokerHosts, zkHosts, katewayHosts map[string]struct{}
	nodeHostMap, hostNodeMap           map[string]string // consul members node->ip
	debug                              bool
}

func (this *Members) Run(args []string) (exitCode int) {
	var (
		zone        string
		showLoadAvg bool
		exec        string
		node        string
		role        string
	)
	cmdFlags := flag.NewFlagSet("members", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.BoolVar(&showLoadAvg, "l", false, "")
	cmdFlags.StringVar(&exec, "exec", "", "")
	cmdFlags.StringVar(&node, "n", "", "")
	cmdFlags.BoolVar(&this.debug, "debug", false, "")
	cmdFlags.StringVar(&role, "r", "", "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if validateArgs(this, this.Ui).
		requireAdminRights("-exec").
		invalid(args) {
		return 2
	}

	zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
	this.fetchAllRunningHostsFromZk(zkzone)

	members := this.consulMembers()
	consulLiveNode := make([]string, 0, len(members))
	for _, member := range members {
		if member.Status == 1 {
			consulLiveNode = append(consulLiveNode, member.Addr)
		} else {
			this.Ui.Warn(fmt.Sprintf("%s %s status:%d", member.Name, member.Addr, member.Status))
		}
	}

	consulLiveMap := make(map[string]struct{})
	brokerN, zkN, katewayN, unknownN := 0, 0, 0, 0
	for _, node := range consulLiveNode {
		_, presentInBroker := this.brokerHosts[node]
		_, presentInZk := this.zkHosts[node]
		_, presentInKateway := this.katewayHosts[node]
		if presentInBroker {
			brokerN++
		}
		if presentInZk {
			zkN++
		}
		if presentInKateway {
			katewayN++
		}

		if !presentInBroker && !presentInZk && !presentInKateway {
			unknownN++
		}

		consulLiveMap[node] = struct{}{}
	}

	// all brokers should run consul
	for broker, _ := range this.brokerHosts {
		if _, present := consulLiveMap[broker]; !present {
			this.Ui.Warn(fmt.Sprintf("- %s", broker))
		}
	}

	switch {
	case showLoadAvg:
		this.displayLoadAvg(role)

	case exec != "":
		if role != "" {
			this.executeOnRole(exec, role)
		} else {
			this.executeOnAll(exec, node)
		}

	default:
		this.displayMembers(members, role)
	}

	// summary
	this.Ui.Output(fmt.Sprintf("Zk:%d Broker:%d Kateway:%d ?:%s => %d",
		zkN, brokerN, katewayN, color.Yellow("%d", unknownN), zkN+brokerN+katewayN+unknownN))

	return
}

func (this *Members) fetchAllRunningHostsFromZk(zkzone *zk.ZkZone) {
	this.brokerHosts = make(map[string]struct{})
	zkzone.ForSortedBrokers(func(cluster string, brokers map[string]*zk.BrokerZnode) {
		for _, brokerInfo := range brokers {
			this.brokerHosts[brokerInfo.Host] = struct{}{}
		}
	})

	this.zkHosts = make(map[string]struct{})
	for _, addr := range zkzone.ZkAddrList() {
		zkNode, _, err := net.SplitHostPort(addr)
		swallow(err)
		this.zkHosts[zkNode] = struct{}{}
	}

	this.katewayHosts = make(map[string]struct{})
	kws, err := zkzone.KatewayInfos()
	swallow(err)
	for _, kw := range kws {
		host, _, err := net.SplitHostPort(kw.PubAddr)
		swallow(err)
		this.katewayHosts[host] = struct{}{}
	}
}

func (this *Members) executeOnRole(execCmd string, role string) {
	this.Ui.Info(fmt.Sprintf("%s ...", execCmd))

	linesByNode := make(map[string][]string)
	var mu sync.Mutex
	var wg sync.WaitGroup
	for _, node := range this.nodesOfRole(role) {
		linesByNode[node] = make([]string, 0)
		wg.Add(1)

		go func(node string) {
			defer wg.Done()

			args := []string{"exec"}
			if node != "" {
				args = append(args, fmt.Sprintf("-node=%s", node))
			}
			args = append(args, strings.Split(execCmd, " ")...)
			if this.debug {
				this.Ui.Output(fmt.Sprintf("consul %+v", args))
			}
			cmd := pipestream.New("consul", args...)
			err := cmd.Open()
			swallow(err)
			defer cmd.Close()

			scanner := bufio.NewScanner(cmd.Reader())
			scanner.Split(bufio.ScanLines)
			for scanner.Scan() {
				line := scanner.Text()
				if strings.Contains(line, "finished with exit code 0") ||
					strings.Contains(line, "completed / acknowledged") {
					continue
				}
				fields := strings.Fields(line)
				if len(fields) == 1 {
					continue
				}

				node := fields[0]
				if strings.HasSuffix(node, ":") {
					node = strings.TrimRight(node, ":")
				}

				host := this.nodeHostMap[node]
				mu.Lock()
				linesByNode[node] = append(linesByNode[node], fmt.Sprintf("%s|%s", host, strings.Join(fields[1:], " ")))
				mu.Unlock()
			}
		}(node)
	}

	wg.Wait()

	sortedNodes := make([]string, 0, len(linesByNode))
	for node, _ := range linesByNode {
		sortedNodes = append(sortedNodes, node)
	}
	sort.Strings(sortedNodes)

	lines := make([]string, 0)
	header := "Host|Result"
	lines = append(lines, header)
	for _, node := range sortedNodes {
		lines = append(lines, linesByNode[node]...)
	}

	if len(lines) > 1 {
		sort.Strings(lines[1:])
		this.Ui.Output(columnize.SimpleFormat(lines))
	}
}

func (this *Members) executeOnAll(execCmd string, node string) {
	args := []string{"exec"}
	if node != "" {
		args = append(args, fmt.Sprintf("-node=%s", node))
	}
	args = append(args, strings.Split(execCmd, " ")...)
	cmd := pipestream.New("consul", args...)
	err := cmd.Open()
	swallow(err)
	defer cmd.Close()

	this.Ui.Info(fmt.Sprintf("%s ...", execCmd))

	lines := make([]string, 0)
	header := "Node|Host|Role|Result"
	lines = append(lines, header)

	scanner := bufio.NewScanner(cmd.Reader())
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		line := scanner.Text()
		if strings.Contains(line, "finished with exit code 0") ||
			strings.Contains(line, "completed / acknowledged") {
			continue
		}
		fields := strings.Fields(line)
		if len(fields) == 1 {
			continue
		}

		node := fields[0]
		if strings.HasSuffix(node, ":") {
			node = strings.TrimRight(node, ":")
		}

		host := this.nodeHostMap[node]
		lines = append(lines, fmt.Sprintf("%s|%s|%s|%s", node, host, this.roleOfHost(host),
			strings.Join(fields[1:], " ")))
	}

	if len(lines) > 1 {
		sort.Strings(lines[1:])
		this.Ui.Output(columnize.SimpleFormat(lines))
	}
}

func (this *Members) displayMembers(members []*consulapi.AgentMember, role string) {
	lines := make([]string, 0)
	header := "Node|Host|Role"
	lines = append(lines, header)
	for _, member := range members {
		hostRole := this.roleOfHost(member.Addr)
		if role != "" && role != hostRole {
			continue
		}

		lines = append(lines, fmt.Sprintf("%s|%s|%s", member.Name, member.Addr, hostRole))
	}

	if len(lines) > 1 {
		sort.Strings(lines[1:])
		this.Ui.Output(columnize.SimpleFormat(lines))
	}
}

func (this *Members) displayLoadAvg(role string) {
	cmd := pipestream.New("consul", "exec",
		"uptime", "|", "grep", "load")
	err := cmd.Open()
	swallow(err)
	defer cmd.Close()

	lines := make([]string, 0)
	header := "Node|Host|Role|Load Avg"
	lines = append(lines, header)

	scanner := bufio.NewScanner(cmd.Reader())
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		line := scanner.Text()
		fields := strings.Fields(line)
		node := fields[0]
		parts := strings.Split(line, "load average:")
		if len(parts) < 2 {
			continue
		}
		if strings.HasSuffix(node, ":") {
			node = strings.TrimRight(node, ":")
		}

		loadAvg := strings.TrimSpace(parts[1])
		if loadAvg[0] > '0' {
			loadAvg += " !"
			if loadAvg[0] > '2' {
				loadAvg += "!"
			}
			if loadAvg[0] > '4' {
				loadAvg += "!"
			}
		}

		host := this.nodeHostMap[node]
		hostRole := this.roleOfHost(host)
		if role != "" && hostRole != role {
			continue
		}

		lines = append(lines, fmt.Sprintf("%s|%s|%s|%s", node, host, hostRole, loadAvg))
	}

	if len(lines) > 1 {
		sort.Strings(lines[1:])
		this.Ui.Output(columnize.SimpleFormat(lines))
	}
}

func (this *Members) roleOfHost(host string) string {
	if _, present := this.brokerHosts[host]; present {
		return "B"
	}
	if _, present := this.zkHosts[host]; present {
		return "Z"
	}
	if _, present := this.katewayHosts[host]; present {
		return "K"
	}
	return "?"
}

func (this *Members) nodesOfRole(role string) []string {
	r := make([]string, 0)
	var container map[string]struct{}
	switch role {
	case "B":
		container = this.brokerHosts
	case "Z":
		container = this.zkHosts
	case "K":
		container = this.katewayHosts
	default:
		return nil
	}

	for host, _ := range container {
		r = append(r, this.hostNodeMap[host])
	}
	return r
}

func (this *Members) consulMembers() []*consulapi.AgentMember {
	cf := consulapi.DefaultConfig()
	client, err := consulapi.NewClient(cf)
	swallow(err)
	members, err := client.Agent().Members(false)
	swallow(err)

	m := make(map[string]*consulapi.AgentMember, len(members))
	sortedName := make([]string, 0, len(members))
	this.nodeHostMap = make(map[string]string, len(members))
	this.hostNodeMap = make(map[string]string, len(members))
	for _, member := range members {
		m[member.Name] = member
		this.nodeHostMap[member.Name] = member.Addr
		this.hostNodeMap[member.Addr] = member.Name

		sortedName = append(sortedName, member.Name)
	}
	sort.Strings(sortedName)

	r := make([]*consulapi.AgentMember, 0, len(sortedName))
	for _, name := range sortedName {
		r = append(r, m[name])
	}

	return r
}

func (*Members) Synopsis() string {
	return "Verify consul members match kafka zone"
}

func (this *Members) Help() string {
	help := fmt.Sprintf(`
Usage: %s members [options]

    Verify consul members match kafka zone

    -z zone

    -r role
      Available roles: [Z|K|B]

    -l
      Display each member load average

    -exec <cmd>
      Execute cmd on all members and print the result by host
      e,g. gk members -exec "ifconfig bond0 | grep 'TX bytes'"

    -n node
      Execute cmd on a single node

    -debug

`, this.Cmd)
	return strings.TrimSpace(help)
}
<|endoftext|>


<commit_before>// Contribution by Tamás Gulácsi

package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"

	"github.com/josephspurrier/goversioninfo"
)

func main() {
	flagExample := flag.Bool("example", false, "dump out an example versioninfo.json to stdout")
	flagOut := flag.String("o", "resource.syso", "output file name")
	flagGo := flag.String("gofile", "", "Go output file name (optional)")
	flagPackage := flag.String("gofilepackage", "main", "Go output package name (optional, requires parameter: 'gofile')")
	flagPlatformSpecific := flag.Bool("platform-specific", false, "output i386,amd64,arm and arm64 named resource.syso, ignores -o")
	flagIcon := flag.String("icon", "", "icon file name")
	flagManifest := flag.String("manifest", "", "manifest file name")

	flagComment := flag.String("comment", "", "StringFileInfo.Comments")
	flagCompany := flag.String("company", "", "StringFileInfo.CompanyName")
	flagDescription := flag.String("description", "", "StringFileInfo.FileDescription")
	flagFileVersion := flag.String("file-version", "", "StringFileInfo.FileVersion")
	flagInternalName := flag.String("internal-name", "", "StringFileInfo.InternalName")
	flagCopyright := flag.String("copyright", "", "StringFileInfo.LegalCopyright")
	flagTrademark := flag.String("trademark", "", "StringFileInfo.LegalTrademarks")
	flagOriginalName := flag.String("original-name", "", "StringFileInfo.OriginalFilename")
	flagPrivateBuild := flag.String("private-build", "", "StringFileInfo.PrivateBuild")
	flagProductName := flag.String("product-name", "", "StringFileInfo.ProductName")
	flagProductVersion := flag.String("product-version", "", "StringFileInfo.ProductVersion")
	flagSpecialBuild := flag.String("special-build", "", "StringFileInfo.SpecialBuild")

	flagTranslation := flag.Int("translation", 0, "translation ID")
	flagCharset := flag.Int("charset", 0, "charset ID")

	flag64 := flag.Bool("64", false, "generate 64-bit binaries")
	flagarm := flag.Bool("arm", false, "generate arm binaries")

	flagVerMajor := flag.Int("ver-major", -1, "FileVersion.Major")
	flagVerMinor := flag.Int("ver-minor", -1, "FileVersion.Minor")
	flagVerPatch := flag.Int("ver-patch", -1, "FileVersion.Patch")
	flagVerBuild := flag.Int("ver-build", -1, "FileVersion.Build")

	flagProductVerMajor := flag.Int("product-ver-major", -1, "ProductVersion.Major")
	flagProductVerMinor := flag.Int("product-ver-minor", -1, "ProductVersion.Minor")
	flagProductVerPatch := flag.Int("product-ver-patch", -1, "ProductVersion.Patch")
	flagProductVerBuild := flag.Int("product-ver-build", -1, "ProductVersion.Build")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [flags] <versioninfo.json>\n\nPossible flags:\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()
	if *flagExample {
		io.WriteString(os.Stdout, example)
		return
	}

	configFile := flag.Arg(0)
	if configFile == "" {
		configFile = "versioninfo.json"
	}
	var err error
	var input = io.ReadCloser(os.Stdin)
	if configFile != "-" {
		if input, err = os.Open(configFile); err != nil {
			log.Printf("Cannot open %q: %v", configFile, err)
			os.Exit(1)
		}
	}

	// Read the config file.
	jsonBytes, err := ioutil.ReadAll(input)
	input.Close()
	if err != nil {
		log.Printf("Error reading %q: %v", configFile, err)
		os.Exit(1)
	}

	// Create a new container.
	vi := &goversioninfo.VersionInfo{}

	// Parse the config.
	if err := vi.ParseJSON(jsonBytes); err != nil {
		log.Printf("Could not parse the .json file: %v", err)
		os.Exit(2)
	}

	// Override from flags.
	if *flagIcon != "" {
		vi.IconPath = *flagIcon
	}
	if *flagManifest != "" {
		vi.ManifestPath = *flagManifest
	}
	if *flagComment != "" {
		vi.StringFileInfo.Comments = *flagComment
	}
	if *flagCompany != "" {
		vi.StringFileInfo.CompanyName = *flagCompany
	}
	if *flagDescription != "" {
		vi.StringFileInfo.FileDescription = *flagDescription
	}
	if *flagFileVersion != "" {
		vi.StringFileInfo.FileVersion = *flagFileVersion
	}
	if *flagInternalName != "" {
		vi.StringFileInfo.InternalName = *flagInternalName
	}
	if *flagCopyright != "" {
		vi.StringFileInfo.LegalCopyright = *flagCopyright
	}
	if *flagTrademark != "" {
		vi.StringFileInfo.LegalTrademarks = *flagTrademark
	}
	if *flagOriginalName != "" {
		vi.StringFileInfo.OriginalFilename = *flagOriginalName
	}
	if *flagPrivateBuild != "" {
		vi.StringFileInfo.PrivateBuild = *flagPrivateBuild
	}
	if *flagProductName != "" {
		vi.StringFileInfo.ProductName = *flagProductName
	}
	if *flagProductVersion != "" {
		vi.StringFileInfo.ProductVersion = *flagProductVersion
	}
	if *flagSpecialBuild != "" {
		vi.StringFileInfo.SpecialBuild = *flagSpecialBuild
	}

	if *flagTranslation > 0 {
		vi.VarFileInfo.Translation.LangID = goversioninfo.LangID(*flagTranslation)
	}
	if *flagCharset > 0 {
		vi.VarFileInfo.Translation.CharsetID = goversioninfo.CharsetID(*flagCharset)
	}

	// File Version flags.
	if *flagVerMajor >= 0 {
		vi.FixedFileInfo.FileVersion.Major = *flagVerMajor
	}
	if *flagVerMinor >= 0 {
		vi.FixedFileInfo.FileVersion.Minor = *flagVerMinor
	}
	if *flagVerPatch >= 0 {
		vi.FixedFileInfo.FileVersion.Patch = *flagVerPatch
	}
	if *flagVerBuild >= 0 {
		vi.FixedFileInfo.FileVersion.Build = *flagVerBuild
	}

	// Product Version flags.
	if *flagProductVerMajor >= 0 {
		vi.FixedFileInfo.ProductVersion.Major = *flagProductVerMajor
	}
	if *flagProductVerMinor >= 0 {
		vi.FixedFileInfo.ProductVersion.Minor = *flagProductVerMinor
	}
	if *flagProductVerPatch >= 0 {
		vi.FixedFileInfo.ProductVersion.Patch = *flagProductVerPatch
	}
	if *flagProductVerBuild >= 0 {
		vi.FixedFileInfo.ProductVersion.Build = *flagProductVerBuild
	}

	// Fill the structures with config data.
	vi.Build()

	// Write the data to a buffer.
	vi.Walk()

	// If the flag is set, then generate the optional Go file.
	if *flagGo != "" {
		vi.WriteGo(*flagGo, *flagPackage)
	}

	// List of the architectures to output.
	var archs []string

	// If platform specific, then output all the architectures for Windows.
	if flagPlatformSpecific != nil && *flagPlatformSpecific {
		archs = []string{"386", "amd64", "arm", "arm64"}
	} else {
		// Set the architecture, defaulted to 386(32-bit x86).
		archs = []string{"386"} // 386(32-bit x86)
		if flagarm != nil && *flagarm {
			if flag64 != nil && *flag64 {
				archs = []string{"arm64"} // arm64(64-bit arm64)
			} else {
				archs = []string{"arm"} // arm(32-bit arm)
			}
		} else {
			if flag64 != nil && *flag64 {
				archs = []string{"amd64"} // amd64(64-bit x86_64)
			}
		}

	}

	// Loop through each artchitecture.
	for _, item := range archs {
		// Create the file using the -o argument.
		fileout := *flagOut

		// If the number of architectures is greater than one, then don't use
		// the -o argument.
		if len(archs) > 1 {
			fileout = fmt.Sprintf("resource_windows_%v.syso", item)
		}

		// Create the file for the specified architecture.
		if err := vi.WriteSyso(fileout, item); err != nil {
			log.Printf("Error writing syso: %v", err)
			os.Exit(3)
		}
	}
}

const example = `{
	"FixedFileInfo": {
		"FileVersion": {
			"Major": 6,
			"Minor": 3,
			"Patch": 9600,
			"Build": 17284
		},
		"ProductVersion": {
			"Major": 6,
			"Minor": 3,
			"Patch": 9600,
			"Build": 17284
		},
		"FileFlagsMask": "3f",
		"FileFlags ": "00",
		"FileOS": "040004",
		"FileType": "01",
		"FileSubType": "00"
	},
	"StringFileInfo": {
		"Comments": "",
		"CompanyName": "Company, Inc.",
		"FileDescription": "",
		"FileVersion": "6.3.9600.17284 (aaa.140822-1915)",
		"InternalName": "goversioninfo",
		"LegalCopyright": "© Author. Licensed under MIT.",
		"LegalTrademarks": "",
		"OriginalFilename": "goversioninfo",
		"PrivateBuild": "",
		"ProductName": "Go Version Info",
		"ProductVersion": "6.3.9600.17284",
		"SpecialBuild": ""
	},
	"VarFileInfo": {
		"Translation": {
			"LangID": "0409",
			"CharsetID": "04B0"
		}
	}
}`
<commit_msg>Fix output formatting<commit_after>// Contribution by Tamás Gulácsi

package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"

	"github.com/josephspurrier/goversioninfo"
)

func main() {
	flagExample := flag.Bool("example", false, "dump out an example versioninfo.json to stdout")
	flagOut := flag.String("o", "resource.syso", "output file name")
	flagGo := flag.String("gofile", "", "Go output file name (optional)")
	flagPackage := flag.String("gofilepackage", "main", "Go output package name (optional, requires parameter: 'gofile')")
	flagPlatformSpecific := flag.Bool("platform-specific", false, "output i386, amd64, arm and arm64 named resource.syso, ignores -o")
	flagIcon := flag.String("icon", "", "icon file name")
	flagManifest := flag.String("manifest", "", "manifest file name")

	flagComment := flag.String("comment", "", "StringFileInfo.Comments")
	flagCompany := flag.String("company", "", "StringFileInfo.CompanyName")
	flagDescription := flag.String("description", "", "StringFileInfo.FileDescription")
	flagFileVersion := flag.String("file-version", "", "StringFileInfo.FileVersion")
	flagInternalName := flag.String("internal-name", "", "StringFileInfo.InternalName")
	flagCopyright := flag.String("copyright", "", "StringFileInfo.LegalCopyright")
	flagTrademark := flag.String("trademark", "", "StringFileInfo.LegalTrademarks")
	flagOriginalName := flag.String("original-name", "", "StringFileInfo.OriginalFilename")
	flagPrivateBuild := flag.String("private-build", "", "StringFileInfo.PrivateBuild")
	flagProductName := flag.String("product-name", "", "StringFileInfo.ProductName")
	flagProductVersion := flag.String("product-version", "", "StringFileInfo.ProductVersion")
	flagSpecialBuild := flag.String("special-build", "", "StringFileInfo.SpecialBuild")

	flagTranslation := flag.Int("translation", 0, "translation ID")
	flagCharset := flag.Int("charset", 0, "charset ID")

	flag64 := flag.Bool("64", false, "generate 64-bit binaries")
	flagarm := flag.Bool("arm", false, "generate arm binaries")

	flagVerMajor := flag.Int("ver-major", -1, "FileVersion.Major")
	flagVerMinor := flag.Int("ver-minor", -1, "FileVersion.Minor")
	flagVerPatch := flag.Int("ver-patch", -1, "FileVersion.Patch")
	flagVerBuild := flag.Int("ver-build", -1, "FileVersion.Build")

	flagProductVerMajor := flag.Int("product-ver-major", -1, "ProductVersion.Major")
	flagProductVerMinor := flag.Int("product-ver-minor", -1, "ProductVersion.Minor")
	flagProductVerPatch := flag.Int("product-ver-patch", -1, "ProductVersion.Patch")
	flagProductVerBuild := flag.Int("product-ver-build", -1, "ProductVersion.Build")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [flags] <versioninfo.json>\n\nPossible flags:\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()
	if *flagExample {
		io.WriteString(os.Stdout, example)
		return
	}

	configFile := flag.Arg(0)
	if configFile == "" {
		configFile = "versioninfo.json"
	}
	var err error
	var input = io.ReadCloser(os.Stdin)
	if configFile != "-" {
		if input, err = os.Open(configFile); err != nil {
			log.Printf("Cannot open %q: %v", configFile, err)
			os.Exit(1)
		}
	}

	// Read the config file.
	jsonBytes, err := ioutil.ReadAll(input)
	input.Close()
	if err != nil {
		log.Printf("Error reading %q: %v", configFile, err)
		os.Exit(1)
	}

	// Create a new container.
	vi := &goversioninfo.VersionInfo{}

	// Parse the config.
	if err := vi.ParseJSON(jsonBytes); err != nil {
		log.Printf("Could not parse the .json file: %v", err)
		os.Exit(2)
	}

	// Override from flags.
	if *flagIcon != "" {
		vi.IconPath = *flagIcon
	}
	if *flagManifest != "" {
		vi.ManifestPath = *flagManifest
	}
	if *flagComment != "" {
		vi.StringFileInfo.Comments = *flagComment
	}
	if *flagCompany != "" {
		vi.StringFileInfo.CompanyName = *flagCompany
	}
	if *flagDescription != "" {
		vi.StringFileInfo.FileDescription = *flagDescription
	}
	if *flagFileVersion != "" {
		vi.StringFileInfo.FileVersion = *flagFileVersion
	}
	if *flagInternalName != "" {
		vi.StringFileInfo.InternalName = *flagInternalName
	}
	if *flagCopyright != "" {
		vi.StringFileInfo.LegalCopyright = *flagCopyright
	}
	if *flagTrademark != "" {
		vi.StringFileInfo.LegalTrademarks = *flagTrademark
	}
	if *flagOriginalName != "" {
		vi.StringFileInfo.OriginalFilename = *flagOriginalName
	}
	if *flagPrivateBuild != "" {
		vi.StringFileInfo.PrivateBuild = *flagPrivateBuild
	}
	if *flagProductName != "" {
		vi.StringFileInfo.ProductName = *flagProductName
	}
	if *flagProductVersion != "" {
		vi.StringFileInfo.ProductVersion = *flagProductVersion
	}
	if *flagSpecialBuild != "" {
		vi.StringFileInfo.SpecialBuild = *flagSpecialBuild
	}

	if *flagTranslation > 0 {
		vi.VarFileInfo.Translation.LangID = goversioninfo.LangID(*flagTranslation)
	}
	if *flagCharset > 0 {
		vi.VarFileInfo.Translation.CharsetID = goversioninfo.CharsetID(*flagCharset)
	}

	// File Version flags.
	if *flagVerMajor >= 0 {
		vi.FixedFileInfo.FileVersion.Major = *flagVerMajor
	}
	if *flagVerMinor >= 0 {
		vi.FixedFileInfo.FileVersion.Minor = *flagVerMinor
	}
	if *flagVerPatch >= 0 {
		vi.FixedFileInfo.FileVersion.Patch = *flagVerPatch
	}
	if *flagVerBuild >= 0 {
		vi.FixedFileInfo.FileVersion.Build = *flagVerBuild
	}

	// Product Version flags.
	if *flagProductVerMajor >= 0 {
		vi.FixedFileInfo.ProductVersion.Major = *flagProductVerMajor
	}
	if *flagProductVerMinor >= 0 {
		vi.FixedFileInfo.ProductVersion.Minor = *flagProductVerMinor
	}
	if *flagProductVerPatch >= 0 {
		vi.FixedFileInfo.ProductVersion.Patch = *flagProductVerPatch
	}
	if *flagProductVerBuild >= 0 {
		vi.FixedFileInfo.ProductVersion.Build = *flagProductVerBuild
	}

	// Fill the structures with config data.
	vi.Build()

	// Write the data to a buffer.
	vi.Walk()

	// If the flag is set, then generate the optional Go file.
	if *flagGo != "" {
		vi.WriteGo(*flagGo, *flagPackage)
	}

	// List of the architectures to output.
	var archs []string

	// If platform specific, then output all the architectures for Windows.
	if flagPlatformSpecific != nil && *flagPlatformSpecific {
		archs = []string{"386", "amd64", "arm", "arm64"}
	} else {
		// Set the architecture, defaulted to 386(32-bit x86).
		archs = []string{"386"} // 386(32-bit x86)
		if flagarm != nil && *flagarm {
			if flag64 != nil && *flag64 {
				archs = []string{"arm64"} // arm64(64-bit arm64)
			} else {
				archs = []string{"arm"} // arm(32-bit arm)
			}
		} else {
			if flag64 != nil && *flag64 {
				archs = []string{"amd64"} // amd64(64-bit x86_64)
			}
		}

	}

	// Loop through each artchitecture.
	for _, item := range archs {
		// Create the file using the -o argument.
		fileout := *flagOut

		// If the number of architectures is greater than one, then don't use
		// the -o argument.
		if len(archs) > 1 {
			fileout = fmt.Sprintf("resource_windows_%v.syso", item)
		}

		// Create the file for the specified architecture.
		if err := vi.WriteSyso(fileout, item); err != nil {
			log.Printf("Error writing syso: %v", err)
			os.Exit(3)
		}
	}
}

const example = `{
	"FixedFileInfo": {
		"FileVersion": {
			"Major": 6,
			"Minor": 3,
			"Patch": 9600,
			"Build": 17284
		},
		"ProductVersion": {
			"Major": 6,
			"Minor": 3,
			"Patch": 9600,
			"Build": 17284
		},
		"FileFlagsMask": "3f",
		"FileFlags ": "00",
		"FileOS": "040004",
		"FileType": "01",
		"FileSubType": "00"
	},
	"StringFileInfo": {
		"Comments": "",
		"CompanyName": "Company, Inc.",
		"FileDescription": "",
		"FileVersion": "6.3.9600.17284 (aaa.140822-1915)",
		"InternalName": "goversioninfo",
		"LegalCopyright": "© Author. Licensed under MIT.",
		"LegalTrademarks": "",
		"OriginalFilename": "goversioninfo",
		"PrivateBuild": "",
		"ProductName": "Go Version Info",
		"ProductVersion": "6.3.9600.17284",
		"SpecialBuild": ""
	},
	"VarFileInfo": {
		"Translation": {
			"LangID": "0409",
			"CharsetID": "04B0"
		}
	}
}`
<|endoftext|>


<commit_before>package hash

import (
	"bytes"
	"encoding/hex"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"os"
	"path"
	"strings"
)

type decoder func(io.ReadCloser, int64) (io.ReadCloser, error)

// Noop does nothong but return the passed in file.
func noop(f io.ReadCloser, s int64) (io.ReadCloser, error) {
	return f, nil
}

func decodeLNX(f io.ReadCloser, s int64) (io.ReadCloser, error) {
	if s < 4 {
		return nil, fmt.Errorf("invalid ROM")
	}
	tmp := make([]byte, 64)
	_, err := io.ReadFull(f, tmp)
	if err != nil {
		return nil, err
	}
	if !bytes.Equal(tmp[:4], []byte("LYNX")) {
		return nil, fmt.Errorf("lnx file missing magic LYNX header, maybe it is a lyx")
	}
	return f, nil
}

func deinterleave(p []byte) []byte {
	l := len(p)
	m := l / 2
	b := make([]byte, l)
	for i, x := range p {
		if i < m {
			b[i*2+1] = x
		} else {
			b[i*2-l] = x
		}
	}
	return b
}

func decodeSMD(f io.ReadCloser, s int64) (io.ReadCloser, error) {
	return decodeMD(f, s, ".smd")
}

func decodeMGD(f io.ReadCloser, s int64) (io.ReadCloser, error) {
	return decodeMD(f, s, ".mgd")
}

func decodeGEN(f io.ReadCloser, s int64) (io.ReadCloser, error) {
	return decodeMD(f, s, ".gen")
}

func decodeMD(f io.ReadCloser, s int64, e string) (io.ReadCloser, error) {
	if s%16384 == 512 {
		tmp := make([]byte, 512)
		_, err := io.ReadFull(f, tmp)
		if err != nil {
			return nil, err
		}
		s -= 512
	}
	if s%16384 != 0 {
		return nil, fmt.Errorf("Invalid MD size")
	}
	b, err := ioutil.ReadAll(f)
	f.Close()
	if err != nil {
		return nil, err
	}
	if bytes.Equal(b[256:260], []byte("SEGA")) {
		return ioutil.NopCloser(bytes.NewReader(b)), nil
	}
	if bytes.Equal(b[8320:8328], []byte("SG EEI  ")) || bytes.Equal(b[8320:8328], []byte("SG EADIE")) {
		for i := 0; int64(i) < (s / int64(16384)); i++ {
			x := i * 16384
			copy(b[x:x+16384], deinterleave(b[x:x+16384]))
		}
		return ioutil.NopCloser(bytes.NewReader(b)), nil
	}
	if bytes.Equal(b[128:135], []byte("EAGNSS ")) || bytes.Equal(b[128:135], []byte("EAMG RV")) {
		b = deinterleave(b)
		return ioutil.NopCloser(bytes.NewReader(b)), nil
	}
	switch e {
	case ".smd":
		for i := 0; int64(i) < (s / int64(16384)); i++ {
			x := i * 16384
			copy(b[x:x+16384], deinterleave(b[x:x+16384]))
		}
		return ioutil.NopCloser(bytes.NewReader(b)), nil
	case ".mgd":
		b = deinterleave(b)
		return ioutil.NopCloser(bytes.NewReader(b)), nil
	case ".gen":
		return ioutil.NopCloser(bytes.NewReader(b)), nil
	}
	return nil, fmt.Errorf("Unknown MD Error")
}

func decodeN64(f io.ReadCloser, s int64) (io.ReadCloser, error) {
	i := 0
	r := swapReader{f, make([]byte, 4), &i, noSwap}
	if s < 4 {
		return nil, fmt.Errorf("invalid ROM")
	}
	_, err := io.ReadFull(r.f, r.b)
	*r.r = 4
	if err != nil {
		return nil, err
	}
	switch {
	case r.b[0] == 0x80:
		r.s = zSwap
	case r.b[3] == 0x80:
		r.s = nSwap
	}
	return r, nil
}

func noSwap(b []byte) {}

func zSwap(b []byte) {
	l := len(b)
	for i := 0; i < l; i += 4 {
		b[i+1], b[i] = b[i], b[i+1]
		b[i+3], b[i+2] = b[i+2], b[i+3]
	}
}

func nSwap(b []byte) {
	l := len(b)
	for i := 0; i < l; i += 4 {
		b[i+2], b[i] = b[i], b[i+2]
		b[i+3], b[i+1] = b[i+1], b[i+3]
	}
}

type swapReader struct {
	f io.ReadCloser
	b []byte
	r *int
	s func([]byte)
}

func (r swapReader) Read(p []byte) (int, error) {
	ll := len(p)
	rl := ll - *r.r
	l := rl + 4 - 1 - (rl-1)%4
	copy(p, r.b[:*r.r])
	if rl <= 0 {
		*r.r = *r.r - ll
		copy(r.b, r.b[ll:])
		return ll, nil
	}
	n := *r.r
	b := make([]byte, l)
	x, err := io.ReadFull(r.f, b)
	if x == 0 || err != nil && err != io.ErrUnexpectedEOF {
		return n, err
	}
	copy(p[n:ll], b[:x])
	n += x
	if ll <= n {
		r.s(p)
		copy(r.b, b[x+ll-n:x])
		*r.r = n - ll
		return ll, nil
	} else {
		r.s(p[:n])
		*r.r = 0
		return n, nil
	}
}

func (r swapReader) Close() error {
	return r.f.Close()
}

type nesReader struct {
	f      io.ReadCloser
	offset int64
	size   int64
	start  int64
	end    int64
}

func (r *nesReader) Read(p []byte) (int, error) {
	n, err := r.f.Read(p)
	if err != nil {
		return n, err
	}
	if r.offset+int64(n) > r.end {
		n = int(r.end - r.offset)
	}
	r.offset += int64(n)
	return n, err
}

func (r *nesReader) Close() error {
	return r.f.Close()
}

func decodeNES(f io.ReadCloser, s int64) (io.ReadCloser, error) {
	header := make([]byte, 16)
	n, err := io.ReadFull(f, header)
	if err != nil {
		return nil, err
	}
	if n < 16 {
		return nil, fmt.Errorf("invalid header")
	}
	prgSize := int64(header[4])
	chrSize := int64(header[5])
	if header[7]&12 == 8 {
		romSize := int64(header[9])
		chrSize = romSize&0x0F<<8 + chrSize
		prgSize = romSize&0xF0<<4 + prgSize
	}
	var prg, chr int64
	prg = 16 * 1024 * prgSize
	chr = 8 * 1024 * chrSize
	hasTrainer := header[6]&4 == 4
	var offset int64
	offset = 16
	if hasTrainer {
		tmp := make([]byte, 512)
		n, err := io.ReadFull(f, tmp)
		if err != nil {
			return nil, err
		}
		offset += int64(n)
	}
	return &nesReader{f, offset, prg + chr, offset, offset + prg + chr}, nil
}

func decodeSNES(f io.ReadCloser, s int64) (io.ReadCloser, error) {
	if s%1024 == 512 {
		tmp := make([]byte, 512)
		_, err := io.ReadFull(f, tmp)
		if err != nil {
			return nil, err
		}
	}
	return f, nil
}

func getDecoder(ext string) (decoder, bool) {
	switch strings.ToLower(ext) {
	case ".bin", ".a26", ".rom", ".cue", ".gdi", ".gb", ".gba", ".gbc", ".lyx", ".32x", ".gg", ".pce", ".sms":
		return noop, true
	case ".lnx":
		return decodeLNX, true
	case ".smd":
		return decodeSMD, true
	case ".mgd":
		return decodeMGD, true
	case ".gen", ".md":
		return decodeGEN, true
	case ".n64", ".v64", ".z64":
		return decodeN64, true
	case ".nes":
		return decodeNES, true
	case ".smc", ".sfc", ".fig", ".swc":
		return decodeSNES, true
	default:
		return noop, false
	}
}

// KnownExt returns true if the ext is recognized.
func KnownExt(ext string) bool {
	_, ok := getDecoder(strings.ToLower(ext))
	return ok
}

// decode takes a path and returns a reader for the inner rom data.
func decode(p string) (io.ReadCloser, error) {
	ext := strings.ToLower(path.Ext(p))
	if ext == ".zip" {
		r, err := decodeZip(p)
		return r, err
	}
	if ext == ".gz" {
		r, err := decodeGZip(p)
		return r, err
	}
	decode, ok := getDecoder(ext)
	if !ok {
		return nil, fmt.Errorf("no registered decoder for extention %s", ext)
	}
	r, err := os.Open(p)
	if err != nil {
		return nil, err
	}
	fi, err := r.Stat()
	if err != nil {
		return nil, err
	}
	ret, err := decode(r, fi.Size())
	return ret, err
}

// Hash returns the hash of a rom given a path to the file and hash function.
func Hash(p string, h hash.Hash) (string, error) {
	r, err := decode(p)
	if err != nil {
		return "", err
	}
	defer r.Close()
	buf := make([]byte, 4*1024*1024)
	for {
		n, err := r.Read(buf)
		if err != nil && err != io.EOF {
			return "", err
		}
		if n == 0 {
			break
		}
		h.Write(buf[:n])
	}
	return hex.EncodeToString(h.Sum(nil)), nil
}
<commit_msg>Add zip and gz to KnownExt function. Fixes Issue #42<commit_after>package hash

import (
	"bytes"
	"encoding/hex"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"os"
	"path"
	"strings"
)

type decoder func(io.ReadCloser, int64) (io.ReadCloser, error)

// Noop does nothong but return the passed in file.
func noop(f io.ReadCloser, s int64) (io.ReadCloser, error) {
	return f, nil
}

func decodeLNX(f io.ReadCloser, s int64) (io.ReadCloser, error) {
	if s < 4 {
		return nil, fmt.Errorf("invalid ROM")
	}
	tmp := make([]byte, 64)
	_, err := io.ReadFull(f, tmp)
	if err != nil {
		return nil, err
	}
	if !bytes.Equal(tmp[:4], []byte("LYNX")) {
		return nil, fmt.Errorf("lnx file missing magic LYNX header, maybe it is a lyx")
	}
	return f, nil
}

func deinterleave(p []byte) []byte {
	l := len(p)
	m := l / 2
	b := make([]byte, l)
	for i, x := range p {
		if i < m {
			b[i*2+1] = x
		} else {
			b[i*2-l] = x
		}
	}
	return b
}

func decodeSMD(f io.ReadCloser, s int64) (io.ReadCloser, error) {
	return decodeMD(f, s, ".smd")
}

func decodeMGD(f io.ReadCloser, s int64) (io.ReadCloser, error) {
	return decodeMD(f, s, ".mgd")
}

func decodeGEN(f io.ReadCloser, s int64) (io.ReadCloser, error) {
	return decodeMD(f, s, ".gen")
}

func decodeMD(f io.ReadCloser, s int64, e string) (io.ReadCloser, error) {
	if s%16384 == 512 {
		tmp := make([]byte, 512)
		_, err := io.ReadFull(f, tmp)
		if err != nil {
			return nil, err
		}
		s -= 512
	}
	if s%16384 != 0 {
		return nil, fmt.Errorf("Invalid MD size")
	}
	b, err := ioutil.ReadAll(f)
	f.Close()
	if err != nil {
		return nil, err
	}
	if bytes.Equal(b[256:260], []byte("SEGA")) {
		return ioutil.NopCloser(bytes.NewReader(b)), nil
	}
	if bytes.Equal(b[8320:8328], []byte("SG EEI  ")) || bytes.Equal(b[8320:8328], []byte("SG EADIE")) {
		for i := 0; int64(i) < (s / int64(16384)); i++ {
			x := i * 16384
			copy(b[x:x+16384], deinterleave(b[x:x+16384]))
		}
		return ioutil.NopCloser(bytes.NewReader(b)), nil
	}
	if bytes.Equal(b[128:135], []byte("EAGNSS ")) || bytes.Equal(b[128:135], []byte("EAMG RV")) {
		b = deinterleave(b)
		return ioutil.NopCloser(bytes.NewReader(b)), nil
	}
	switch e {
	case ".smd":
		for i := 0; int64(i) < (s / int64(16384)); i++ {
			x := i * 16384
			copy(b[x:x+16384], deinterleave(b[x:x+16384]))
		}
		return ioutil.NopCloser(bytes.NewReader(b)), nil
	case ".mgd":
		b = deinterleave(b)
		return ioutil.NopCloser(bytes.NewReader(b)), nil
	case ".gen":
		return ioutil.NopCloser(bytes.NewReader(b)), nil
	}
	return nil, fmt.Errorf("Unknown MD Error")
}

func decodeN64(f io.ReadCloser, s int64) (io.ReadCloser, error) {
	i := 0
	r := swapReader{f, make([]byte, 4), &i, noSwap}
	if s < 4 {
		return nil, fmt.Errorf("invalid ROM")
	}
	_, err := io.ReadFull(r.f, r.b)
	*r.r = 4
	if err != nil {
		return nil, err
	}
	switch {
	case r.b[0] == 0x80:
		r.s = zSwap
	case r.b[3] == 0x80:
		r.s = nSwap
	}
	return r, nil
}

func noSwap(b []byte) {}

func zSwap(b []byte) {
	l := len(b)
	for i := 0; i < l; i += 4 {
		b[i+1], b[i] = b[i], b[i+1]
		b[i+3], b[i+2] = b[i+2], b[i+3]
	}
}

func nSwap(b []byte) {
	l := len(b)
	for i := 0; i < l; i += 4 {
		b[i+2], b[i] = b[i], b[i+2]
		b[i+3], b[i+1] = b[i+1], b[i+3]
	}
}

type swapReader struct {
	f io.ReadCloser
	b []byte
	r *int
	s func([]byte)
}

func (r swapReader) Read(p []byte) (int, error) {
	ll := len(p)
	rl := ll - *r.r
	l := rl + 4 - 1 - (rl-1)%4
	copy(p, r.b[:*r.r])
	if rl <= 0 {
		*r.r = *r.r - ll
		copy(r.b, r.b[ll:])
		return ll, nil
	}
	n := *r.r
	b := make([]byte, l)
	x, err := io.ReadFull(r.f, b)
	if x == 0 || err != nil && err != io.ErrUnexpectedEOF {
		return n, err
	}
	copy(p[n:ll], b[:x])
	n += x
	if ll <= n {
		r.s(p)
		copy(r.b, b[x+ll-n:x])
		*r.r = n - ll
		return ll, nil
	} else {
		r.s(p[:n])
		*r.r = 0
		return n, nil
	}
}

func (r swapReader) Close() error {
	return r.f.Close()
}

type nesReader struct {
	f      io.ReadCloser
	offset int64
	size   int64
	start  int64
	end    int64
}

func (r *nesReader) Read(p []byte) (int, error) {
	n, err := r.f.Read(p)
	if err != nil {
		return n, err
	}
	if r.offset+int64(n) > r.end {
		n = int(r.end - r.offset)
	}
	r.offset += int64(n)
	return n, err
}

func (r *nesReader) Close() error {
	return r.f.Close()
}

func decodeNES(f io.ReadCloser, s int64) (io.ReadCloser, error) {
	header := make([]byte, 16)
	n, err := io.ReadFull(f, header)
	if err != nil {
		return nil, err
	}
	if n < 16 {
		return nil, fmt.Errorf("invalid header")
	}
	prgSize := int64(header[4])
	chrSize := int64(header[5])
	if header[7]&12 == 8 {
		romSize := int64(header[9])
		chrSize = romSize&0x0F<<8 + chrSize
		prgSize = romSize&0xF0<<4 + prgSize
	}
	var prg, chr int64
	prg = 16 * 1024 * prgSize
	chr = 8 * 1024 * chrSize
	hasTrainer := header[6]&4 == 4
	var offset int64
	offset = 16
	if hasTrainer {
		tmp := make([]byte, 512)
		n, err := io.ReadFull(f, tmp)
		if err != nil {
			return nil, err
		}
		offset += int64(n)
	}
	return &nesReader{f, offset, prg + chr, offset, offset + prg + chr}, nil
}

func decodeSNES(f io.ReadCloser, s int64) (io.ReadCloser, error) {
	if s%1024 == 512 {
		tmp := make([]byte, 512)
		_, err := io.ReadFull(f, tmp)
		if err != nil {
			return nil, err
		}
	}
	return f, nil
}

func getDecoder(ext string) (decoder, bool) {
	switch strings.ToLower(ext) {
	case ".bin", ".a26", ".rom", ".cue", ".gdi", ".gb", ".gba", ".gbc", ".lyx", ".32x", ".gg", ".pce", ".sms":
		return noop, true
	case ".lnx":
		return decodeLNX, true
	case ".smd":
		return decodeSMD, true
	case ".mgd":
		return decodeMGD, true
	case ".gen", ".md":
		return decodeGEN, true
	case ".n64", ".v64", ".z64":
		return decodeN64, true
	case ".nes":
		return decodeNES, true
	case ".smc", ".sfc", ".fig", ".swc":
		return decodeSNES, true
	default:
		return noop, false
	}
}

// KnownExt returns true if the ext is recognized.
func KnownExt(ext string) bool {
	ext = strings.ToLower(ext)
	if ext == ".zip" {
		return true
	}
	if ext == ".gz" {
		return true
	}
	_, ok := getDecoder(ext)
	return ok
}

// decode takes a path and returns a reader for the inner rom data.
func decode(p string) (io.ReadCloser, error) {
	ext := strings.ToLower(path.Ext(p))
	if ext == ".zip" {
		r, err := decodeZip(p)
		return r, err
	}
	if ext == ".gz" {
		r, err := decodeGZip(p)
		return r, err
	}
	decode, ok := getDecoder(ext)
	if !ok {
		return nil, fmt.Errorf("no registered decoder for extention %s", ext)
	}
	r, err := os.Open(p)
	if err != nil {
		return nil, err
	}
	fi, err := r.Stat()
	if err != nil {
		return nil, err
	}
	ret, err := decode(r, fi.Size())
	return ret, err
}

// Hash returns the hash of a rom given a path to the file and hash function.
func Hash(p string, h hash.Hash) (string, error) {
	r, err := decode(p)
	if err != nil {
		return "", err
	}
	defer r.Close()
	buf := make([]byte, 4*1024*1024)
	for {
		n, err := r.Read(buf)
		if err != nil && err != io.EOF {
			return "", err
		}
		if n == 0 {
			break
		}
		h.Write(buf[:n])
	}
	return hex.EncodeToString(h.Sum(nil)), nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"os"
	"os/exec"
	_ "path/filepath"
	"strings"
)

func main() {

	var repo = flag.String("repo", "", "...")
	var start_commit = flag.String("start-commit", "", "...")
	var stop_commit = flag.String("stop-commit", "", "...")

	flag.Parse()

	_, err := os.Stat(*repo)

	if os.IsNotExist(err) {
		log.Fatal("Repo does not exist", *repo)
	}

	cwd, err := os.Getwd()

	if err != nil {
		log.Fatal(err)
	}

	// I wish this wasn't necessary. I wish I could make sense of the
	// libgit2 documentation... (20161222/thisisaaronland)

	err = os.Chdir(*repo)

	if err != nil {
		log.Fatal(err)
	}

	// https://git-scm.com/docs/git-diff

	git_args := []string{
		"diff", "--pretty=format:", "--name-only",
	}

	if *start_commit != "" {
		git_args = append(git_args, *start_commit)
	}

	if *stop_commit != "" {
		git_args = append(git_args, *stop_commit)
	}

	log.Println(strings.Join(git_args, " "))

	cmd := exec.Command("git", git_args...)

	out, err := cmd.Output()

	os.Chdir(cwd)

	if err != nil {
		log.Fatal(err)
	}

	log.Printf("%s", out)
}
<commit_msg>make the git diff stuff work<commit_after>package main

import (
	"flag"
	"log"
	"os"
	"os/exec"
	_ "path/filepath"
	"strings"
	_ "time"
)

func main() {

	var repo = flag.String("repo", "", "...")
	var start_commit = flag.String("start-commit", "", "...")
	var stop_commit = flag.String("stop-commit", "", "...")

	flag.Parse()

	_, err := os.Stat(*repo)

	if os.IsNotExist(err) {
		log.Fatal("Repo does not exist", *repo)
	}

	cwd, err := os.Getwd()

	if err != nil {
		log.Fatal(err)
	}

	// I wish this wasn't necessary. I wish I could make sense of the
	// libgit2 documentation... (20161222/thisisaaronland)

	err = os.Chdir(*repo)

	if err != nil {
		log.Fatal(err)
	}

	// https://git-scm.com/docs/git-diff

	git_args := []string{
		"diff", "--pretty=format:", "--name-only",
	}

	if *start_commit != "" {
		git_args = append(git_args, *start_commit)
	}

	if *stop_commit != "" {
		git_args = append(git_args, *stop_commit)
	}

	log.Println(strings.Join(git_args, " "))

	cmd := exec.Command("git", git_args...)
	out, err := cmd.Output()

	os.Chdir(cwd)

	if err != nil {
		log.Fatal(err)
	}

	files := make([]string, 0)

	for _, path := range strings.Split(string(out), "\n") {

		if strings.HasSuffix(path, ".geojson") {
			files = append(files, path)
		}
	}

	log.Printf("%s", files)
}
<|endoftext|>


<commit_before>package collector

import (
	"errors"
	"fmt"
	"github.com/antonholmquist/jason"
	"github.com/linkedin/goavro"
	"github.com/mesosphere/dcos-stats/collector/metrics-schema"
	"io/ioutil"
	"log"
	"os/exec"
	"strings"
	"time"
)

var (
	agentIpCommandFlag = StringEnvFlag("agent-ip-command", "/opt/mesosphere/bin/detect_ip",
		"A command to execute which writes the local Mesos Agent's IP to stdout")
	agentPortFlag = IntEnvFlag("agent-port", 5051,
		"HTTP port to use when querying the local Mesos Agent")
	agentPollPeriodFlag = IntEnvFlag("agent-poll-period", 15,
		"Period between retrievals of data from the local Mesos Agent, in seconds")
	agentMetricsTopicFlag = StringEnvFlag("agent-metrics-topic", "agent",
		"Topic to use for local Mesos Agent statistics. Will be prefixed with -kafka-topic-prefix")
	agentTestStateFileFlag = StringEnvFlag("agent-test-state-file", "",
		"JSON file containing the agent state to be used, for debugging")
	agentTestSystemFileFlag = StringEnvFlag("agent-test-system-file", "",
		"JSON file containing the agent system metrics to be used, for debugging")
	agentTestContainersFileFlag = StringEnvFlag("agent-test-containers-file", "",
		"JSON file containing the container usage metrics to be used, for debugging")

	datapointNamespace = goavro.RecordEnclosingNamespace(metrics_schema.DatapointNamespace)
	datapointSchema    = goavro.RecordSchema(metrics_schema.DatapointSchema)

	metricListNamespace = goavro.RecordEnclosingNamespace(metrics_schema.MetricListNamespace)
	metricListSchema    = goavro.RecordSchema(metrics_schema.MetricListSchema)

	tagNamespace = goavro.RecordEnclosingNamespace(metrics_schema.TagNamespace)
	tagSchema    = goavro.RecordSchema(metrics_schema.TagSchema)
)

const (
	containerMetricPrefix = "dcos.metrics.container."
	systemMetricPrefix    = "dcos.metrics.agent."

	// same name in both agent json and metrics tags:
	timestampKey   = "timestamp"
	containerIdKey = "container_id"
	executorIdKey  = "executor_id"
	frameworkIdKey = "framework_id"
)

type AgentState struct {
	// agent_id
	agentId string
	// framework_id => framework_name
	frameworkNames map[string]string
}

// Runs an Agent Poller which periodically produces data retrieved from a local Mesos Agent.
// This function should be run as a gofunc.
func RunAgentPoller(recordsChan chan<- *AvroData, agentStateChan chan<- *AgentState, stats chan<- StatsEvent) {
	// do one poll immediately upon starting, to ensure that agent metadata is populated early:
	pollAgent(recordsChan, agentStateChan, stats)
	ticker := time.NewTicker(time.Second * time.Duration(*agentPollPeriodFlag))
	for {
		select {
		case _ = <-ticker.C:
			pollAgent(recordsChan, agentStateChan, stats)
		}
	}
}

// ---

func pollAgent(recordsChan chan<- *AvroData, agentStateChan chan<- *AgentState, stats chan<- StatsEvent) {
	// refresh agent ip (probably don't need to refresh constantly, but just in case..)
	var agentIp string = ""
	if len(*agentTestStateFileFlag) == 0 ||
		len(*agentTestSystemFileFlag) == 0 ||
		len(*agentTestContainersFileFlag) == 0 {
		// only get the ip if actually needed
		agentIp = getAgentIp(stats)
	}
	// fetch/emit agent state first: downstream can use it when handling agent metrics
	agentState, err := getAgentState(agentIp, stats)
	if err == nil {
		agentStateChan <- agentState
	} else {
		log.Printf("Failed to retrieve state from agent at %s: %s", agentIp, err)
	}
	systemMetricsList, err := getSystemMetrics(agentIp, agentState, stats)
	if err == nil {
		if systemMetricsList != nil {
			recordsChan <- systemMetricsList
		}
	} else {
		log.Printf("Failed to retrieve system metrics from agent at %s: %s", agentIp, err)
	}
	containerMetricsLists, err := getContainerMetrics(agentIp, agentState, stats)
	if err == nil {
		for _, metricList := range containerMetricsLists {
			recordsChan <- metricList
		}
	} else {
		log.Printf("Failed to retrieve container metrics from agent at %s: %s", agentIp, err)
	}
}

// runs detect_ip => "10.0.3.26\n"
func getAgentIp(stats chan<- StatsEvent) string {
	stats <- MakeEvent(AgentIpLookup)
	cmdWithArgs := strings.Split(*agentIpCommandFlag, " ")
	ipBytes, err := exec.Command(cmdWithArgs[0], cmdWithArgs[1:]...).Output()
	if err != nil {
		stats <- MakeEvent(AgentIpLookupFailed)
		log.Fatalf("Fetching Agent IP with -agent-ip-command='%s' failed: %s", *agentIpCommandFlag, err)
	}
	ip := strings.TrimSpace(string(ipBytes))
	if len(ip) == 0 {
		stats <- MakeEvent(AgentIpLookupEmpty)
		log.Fatalf("Agent IP fetched with -agent-ip-command='%s' is empty", *agentIpCommandFlag)
	}
	//log.Printf("Agent IP obtained with -agent-ip-command='%s': %s\n", *agentIpCommandFlag, ip)
	return ip
}

// fetches container-level resource metrics from the agent (via /containers)
func getContainerMetrics(agentIp string, agentState *AgentState, stats chan<- StatsEvent) ([]*AvroData, error) {
	rootJson, err := getJsonFromAgent(agentIp, "/containers", agentTestContainersFileFlag, stats)
	if err != nil {
		return nil, err
	}

	containersArray, err := rootJson.ObjectArray()
	if err != nil {
		stats <- MakeEvent(AgentQueryBadData)
		return nil, err
	}

	metricLists := make([]*AvroData, 0)
	// collect datapoints. expecting a list of dicts, where each dict has:
	// - tags: 'container_id'/'executor_id'/'framework_id' strings
	// - datapoints/timestamp: 'statistics' dict of string=>int/dbl (incl a dbl 'timestamp')
	for _, containerObj := range containersArray {

		statisticsObj, err := containerObj.GetObject("statistics")
		if err != nil {
			stats <- MakeEvent(AgentQueryBadData)
			return nil, err
		}

		// extract timestamp from statistics
		timestampRaw, err := statisticsObj.GetFloat64(timestampKey)
		if err != nil {
			stats <- MakeEvent(AgentQueryBadData)
			return nil, err
		}
		timestampMillis := int64(timestampRaw * 1000)

		// create datapoints from statistics (excluding timestamp itself)
		datapoints := make([]interface{}, 0)
		for key, valRaw := range statisticsObj.Map() {
			if key == timestampKey {
				continue
			}
			valFloat, err := valRaw.Float64()
			if err != nil {
				stats <- MakeEvent(AgentMetricsValueUnsupported)
				log.Printf("Failed to convert value %s to float64: %+v", key, valRaw)
				continue
			}
			datapoint, err := goavro.NewRecord(datapointNamespace, datapointSchema)
			if err != nil {
				log.Fatalf("Failed to create Datapoint record for topic %s (agent %s): %s",
					*agentMetricsTopicFlag, agentState.agentId, err)
			}
			datapoint.Set("name", containerMetricPrefix+key)
			datapoint.Set("time_ms", timestampMillis)
			datapoint.Set("value", valFloat)
			datapoints = append(datapoints, datapoint)
		}
		stats <- MakeEventCount(AgentMetricsValue, len(datapoints))
		if len(datapoints) == 0 {
			// no data, exit early
			continue
		}

		// create tags
		tags := make([]interface{}, 0)
		// container_id
		tagVal, err := containerObj.GetString(containerIdKey)
		if err != nil {
			stats <- MakeEvent(AgentQueryBadData)
			return nil, err
		}
		tags = addTag(tags, containerIdKey, tagVal)
		// executor_id
		tagVal, err = containerObj.GetString(executorIdKey)
		if err != nil {
			stats <- MakeEvent(AgentQueryBadData)
			return nil, err
		}
		tags = addTag(tags, executorIdKey, tagVal)
		// framework_id
		tagVal, err = containerObj.GetString(frameworkIdKey)
		if err != nil {
			stats <- MakeEvent(AgentQueryBadData)
			return nil, err
		}
		tags = addTag(tags, frameworkIdKey, tagVal)

		// note: agent_id/framework_name tags are automatically added downstream
		metricListRec, err := goavro.NewRecord(metricListNamespace, metricListSchema)
		if err != nil {
			log.Fatalf("Failed to create MetricList record for topic %s (agent %s): %s",
				*agentMetricsTopicFlag, agentState.agentId, err)
		}
		metricListRec.Set("topic", *agentMetricsTopicFlag)
		metricListRec.Set("datapoints", datapoints)
		metricListRec.Set("tags", tags)
		metricLists = append(metricLists, &AvroData{metricListRec, *agentMetricsTopicFlag})
	}
	return metricLists, nil
}

// fetches system-level metrics from the agent (via /metrics/snapshot)
func getSystemMetrics(agentIp string, agentState *AgentState, stats chan<- StatsEvent) (*AvroData, error) {
	rootJson, err := getJsonFromAgent(agentIp, "/metrics/snapshot", agentTestSystemFileFlag, stats)
	if err != nil {
		return nil, err
	}

	json, err := rootJson.Object()
	if err != nil {
		stats <- MakeEvent(AgentQueryBadData)
		return nil, err
	}

	nowMillis := time.Now().UnixNano() / 1000000
	// collect datapoints
	// expecting a single dict containing 'string => floatval' entries
	datapoints := make([]interface{}, 0)
	for key, valRaw := range json.Map() {
		valFloat, err := valRaw.Float64()
		if err != nil {
			stats <- MakeEvent(AgentMetricsValueUnsupported)
			log.Printf("Failed to convert value %s to float64: %+v", key, valRaw)
			continue
		}
		datapoint, err := goavro.NewRecord(datapointNamespace, datapointSchema)
		if err != nil {
			log.Fatalf("Failed to create Datapoint record for topic %s (agent %s): %s",
				*agentMetricsTopicFlag, agentState.agentId, err)
		}
		datapoint.Set("name", systemMetricPrefix+strings.Replace(key, "/", ".", -1)) // "key/path" => "key.path"
		datapoint.Set("time_ms", nowMillis)
		datapoint.Set("value", valFloat)
		datapoints = append(datapoints, datapoint)
	}
	stats <- MakeEventCount(AgentMetricsValue, len(datapoints))
	if len(datapoints) == 0 {
		return nil, errors.New("No datapoints found in agent metrics")
	}

	metricListRec, err := goavro.NewRecord(metricListNamespace, metricListSchema)
	if err != nil {
		log.Fatalf("Failed to create MetricList record for topic %s (agent %s): %s",
			*agentMetricsTopicFlag, agentState.agentId, err)
	}
	metricListRec.Set("topic", *agentMetricsTopicFlag)
	metricListRec.Set("datapoints", datapoints)
	// note: agent_id tag is automatically added downstream
	metricListRec.Set("tags", make([]interface{}, 0))
	return &AvroData{metricListRec, *agentMetricsTopicFlag}, nil
}

func getAgentState(agentIp string, stats chan<- StatsEvent) (*AgentState, error) {
	rootJson, err := getJsonFromAgent(agentIp, "/state", agentTestStateFileFlag, stats)
	if err != nil {
		return nil, err
	}

	json, err := rootJson.Object()
	if err != nil {
		stats <- MakeEvent(AgentQueryBadData)
		return nil, err
	}

	// state["id"] (agent_id)
	agentId, err := json.GetString("id")
	if err != nil {
		stats <- MakeEvent(AgentQueryBadData)
		return nil, err
	}

	// state["frameworks"][0]["id"] (framework_id) => state["frameworks"][0]["name"] (framework_name)
	frameworks, err := json.GetObjectArray("frameworks")
	if err != nil {
		stats <- MakeEvent(AgentQueryBadData)
		return nil, err
	}
	frameworkNames := make(map[string]string, len(frameworks))
	for _, framework := range frameworks {
		frameworkId, err := framework.GetString("id")
		if err != nil {
			stats <- MakeEvent(AgentQueryBadData)
			return nil, err
		}
		frameworkName, err := framework.GetString("name")
		if err != nil {
			stats <- MakeEvent(AgentQueryBadData)
			return nil, err
		}
		frameworkNames[frameworkId] = frameworkName
	}

	return &AgentState{agentId, frameworkNames}, nil
}

func getJsonFromAgent(agentIp string, urlPath string, testFileFlag *string, stats chan<- StatsEvent) (*jason.Value, error) {
	stats <- MakeEvent(AgentQuery)
	var rawJson []byte = nil
	var err error = nil
	if len(*testFileFlag) == 0 {
		rawJson, err = HttpGet(fmt.Sprintf("http://%s:%d%s", agentIp, *agentPortFlag, urlPath))
	} else {
		rawJson, err = ioutil.ReadFile(*testFileFlag)
	}
	if err != nil {
		stats <- MakeEvent(AgentQueryFailed)
		return nil, err
	}

	json, err := jason.NewValueFromBytes(rawJson)
	if err != nil {
		stats <- MakeEvent(AgentQueryBadData)
		return nil, err
	}
	return json, nil
}
<commit_msg>Only fetch agent IP once<commit_after>package collector

import (
	"errors"
	"fmt"
	"github.com/antonholmquist/jason"
	"github.com/linkedin/goavro"
	"github.com/mesosphere/dcos-stats/collector/metrics-schema"
	"io/ioutil"
	"log"
	"os/exec"
	"strings"
	"time"
)

var (
	agentIpCommandFlag = StringEnvFlag("agent-ip-command", "/opt/mesosphere/bin/detect_ip",
		"A command to execute which writes the local Mesos Agent's IP to stdout")
	agentPortFlag = IntEnvFlag("agent-port", 5051,
		"HTTP port to use when querying the local Mesos Agent")
	agentPollPeriodFlag = IntEnvFlag("agent-poll-period", 15,
		"Period between retrievals of data from the local Mesos Agent, in seconds")
	agentMetricsTopicFlag = StringEnvFlag("agent-metrics-topic", "agent",
		"Topic to use for local Mesos Agent statistics. Will be prefixed with -kafka-topic-prefix")
	agentTestStateFileFlag = StringEnvFlag("agent-test-state-file", "",
		"JSON file containing the agent state to be used, for debugging")
	agentTestSystemFileFlag = StringEnvFlag("agent-test-system-file", "",
		"JSON file containing the agent system metrics to be used, for debugging")
	agentTestContainersFileFlag = StringEnvFlag("agent-test-containers-file", "",
		"JSON file containing the container usage metrics to be used, for debugging")

	datapointNamespace = goavro.RecordEnclosingNamespace(metrics_schema.DatapointNamespace)
	datapointSchema    = goavro.RecordSchema(metrics_schema.DatapointSchema)

	metricListNamespace = goavro.RecordEnclosingNamespace(metrics_schema.MetricListNamespace)
	metricListSchema    = goavro.RecordSchema(metrics_schema.MetricListSchema)

	tagNamespace = goavro.RecordEnclosingNamespace(metrics_schema.TagNamespace)
	tagSchema    = goavro.RecordSchema(metrics_schema.TagSchema)
)

const (
	containerMetricPrefix = "dcos.metrics.container."
	systemMetricPrefix    = "dcos.metrics.agent."

	// same name in both agent json and metrics tags:
	timestampKey   = "timestamp"
	containerIdKey = "container_id"
	executorIdKey  = "executor_id"
	frameworkIdKey = "framework_id"
)

type AgentState struct {
	// agent_id
	agentId string
	// framework_id => framework_name
	frameworkNames map[string]string
}

// Runs an Agent Poller which periodically produces data retrieved from a local Mesos Agent.
// This function should be run as a gofunc.
func RunAgentPoller(recordsChan chan<- *AvroData, agentStateChan chan<- *AgentState, stats chan<- StatsEvent) {
	// fetch agent ip once. per DC/OS docs, changing a node IP is unsupported
	var agentIp string = ""
	if len(*agentTestStateFileFlag) == 0 ||
		len(*agentTestSystemFileFlag) == 0 ||
		len(*agentTestContainersFileFlag) == 0 {
		// only get the ip if actually needed
		agentIp = getAgentIp(stats)
	}

	// do one poll immediately upon starting, to ensure that agent metadata is populated early:
	pollAgent(agentIp, recordsChan, agentStateChan, stats)
	ticker := time.NewTicker(time.Second * time.Duration(*agentPollPeriodFlag))
	for {
		select {
		case _ = <-ticker.C:
			pollAgent(agentIp, recordsChan, agentStateChan, stats)
		}
	}
}

// ---

func pollAgent(agentIp string, recordsChan chan<- *AvroData, agentStateChan chan<- *AgentState, stats chan<- StatsEvent) {
	// always fetch/emit agent state first: downstream will use it for tagging metrics
	agentState, err := getAgentState(agentIp, stats)
	if err == nil {
		agentStateChan <- agentState
	} else {
		log.Printf("Failed to retrieve state from agent at %s: %s", agentIp, err)
	}

	systemMetricsList, err := getSystemMetrics(agentIp, agentState, stats)
	if err == nil {
		if systemMetricsList != nil {
			recordsChan <- systemMetricsList
		}
	} else {
		log.Printf("Failed to retrieve system metrics from agent at %s: %s", agentIp, err)
	}

	containerMetricsLists, err := getContainerMetrics(agentIp, agentState, stats)
	if err == nil {
		for _, metricList := range containerMetricsLists {
			recordsChan <- metricList
		}
	} else {
		log.Printf("Failed to retrieve container metrics from agent at %s: %s", agentIp, err)
	}
}

// runs detect_ip => "10.0.3.26\n"
func getAgentIp(stats chan<- StatsEvent) string {
	stats <- MakeEvent(AgentIpLookup)
	cmdWithArgs := strings.Split(*agentIpCommandFlag, " ")
	ipBytes, err := exec.Command(cmdWithArgs[0], cmdWithArgs[1:]...).Output()
	if err != nil {
		stats <- MakeEvent(AgentIpLookupFailed)
		log.Fatalf("Fetching Agent IP with -agent-ip-command='%s' failed: %s", *agentIpCommandFlag, err)
	}
	ip := strings.TrimSpace(string(ipBytes))
	if len(ip) == 0 {
		stats <- MakeEvent(AgentIpLookupEmpty)
		log.Fatalf("Agent IP fetched with -agent-ip-command='%s' is empty", *agentIpCommandFlag)
	}
	//log.Printf("Agent IP obtained with -agent-ip-command='%s': %s\n", *agentIpCommandFlag, ip)
	return ip
}

// fetches container-level resource metrics from the agent (via /containers)
func getContainerMetrics(agentIp string, agentState *AgentState, stats chan<- StatsEvent) ([]*AvroData, error) {
	rootJson, err := getJsonFromAgent(agentIp, "/containers", agentTestContainersFileFlag, stats)
	if err != nil {
		return nil, err
	}

	containersArray, err := rootJson.ObjectArray()
	if err != nil {
		stats <- MakeEvent(AgentQueryBadData)
		return nil, err
	}

	metricLists := make([]*AvroData, 0)
	// collect datapoints. expecting a list of dicts, where each dict has:
	// - tags: 'container_id'/'executor_id'/'framework_id' strings
	// - datapoints/timestamp: 'statistics' dict of string=>int/dbl (incl a dbl 'timestamp')
	for _, containerObj := range containersArray {

		statisticsObj, err := containerObj.GetObject("statistics")
		if err != nil {
			stats <- MakeEvent(AgentQueryBadData)
			return nil, err
		}

		// extract timestamp from statistics
		timestampRaw, err := statisticsObj.GetFloat64(timestampKey)
		if err != nil {
			stats <- MakeEvent(AgentQueryBadData)
			return nil, err
		}
		timestampMillis := int64(timestampRaw * 1000)

		// create datapoints from statistics (excluding timestamp itself)
		datapoints := make([]interface{}, 0)
		for key, valRaw := range statisticsObj.Map() {
			if key == timestampKey {
				continue
			}
			valFloat, err := valRaw.Float64()
			if err != nil {
				stats <- MakeEvent(AgentMetricsValueUnsupported)
				log.Printf("Failed to convert value %s to float64: %+v", key, valRaw)
				continue
			}
			datapoint, err := goavro.NewRecord(datapointNamespace, datapointSchema)
			if err != nil {
				log.Fatalf("Failed to create Datapoint record for topic %s (agent %s): %s",
					*agentMetricsTopicFlag, agentState.agentId, err)
			}
			datapoint.Set("name", containerMetricPrefix+key)
			datapoint.Set("time_ms", timestampMillis)
			datapoint.Set("value", valFloat)
			datapoints = append(datapoints, datapoint)
		}
		stats <- MakeEventCount(AgentMetricsValue, len(datapoints))
		if len(datapoints) == 0 {
			// no data, exit early
			continue
		}

		// create tags
		tags := make([]interface{}, 0)
		// container_id
		tagVal, err := containerObj.GetString(containerIdKey)
		if err != nil {
			stats <- MakeEvent(AgentQueryBadData)
			return nil, err
		}
		tags = addTag(tags, containerIdKey, tagVal)
		// executor_id
		tagVal, err = containerObj.GetString(executorIdKey)
		if err != nil {
			stats <- MakeEvent(AgentQueryBadData)
			return nil, err
		}
		tags = addTag(tags, executorIdKey, tagVal)
		// framework_id
		tagVal, err = containerObj.GetString(frameworkIdKey)
		if err != nil {
			stats <- MakeEvent(AgentQueryBadData)
			return nil, err
		}
		tags = addTag(tags, frameworkIdKey, tagVal)

		// note: agent_id/framework_name tags are automatically added downstream
		metricListRec, err := goavro.NewRecord(metricListNamespace, metricListSchema)
		if err != nil {
			log.Fatalf("Failed to create MetricList record for topic %s (agent %s): %s",
				*agentMetricsTopicFlag, agentState.agentId, err)
		}
		metricListRec.Set("topic", *agentMetricsTopicFlag)
		metricListRec.Set("datapoints", datapoints)
		metricListRec.Set("tags", tags)
		metricLists = append(metricLists, &AvroData{metricListRec, *agentMetricsTopicFlag})
	}
	return metricLists, nil
}

// fetches system-level metrics from the agent (via /metrics/snapshot)
func getSystemMetrics(agentIp string, agentState *AgentState, stats chan<- StatsEvent) (*AvroData, error) {
	rootJson, err := getJsonFromAgent(agentIp, "/metrics/snapshot", agentTestSystemFileFlag, stats)
	if err != nil {
		return nil, err
	}

	json, err := rootJson.Object()
	if err != nil {
		stats <- MakeEvent(AgentQueryBadData)
		return nil, err
	}

	nowMillis := time.Now().UnixNano() / 1000000
	// collect datapoints
	// expecting a single dict containing 'string => floatval' entries
	datapoints := make([]interface{}, 0)
	for key, valRaw := range json.Map() {
		valFloat, err := valRaw.Float64()
		if err != nil {
			stats <- MakeEvent(AgentMetricsValueUnsupported)
			log.Printf("Failed to convert value %s to float64: %+v", key, valRaw)
			continue
		}
		datapoint, err := goavro.NewRecord(datapointNamespace, datapointSchema)
		if err != nil {
			log.Fatalf("Failed to create Datapoint record for topic %s (agent %s): %s",
				*agentMetricsTopicFlag, agentState.agentId, err)
		}
		datapoint.Set("name", systemMetricPrefix+strings.Replace(key, "/", ".", -1)) // "key/path" => "key.path"
		datapoint.Set("time_ms", nowMillis)
		datapoint.Set("value", valFloat)
		datapoints = append(datapoints, datapoint)
	}
	stats <- MakeEventCount(AgentMetricsValue, len(datapoints))
	if len(datapoints) == 0 {
		return nil, errors.New("No datapoints found in agent metrics")
	}

	metricListRec, err := goavro.NewRecord(metricListNamespace, metricListSchema)
	if err != nil {
		log.Fatalf("Failed to create MetricList record for topic %s (agent %s): %s",
			*agentMetricsTopicFlag, agentState.agentId, err)
	}
	metricListRec.Set("topic", *agentMetricsTopicFlag)
	metricListRec.Set("datapoints", datapoints)
	// note: agent_id tag is automatically added downstream
	metricListRec.Set("tags", make([]interface{}, 0))
	return &AvroData{metricListRec, *agentMetricsTopicFlag}, nil
}

func getAgentState(agentIp string, stats chan<- StatsEvent) (*AgentState, error) {
	rootJson, err := getJsonFromAgent(agentIp, "/state", agentTestStateFileFlag, stats)
	if err != nil {
		return nil, err
	}

	json, err := rootJson.Object()
	if err != nil {
		stats <- MakeEvent(AgentQueryBadData)
		return nil, err
	}

	// state["id"] (agent_id)
	agentId, err := json.GetString("id")
	if err != nil {
		stats <- MakeEvent(AgentQueryBadData)
		return nil, err
	}

	// state["frameworks"][0]["id"] (framework_id) => state["frameworks"][0]["name"] (framework_name)
	frameworks, err := json.GetObjectArray("frameworks")
	if err != nil {
		stats <- MakeEvent(AgentQueryBadData)
		return nil, err
	}
	frameworkNames := make(map[string]string, len(frameworks))
	for _, framework := range frameworks {
		frameworkId, err := framework.GetString("id")
		if err != nil {
			stats <- MakeEvent(AgentQueryBadData)
			return nil, err
		}
		frameworkName, err := framework.GetString("name")
		if err != nil {
			stats <- MakeEvent(AgentQueryBadData)
			return nil, err
		}
		frameworkNames[frameworkId] = frameworkName
	}

	return &AgentState{agentId, frameworkNames}, nil
}

func getJsonFromAgent(agentIp string, urlPath string, testFileFlag *string, stats chan<- StatsEvent) (*jason.Value, error) {
	stats <- MakeEvent(AgentQuery)
	var rawJson []byte = nil
	var err error = nil
	if len(*testFileFlag) == 0 {
		rawJson, err = HttpGet(fmt.Sprintf("http://%s:%d%s", agentIp, *agentPortFlag, urlPath))
	} else {
		rawJson, err = ioutil.ReadFile(*testFileFlag)
	}
	if err != nil {
		stats <- MakeEvent(AgentQueryFailed)
		return nil, err
	}

	json, err := jason.NewValueFromBytes(rawJson)
	if err != nil {
		stats <- MakeEvent(AgentQueryBadData)
		return nil, err
	}
	return json, nil
}
<|endoftext|>


<commit_before>package commands

import (
	"github.com/github/git-media/gitmedia"
	"github.com/github/git-media/pointer"
	"github.com/spf13/cobra"
	"os"
)

var (
	cleanCmd = &cobra.Command{
		Use:   "clean",
		Short: "Implements the Git clean filter",
		Run:   cleanCommand,
	}
)

func cleanCommand(cmd *cobra.Command, args []string) {
	gitmedia.InstallHooks()

	var filename string
	var cb gitmedia.CopyCallback
	var file *os.File
	var fileSize int64
	if len(args) > 0 {
		filename = args[0]

		stat, err := os.Stat(filename)
		if err == nil && stat != nil {
			fileSize = stat.Size()

			localCb, localFile, err := gitmedia.CopyCallbackFile("clean", filename, 1, 1)
			if err != nil {
				Error(err.Error())
			} else {
				cb = localCb
				file = localFile
			}
		}
	}

	cleaned, err := pointer.Clean(os.Stdin, fileSize, cb)
	if file != nil {
		file.Close()
	}

	cleaned.Close()
	defer cleaned.Teardown()
	if err != nil {
		Panic(err, "Error cleaning asset.")
	}

	tmpfile := cleaned.File.Name()
	mediafile, err := gitmedia.LocalMediaPath(cleaned.Oid)
	if err != nil {
		Panic(err, "Unable to get local media path.")
	}

	if stat, _ := os.Stat(mediafile); stat != nil {
		if stat.Size() != cleaned.Size {
			Exit("Files don't match:\n%s\n%s", mediafile, tmpfile)
		}
		Debug("%s exists", mediafile)
	} else {
		if err := os.Rename(tmpfile, mediafile); err != nil {
			Panic(err, "Unable to move %s to %s\n", tmpfile, mediafile)
		}

		if err = gitmedia.QueueUpload(cleaned.Oid, filename); err != nil {
			Panic(err, "Unable to add %s to queue", cleaned.Oid)
		}
		Debug("Writing %s", mediafile)
	}

	err = cleaned.CreateLink()
	if err != nil {
		Panic(err, "Unable to write link file %s", err)
	}

	pointer.Encode(os.Stdout, cleaned.Pointer)
}

func init() {
	RootCmd.AddCommand(cleanCmd)
}
<commit_msg>Don't queue the file when cleaning<commit_after>package commands

import (
	"github.com/github/git-media/gitmedia"
	"github.com/github/git-media/pointer"
	"github.com/spf13/cobra"
	"os"
)

var (
	cleanCmd = &cobra.Command{
		Use:   "clean",
		Short: "Implements the Git clean filter",
		Run:   cleanCommand,
	}
)

func cleanCommand(cmd *cobra.Command, args []string) {
	gitmedia.InstallHooks()

	var filename string
	var cb gitmedia.CopyCallback
	var file *os.File
	var fileSize int64
	if len(args) > 0 {
		filename = args[0]

		stat, err := os.Stat(filename)
		if err == nil && stat != nil {
			fileSize = stat.Size()

			localCb, localFile, err := gitmedia.CopyCallbackFile("clean", filename, 1, 1)
			if err != nil {
				Error(err.Error())
			} else {
				cb = localCb
				file = localFile
			}
		}
	}

	cleaned, err := pointer.Clean(os.Stdin, fileSize, cb)
	if file != nil {
		file.Close()
	}

	cleaned.Close()
	defer cleaned.Teardown()
	if err != nil {
		Panic(err, "Error cleaning asset.")
	}

	tmpfile := cleaned.File.Name()
	mediafile, err := gitmedia.LocalMediaPath(cleaned.Oid)
	if err != nil {
		Panic(err, "Unable to get local media path.")
	}

	if stat, _ := os.Stat(mediafile); stat != nil {
		if stat.Size() != cleaned.Size {
			Exit("Files don't match:\n%s\n%s", mediafile, tmpfile)
		}
		Debug("%s exists", mediafile)
	} else {
		if err := os.Rename(tmpfile, mediafile); err != nil {
			Panic(err, "Unable to move %s to %s\n", tmpfile, mediafile)
		}

		Debug("Writing %s", mediafile)
	}

	err = cleaned.CreateLink()
	if err != nil {
		Panic(err, "Unable to write link file %s", err)
	}

	pointer.Encode(os.Stdout, cleaned.Pointer)
}

func init() {
	RootCmd.AddCommand(cleanCmd)
}
<|endoftext|>


<commit_before>package commands

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/github/git-lfs/lfs"
	"github.com/github/git-lfs/vendor/_nuts/github.com/spf13/cobra"
)

var (
	trackCmd = &cobra.Command{
		Use:   "track",
		Short: "Manipulate .gitattributes",
		Run:   trackCommand,
	}
)

func trackCommand(cmd *cobra.Command, args []string) {
	if lfs.LocalGitDir == "" {
		Print("Not a git repository.")
		os.Exit(128)
	}

	lfs.InstallHooks(false)
	knownPaths := findPaths()

	if len(args) == 0 {
		Print("Listing tracked paths")
		for _, t := range knownPaths {
			Print("    %s (%s)", t.Path, t.Source)
		}
		return
	}

	addTrailingLinebreak := needsTrailingLinebreak(".gitattributes")
	attributesFile, err := os.OpenFile(".gitattributes", os.O_RDWR|os.O_APPEND|os.O_CREATE, 0660)
	if err != nil {
		Print("Error opening .gitattributes file")
		return
	}

	if addTrailingLinebreak {
		if _, err := attributesFile.WriteString("\n"); err != nil {
			Print("Error writing to .gitattributes")
		}
	}

	for _, t := range args {
		isKnownPath := false
		for _, k := range knownPaths {
			if t == k.Path {
				isKnownPath = true
				break
			}
		}

		if isKnownPath {
			Print("%s already supported", t)
			continue
		}

		encodedArg := strings.Replace(t, " ", "[[:space:]]", -1)
		_, err := attributesFile.WriteString(fmt.Sprintf("%s filter=lfs diff=lfs merge=lfs -crlf\n", encodedArg))
		if err != nil {
			Print("Error adding path %s", t)
			continue
		}
		Print("Tracking %s", t)
	}

	attributesFile.Close()
}

type mediaPath struct {
	Path   string
	Source string
}

func findAttributeFiles() []string {
	paths := make([]string, 0)

	repoAttributes := filepath.Join(lfs.LocalGitDir, "info", "attributes")
	if _, err := os.Stat(repoAttributes); err == nil {
		paths = append(paths, repoAttributes)
	}

	filepath.Walk(lfs.LocalWorkingDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if !info.IsDir() && (filepath.Base(path) == ".gitattributes") {
			paths = append(paths, path)
		}
		return nil
	})

	return paths
}

func findPaths() []mediaPath {
	paths := make([]mediaPath, 0)
	wd, _ := os.Getwd()

	for _, path := range findAttributeFiles() {
		attributes, err := os.Open(path)
		if err != nil {
			return paths
		}

		scanner := bufio.NewScanner(attributes)
		for scanner.Scan() {
			line := scanner.Text()
			if line == "" {
				continue
			}

			if strings.Contains(line, "filter=lfs") {
				fields := strings.Fields(line)
				relPath, _ := filepath.Rel(wd, path)
				paths = append(paths, mediaPath{Path: fields[0], Source: relPath})
			}
		}
	}

	return paths
}

func needsTrailingLinebreak(filename string) bool {
	file, err := os.Open(filename)
	if err != nil {
		return false
	}
	defer file.Close()

	buf := make([]byte, 16384)
	bytesRead := 0
	for {
		n, err := file.Read(buf)
		if err == io.EOF {
			break
		} else if err != nil {
			return false
		}
		bytesRead = n
	}

	return !strings.HasSuffix(string(buf[0:bytesRead]), "\n")
}

func init() {
	RootCmd.AddCommand(trackCmd)
}
<commit_msg>アアー アアアア アー<commit_after>package commands

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/github/git-lfs/lfs"
	"github.com/github/git-lfs/vendor/_nuts/github.com/spf13/cobra"
)

var (
	trackCmd = &cobra.Command{
		Use:   "track",
		Short: "Manipulate .gitattributes",
		Run:   trackCommand,
	}
)

func trackCommand(cmd *cobra.Command, args []string) {
	if lfs.LocalGitDir == "" {
		Print("Not a git repository.")
		os.Exit(128)
	}

	lfs.InstallHooks(false)
	knownPaths := findPaths()

	if len(args) == 0 {
		Print("Listing tracked paths")
		for _, t := range knownPaths {
			Print("    %s (%s)", t.Path, t.Source)
		}
		return
	}

	addTrailingLinebreak := needsTrailingLinebreak(".gitattributes")
	attributesFile, err := os.OpenFile(".gitattributes", os.O_RDWR|os.O_APPEND|os.O_CREATE, 0660)
	if err != nil {
		Print("Error opening .gitattributes file")
		return
	}

	if addTrailingLinebreak {
		if _, err := attributesFile.WriteString("\n"); err != nil {
			Print("Error writing to .gitattributes")
		}
	}

	for _, t := range args {
		isKnownPath := false
		for _, k := range knownPaths {
			if t == k.Path {
				isKnownPath = true
				break
			}
		}

		if isKnownPath {
			Print("%s already supported", t)
			continue
		}

		encodedArg := strings.Replace(t, " ", "[[:space:]]", -1)
		_, err := attributesFile.WriteString(fmt.Sprintf("%s filter=lfs diff=lfs merge=lfs -crlf\n", encodedArg))
		if err != nil {
			Print("Error adding path %s", t)
			continue
		}
		Print("Tracking %s", t)
	}

	attributesFile.Close()
}

type mediaPath struct {
	Path   string
	Source string
}

func findPaths() []mediaPath {
	paths := make([]mediaPath, 0)
	wd, _ := os.Getwd()

	for _, path := range findAttributeFiles() {
		attributes, err := os.Open(path)
		if err != nil {
			return paths
		}

		scanner := bufio.NewScanner(attributes)
		for scanner.Scan() {
			line := scanner.Text()
			if line == "" {
				continue
			}

			if strings.Contains(line, "filter=lfs") {
				fields := strings.Fields(line)
				relPath, _ := filepath.Rel(wd, path)
				paths = append(paths, mediaPath{Path: fields[0], Source: relPath})
			}
		}
	}

	return paths
}

func findAttributeFiles() []string {
	paths := make([]string, 0)

	repoAttributes := filepath.Join(lfs.LocalGitDir, "info", "attributes")
	if _, err := os.Stat(repoAttributes); err == nil {
		paths = append(paths, repoAttributes)
	}

	filepath.Walk(lfs.LocalWorkingDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if !info.IsDir() && (filepath.Base(path) == ".gitattributes") {
			paths = append(paths, path)
		}
		return nil
	})

	return paths
}

func needsTrailingLinebreak(filename string) bool {
	file, err := os.Open(filename)
	if err != nil {
		return false
	}
	defer file.Close()

	buf := make([]byte, 16384)
	bytesRead := 0
	for {
		n, err := file.Read(buf)
		if err == io.EOF {
			break
		} else if err != nil {
			return false
		}
		bytesRead = n
	}

	return !strings.HasSuffix(string(buf[0:bytesRead]), "\n")
}

func init() {
	RootCmd.AddCommand(trackCmd)
}
<|endoftext|>


<commit_before>// Package router provides an interface for micro network router
package router

// Router is micro network router
type Router interface {
	// Init initializes the router with options
	Init(...Option) error
	// Options returns the router options
	Options() Options
	// ID returns the id of the router
	ID() string
	// Table returns the routing table
	Table() Table
	// Address returns the router adddress
	Address() string
	// Network returns the network address of the router
	Network() string
	// Advertise starts advertising the routes to the network
	Advertise() error
	// Stop stops the router
	Stop() error
	// String returns debug info
	String() string
}

// Option used by the router
type Option func(*Options)

// NewRouter creates new Router and returns it
func NewRouter(opts ...Option) Router {
	return newRouter(opts...)
}
<commit_msg>Update comments<commit_after>// Package router provides a network routing control plane
package router

// Router is an interface for a routing control plane
type Router interface {
	// Init initializes the router with options
	Init(...Option) error
	// Options returns the router options
	Options() Options
	// ID returns the id of the router
	ID() string
	// Table returns the routing table
	Table() Table
	// Address returns the router adddress
	Address() string
	// Network returns the network address of the router
	Network() string
	// Advertise starts advertising the routes to the network
	Advertise() error
	// Stop stops the router
	Stop() error
	// String returns debug info
	String() string
}

// Option used by the router
type Option func(*Options)

// NewRouter creates new Router and returns it
func NewRouter(opts ...Option) Router {
	return newRouter(opts...)
}
<|endoftext|>


<commit_before>package analysis

import (
	"go/ast"
	"go/token"

	"github.com/gopherjs/gopherjs/compiler/astutil"
	"github.com/gopherjs/gopherjs/compiler/typesutil"

	"golang.org/x/tools/go/types"
)

type continueStmt struct {
	forStmt      *ast.ForStmt
	analyzeStack []ast.Node
}

type Info struct {
	*types.Info
	Pkg           *types.Package
	IsBlocking    func(*types.Func) bool
	FuncDeclInfos map[*types.Func]*FuncInfo
	FuncLitInfos  map[*ast.FuncLit]*FuncInfo
	InitFuncInfo  *FuncInfo
	allInfos      []*FuncInfo
	comments      ast.CommentMap
}

type FuncInfo struct {
	HasDefer      bool
	Flattened     map[ast.Node]bool
	Blocking      map[ast.Node]bool
	GotoLabel     map[*types.Label]bool
	LocalCalls    map[*types.Func][][]ast.Node
	ContinueStmts []continueStmt
	p             *Info
	analyzeStack  []ast.Node
}

func (info *Info) newFuncInfo() *FuncInfo {
	funcInfo := &FuncInfo{
		p:          info,
		Flattened:  make(map[ast.Node]bool),
		Blocking:   make(map[ast.Node]bool),
		GotoLabel:  make(map[*types.Label]bool),
		LocalCalls: make(map[*types.Func][][]ast.Node),
	}
	info.allInfos = append(info.allInfos, funcInfo)
	return funcInfo
}

func AnalyzePkg(files []*ast.File, fileSet *token.FileSet, typesInfo *types.Info, typesPkg *types.Package, isBlocking func(*types.Func) bool) *Info {
	info := &Info{
		Info:          typesInfo,
		Pkg:           typesPkg,
		comments:      make(ast.CommentMap),
		IsBlocking:    isBlocking,
		FuncDeclInfos: make(map[*types.Func]*FuncInfo),
		FuncLitInfos:  make(map[*ast.FuncLit]*FuncInfo),
	}
	info.InitFuncInfo = info.newFuncInfo()

	for _, file := range files {
		for k, v := range ast.NewCommentMap(fileSet, file, file.Comments) {
			info.comments[k] = v
		}
		ast.Walk(info.InitFuncInfo, file)
	}

	for {
		done := true
		for _, funcInfo := range info.allInfos {
			for obj, calls := range funcInfo.LocalCalls {
				if len(info.FuncDeclInfos[obj].Blocking) != 0 {
					for _, call := range calls {
						funcInfo.markBlocking(call)
					}
					delete(funcInfo.LocalCalls, obj)
					done = false
				}
			}
		}
		if done {
			break
		}
	}

	for _, funcInfo := range info.allInfos {
		for _, continueStmt := range funcInfo.ContinueStmts {
			if funcInfo.Blocking[continueStmt.forStmt.Post] {
				funcInfo.markBlocking(continueStmt.analyzeStack)
			}
		}
	}

	return info
}

func (c *FuncInfo) Visit(node ast.Node) ast.Visitor {
	if node == nil {
		if len(c.analyzeStack) != 0 {
			c.analyzeStack = c.analyzeStack[:len(c.analyzeStack)-1]
		}
		return nil
	}
	c.analyzeStack = append(c.analyzeStack, node)

	switch n := node.(type) {
	case *ast.FuncDecl:
		newInfo := c.p.newFuncInfo()
		c.p.FuncDeclInfos[c.p.Defs[n.Name].(*types.Func)] = newInfo
		return newInfo
	case *ast.FuncLit:
		newInfo := c.p.newFuncInfo()
		c.p.FuncLitInfos[n] = newInfo
		return newInfo
	case *ast.BranchStmt:
		switch n.Tok {
		case token.GOTO:
			for _, n2 := range c.analyzeStack {
				c.Flattened[n2] = true
			}
			c.GotoLabel[c.p.Uses[n.Label].(*types.Label)] = true
		case token.CONTINUE:
			if n.Label != nil {
				label := c.p.Uses[n.Label].(*types.Label)
				for i := len(c.analyzeStack) - 1; i >= 0; i-- {
					if labelStmt, ok := c.analyzeStack[i].(*ast.LabeledStmt); ok && c.p.Defs[labelStmt.Label] == label {
						if _, ok := labelStmt.Stmt.(*ast.RangeStmt); ok {
							return nil
						}
						stack := make([]ast.Node, len(c.analyzeStack))
						copy(stack, c.analyzeStack)
						c.ContinueStmts = append(c.ContinueStmts, continueStmt{labelStmt.Stmt.(*ast.ForStmt), stack})
						return nil
					}
				}
				return nil
			}
			for i := len(c.analyzeStack) - 1; i >= 0; i-- {
				if _, ok := c.analyzeStack[i].(*ast.RangeStmt); ok {
					return nil
				}
				if forStmt, ok := c.analyzeStack[i].(*ast.ForStmt); ok {
					stack := make([]ast.Node, len(c.analyzeStack))
					copy(stack, c.analyzeStack)
					c.ContinueStmts = append(c.ContinueStmts, continueStmt{forStmt, stack})
					return nil
				}
			}
		}
	case *ast.CallExpr:
		callTo := func(obj types.Object) {
			switch o := obj.(type) {
			case *types.Func:
				if recv := o.Type().(*types.Signature).Recv(); recv != nil {
					if _, ok := recv.Type().Underlying().(*types.Interface); ok {
						c.markBlocking(c.analyzeStack)
						return
					}
				}
				if o.Pkg() != c.p.Pkg {
					if c.p.IsBlocking(o) {
						c.markBlocking(c.analyzeStack)
					}
					return
				}
				stack := make([]ast.Node, len(c.analyzeStack))
				copy(stack, c.analyzeStack)
				c.LocalCalls[o] = append(c.LocalCalls[o], stack)
			case *types.Var:
				c.markBlocking(c.analyzeStack)
			}
		}
		switch f := astutil.RemoveParens(n.Fun).(type) {
		case *ast.Ident:
			callTo(c.p.Uses[f])
		case *ast.SelectorExpr:
			if sel := c.p.Selections[f]; sel != nil && typesutil.IsJsObject(sel.Recv()) {
				break
			}
			callTo(c.p.Uses[f.Sel])
		default:
			if !astutil.IsTypeExpr(f, c.p.Info) {
				c.markBlocking(c.analyzeStack)
			}
		}
	case *ast.SendStmt:
		c.markBlocking(c.analyzeStack)
	case *ast.UnaryExpr:
		if n.Op == token.ARROW {
			c.markBlocking(c.analyzeStack)
		}
	case *ast.RangeStmt:
		if _, ok := c.p.Types[n.X].Type.Underlying().(*types.Chan); ok {
			c.markBlocking(c.analyzeStack)
		}
	case *ast.SelectStmt:
		for _, s := range n.Body.List {
			if s.(*ast.CommClause).Comm == nil { // default clause
				return c
			}
		}
		c.markBlocking(c.analyzeStack)
	case *ast.CommClause:
		for _, s := range n.Body {
			ast.Walk(c, s)
		}
		return nil
	case *ast.GoStmt:
		ast.Walk(c, n.Call.Fun)
		for _, arg := range n.Call.Args {
			ast.Walk(c, arg)
		}
		return nil
	case *ast.DeferStmt:
		c.HasDefer = true
		if funcLit, ok := n.Call.Fun.(*ast.FuncLit); ok {
			ast.Walk(c, funcLit.Body)
		}
	}
	return c
}

func (c *FuncInfo) markBlocking(stack []ast.Node) {
	for _, n := range stack {
		c.Blocking[n] = true
		c.Flattened[n] = true
	}
}
<commit_msg>improved blocking analysis on calls of func literals (mainly used with "defer")<commit_after>package analysis

import (
	"go/ast"
	"go/token"

	"github.com/gopherjs/gopherjs/compiler/astutil"
	"github.com/gopherjs/gopherjs/compiler/typesutil"

	"golang.org/x/tools/go/types"
)

type continueStmt struct {
	forStmt      *ast.ForStmt
	analyzeStack []ast.Node
}

type Info struct {
	*types.Info
	Pkg           *types.Package
	IsBlocking    func(*types.Func) bool
	FuncDeclInfos map[*types.Func]*FuncInfo
	FuncLitInfos  map[*ast.FuncLit]*FuncInfo
	InitFuncInfo  *FuncInfo
	allInfos      []*FuncInfo
	comments      ast.CommentMap
}

type FuncInfo struct {
	HasDefer      bool
	Flattened     map[ast.Node]bool
	Blocking      map[ast.Node]bool
	GotoLabel     map[*types.Label]bool
	LocalCalls    map[*types.Func][][]ast.Node
	ContinueStmts []continueStmt
	p             *Info
	analyzeStack  []ast.Node
}

func (info *Info) newFuncInfo() *FuncInfo {
	funcInfo := &FuncInfo{
		p:          info,
		Flattened:  make(map[ast.Node]bool),
		Blocking:   make(map[ast.Node]bool),
		GotoLabel:  make(map[*types.Label]bool),
		LocalCalls: make(map[*types.Func][][]ast.Node),
	}
	info.allInfos = append(info.allInfos, funcInfo)
	return funcInfo
}

func AnalyzePkg(files []*ast.File, fileSet *token.FileSet, typesInfo *types.Info, typesPkg *types.Package, isBlocking func(*types.Func) bool) *Info {
	info := &Info{
		Info:          typesInfo,
		Pkg:           typesPkg,
		comments:      make(ast.CommentMap),
		IsBlocking:    isBlocking,
		FuncDeclInfos: make(map[*types.Func]*FuncInfo),
		FuncLitInfos:  make(map[*ast.FuncLit]*FuncInfo),
	}
	info.InitFuncInfo = info.newFuncInfo()

	for _, file := range files {
		for k, v := range ast.NewCommentMap(fileSet, file, file.Comments) {
			info.comments[k] = v
		}
		ast.Walk(info.InitFuncInfo, file)
	}

	for {
		done := true
		for _, funcInfo := range info.allInfos {
			for obj, calls := range funcInfo.LocalCalls {
				if len(info.FuncDeclInfos[obj].Blocking) != 0 {
					for _, call := range calls {
						funcInfo.markBlocking(call)
					}
					delete(funcInfo.LocalCalls, obj)
					done = false
				}
			}
		}
		if done {
			break
		}
	}

	for _, funcInfo := range info.allInfos {
		for _, continueStmt := range funcInfo.ContinueStmts {
			if funcInfo.Blocking[continueStmt.forStmt.Post] {
				funcInfo.markBlocking(continueStmt.analyzeStack)
			}
		}
	}

	return info
}

func (c *FuncInfo) Visit(node ast.Node) ast.Visitor {
	if node == nil {
		if len(c.analyzeStack) != 0 {
			c.analyzeStack = c.analyzeStack[:len(c.analyzeStack)-1]
		}
		return nil
	}
	c.analyzeStack = append(c.analyzeStack, node)

	switch n := node.(type) {
	case *ast.FuncDecl:
		newInfo := c.p.newFuncInfo()
		c.p.FuncDeclInfos[c.p.Defs[n.Name].(*types.Func)] = newInfo
		return newInfo
	case *ast.FuncLit:
		newInfo := c.p.newFuncInfo()
		c.p.FuncLitInfos[n] = newInfo
		return newInfo
	case *ast.BranchStmt:
		switch n.Tok {
		case token.GOTO:
			for _, n2 := range c.analyzeStack {
				c.Flattened[n2] = true
			}
			c.GotoLabel[c.p.Uses[n.Label].(*types.Label)] = true
		case token.CONTINUE:
			if n.Label != nil {
				label := c.p.Uses[n.Label].(*types.Label)
				for i := len(c.analyzeStack) - 1; i >= 0; i-- {
					if labelStmt, ok := c.analyzeStack[i].(*ast.LabeledStmt); ok && c.p.Defs[labelStmt.Label] == label {
						if _, ok := labelStmt.Stmt.(*ast.RangeStmt); ok {
							return nil
						}
						stack := make([]ast.Node, len(c.analyzeStack))
						copy(stack, c.analyzeStack)
						c.ContinueStmts = append(c.ContinueStmts, continueStmt{labelStmt.Stmt.(*ast.ForStmt), stack})
						return nil
					}
				}
				return nil
			}
			for i := len(c.analyzeStack) - 1; i >= 0; i-- {
				if _, ok := c.analyzeStack[i].(*ast.RangeStmt); ok {
					return nil
				}
				if forStmt, ok := c.analyzeStack[i].(*ast.ForStmt); ok {
					stack := make([]ast.Node, len(c.analyzeStack))
					copy(stack, c.analyzeStack)
					c.ContinueStmts = append(c.ContinueStmts, continueStmt{forStmt, stack})
					return nil
				}
			}
		}
	case *ast.CallExpr:
		callTo := func(obj types.Object) {
			switch o := obj.(type) {
			case *types.Func:
				if recv := o.Type().(*types.Signature).Recv(); recv != nil {
					if _, ok := recv.Type().Underlying().(*types.Interface); ok {
						c.markBlocking(c.analyzeStack)
						return
					}
				}
				if o.Pkg() != c.p.Pkg {
					if c.p.IsBlocking(o) {
						c.markBlocking(c.analyzeStack)
					}
					return
				}
				stack := make([]ast.Node, len(c.analyzeStack))
				copy(stack, c.analyzeStack)
				c.LocalCalls[o] = append(c.LocalCalls[o], stack)
			case *types.Var:
				c.markBlocking(c.analyzeStack)
			}
		}
		switch f := astutil.RemoveParens(n.Fun).(type) {
		case *ast.Ident:
			callTo(c.p.Uses[f])
		case *ast.SelectorExpr:
			if sel := c.p.Selections[f]; sel != nil && typesutil.IsJsObject(sel.Recv()) {
				break
			}
			callTo(c.p.Uses[f.Sel])
		case *ast.FuncLit:
			ast.Walk(c, n.Fun)
			for _, arg := range n.Args {
				ast.Walk(c, arg)
			}
			if len(c.p.FuncLitInfos[f].Blocking) != 0 {
				c.markBlocking(c.analyzeStack)
			}
			return nil
		default:
			if !astutil.IsTypeExpr(f, c.p.Info) {
				c.markBlocking(c.analyzeStack)
			}
		}
	case *ast.SendStmt:
		c.markBlocking(c.analyzeStack)
	case *ast.UnaryExpr:
		if n.Op == token.ARROW {
			c.markBlocking(c.analyzeStack)
		}
	case *ast.RangeStmt:
		if _, ok := c.p.Types[n.X].Type.Underlying().(*types.Chan); ok {
			c.markBlocking(c.analyzeStack)
		}
	case *ast.SelectStmt:
		for _, s := range n.Body.List {
			if s.(*ast.CommClause).Comm == nil { // default clause
				return c
			}
		}
		c.markBlocking(c.analyzeStack)
	case *ast.CommClause:
		for _, s := range n.Body {
			ast.Walk(c, s)
		}
		return nil
	case *ast.GoStmt:
		ast.Walk(c, n.Call.Fun)
		for _, arg := range n.Call.Args {
			ast.Walk(c, arg)
		}
		return nil
	case *ast.DeferStmt:
		c.HasDefer = true
		if funcLit, ok := n.Call.Fun.(*ast.FuncLit); ok {
			ast.Walk(c, funcLit.Body)
		}
	}
	return c
}

func (c *FuncInfo) markBlocking(stack []ast.Node) {
	for _, n := range stack {
		c.Blocking[n] = true
		c.Flattened[n] = true
	}
}
<|endoftext|>


<commit_before>// +build !go1.11
// +build go1.10

package compiler

const ___GOPHERJS_REQUIRES_GO_VERSION_1_10___ = true

// Version is the GopherJS compiler version string.
const Version = "1.10-2"
<commit_msg>compiler: Bump version to GopherJS 1.10-3.<commit_after>// +build !go1.11
// +build go1.10

package compiler

const ___GOPHERJS_REQUIRES_GO_VERSION_1_10___ = true

// Version is the GopherJS compiler version string.
const Version = "1.10-3"
<|endoftext|>