<commit_before>package command

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
	"github.com/funkygao/golib/gofmt"
	"github.com/funkygao/golib/pipestream"
	"github.com/ryanuber/columnize"
)

type Brokers struct {
	Ui  cli.Ui
	Cmd string

	staleOnly    bool
	showVersions bool
	ipInNumber   bool
	cluster      string
}

func (this *Brokers) Run(args []string) (exitCode int) {
	var (
		zone  string
		debug bool
	)
	cmdFlags := flag.NewFlagSet("brokers", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", "", "")
	cmdFlags.StringVar(&this.cluster, "c", "", "")
	cmdFlags.BoolVar(&debug, "debug", false, "")
	cmdFlags.BoolVar(&this.ipInNumber, "n", false, "")
	cmdFlags.BoolVar(&this.staleOnly, "stale", false, "")
	cmdFlags.BoolVar(&this.showVersions, "versions", false, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if this.showVersions {
		this.doShowVersions()
		return
	}

	if debug {
		sarama.Logger = log.New(os.Stderr, color.Magenta("[sarama]"), log.LstdFlags)
	}

	if zone != "" {
		ensureZoneValid(zone)

		zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
		this.displayZoneBrokers(zkzone)

		return
	}

	// print all brokers on all zones by default
	forSortedZones(func(zkzone *zk.ZkZone) {
		this.displayZoneBrokers(zkzone)
	})

	return
}

func (this *Brokers) maxBrokerId(zkzone *zk.ZkZone, clusterName string) int {
	var maxBrokerId int
	zkzone.ForSortedBrokers(func(cluster string, liveBrokers map[string]*zk.BrokerZnode) {
		if cluster == clusterName {
			for _, b := range liveBrokers {
				id, _ := strconv.Atoi(b.Id)
				if id > maxBrokerId {
					maxBrokerId = id
				}
			}
		}
	})

	return maxBrokerId
}

func (this *Brokers) displayZoneBrokers(zkzone *zk.ZkZone) {
	lines := make([]string, 0)
	header := "Zone|Cluster|Id|Broker|Uptime"
	lines = append(lines, header)

	n := 0
	zkzone.ForSortedBrokers(func(cluster string, liveBrokers map[string]*zk.BrokerZnode) {
		outputs := this.clusterBrokers(zkzone.Name(), cluster, liveBrokers)
		n += len(outputs)
		lines = append(lines, outputs...)
	})
	if this.staleOnly {
		this.Ui.Info(fmt.Sprintf("%d problematic brokers in zone[%s]", n, zkzone.Name()))
	} else {
		this.Ui.Info(fmt.Sprintf("%d brokers in zone[%s]", n, zkzone.Name()))
	}
	if len(lines) > 1 {
		// lines has header
		this.Ui.Output(columnize.SimpleFormat(lines))
	}
}

func (this *Brokers) clusterBrokers(zone, cluster string, brokers map[string]*zk.BrokerZnode) []string {
	if !patternMatched(cluster, this.cluster) {
		return nil
	}

	if brokers == nil || len(brokers) == 0 {
		return []string{fmt.Sprintf("%s|%s|%s|%s|%s",
			zone, cluster, " ", color.Red("empty brokers"), " ")}
	}

	lines := make([]string, 0, len(brokers))
	if this.staleOnly {
		// try each broker's aliveness
		for brokerId, broker := range brokers {
			cf := sarama.NewConfig()
			cf.Net.ReadTimeout = time.Second * 4
			cf.Net.WriteTimeout = time.Second * 4
			kfk, err := sarama.NewClient([]string{broker.Addr()}, cf)
			if err != nil {
				lines = append(lines, fmt.Sprintf("%s|%s|%s|%s|%s",
					zone, cluster,
					brokerId, broker.Addr(),
					gofmt.PrettySince(broker.Uptime())))
			} else {
				kfk.Close()
			}
		}

		return lines
	}

	// sort by broker id
	sortedBrokerIds := make([]string, 0, len(brokers))
	for brokerId, _ := range brokers {
		sortedBrokerIds = append(sortedBrokerIds, brokerId)
	}
	sort.Strings(sortedBrokerIds)

	for _, brokerId := range sortedBrokerIds {
		b := brokers[brokerId]
		uptime := gofmt.PrettySince(b.Uptime())
		if time.Since(b.Uptime()) < time.Hour*24*7 {
			uptime = color.Green(uptime)
		}
		if this.ipInNumber {
			lines = append(lines, fmt.Sprintf("%s|%s|%s|%s|%s",
				zone, cluster,
				brokerId, b.Addr(),
				gofmt.PrettySince(b.Uptime())))
		} else {
			lines = append(lines, fmt.Sprintf("%s|%s|%s|%s|%s",
				zone, cluster,
				brokerId, b.NamedAddr(),
				gofmt.PrettySince(b.Uptime())))
		}

	}
	return lines
}

func (this *Brokers) doShowVersions() {
	kafkaVerExp := regexp.MustCompile(`/kafka_(?P<ver>[-\d.]*)\.jar`)
	processExp := regexp.MustCompile(`kfk_(?P<process>\S*)/config/server.properties`)

	cmd := pipestream.New("/usr/bin/consul", "exec",
		"pgrep", "-lf", "java",
		"|", "grep", "-w", "kafka",
		"|", "grep", "-vw", "grep")
	err := cmd.Open()
	swallow(err)
	defer cmd.Close()

	scanner := bufio.NewScanner(cmd.Reader())
	scanner.Split(bufio.ScanLines)

	var (
		line     string
		lastLine string
	)
	hosts := make(map[string]struct{})
	lines := make([]string, 0)
	header := "Process|Host|Version"
	lines = append(lines, header)
	records := make(map[string]map[string]string) // {process: {host: ver}}
	for scanner.Scan() {
		line = scanner.Text()
		if strings.Contains(line, "finished with exit code") {
			continue
		}
		if strings.Contains(line, "node(s) completed") {
			continue
		}

		fields := strings.Fields(line)
		if len(fields) < 2 {
			continue
		}

		if _, err := strconv.Atoi(fields[1]); err != nil {
			// field 1 should be pid
			// if not pid, it continues with last line
			line = lastLine + strings.Join(fields[1:], " ")

			// redo fields
			fields := strings.Fields(line)
			if len(fields) < 2 {
				continue
			}
		}

		lastLine = line

		// version
		matched := kafkaVerExp.FindStringSubmatch(line)
		if len(matched) < 2 {
			continue
		}
		ver := matched[1]

		// process name
		matched = processExp.FindStringSubmatch(line)
		if len(matched) < 2 {
			continue
		}
		process := matched[1]

		// got a valid process record
		if _, present := records[process]; !present {
			records[process] = make(map[string]string)
		}
		host := fields[0][0 : len(fields[0])-1] // discard the ending ':'
		hosts[host] = struct{}{}
		records[process][host] = ver
	}
	swallow(scanner.Err())

	sortedProceses := make([]string, 0, len(records))
	for proc, _ := range records {
		sortedProceses = append(sortedProceses, proc)
	}
	sort.Strings(sortedProceses)

	procsWithSingleInstance := make([]string, 0)
	for _, proc := range sortedProceses {
		sortedHosts := make([]string, 0, len(records[proc]))
		for host, _ := range records[proc] {
			sortedHosts = append(sortedHosts, host)
		}
		sort.Strings(sortedHosts)

		if len(sortedHosts) < 2 {
			procsWithSingleInstance = append(procsWithSingleInstance, proc)
		}

		for _, host := range sortedHosts {
			lines = append(lines, fmt.Sprintf("%s|%s|%s", proc, host, records[proc][host]))
		}
	}

	this.Ui.Output(columnize.SimpleFormat(lines))
	this.Ui.Output("")
	this.Ui.Output(fmt.Sprintf("TOTAL %d processes running on %d hosts",
		len(lines)-1, len(hosts)))
	if len(procsWithSingleInstance) > 0 {
		this.Ui.Output(fmt.Sprintf("\nProcess with 1 SPOF: "))
		this.Ui.Warn(fmt.Sprintf("%v", procsWithSingleInstance))
	}
}

func (*Brokers) Synopsis() string {
	return "Print online brokers from Zookeeper"
}

func (this *Brokers) Help() string {
	help := fmt.Sprintf(`
Usage: %s brokers [options]

    Print online brokers from Zookeeper

Options:

    -z zone
      Only print brokers within a zone

    -c cluster name
      Only print brokers of this cluster

    -versions
      Display kafka instances versions by host
      Precondition: you MUST install consul on each broker host

    -debug

    -n
      Show network addresses as numbers

    -stale
      Only print stale brokers: found in zk but not connectable

`, this.Cmd)
	return strings.TrimSpace(help)
}
<commit_msg>display stale broker err info<commit_after>package command

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
	"github.com/funkygao/golib/gofmt"
	"github.com/funkygao/golib/pipestream"
	"github.com/ryanuber/columnize"
)

type Brokers struct {
	Ui  cli.Ui
	Cmd string

	staleOnly    bool
	showVersions bool
	ipInNumber   bool
	cluster      string
}

func (this *Brokers) Run(args []string) (exitCode int) {
	var (
		zone  string
		debug bool
	)
	cmdFlags := flag.NewFlagSet("brokers", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", "", "")
	cmdFlags.StringVar(&this.cluster, "c", "", "")
	cmdFlags.BoolVar(&debug, "debug", false, "")
	cmdFlags.BoolVar(&this.ipInNumber, "n", false, "")
	cmdFlags.BoolVar(&this.staleOnly, "stale", false, "")
	cmdFlags.BoolVar(&this.showVersions, "versions", false, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if this.showVersions {
		this.doShowVersions()
		return
	}

	if debug {
		sarama.Logger = log.New(os.Stderr, color.Magenta("[sarama]"), log.LstdFlags)
	}

	if zone != "" {
		ensureZoneValid(zone)

		zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
		this.displayZoneBrokers(zkzone)

		return
	}

	// print all brokers on all zones by default
	forSortedZones(func(zkzone *zk.ZkZone) {
		this.displayZoneBrokers(zkzone)
	})

	return
}

func (this *Brokers) maxBrokerId(zkzone *zk.ZkZone, clusterName string) int {
	var maxBrokerId int
	zkzone.ForSortedBrokers(func(cluster string, liveBrokers map[string]*zk.BrokerZnode) {
		if cluster == clusterName {
			for _, b := range liveBrokers {
				id, _ := strconv.Atoi(b.Id)
				if id > maxBrokerId {
					maxBrokerId = id
				}
			}
		}
	})

	return maxBrokerId
}

func (this *Brokers) displayZoneBrokers(zkzone *zk.ZkZone) {
	lines := make([]string, 0)
	header := "Zone|Cluster|Id|Broker|Uptime"
	lines = append(lines, header)

	n := 0
	zkzone.ForSortedBrokers(func(cluster string, liveBrokers map[string]*zk.BrokerZnode) {
		outputs := this.clusterBrokers(zkzone.Name(), cluster, liveBrokers)
		n += len(outputs)
		lines = append(lines, outputs...)
	})
	if this.staleOnly {
		this.Ui.Info(fmt.Sprintf("%d problematic brokers in zone[%s]", n, zkzone.Name()))
	} else {
		this.Ui.Info(fmt.Sprintf("%d brokers in zone[%s]", n, zkzone.Name()))
	}
	if len(lines) > 1 {
		// lines has header
		this.Ui.Output(columnize.SimpleFormat(lines))
	}
}

func (this *Brokers) clusterBrokers(zone, cluster string, brokers map[string]*zk.BrokerZnode) []string {
	if !patternMatched(cluster, this.cluster) {
		return nil
	}

	if brokers == nil || len(brokers) == 0 {
		return []string{fmt.Sprintf("%s|%s|%s|%s|%s",
			zone, cluster, " ", color.Red("empty brokers"), " ")}
	}

	lines := make([]string, 0, len(brokers))
	if this.staleOnly {
		// try each broker's aliveness
		for brokerId, broker := range brokers {
			cf := sarama.NewConfig()
			cf.Net.ReadTimeout = time.Second * 4
			cf.Net.WriteTimeout = time.Second * 4
			kfk, err := sarama.NewClient([]string{broker.Addr()}, cf)
			if err != nil {
				lines = append(lines, fmt.Sprintf("%s|%s|%s|%s|%s",
					zone, cluster,
					brokerId, broker.Addr(),
					fmt.Sprintf("%s: %v", gofmt.PrettySince(broker.Uptime()), err)))
			} else {
				kfk.Close()
			}
		}

		return lines
	}

	// sort by broker id
	sortedBrokerIds := make([]string, 0, len(brokers))
	for brokerId, _ := range brokers {
		sortedBrokerIds = append(sortedBrokerIds, brokerId)
	}
	sort.Strings(sortedBrokerIds)

	for _, brokerId := range sortedBrokerIds {
		b := brokers[brokerId]
		uptime := gofmt.PrettySince(b.Uptime())
		if time.Since(b.Uptime()) < time.Hour*24*7 {
			uptime = color.Green(uptime)
		}
		if this.ipInNumber {
			lines = append(lines, fmt.Sprintf("%s|%s|%s|%s|%s",
				zone, cluster,
				brokerId, b.Addr(),
				gofmt.PrettySince(b.Uptime())))
		} else {
			lines = append(lines, fmt.Sprintf("%s|%s|%s|%s|%s",
				zone, cluster,
				brokerId, b.NamedAddr(),
				gofmt.PrettySince(b.Uptime())))
		}

	}
	return lines
}

func (this *Brokers) doShowVersions() {
	kafkaVerExp := regexp.MustCompile(`/kafka_(?P<ver>[-\d.]*)\.jar`)
	processExp := regexp.MustCompile(`kfk_(?P<process>\S*)/config/server.properties`)

	cmd := pipestream.New("/usr/bin/consul", "exec",
		"pgrep", "-lf", "java",
		"|", "grep", "-w", "kafka",
		"|", "grep", "-vw", "grep")
	err := cmd.Open()
	swallow(err)
	defer cmd.Close()

	scanner := bufio.NewScanner(cmd.Reader())
	scanner.Split(bufio.ScanLines)

	var (
		line     string
		lastLine string
	)
	hosts := make(map[string]struct{})
	lines := make([]string, 0)
	header := "Process|Host|Version"
	lines = append(lines, header)
	records := make(map[string]map[string]string) // {process: {host: ver}}
	for scanner.Scan() {
		line = scanner.Text()
		if strings.Contains(line, "finished with exit code") {
			continue
		}
		if strings.Contains(line, "node(s) completed") {
			continue
		}

		fields := strings.Fields(line)
		if len(fields) < 2 {
			continue
		}

		if _, err := strconv.Atoi(fields[1]); err != nil {
			// field 1 should be pid
			// if not pid, it continues with last line
			line = lastLine + strings.Join(fields[1:], " ")

			// redo fields
			fields := strings.Fields(line)
			if len(fields) < 2 {
				continue
			}
		}

		lastLine = line

		// version
		matched := kafkaVerExp.FindStringSubmatch(line)
		if len(matched) < 2 {
			continue
		}
		ver := matched[1]

		// process name
		matched = processExp.FindStringSubmatch(line)
		if len(matched) < 2 {
			continue
		}
		process := matched[1]

		// got a valid process record
		if _, present := records[process]; !present {
			records[process] = make(map[string]string)
		}
		host := fields[0][0 : len(fields[0])-1] // discard the ending ':'
		hosts[host] = struct{}{}
		records[process][host] = ver
	}
	swallow(scanner.Err())

	sortedProceses := make([]string, 0, len(records))
	for proc, _ := range records {
		sortedProceses = append(sortedProceses, proc)
	}
	sort.Strings(sortedProceses)

	procsWithSingleInstance := make([]string, 0)
	for _, proc := range sortedProceses {
		sortedHosts := make([]string, 0, len(records[proc]))
		for host, _ := range records[proc] {
			sortedHosts = append(sortedHosts, host)
		}
		sort.Strings(sortedHosts)

		if len(sortedHosts) < 2 {
			procsWithSingleInstance = append(procsWithSingleInstance, proc)
		}

		for _, host := range sortedHosts {
			lines = append(lines, fmt.Sprintf("%s|%s|%s", proc, host, records[proc][host]))
		}
	}

	this.Ui.Output(columnize.SimpleFormat(lines))
	this.Ui.Output("")
	this.Ui.Output(fmt.Sprintf("TOTAL %d processes running on %d hosts",
		len(lines)-1, len(hosts)))
	if len(procsWithSingleInstance) > 0 {
		this.Ui.Output(fmt.Sprintf("\nProcess with 1 SPOF: "))
		this.Ui.Warn(fmt.Sprintf("%v", procsWithSingleInstance))
	}
}

func (*Brokers) Synopsis() string {
	return "Print online brokers from Zookeeper"
}

func (this *Brokers) Help() string {
	help := fmt.Sprintf(`
Usage: %s brokers [options]

    Print online brokers from Zookeeper

Options:

    -z zone
      Only print brokers within a zone

    -c cluster name
      Only print brokers of this cluster

    -versions
      Display kafka instances versions by host
      Precondition: you MUST install consul on each broker host

    -debug

    -n
      Show network addresses as numbers

    -stale
      Only print stale brokers: found in zk but not connectable

`, this.Cmd)
	return strings.TrimSpace(help)
}
<|endoftext|>


<commit_before>package command

import (
	"bufio"
	"flag"
	"fmt"
	"net"
	"strings"

	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
	"github.com/funkygao/golib/pipestream"
	"github.com/ryanuber/columnize"
)

// consul members will include:
// - zk cluster as server
// - agents
//   - brokers
//   - kateway
type Members struct {
	Ui  cli.Ui
	Cmd string

	brokerHosts, zkHosts, katewayHosts map[string]struct{}
	nodeHostMap                        map[string]string // consul members node->ip
}

func (this *Members) Run(args []string) (exitCode int) {
	var (
		zone        string
		showLoadAvg bool
	)
	cmdFlags := flag.NewFlagSet("members", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.BoolVar(&showLoadAvg, "l", false, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
	this.fillTheHosts(zkzone)

	consulLiveNode, consulDeadNodes := this.consulMembers()
	for _, node := range consulDeadNodes {
		this.Ui.Error(fmt.Sprintf("%s consul dead", node))
	}

	consulLiveMap := make(map[string]struct{})
	brokerN, zkN, katewayN, unknownN := 0, 0, 0, 0
	for _, node := range consulLiveNode {
		_, presentInBroker := this.brokerHosts[node]
		_, presentInZk := this.zkHosts[node]
		_, presentInKateway := this.katewayHosts[node]
		if presentInBroker {
			brokerN++
		}
		if presentInZk {
			zkN++
		}
		if presentInKateway {
			katewayN++
		}

		if !presentInBroker && !presentInZk && !presentInKateway {
			unknownN++

			this.Ui.Info(fmt.Sprintf("? %s", node))
		}

		consulLiveMap[node] = struct{}{}
	}

	// all brokers should run consul
	for broker, _ := range this.brokerHosts {
		if _, present := consulLiveMap[broker]; !present {
			this.Ui.Warn(fmt.Sprintf("- %s", broker))
		}
	}

	if showLoadAvg {
		this.displayLoadAvg()
	}

	this.Ui.Output(fmt.Sprintf("zk:%s broker:%s kateway:%s ?:%s",
		color.Magenta("%d", zkN),
		color.Magenta("%d", brokerN),
		color.Magenta("%d", katewayN),
		color.Green("%d", unknownN)))

	return
}

func (this *Members) fillTheHosts(zkzone *zk.ZkZone) {
	this.brokerHosts = make(map[string]struct{})
	zkzone.ForSortedBrokers(func(cluster string, brokers map[string]*zk.BrokerZnode) {
		for _, brokerInfo := range brokers {
			this.brokerHosts[brokerInfo.Host] = struct{}{}
		}
	})

	this.zkHosts = make(map[string]struct{})
	for _, addr := range zkzone.ZkAddrList() {
		zkNode, _, err := net.SplitHostPort(addr)
		swallow(err)
		this.zkHosts[zkNode] = struct{}{}
	}

	this.katewayHosts = make(map[string]struct{})
	kws, err := zkzone.KatewayInfos()
	swallow(err)
	for _, kw := range kws {
		host, _, err := net.SplitHostPort(kw.PubAddr)
		swallow(err)
		this.katewayHosts[host] = struct{}{}
	}
}

func (this *Members) displayLoadAvg() {
	cmd := pipestream.New("consul", "exec",
		"uptime", "|", "grep", "load")
	err := cmd.Open()
	swallow(err)
	defer cmd.Close()

	lines := make([]string, 0)
	header := "Node|Host|Role|Load Avg"
	lines = append(lines, header)

	scanner := bufio.NewScanner(cmd.Reader())
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		line := scanner.Text()
		fields := strings.Fields(line)
		node := fields[0]
		parts := strings.Split(line, "load average:")
		if len(parts) < 2 {
			continue
		}
		if strings.HasSuffix(node, ":") {
			node = strings.TrimRight(node, ":")
		}

		host := this.nodeHostMap[node]
		lines = append(lines, fmt.Sprintf("%s|%s|%s|%s", node, host, this.roleOfHost(host), parts[1]))
	}

	if len(lines) > 1 {
		this.Ui.Output(columnize.SimpleFormat(lines))
	}
}

func (this *Members) roleOfHost(host string) string {
	if _, present := this.brokerHosts[host]; present {
		return "B"
	}
	if _, present := this.zkHosts[host]; present {
		return "Z"
	}
	if _, present := this.katewayHosts[host]; present {
		return "K"
	}
	return "?"
}

func (this *Members) consulMembers() ([]string, []string) {
	cmd := pipestream.New("consul", "members")
	err := cmd.Open()
	swallow(err)
	defer cmd.Close()

	liveHosts, deadHosts := []string{}, []string{}
	scanner := bufio.NewScanner(cmd.Reader())
	scanner.Split(bufio.ScanLines)
	this.nodeHostMap = make(map[string]string)
	for scanner.Scan() {
		if strings.Contains(scanner.Text(), "Protocol") {
			// the header
			continue
		}

		fields := strings.Fields(scanner.Text())
		node, addr, alive := fields[0], fields[1], fields[2]
		host, _, err := net.SplitHostPort(addr)
		swallow(err)

		this.nodeHostMap[node] = host

		if alive == "alive" {
			liveHosts = append(liveHosts, host)
		} else {
			deadHosts = append(deadHosts, host)
		}
	}

	return liveHosts, deadHosts
}

func (*Members) Synopsis() string {
	return "Verify consul members match kafka zone"
}

func (this *Members) Help() string {
	help := fmt.Sprintf(`
Usage: %s members [options]

    Verify consul members match kafka zone

    -z zone

    -l
      Display each member load average

`, this.Cmd)
	return strings.TrimSpace(help)
}
<commit_msg>member load avg sort by node name<commit_after>package command

import (
	"bufio"
	"flag"
	"fmt"
	"net"
	"sort"
	"strings"

	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
	"github.com/funkygao/golib/pipestream"
	"github.com/ryanuber/columnize"
)

// consul members will include:
// - zk cluster as server
// - agents
//   - brokers
//   - kateway
type Members struct {
	Ui  cli.Ui
	Cmd string

	brokerHosts, zkHosts, katewayHosts map[string]struct{}
	nodeHostMap                        map[string]string // consul members node->ip
}

func (this *Members) Run(args []string) (exitCode int) {
	var (
		zone        string
		showLoadAvg bool
	)
	cmdFlags := flag.NewFlagSet("members", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.BoolVar(&showLoadAvg, "l", false, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
	this.fillTheHosts(zkzone)

	consulLiveNode, consulDeadNodes := this.consulMembers()
	for _, node := range consulDeadNodes {
		this.Ui.Error(fmt.Sprintf("%s consul dead", node))
	}

	consulLiveMap := make(map[string]struct{})
	brokerN, zkN, katewayN, unknownN := 0, 0, 0, 0
	for _, node := range consulLiveNode {
		_, presentInBroker := this.brokerHosts[node]
		_, presentInZk := this.zkHosts[node]
		_, presentInKateway := this.katewayHosts[node]
		if presentInBroker {
			brokerN++
		}
		if presentInZk {
			zkN++
		}
		if presentInKateway {
			katewayN++
		}

		if !presentInBroker && !presentInZk && !presentInKateway {
			unknownN++
		}

		consulLiveMap[node] = struct{}{}
	}

	// all brokers should run consul
	for broker, _ := range this.brokerHosts {
		if _, present := consulLiveMap[broker]; !present {
			this.Ui.Warn(fmt.Sprintf("- %s", broker))
		}
	}

	if showLoadAvg {
		this.displayLoadAvg()
	}

	this.Ui.Output(fmt.Sprintf("zk:%s broker:%s kateway:%s ?:%s",
		color.Magenta("%d", zkN),
		color.Magenta("%d", brokerN),
		color.Magenta("%d", katewayN),
		color.Green("%d", unknownN)))

	return
}

func (this *Members) fillTheHosts(zkzone *zk.ZkZone) {
	this.brokerHosts = make(map[string]struct{})
	zkzone.ForSortedBrokers(func(cluster string, brokers map[string]*zk.BrokerZnode) {
		for _, brokerInfo := range brokers {
			this.brokerHosts[brokerInfo.Host] = struct{}{}
		}
	})

	this.zkHosts = make(map[string]struct{})
	for _, addr := range zkzone.ZkAddrList() {
		zkNode, _, err := net.SplitHostPort(addr)
		swallow(err)
		this.zkHosts[zkNode] = struct{}{}
	}

	this.katewayHosts = make(map[string]struct{})
	kws, err := zkzone.KatewayInfos()
	swallow(err)
	for _, kw := range kws {
		host, _, err := net.SplitHostPort(kw.PubAddr)
		swallow(err)
		this.katewayHosts[host] = struct{}{}
	}
}

func (this *Members) displayLoadAvg() {
	cmd := pipestream.New("consul", "exec",
		"uptime", "|", "grep", "load")
	err := cmd.Open()
	swallow(err)
	defer cmd.Close()

	lines := make([]string, 0)
	header := "Node|Host|Role|Load Avg"
	lines = append(lines, header)

	scanner := bufio.NewScanner(cmd.Reader())
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		line := scanner.Text()
		fields := strings.Fields(line)
		node := fields[0]
		parts := strings.Split(line, "load average:")
		if len(parts) < 2 {
			continue
		}
		if strings.HasSuffix(node, ":") {
			node = strings.TrimRight(node, ":")
		}

		host := this.nodeHostMap[node]
		lines = append(lines, fmt.Sprintf("%s|%s|%s|%s", node, host, this.roleOfHost(host), parts[1]))
	}

	if len(lines) > 1 {
		sort.Strings(lines[1:])
		this.Ui.Output(columnize.SimpleFormat(lines))
	}
}

func (this *Members) roleOfHost(host string) string {
	if _, present := this.brokerHosts[host]; present {
		return "B"
	}
	if _, present := this.zkHosts[host]; present {
		return "Z"
	}
	if _, present := this.katewayHosts[host]; present {
		return "K"
	}
	return "?"
}

func (this *Members) consulMembers() ([]string, []string) {
	cmd := pipestream.New("consul", "members")
	err := cmd.Open()
	swallow(err)
	defer cmd.Close()

	liveHosts, deadHosts := []string{}, []string{}
	scanner := bufio.NewScanner(cmd.Reader())
	scanner.Split(bufio.ScanLines)
	this.nodeHostMap = make(map[string]string)
	for scanner.Scan() {
		if strings.Contains(scanner.Text(), "Protocol") {
			// the header
			continue
		}

		fields := strings.Fields(scanner.Text())
		node, addr, alive := fields[0], fields[1], fields[2]
		host, _, err := net.SplitHostPort(addr)
		swallow(err)

		this.nodeHostMap[node] = host

		if alive == "alive" {
			liveHosts = append(liveHosts, host)
		} else {
			deadHosts = append(deadHosts, host)
		}
	}

	return liveHosts, deadHosts
}

func (*Members) Synopsis() string {
	return "Verify consul members match kafka zone"
}

func (this *Members) Help() string {
	help := fmt.Sprintf(`
Usage: %s members [options]

    Verify consul members match kafka zone

    -z zone

    -l
      Display each member load average

`, this.Cmd)
	return strings.TrimSpace(help)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/binary"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"text/tabwriter"
	"time"

	"github.com/golang/snappy"
	"github.com/influxdb/influxdb/tsdb"
	"github.com/influxdb/influxdb/tsdb/engine/tsm1"
)

type tsdmDumpOpts struct {
	dumpIndex  bool
	dumpBlocks bool
	filterKey  string
	path       string
}

type tsmIndex struct {
	series  int
	offset  int64
	minTime time.Time
	maxTime time.Time
	blocks  []*block
}

type block struct {
	id     uint64
	offset int64
}

type blockStats struct {
	min, max int
	counts   [][]int
}

func (b *blockStats) inc(typ int, enc byte) {
	for len(b.counts) <= typ {
		b.counts = append(b.counts, []int{})
	}
	for len(b.counts[typ]) <= int(enc) {
		b.counts[typ] = append(b.counts[typ], 0)
	}
	b.counts[typ][enc]++
}

func (b *blockStats) size(sz int) {
	if b.min == 0 || sz < b.min {
		b.min = sz
	}
	if b.min == 0 || sz > b.max {
		b.max = sz
	}
}

var (
	fieldType = []string{
		"timestamp", "float", "int", "bool", "string",
	}
	blockTypes = []string{
		"float64", "int64", "bool", "string",
	}
	timeEnc = []string{
		"none", "s8b", "rle",
	}
	floatEnc = []string{
		"none", "gor",
	}
	intEnc = []string{
		"none", "s8b", "rle",
	}
	boolEnc = []string{
		"none", "bp",
	}
	stringEnc = []string{
		"none", "snpy",
	}
	encDescs = [][]string{
		timeEnc, floatEnc, intEnc, boolEnc, stringEnc,
	}
)

func readFields(path string) (map[string]*tsdb.MeasurementFields, error) {
	fields := make(map[string]*tsdb.MeasurementFields)

	f, err := os.OpenFile(filepath.Join(path, tsm1.FieldsFileExtension), os.O_RDONLY, 0666)
	if os.IsNotExist(err) {
		return fields, nil
	} else if err != nil {
		return nil, err
	}
	b, err := ioutil.ReadAll(f)
	if err != nil {
		return nil, err
	}

	data, err := snappy.Decode(nil, b)
	if err != nil {
		return nil, err
	}

	if err := json.Unmarshal(data, &fields); err != nil {
		return nil, err
	}
	return fields, nil
}

func readSeries(path string) (map[string]*tsdb.Series, error) {
	series := make(map[string]*tsdb.Series)

	f, err := os.OpenFile(filepath.Join(path, tsm1.SeriesFileExtension), os.O_RDONLY, 0666)
	if os.IsNotExist(err) {
		return series, nil
	} else if err != nil {
		return nil, err
	}
	defer f.Close()
	b, err := ioutil.ReadAll(f)
	if err != nil {
		return nil, err
	}

	data, err := snappy.Decode(nil, b)
	if err != nil {
		return nil, err
	}

	if err := json.Unmarshal(data, &series); err != nil {
		return nil, err
	}

	return series, nil
}

func readIds(path string) (map[string]uint64, error) {
	f, err := os.OpenFile(filepath.Join(path, tsm1.IDsFileExtension), os.O_RDONLY, 0666)
	if os.IsNotExist(err) {
		return nil, nil
	} else if err != nil {
		return nil, err
	}
	b, err := ioutil.ReadAll(f)
	if err != nil {
		return nil, err
	}

	b, err = snappy.Decode(nil, b)
	if err != nil {
		return nil, err
	}

	ids := make(map[string]uint64)
	if b != nil {
		if err := json.Unmarshal(b, &ids); err != nil {
			return nil, err
		}
	}
	return ids, err
}
func readIndex(f *os.File) (*tsmIndex, error) {
	// Get the file size
	stat, err := f.Stat()
	if err != nil {
		return nil, err
	}

	// Seek to the series count
	f.Seek(-4, os.SEEK_END)
	b := make([]byte, 8)
	_, err = f.Read(b[:4])
	if err != nil {
		return nil, err
	}

	seriesCount := binary.BigEndian.Uint32(b)

	// Get the min time
	f.Seek(-20, os.SEEK_END)
	f.Read(b)
	minTime := time.Unix(0, int64(btou64(b)))

	// Get max time
	f.Seek(-12, os.SEEK_END)
	f.Read(b)
	maxTime := time.Unix(0, int64(btou64(b)))

	// Figure out where the index starts
	indexStart := stat.Size() - int64(seriesCount*12+20)

	// Seek to the start of the index
	f.Seek(indexStart, os.SEEK_SET)
	count := int(seriesCount)
	index := &tsmIndex{
		offset:  indexStart,
		minTime: minTime,
		maxTime: maxTime,
		series:  count,
	}

	if indexStart < 0 {
		return nil, fmt.Errorf("index corrupt: offset=%d", indexStart)
	}

	// Read the index entries
	for i := 0; i < count; i++ {
		f.Read(b)
		id := binary.BigEndian.Uint64(b)
		f.Read(b[:4])
		pos := binary.BigEndian.Uint32(b[:4])
		index.blocks = append(index.blocks, &block{id: id, offset: int64(pos)})
	}

	return index, nil
}

func cmdDumpTsm1(opts *tsdmDumpOpts) {
	var errors []error

	f, err := os.Open(opts.path)
	if err != nil {
		println(err.Error())
		os.Exit(1)
	}

	// Get the file size
	stat, err := f.Stat()
	if err != nil {
		println(err.Error())
		os.Exit(1)
	}

	b := make([]byte, 8)
	f.Read(b[:4])

	// Verify magic number
	if binary.BigEndian.Uint32(b[:4]) != 0x16D116D1 {
		println("Not a tsm1 file.")
		os.Exit(1)
	}

	ids, err := readIds(filepath.Dir(opts.path))
	if err != nil {
		println("Failed to read series:", err.Error())
		os.Exit(1)
	}

	invIds := map[uint64]string{}
	for k, v := range ids {
		invIds[v] = k
	}

	index, err := readIndex(f)
	if err != nil {
		println("Failed to readIndex:", err.Error())

		// Create a stubbed out index so we can still try and read the block data directly
		// w/o panicing ourselves.
		index = &tsmIndex{
			minTime: time.Unix(0, 0),
			maxTime: time.Unix(0, 0),
			offset:  stat.Size(),
		}
	}

	blockStats := &blockStats{}

	println("Summary:")
	fmt.Printf("  File: %s\n", opts.path)
	fmt.Printf("  Time Range: %s - %s\n",
		index.minTime.UTC().Format(time.RFC3339Nano),
		index.maxTime.UTC().Format(time.RFC3339Nano),
	)
	fmt.Printf("  Duration: %s ", index.maxTime.Sub(index.minTime))
	fmt.Printf("  Series: %d ", index.series)
	fmt.Printf("  File Size: %d\n", stat.Size())
	println()

	tw := tabwriter.NewWriter(os.Stdout, 8, 8, 1, '\t', 0)
	fmt.Fprintln(tw, "  "+strings.Join([]string{"Pos", "ID", "Ofs", "Key", "Field"}, "\t"))
	for i, block := range index.blocks {
		key := invIds[block.id]
		split := strings.Split(key, "#!~#")

		// We dont' know know if we have fields so use an informative default
		var measurement, field string = "UNKNOWN", "UNKNOWN"

		// We read some IDs from the ids file
		if len(invIds) > 0 {
			// Change the default to error until we know we have a valid key
			measurement = "ERR"
			field = "ERR"

			// Possible corruption? Try to read as much as we can and point to the problem.
			if key == "" {
				errors = append(errors, fmt.Errorf("index pos %d, field id: %d, missing key for id", i, block.id))
			} else if len(split) < 2 {
				errors = append(errors, fmt.Errorf("index pos %d, field id: %d, key corrupt: got '%v'", i, block.id, key))
			} else {
				measurement = split[0]
				field = split[1]
			}
		}

		if opts.filterKey != "" && !strings.Contains(key, opts.filterKey) {
			continue
		}
		fmt.Fprintln(tw, "  "+strings.Join([]string{
			strconv.FormatInt(int64(i), 10),
			strconv.FormatUint(block.id, 10),
			strconv.FormatInt(int64(block.offset), 10),
			measurement,
			field,
		}, "\t"))
	}

	if opts.dumpIndex {
		println("Index:")
		tw.Flush()
		println()
	}

	tw = tabwriter.NewWriter(os.Stdout, 8, 8, 1, '\t', 0)
	fmt.Fprintln(tw, "  "+strings.Join([]string{"Blk", "Ofs", "Len", "ID", "Type", "Min Time", "Points", "Enc [T/V]", "Len [T/V]"}, "\t"))

	// Staring at 4 because the magic number is 4 bytes
	i := int64(4)
	var blockCount, pointCount, blockSize int64
	indexSize := stat.Size() - index.offset

	// Start at the beginning and read every block
	for i < index.offset {
		f.Seek(int64(i), 0)

		f.Read(b)
		id := btou64(b)
		f.Read(b[:4])
		length := binary.BigEndian.Uint32(b[:4])
		buf := make([]byte, length)
		f.Read(buf)

		blockSize += int64(len(buf)) + 12

		startTime := time.Unix(0, int64(btou64(buf[:8])))
		blockType := buf[8]

		encoded := buf[9:]

		var v []tsm1.Value
		err := tsm1.DecodeBlock(buf, &v)
		if err != nil {
			fmt.Printf("error: %v\n", err.Error())
			os.Exit(1)
		}

		pointCount += int64(len(v))

		// Length of the timestamp block
		tsLen, j := binary.Uvarint(encoded)

		// Unpack the timestamp bytes
		ts := encoded[int(j) : int(j)+int(tsLen)]

		// Unpack the value bytes
		values := encoded[int(j)+int(tsLen):]

		tsEncoding := timeEnc[int(ts[0]>>4)]
		vEncoding := encDescs[int(blockType+1)][values[0]>>4]

		typeDesc := blockTypes[blockType]

		blockStats.inc(0, ts[0]>>4)
		blockStats.inc(int(blockType+1), values[0]>>4)
		blockStats.size(len(buf))

		if opts.filterKey != "" && !strings.Contains(invIds[id], opts.filterKey) {
			i += (12 + int64(length))
			blockCount++
			continue
		}

		fmt.Fprintln(tw, "  "+strings.Join([]string{
			strconv.FormatInt(blockCount, 10),
			strconv.FormatInt(i, 10),
			strconv.FormatInt(int64(len(buf)), 10),
			strconv.FormatUint(id, 10),
			typeDesc,
			startTime.UTC().Format(time.RFC3339Nano),
			strconv.FormatInt(int64(len(v)), 10),
			fmt.Sprintf("%s/%s", tsEncoding, vEncoding),
			fmt.Sprintf("%d/%d", len(ts), len(values)),
		}, "\t"))

		i += (12 + int64(length))
		blockCount++
	}
	if opts.dumpBlocks {
		println("Blocks:")
		tw.Flush()
		println()
	}

	fmt.Printf("Statistics\n")
	fmt.Printf("  Blocks:\n")
	fmt.Printf("    Total: %d Size: %d Min: %d Max: %d Avg: %d\n",
		blockCount, blockSize, blockStats.min, blockStats.max, blockSize/blockCount)
	fmt.Printf("  Index:\n")
	fmt.Printf("    Total: %d Size: %d\n", len(index.blocks), indexSize)
	fmt.Printf("  Points:\n")
	fmt.Printf("    Total: %d", pointCount)
	println()

	println("  Encoding:")
	for i, counts := range blockStats.counts {
		if len(counts) == 0 {
			continue
		}
		fmt.Printf("    %s: ", strings.Title(fieldType[i]))
		for j, v := range counts {
			fmt.Printf("\t%s: %d (%d%%) ", encDescs[i][j], v, int(float64(v)/float64(blockCount)*100))
		}
		println()
	}
	fmt.Printf("  Compression:\n")
	fmt.Printf("    Per block: %0.2f bytes/point\n", float64(blockSize)/float64(pointCount))
	fmt.Printf("    Total: %0.2f bytes/point\n", float64(stat.Size())/float64(pointCount))

	if len(errors) > 0 {
		println()
		fmt.Printf("Errors (%d):\n", len(errors))
		for _, err := range errors {
			fmt.Printf("  * %v\n", err)
		}
		println()
	}
}
<commit_msg>Update influx_inspect to use new DecodeBlock interface<commit_after>package main

import (
	"encoding/binary"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"text/tabwriter"
	"time"

	"github.com/golang/snappy"
	"github.com/influxdb/influxdb/tsdb"
	"github.com/influxdb/influxdb/tsdb/engine/tsm1"
)

type tsdmDumpOpts struct {
	dumpIndex  bool
	dumpBlocks bool
	filterKey  string
	path       string
}

type tsmIndex struct {
	series  int
	offset  int64
	minTime time.Time
	maxTime time.Time
	blocks  []*block
}

type block struct {
	id     uint64
	offset int64
}

type blockStats struct {
	min, max int
	counts   [][]int
}

func (b *blockStats) inc(typ int, enc byte) {
	for len(b.counts) <= typ {
		b.counts = append(b.counts, []int{})
	}
	for len(b.counts[typ]) <= int(enc) {
		b.counts[typ] = append(b.counts[typ], 0)
	}
	b.counts[typ][enc]++
}

func (b *blockStats) size(sz int) {
	if b.min == 0 || sz < b.min {
		b.min = sz
	}
	if b.min == 0 || sz > b.max {
		b.max = sz
	}
}

var (
	fieldType = []string{
		"timestamp", "float", "int", "bool", "string",
	}
	blockTypes = []string{
		"float64", "int64", "bool", "string",
	}
	timeEnc = []string{
		"none", "s8b", "rle",
	}
	floatEnc = []string{
		"none", "gor",
	}
	intEnc = []string{
		"none", "s8b", "rle",
	}
	boolEnc = []string{
		"none", "bp",
	}
	stringEnc = []string{
		"none", "snpy",
	}
	encDescs = [][]string{
		timeEnc, floatEnc, intEnc, boolEnc, stringEnc,
	}
)

func readFields(path string) (map[string]*tsdb.MeasurementFields, error) {
	fields := make(map[string]*tsdb.MeasurementFields)

	f, err := os.OpenFile(filepath.Join(path, tsm1.FieldsFileExtension), os.O_RDONLY, 0666)
	if os.IsNotExist(err) {
		return fields, nil
	} else if err != nil {
		return nil, err
	}
	b, err := ioutil.ReadAll(f)
	if err != nil {
		return nil, err
	}

	data, err := snappy.Decode(nil, b)
	if err != nil {
		return nil, err
	}

	if err := json.Unmarshal(data, &fields); err != nil {
		return nil, err
	}
	return fields, nil
}

func readSeries(path string) (map[string]*tsdb.Series, error) {
	series := make(map[string]*tsdb.Series)

	f, err := os.OpenFile(filepath.Join(path, tsm1.SeriesFileExtension), os.O_RDONLY, 0666)
	if os.IsNotExist(err) {
		return series, nil
	} else if err != nil {
		return nil, err
	}
	defer f.Close()
	b, err := ioutil.ReadAll(f)
	if err != nil {
		return nil, err
	}

	data, err := snappy.Decode(nil, b)
	if err != nil {
		return nil, err
	}

	if err := json.Unmarshal(data, &series); err != nil {
		return nil, err
	}

	return series, nil
}

func readIds(path string) (map[string]uint64, error) {
	f, err := os.OpenFile(filepath.Join(path, tsm1.IDsFileExtension), os.O_RDONLY, 0666)
	if os.IsNotExist(err) {
		return nil, nil
	} else if err != nil {
		return nil, err
	}
	b, err := ioutil.ReadAll(f)
	if err != nil {
		return nil, err
	}

	b, err = snappy.Decode(nil, b)
	if err != nil {
		return nil, err
	}

	ids := make(map[string]uint64)
	if b != nil {
		if err := json.Unmarshal(b, &ids); err != nil {
			return nil, err
		}
	}
	return ids, err
}
func readIndex(f *os.File) (*tsmIndex, error) {
	// Get the file size
	stat, err := f.Stat()
	if err != nil {
		return nil, err
	}

	// Seek to the series count
	f.Seek(-4, os.SEEK_END)
	b := make([]byte, 8)
	_, err = f.Read(b[:4])
	if err != nil {
		return nil, err
	}

	seriesCount := binary.BigEndian.Uint32(b)

	// Get the min time
	f.Seek(-20, os.SEEK_END)
	f.Read(b)
	minTime := time.Unix(0, int64(btou64(b)))

	// Get max time
	f.Seek(-12, os.SEEK_END)
	f.Read(b)
	maxTime := time.Unix(0, int64(btou64(b)))

	// Figure out where the index starts
	indexStart := stat.Size() - int64(seriesCount*12+20)

	// Seek to the start of the index
	f.Seek(indexStart, os.SEEK_SET)
	count := int(seriesCount)
	index := &tsmIndex{
		offset:  indexStart,
		minTime: minTime,
		maxTime: maxTime,
		series:  count,
	}

	if indexStart < 0 {
		return nil, fmt.Errorf("index corrupt: offset=%d", indexStart)
	}

	// Read the index entries
	for i := 0; i < count; i++ {
		f.Read(b)
		id := binary.BigEndian.Uint64(b)
		f.Read(b[:4])
		pos := binary.BigEndian.Uint32(b[:4])
		index.blocks = append(index.blocks, &block{id: id, offset: int64(pos)})
	}

	return index, nil
}

func cmdDumpTsm1(opts *tsdmDumpOpts) {
	var errors []error

	f, err := os.Open(opts.path)
	if err != nil {
		println(err.Error())
		os.Exit(1)
	}

	// Get the file size
	stat, err := f.Stat()
	if err != nil {
		println(err.Error())
		os.Exit(1)
	}

	b := make([]byte, 8)
	f.Read(b[:4])

	// Verify magic number
	if binary.BigEndian.Uint32(b[:4]) != 0x16D116D1 {
		println("Not a tsm1 file.")
		os.Exit(1)
	}

	ids, err := readIds(filepath.Dir(opts.path))
	if err != nil {
		println("Failed to read series:", err.Error())
		os.Exit(1)
	}

	invIds := map[uint64]string{}
	for k, v := range ids {
		invIds[v] = k
	}

	index, err := readIndex(f)
	if err != nil {
		println("Failed to readIndex:", err.Error())

		// Create a stubbed out index so we can still try and read the block data directly
		// w/o panicing ourselves.
		index = &tsmIndex{
			minTime: time.Unix(0, 0),
			maxTime: time.Unix(0, 0),
			offset:  stat.Size(),
		}
	}

	blockStats := &blockStats{}

	println("Summary:")
	fmt.Printf("  File: %s\n", opts.path)
	fmt.Printf("  Time Range: %s - %s\n",
		index.minTime.UTC().Format(time.RFC3339Nano),
		index.maxTime.UTC().Format(time.RFC3339Nano),
	)
	fmt.Printf("  Duration: %s ", index.maxTime.Sub(index.minTime))
	fmt.Printf("  Series: %d ", index.series)
	fmt.Printf("  File Size: %d\n", stat.Size())
	println()

	tw := tabwriter.NewWriter(os.Stdout, 8, 8, 1, '\t', 0)
	fmt.Fprintln(tw, "  "+strings.Join([]string{"Pos", "ID", "Ofs", "Key", "Field"}, "\t"))
	for i, block := range index.blocks {
		key := invIds[block.id]
		split := strings.Split(key, "#!~#")

		// We dont' know know if we have fields so use an informative default
		var measurement, field string = "UNKNOWN", "UNKNOWN"

		// We read some IDs from the ids file
		if len(invIds) > 0 {
			// Change the default to error until we know we have a valid key
			measurement = "ERR"
			field = "ERR"

			// Possible corruption? Try to read as much as we can and point to the problem.
			if key == "" {
				errors = append(errors, fmt.Errorf("index pos %d, field id: %d, missing key for id", i, block.id))
			} else if len(split) < 2 {
				errors = append(errors, fmt.Errorf("index pos %d, field id: %d, key corrupt: got '%v'", i, block.id, key))
			} else {
				measurement = split[0]
				field = split[1]
			}
		}

		if opts.filterKey != "" && !strings.Contains(key, opts.filterKey) {
			continue
		}
		fmt.Fprintln(tw, "  "+strings.Join([]string{
			strconv.FormatInt(int64(i), 10),
			strconv.FormatUint(block.id, 10),
			strconv.FormatInt(int64(block.offset), 10),
			measurement,
			field,
		}, "\t"))
	}

	if opts.dumpIndex {
		println("Index:")
		tw.Flush()
		println()
	}

	tw = tabwriter.NewWriter(os.Stdout, 8, 8, 1, '\t', 0)
	fmt.Fprintln(tw, "  "+strings.Join([]string{"Blk", "Ofs", "Len", "ID", "Type", "Min Time", "Points", "Enc [T/V]", "Len [T/V]"}, "\t"))

	// Staring at 4 because the magic number is 4 bytes
	i := int64(4)
	var blockCount, pointCount, blockSize int64
	indexSize := stat.Size() - index.offset

	// Start at the beginning and read every block
	for i < index.offset {
		f.Seek(int64(i), 0)

		f.Read(b)
		id := btou64(b)
		f.Read(b[:4])
		length := binary.BigEndian.Uint32(b[:4])
		buf := make([]byte, length)
		f.Read(buf)

		blockSize += int64(len(buf)) + 12

		startTime := time.Unix(0, int64(btou64(buf[:8])))
		blockType := buf[8]

		encoded := buf[9:]

		var v []tsm1.Value
		v, err := tsm1.DecodeBlock(buf, v)
		if err != nil {
			fmt.Printf("error: %v\n", err.Error())
			os.Exit(1)
		}

		pointCount += int64(len(v))

		// Length of the timestamp block
		tsLen, j := binary.Uvarint(encoded)

		// Unpack the timestamp bytes
		ts := encoded[int(j) : int(j)+int(tsLen)]

		// Unpack the value bytes
		values := encoded[int(j)+int(tsLen):]

		tsEncoding := timeEnc[int(ts[0]>>4)]
		vEncoding := encDescs[int(blockType+1)][values[0]>>4]

		typeDesc := blockTypes[blockType]

		blockStats.inc(0, ts[0]>>4)
		blockStats.inc(int(blockType+1), values[0]>>4)
		blockStats.size(len(buf))

		if opts.filterKey != "" && !strings.Contains(invIds[id], opts.filterKey) {
			i += (12 + int64(length))
			blockCount++
			continue
		}

		fmt.Fprintln(tw, "  "+strings.Join([]string{
			strconv.FormatInt(blockCount, 10),
			strconv.FormatInt(i, 10),
			strconv.FormatInt(int64(len(buf)), 10),
			strconv.FormatUint(id, 10),
			typeDesc,
			startTime.UTC().Format(time.RFC3339Nano),
			strconv.FormatInt(int64(len(v)), 10),
			fmt.Sprintf("%s/%s", tsEncoding, vEncoding),
			fmt.Sprintf("%d/%d", len(ts), len(values)),
		}, "\t"))

		i += (12 + int64(length))
		blockCount++
	}
	if opts.dumpBlocks {
		println("Blocks:")
		tw.Flush()
		println()
	}

	fmt.Printf("Statistics\n")
	fmt.Printf("  Blocks:\n")
	fmt.Printf("    Total: %d Size: %d Min: %d Max: %d Avg: %d\n",
		blockCount, blockSize, blockStats.min, blockStats.max, blockSize/blockCount)
	fmt.Printf("  Index:\n")
	fmt.Printf("    Total: %d Size: %d\n", len(index.blocks), indexSize)
	fmt.Printf("  Points:\n")
	fmt.Printf("    Total: %d", pointCount)
	println()

	println("  Encoding:")
	for i, counts := range blockStats.counts {
		if len(counts) == 0 {
			continue
		}
		fmt.Printf("    %s: ", strings.Title(fieldType[i]))
		for j, v := range counts {
			fmt.Printf("\t%s: %d (%d%%) ", encDescs[i][j], v, int(float64(v)/float64(blockCount)*100))
		}
		println()
	}
	fmt.Printf("  Compression:\n")
	fmt.Printf("    Per block: %0.2f bytes/point\n", float64(blockSize)/float64(pointCount))
	fmt.Printf("    Total: %0.2f bytes/point\n", float64(stat.Size())/float64(pointCount))

	if len(errors) > 0 {
		println()
		fmt.Printf("Errors (%d):\n", len(errors))
		for _, err := range errors {
			fmt.Printf("  * %v\n", err)
		}
		println()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/config"
	"launchpad.net/juju-core/juju/testing"
	"launchpad.net/juju-core/state"
	coretesting "launchpad.net/juju-core/testing"
	"launchpad.net/juju-core/version"
	"launchpad.net/tomb"
	"net/http"
	"path/filepath"
	"time"
)

var _ = Suite(&upgraderSuite{})

type upgraderSuite struct {
	testing.JujuConnSuite
	oldVarDir string
}

func (s *upgraderSuite) SetUpTest(c *C) {
	s.JujuConnSuite.SetUpTest(c)
	s.oldVarDir = environs.VarDir
	environs.VarDir = c.MkDir()
}

func (s *upgraderSuite) TearDownTest(c *C) {
	environs.VarDir = s.oldVarDir
	invalidVersion = func() {}
	sameVersion = func() {}
	s.JujuConnSuite.TearDownTest(c)
}

func (s *upgraderSuite) TestUpgraderError(c *C) {
	st, err := state.Open(s.StateInfo(c))
	c.Assert(err, IsNil)
	_, as, upgraderDone := startUpgrader(st)
	// We have no installed tools, so the logic should set the agent
	// tools anyway, but with no URL.
	assertEvent(c, as.event, fmt.Sprintf("SetAgentTools %s ", version.Current))

	// Close the state under the watcher and check that the upgrader dies.
	st.Close()
	select {
	case err := <-upgraderDone:
		c.Assert(err, Not(FitsTypeOf), &UpgradedError{})
		c.Assert(err, NotNil)
	case <-time.After(500 * time.Millisecond):
		c.Fatalf("upgrader did not stop as expected")
	}
}

func (s *upgraderSuite) TestUpgraderStop(c *C) {
	u, as, upgraderDone := startUpgrader(s.State)
	assertEvent(c, as.event, fmt.Sprintf("SetAgentTools %s ", version.Current))

	err := u.Stop()
	c.Assert(err, IsNil)

	select {
	case err := <-upgraderDone:
		c.Assert(err, IsNil)
	case <-time.After(500 * time.Millisecond):
		c.Fatalf("upgrader did not stop as expected")
	}
}

// startUpgrader starts the upgrader using the given machine
// for observing and changing agent tools.
func startUpgrader(st *state.State) (u *Upgrader, as *testAgentState, upgraderDone <-chan error) {
	as = newTestAgentState()
	u = NewUpgrader(st, "testagent", as)
	done := make(chan error, 1)
	go func() {
		done <- u.Wait()
	}()
	upgraderDone = done
	return
}

func (s *upgraderSuite) proposeVersion(c *C, vers version.Number) {
	cfg, err := s.State.EnvironConfig()
	c.Assert(err, IsNil)
	attrs := cfg.AllAttrs()
	attrs["agent-version"] = vers.String()
	newCfg, err := config.New(attrs)
	c.Assert(err, IsNil)
	err = s.State.SetEnvironConfig(newCfg)
	c.Assert(err, IsNil)
}

func (s *upgraderSuite) uploadTools(c *C, vers version.Binary) (path string, tools *state.Tools) {
	tgz := coretesting.TarGz(
		coretesting.NewTarFile("juju", 0777, "juju contents "+vers.String()),
		coretesting.NewTarFile("jujuc", 0777, "jujuc contents "+vers.String()),
		coretesting.NewTarFile("jujud", 0777, "jujud contents "+vers.String()),
	)
	storage := s.Conn.Environ.Storage()
	err := storage.Put(environs.ToolsStoragePath(vers), bytes.NewReader(tgz), int64(len(tgz)))
	c.Assert(err, IsNil)
	path = environs.ToolsStoragePath(vers)
	url, err := s.Conn.Environ.Storage().URL(path)
	c.Assert(err, IsNil)
	return path, &state.Tools{URL: url, Binary: vers}
}

func (s *upgraderSuite) TestUpgrader(c *C) {
	// Set up the test hooks.
	sameVersionEvent := make(chan struct{}, 10)
	sameVersion = func() {
		sameVersionEvent <- struct{}{}
	}
	invalidVersionEvent := make(chan struct{}, 10)
	invalidVersion = func() {
		invalidVersionEvent <- struct{}{}
	}

	// Set up the current version and tools.
	version.Current = version.MustParseBinary("1.0.1-foo-bar")
	v1path, v1tools := s.uploadTools(c, version.Current)

	// Unpack the "current" version of the tools, and delete them from
	// the storage so that we're sure that the uploader isn't trying
	// to fetch them.
	resp, err := http.Get(v1tools.URL)
	c.Assert(err, IsNil)
	err = environs.UnpackTools(v1tools, resp.Body)
	c.Assert(err, IsNil)
	err = s.Conn.Environ.Storage().Remove(v1path)
	c.Assert(err, IsNil)

	// Start the upgrader going and check that the tools are those
	// that we set up.
	_, as, upgraderDone := startUpgrader(s.State)
	assertEvent(c, as.event, "SetAgentTools 1.0.1-foo-bar "+v1tools.URL)

	// Propose some tools that are not there, and check that it saw
	// the change.
	s.proposeVersion(c, version.MustParse("1.0.2"))
	<-invalidVersionEvent

	// Upload the current tools again, and check that it saw the change.
	v1path, v1tools = s.uploadTools(c, version.Current)
	s.proposeVersion(c, version.MustParse("1.0.3"))
	<-sameVersionEvent

	// Upload a two new versions of the tools. We'll test upgrading to these tools.
	_, v5tools := s.uploadTools(c, version.MustParseBinary("1.0.5-foo-bar"))
	_, v6tools := s.uploadTools(c, version.MustParseBinary("1.0.6-foo-bar"))

	// Check that it won't choose tools with a greater version number.
	s.proposeVersion(c, version.MustParse("1.0.4"))
	<-sameVersionEvent

	s.proposeVersion(c, v6tools.Number)
	select {
	case err := <-upgraderDone:
		c.Assert(err, DeepEquals, &UpgradedError{v6tools})
	case <-time.After(500 * time.Millisecond):
		c.Fatalf("upgrader did not stop as expected")
	}

	// Check that the upgraded version was really downloaded.
	data, err := ioutil.ReadFile(filepath.Join(environs.ToolsDir(v6tools.Binary), "jujud"))
	c.Assert(err, IsNil)
	c.Assert(string(data), Equals, "jujud contents 1.0.6-foo-bar")

	version.Current = v6tools.Binary
	// Check that we can start again.
	_, as, upgraderDone = startUpgrader(s.State)
	assertEvent(c, as.event, "SetAgentTools 1.0.6-foo-bar "+v6tools.URL)

	// Check that we can downgrade.
	s.proposeVersion(c, v5tools.Number)

	select {
	case tools := <-upgraderDone:
		c.Assert(tools, DeepEquals, &UpgradedError{v5tools})
	case <-time.After(500 * time.Millisecond):
		c.Fatalf("upgrader did not stop as expected")
	}
}

func assertEvent(c *C, event <-chan string, want string) {
	select {
	case got := <-event:
		c.Assert(got, Equals, want)
	case <-time.After(500 * time.Millisecond):
		c.Fatalf("no event received; expected %q", want)
	}
}

type testAgentState struct {
	tomb.Tomb
	event chan string
}

func newTestAgentState() *testAgentState {
	return &testAgentState{
		event: make(chan string),
	}
}

func (t *testAgentState) SetAgentTools(tools *state.Tools) error {
	t.event <- fmt.Sprintf("SetAgentTools %v %s", tools.Binary, tools.URL)
	return nil
}
<commit_msg>cmd/jujud: fix comment<commit_after>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/config"
	"launchpad.net/juju-core/juju/testing"
	"launchpad.net/juju-core/state"
	coretesting "launchpad.net/juju-core/testing"
	"launchpad.net/juju-core/version"
	"launchpad.net/tomb"
	"net/http"
	"path/filepath"
	"time"
)

var _ = Suite(&upgraderSuite{})

type upgraderSuite struct {
	testing.JujuConnSuite
	oldVarDir string
}

func (s *upgraderSuite) SetUpTest(c *C) {
	s.JujuConnSuite.SetUpTest(c)
	s.oldVarDir = environs.VarDir
	environs.VarDir = c.MkDir()
}

func (s *upgraderSuite) TearDownTest(c *C) {
	environs.VarDir = s.oldVarDir
	invalidVersion = func() {}
	sameVersion = func() {}
	s.JujuConnSuite.TearDownTest(c)
}

func (s *upgraderSuite) TestUpgraderError(c *C) {
	st, err := state.Open(s.StateInfo(c))
	c.Assert(err, IsNil)
	_, as, upgraderDone := startUpgrader(st)
	// We have no installed tools, so the logic should set the agent
	// tools anyway, but with no URL.
	assertEvent(c, as.event, fmt.Sprintf("SetAgentTools %s ", version.Current))

	// Close the state under the watcher and check that the upgrader dies.
	st.Close()
	select {
	case err := <-upgraderDone:
		c.Assert(err, Not(FitsTypeOf), &UpgradedError{})
		c.Assert(err, NotNil)
	case <-time.After(500 * time.Millisecond):
		c.Fatalf("upgrader did not stop as expected")
	}
}

func (s *upgraderSuite) TestUpgraderStop(c *C) {
	u, as, upgraderDone := startUpgrader(s.State)
	assertEvent(c, as.event, fmt.Sprintf("SetAgentTools %s ", version.Current))

	err := u.Stop()
	c.Assert(err, IsNil)

	select {
	case err := <-upgraderDone:
		c.Assert(err, IsNil)
	case <-time.After(500 * time.Millisecond):
		c.Fatalf("upgrader did not stop as expected")
	}
}

// startUpgrader starts the upgrader using the given machine
// for observing and changing agent tools.
func startUpgrader(st *state.State) (u *Upgrader, as *testAgentState, upgraderDone <-chan error) {
	as = newTestAgentState()
	u = NewUpgrader(st, "testagent", as)
	done := make(chan error, 1)
	go func() {
		done <- u.Wait()
	}()
	upgraderDone = done
	return
}

func (s *upgraderSuite) proposeVersion(c *C, vers version.Number) {
	cfg, err := s.State.EnvironConfig()
	c.Assert(err, IsNil)
	attrs := cfg.AllAttrs()
	attrs["agent-version"] = vers.String()
	newCfg, err := config.New(attrs)
	c.Assert(err, IsNil)
	err = s.State.SetEnvironConfig(newCfg)
	c.Assert(err, IsNil)
}

func (s *upgraderSuite) uploadTools(c *C, vers version.Binary) (path string, tools *state.Tools) {
	tgz := coretesting.TarGz(
		coretesting.NewTarFile("juju", 0777, "juju contents "+vers.String()),
		coretesting.NewTarFile("jujuc", 0777, "jujuc contents "+vers.String()),
		coretesting.NewTarFile("jujud", 0777, "jujud contents "+vers.String()),
	)
	storage := s.Conn.Environ.Storage()
	err := storage.Put(environs.ToolsStoragePath(vers), bytes.NewReader(tgz), int64(len(tgz)))
	c.Assert(err, IsNil)
	path = environs.ToolsStoragePath(vers)
	url, err := s.Conn.Environ.Storage().URL(path)
	c.Assert(err, IsNil)
	return path, &state.Tools{URL: url, Binary: vers}
}

func (s *upgraderSuite) TestUpgrader(c *C) {
	// Set up the test hooks.
	sameVersionEvent := make(chan struct{}, 10)
	sameVersion = func() {
		sameVersionEvent <- struct{}{}
	}
	invalidVersionEvent := make(chan struct{}, 10)
	invalidVersion = func() {
		invalidVersionEvent <- struct{}{}
	}

	// Set up the current version and tools.
	version.Current = version.MustParseBinary("1.0.1-foo-bar")
	v1path, v1tools := s.uploadTools(c, version.Current)

	// Unpack the "current" version of the tools, and delete them from
	// the storage so that we're sure that the uploader isn't trying
	// to fetch them.
	resp, err := http.Get(v1tools.URL)
	c.Assert(err, IsNil)
	err = environs.UnpackTools(v1tools, resp.Body)
	c.Assert(err, IsNil)
	err = s.Conn.Environ.Storage().Remove(v1path)
	c.Assert(err, IsNil)

	// Start the upgrader going and check that the tools are those
	// that we set up.
	_, as, upgraderDone := startUpgrader(s.State)
	assertEvent(c, as.event, "SetAgentTools 1.0.1-foo-bar "+v1tools.URL)

	// Propose some tools that are not there, and check that it saw
	// the change.
	s.proposeVersion(c, version.MustParse("1.0.2"))
	<-invalidVersionEvent

	// Upload the current tools again, and check that it saw the change.
	v1path, v1tools = s.uploadTools(c, version.Current)
	s.proposeVersion(c, version.MustParse("1.0.3"))
	<-sameVersionEvent

	// Upload two new versions of the tools. We'll test upgrading to these tools.
	_, v5tools := s.uploadTools(c, version.MustParseBinary("1.0.5-foo-bar"))
	_, v6tools := s.uploadTools(c, version.MustParseBinary("1.0.6-foo-bar"))

	// Check that it won't choose tools with a greater version number.
	s.proposeVersion(c, version.MustParse("1.0.4"))
	<-sameVersionEvent

	s.proposeVersion(c, v6tools.Number)
	select {
	case err := <-upgraderDone:
		c.Assert(err, DeepEquals, &UpgradedError{v6tools})
	case <-time.After(500 * time.Millisecond):
		c.Fatalf("upgrader did not stop as expected")
	}

	// Check that the upgraded version was really downloaded.
	data, err := ioutil.ReadFile(filepath.Join(environs.ToolsDir(v6tools.Binary), "jujud"))
	c.Assert(err, IsNil)
	c.Assert(string(data), Equals, "jujud contents 1.0.6-foo-bar")

	version.Current = v6tools.Binary
	// Check that we can start again.
	_, as, upgraderDone = startUpgrader(s.State)
	assertEvent(c, as.event, "SetAgentTools 1.0.6-foo-bar "+v6tools.URL)

	// Check that we can downgrade.
	s.proposeVersion(c, v5tools.Number)

	select {
	case tools := <-upgraderDone:
		c.Assert(tools, DeepEquals, &UpgradedError{v5tools})
	case <-time.After(500 * time.Millisecond):
		c.Fatalf("upgrader did not stop as expected")
	}
}

func assertEvent(c *C, event <-chan string, want string) {
	select {
	case got := <-event:
		c.Assert(got, Equals, want)
	case <-time.After(500 * time.Millisecond):
		c.Fatalf("no event received; expected %q", want)
	}
}

type testAgentState struct {
	tomb.Tomb
	event chan string
}

func newTestAgentState() *testAgentState {
	return &testAgentState{
		event: make(chan string),
	}
}

func (t *testAgentState) SetAgentTools(tools *state.Tools) error {
	t.event <- fmt.Sprintf("SetAgentTools %v %s", tools.Binary, tools.URL)
	return nil
}
<|endoftext|>


<commit_before>/**
 *  Copyright 2014 Paul Querna
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 */

package ff

type SweetInterface interface {
	Cats() int
}

type Cats struct {
	FieldOnCats int
}

func (c *Cats) Cats() int {
	return 42
}

type Embed struct {
	SuperBool bool
}

type Everything struct {
	Embed
	Bool             bool
	Int              int
	Int8             int8
	Int16            int16
	Int32            int32
	Int64            int64
	Uint             uint
	Uint8            uint8
	Uint16           uint16
	Uint32           uint32
	Uint64           uint64
	Uintptr          uintptr
	Float32          float32
	Float64          float64
	Array            []int
	Map              map[string]int
	String           string
	StringPointer    *string
	Int64Pointer     *int64
	FooStruct        *Foo
	MySweetInterface SweetInterface
	nonexported
}

type nonexported struct {
	Something int8
}

type Foo struct {
	Bar int
}

func NewEverything(e *Everything) {
	e.SuperBool = true
	e.Bool = true
	e.Int = 1
	e.Int8 = 2
	e.Int16 = 3
	e.Int32 = -4
	e.Int64 = 2 ^ 59
	e.Uint = 100
	e.Uint8 = 101
	e.Uint16 = 102
	e.Uint64 = 103
	e.Uintptr = 104
	e.Float32 = 3.14
	e.Float64 = 3.15
	e.Array = []int{1, 2, 3}
	e.Map = map[string]int{
		"foo": 1,
		"bar": 2,
	}
	e.String = "snowman->☃"
	e.FooStruct = &Foo{Bar: 1}
	e.Something = 99
	e.MySweetInterface = &Cats{}
}
<commit_msg>Add array case to "everything" test<commit_after>/**
 *  Copyright 2014 Paul Querna
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 */

package ff

type SweetInterface interface {
	Cats() int
}

type Cats struct {
	FieldOnCats int
}

func (c *Cats) Cats() int {
	return 42
}

type Embed struct {
	SuperBool bool
}

type Everything struct {
	Embed
	Bool             bool
	Int              int
	Int8             int8
	Int16            int16
	Int32            int32
	Int64            int64
	Uint             uint
	Uint8            uint8
	Uint16           uint16
	Uint32           uint32
	Uint64           uint64
	Uintptr          uintptr
	Float32          float32
	Float64          float64
	Array            [2]int
	Slice            []int
	Map              map[string]int
	String           string
	StringPointer    *string
	Int64Pointer     *int64
	FooStruct        *Foo
	MySweetInterface SweetInterface
	nonexported
}

type nonexported struct {
	Something int8
}

type Foo struct {
	Bar int
}

func NewEverything(e *Everything) {
	e.SuperBool = true
	e.Bool = true
	e.Int = 1
	e.Int8 = 2
	e.Int16 = 3
	e.Int32 = -4
	e.Int64 = 2 ^ 59
	e.Uint = 100
	e.Uint8 = 101
	e.Uint16 = 102
	e.Uint64 = 103
	e.Uintptr = 104
	e.Float32 = 3.14
	e.Float64 = 3.15
	e.Array = [2]int{11, 12}
	e.Slice = []int{1, 2, 3}
	e.Map = map[string]int{
		"foo": 1,
		"bar": 2,
	}
	e.String = "snowman->☃"
	e.FooStruct = &Foo{Bar: 1}
	e.Something = 99
	e.MySweetInterface = &Cats{}
}
<|endoftext|>


<commit_before>// +build !windows

package libcrypto

// #include <openssl/evp.h>
// #include <openssl/dh.h>
// #include <openssl/ec.h>
// #include <openssl/obj_mac.h>
import "C"
import (
	"errors"
	"github.com/mkobetic/okapi"
	"unsafe"
)

func init() {
	okapi.DH = DH.constructor()
}

type dhParameters struct {
	ecc bool
}

var (
	DH         = dhParameters{ecc: false}
	ECDH       = dhParameters{ecc: true}
	size2curve = map[int]C.int{224: C.NID_secp224r1, 384: C.NID_secp384r1, 521: C.NID_secp521r1}
)

func (p dhParameters) constructor() okapi.KeyConstructor {
	return func(keyParameters interface{}) (okapi.PrivateKey, error) {
		return NewPKey(keyParameters, p)
	}
}

func (p dhParameters) configure(key *PKey) {
	key.parameters = p
	if !key.public {
		check1(C.EVP_PKEY_derive_init(key.ctx))
	}
}

func (p dhParameters) isForEncryption() bool   { return false }
func (p dhParameters) isForSigning() bool      { return false }
func (p dhParameters) isForKeyAgreement() bool { return true }

func (p dhParameters) toPublic(pri *PKey) (pub *PKey, err error) {
	if p.ecc {
		return nil, errors.New("TODO")
	}
	// This is butt ugly, but it seems that the only way to create
	// a public EVP_PKEY for DH is to create a key from the parameters
	// and then manually copy the pub_key member of the internal DH key over.
	dh1 := (*C.DH)(C.EVP_PKEY_get1_DH(pri.pkey))
	if dh1 == nil {
		return nil, errors.New(libcryptoError())
	}
	size := C.i2d_DHparams(dh1, nil)
	bytes := make([]byte, int(size))
	bytesp := (*C.uchar)(&bytes[0])
	size = C.i2d_DHparams(dh1, &bytesp)
	bytesp = (*C.uchar)(&bytes[0])
	dh2 := C.d2i_DHparams(nil, &bytesp, (C.long)(size))
	if dh2 == nil {
		return nil, errors.New(libcryptoError())
	}
	dh2.pub_key = C.BN_dup(dh1.pub_key)
	pkey := C.EVP_PKEY_new()
	// err := error1(C.EVP_PKEY_assign_DH(pkey, dh2))
	err = error1(C.EVP_PKEY_assign(pkey, C.EVP_PKEY_DH, unsafe.Pointer(dh2)))
	if err != nil {
		return nil, err
	}
	pub = &PKey{pkey: pkey, public: true, parameters: pri.parameters}
	ctx := C.EVP_PKEY_CTX_new(pkey, nil)
	if ctx == nil {
		C.EVP_PKEY_free(pkey)
		return nil, errors.New(libcryptoError())
	}
	pub.ctx = ctx
	pri.parameters.configure(pub)
	return pub, nil
}

func (p dhParameters) keyType() C.int {
	if p.ecc {
		return C.EVP_PKEY_EC
	} else {
		return C.EVP_PKEY_DH
	}
}

func (p dhParameters) generate(size int) (*PKey, error) {
	pkey, err := newDHParams(size, p.keyType())
	if err != nil {
		return nil, err
	}
	return newPKeyFromParams(pkey)
}

func newDHParams(size int, keyType C.int) (*C.EVP_PKEY, error) {
	ctx := C.EVP_PKEY_CTX_new_id(keyType, nil)
	if ctx == nil {
		return nil, errors.New("Failed EVP_PKEY_CTX_new_id")
	}
	defer C.EVP_PKEY_CTX_free(ctx)
	err := error1(C.EVP_PKEY_paramgen_init(ctx))
	if err != nil {
		return nil, err
	}
	if keyType == C.EVP_PKEY_EC {
		// Following macro didn't work:
		// err = error1(C.EVP_PKEY_CTX_set_ec_paramgen_curve_nid(pctx, size2curve[size]))
		err = error1(C.EVP_PKEY_CTX_ctrl(ctx, C.EVP_PKEY_EC, C.EVP_PKEY_OP_PARAMGEN, C.EVP_PKEY_CTRL_EC_PARAMGEN_CURVE_NID, size2curve[size], nil))
	} else {
		// Following macro didn't work:
		// err = error1(C.EVP_PKEY_CTX_set_dh_paramgen_prime_len(ctx, size))
		err = error1(C.EVP_PKEY_CTX_ctrl(ctx, C.EVP_PKEY_DH, C.EVP_PKEY_OP_PARAMGEN, C.EVP_PKEY_CTRL_DH_PARAMGEN_PRIME_LEN, C.int(size), nil))
	}
	if err != nil {
		return nil, err
	}
	var pkey *C.EVP_PKEY
	err = error1(C.EVP_PKEY_paramgen(ctx, &pkey))
	if err != nil {
		return nil, err
	}
	return pkey, nil
}
<commit_msg>minor fix<commit_after>// +build !windows

package libcrypto

// #include <openssl/evp.h>
// #include <openssl/dh.h>
// #include <openssl/ec.h>
// #include <openssl/obj_mac.h>
import "C"
import (
	"errors"
	"github.com/mkobetic/okapi"
	"unsafe"
)

func init() {
	okapi.DH = DH.constructor()
}

type dhParameters struct {
	ecc bool
}

var (
	DH   = dhParameters{ecc: false}
	ECDH = dhParameters{ecc: true}
	// size2curve = map[int]C.int{224: C.NID_secp224r1, 384: C.NID_secp384r1, 521: C.NID_secp521r1}
)

func (p dhParameters) constructor() okapi.KeyConstructor {
	return func(keyParameters interface{}) (okapi.PrivateKey, error) {
		return NewPKey(keyParameters, p)
	}
}

func (p dhParameters) configure(key *PKey) {
	key.parameters = p
	if !key.public {
		check1(C.EVP_PKEY_derive_init(key.ctx))
	}
}

func (p dhParameters) isForEncryption() bool   { return false }
func (p dhParameters) isForSigning() bool      { return false }
func (p dhParameters) isForKeyAgreement() bool { return true }

func (p dhParameters) toPublic(pri *PKey) (pub *PKey, err error) {
	if p.ecc {
		return nil, errors.New("TODO")
	}
	// This is butt ugly, but it seems that the only way to create
	// a public EVP_PKEY for DH is to create a key from the parameters
	// and then manually copy the pub_key member of the internal DH key over.
	dh1 := (*C.DH)(C.EVP_PKEY_get1_DH(pri.pkey))
	if dh1 == nil {
		return nil, errors.New(libcryptoError())
	}
	size := C.i2d_DHparams(dh1, nil)
	bytes := make([]byte, int(size))
	bytesp := (*C.uchar)(&bytes[0])
	size = C.i2d_DHparams(dh1, &bytesp)
	bytesp = (*C.uchar)(&bytes[0])
	dh2 := C.d2i_DHparams(nil, &bytesp, (C.long)(size))
	if dh2 == nil {
		return nil, errors.New(libcryptoError())
	}
	dh2.pub_key = C.BN_dup(dh1.pub_key)
	pkey := C.EVP_PKEY_new()
	// err := error1(C.EVP_PKEY_assign_DH(pkey, dh2))
	err = error1(C.EVP_PKEY_assign(pkey, C.EVP_PKEY_DH, unsafe.Pointer(dh2)))
	if err != nil {
		return nil, err
	}
	pub = &PKey{pkey: pkey, public: true, parameters: pri.parameters}
	ctx := C.EVP_PKEY_CTX_new(pkey, nil)
	if ctx == nil {
		C.EVP_PKEY_free(pkey)
		return nil, errors.New(libcryptoError())
	}
	pub.ctx = ctx
	pri.parameters.configure(pub)
	return pub, nil
}

func (p dhParameters) keyType() C.int {
	if p.ecc {
		return C.EVP_PKEY_EC
	} else {
		return C.EVP_PKEY_DH
	}
}

func (p dhParameters) generate(size int) (*PKey, error) {
	pkey, err := newDHParams(size, p.keyType())
	if err != nil {
		return nil, err
	}
	return newPKeyFromParams(pkey)
}

func newDHParams(size int, keyType C.int) (*C.EVP_PKEY, error) {
	ctx := C.EVP_PKEY_CTX_new_id(keyType, nil)
	if ctx == nil {
		return nil, errors.New("Failed EVP_PKEY_CTX_new_id")
	}
	defer C.EVP_PKEY_CTX_free(ctx)
	err := error1(C.EVP_PKEY_paramgen_init(ctx))
	if err != nil {
		return nil, err
	}
	if keyType == C.EVP_PKEY_EC {
		// Following macro didn't work:
		// err = error1(C.EVP_PKEY_CTX_set_ec_paramgen_curve_nid(pctx, size2curve[size]))
		err = error1(C.EVP_PKEY_CTX_ctrl(ctx, C.EVP_PKEY_EC, C.EVP_PKEY_OP_PARAMGEN, C.EVP_PKEY_CTRL_EC_PARAMGEN_CURVE_NID, size2curve[size], nil))
	} else {
		// Following macro didn't work:
		// err = error1(C.EVP_PKEY_CTX_set_dh_paramgen_prime_len(ctx, size))
		err = error1(C.EVP_PKEY_CTX_ctrl(ctx, C.EVP_PKEY_DH, C.EVP_PKEY_OP_PARAMGEN, C.EVP_PKEY_CTRL_DH_PARAMGEN_PRIME_LEN, C.int(size), nil))
	}
	if err != nil {
		return nil, err
	}
	var pkey *C.EVP_PKEY
	err = error1(C.EVP_PKEY_paramgen(ctx, &pkey))
	if err != nil {
		return nil, err
	}
	return pkey, nil
}
<|endoftext|>


<commit_before><commit_msg>Resolve escaped newline<commit_after><|endoftext|>


<commit_before>/*
   Copyright 2020 Docker, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package convert

import (
	"fmt"
	"net/url"
	"strings"

	"github.com/pkg/errors"

	"github.com/compose-spec/compose-go/types"

	"github.com/docker/api/errdefs"
)

// GetRunVolumes return volume configurations for a project and a single service
// this is meant to be used as a compose project of a single service
func GetRunVolumes(volumes []string) (map[string]types.VolumeConfig, []types.ServiceVolumeConfig, error) {
	var serviceConfigVolumes []types.ServiceVolumeConfig
	projectVolumes := make(map[string]types.VolumeConfig, len(volumes))
	for i, v := range volumes {
		var vi volumeInput
		err := vi.parse(fmt.Sprintf("volume-%d", i), v)
		if err != nil {
			return nil, nil, err
		}
		projectVolumes[vi.name] = types.VolumeConfig{
			Name:   vi.name,
			Driver: azureFileDriverName,
			DriverOpts: map[string]string{
				volumeDriveroptsAccountNameKey: vi.username,
				volumeDriveroptsAccountKeyKey:  vi.key,
				volumeDriveroptsShareNameKey:   vi.share,
			},
		}
		sv := types.ServiceVolumeConfig{
			Type:   azureFileDriverName,
			Source: vi.name,
			Target: vi.target,
		}
		serviceConfigVolumes = append(serviceConfigVolumes, sv)
	}

	return projectVolumes, serviceConfigVolumes, nil
}

type volumeInput struct {
	name     string
	username string
	key      string
	share    string
	target   string
}

func escapeKeySlashes(rawURL string) (string, error) {
	urlSplit := strings.Split(rawURL, "@")
	if len(urlSplit) < 1 {
		return "", errors.Wrap(errdefs.ErrParsingFailed, "invalid url format "+rawURL)
	}
	userPasswd := strings.ReplaceAll(urlSplit[0], "/", "_")

	atIndex := strings.Index(rawURL, "@")
	if atIndex < 0 {
		return "", errors.Wrap(errdefs.ErrParsingFailed, "no share specified in "+rawURL)
	}

	scaped := userPasswd + rawURL[atIndex:]

	return scaped, nil
}

func unescapeKey(key string) string {
	return strings.ReplaceAll(key, "_", "/")
}

// Removes the second ':' that separates the source from target
func volumeURL(pathURL string) (*url.URL, error) {
	scapedURL, err := escapeKeySlashes(pathURL)
	if err != nil {
		return nil, err
	}
	pathURL = "//" + scapedURL

	count := strings.Count(pathURL, ":")
	if count > 2 {
		return nil, errors.Wrap(errdefs.ErrParsingFailed, fmt.Sprintf("unable to parse volume mount %q", pathURL))
	}
	if count == 2 {
		tokens := strings.Split(pathURL, ":")
		pathURL = fmt.Sprintf("%s:%s%s", tokens[0], tokens[1], tokens[2])
	}
	return url.Parse(pathURL)
}

func (v *volumeInput) parse(name string, s string) error {
	volumeURL, err := volumeURL(s)
	if err != nil {
		return errors.Wrap(errdefs.ErrParsingFailed, fmt.Sprintf("volume specification %q could not be parsed %q", s, err))
	}
	v.username = volumeURL.User.Username()
	if v.username == "" {
		return errors.Wrap(errdefs.ErrParsingFailed, fmt.Sprintf("volume specification %q does not include a storage username", v))
	}
	key, ok := volumeURL.User.Password()
	if !ok || key == "" {
		return errors.Wrap(errdefs.ErrParsingFailed, fmt.Sprintf("volume specification %q does not include a storage key", v))
	}
	v.key = unescapeKey(key)
	v.share = volumeURL.Host
	if v.share == "" {
		return errors.Wrap(errdefs.ErrParsingFailed, fmt.Sprintf("volume specification %q does not include a storage file share", v))
	}
	v.name = name
	v.target = volumeURL.Path
	if v.target == "" {
		// Do not use filepath.Join, on Windows it will replace / by \
		v.target = "/run/volumes/" + v.share
	}
	return nil
}
<commit_msg>azure: Clean up volume parsing functions<commit_after>/*
   Copyright 2020 Docker, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package convert

import (
	"fmt"
	"net/url"
	"strings"

	"github.com/pkg/errors"

	"github.com/compose-spec/compose-go/types"

	"github.com/docker/api/errdefs"
)

// GetRunVolumes return volume configurations for a project and a single service
// this is meant to be used as a compose project of a single service
func GetRunVolumes(volumes []string) (map[string]types.VolumeConfig, []types.ServiceVolumeConfig, error) {
	var serviceConfigVolumes []types.ServiceVolumeConfig
	projectVolumes := make(map[string]types.VolumeConfig, len(volumes))
	for i, v := range volumes {
		var vi volumeInput
		err := vi.parse(fmt.Sprintf("volume-%d", i), v)
		if err != nil {
			return nil, nil, err
		}
		projectVolumes[vi.name] = types.VolumeConfig{
			Name:   vi.name,
			Driver: azureFileDriverName,
			DriverOpts: map[string]string{
				volumeDriveroptsAccountNameKey: vi.username,
				volumeDriveroptsAccountKeyKey:  vi.key,
				volumeDriveroptsShareNameKey:   vi.share,
			},
		}
		sv := types.ServiceVolumeConfig{
			Type:   azureFileDriverName,
			Source: vi.name,
			Target: vi.target,
		}
		serviceConfigVolumes = append(serviceConfigVolumes, sv)
	}

	return projectVolumes, serviceConfigVolumes, nil
}

type volumeInput struct {
	name     string
	username string
	key      string
	share    string
	target   string
}

func escapeKeySlashes(rawURL string) (string, error) {
	urlSplit := strings.Split(rawURL, "@")
	if len(urlSplit) < 1 {
		return "", fmt.Errorf("invalid URL format: %s", rawURL)
	}
	userPasswd := strings.ReplaceAll(urlSplit[0], "/", "_")

	atIndex := strings.Index(rawURL, "@")
	if atIndex < 0 {
		return "", fmt.Errorf("no share specified in: %s", rawURL)
	}

	scaped := userPasswd + rawURL[atIndex:]

	return scaped, nil
}

func unescapeKey(key string) string {
	return strings.ReplaceAll(key, "_", "/")
}

// Removes the second ':' that separates the source from target
func volumeURL(pathURL string) (*url.URL, error) {
	scapedURL, err := escapeKeySlashes(pathURL)
	if err != nil {
		return nil, err
	}
	pathURL = "//" + scapedURL

	count := strings.Count(pathURL, ":")
	if count > 2 {
		return nil, fmt.Errorf("invalid path URL: %s", pathURL)
	}
	if count == 2 {
		tokens := strings.Split(pathURL, ":")
		pathURL = fmt.Sprintf("%s:%s%s", tokens[0], tokens[1], tokens[2])
	}
	return url.Parse(pathURL)
}

func (v *volumeInput) parse(name string, s string) error {
	volumeURL, err := volumeURL(s)
	if err != nil {
		return errors.Wrapf(errdefs.ErrParsingFailed, "unable to parse volume specification: %s", err.Error())
	}
	v.username = volumeURL.User.Username()
	if v.username == "" {
		return errors.Wrapf(errdefs.ErrParsingFailed, "volume specification %q does not include a storage username", v)
	}
	key, ok := volumeURL.User.Password()
	if !ok || key == "" {
		return errors.Wrapf(errdefs.ErrParsingFailed, "volume specification %q does not include a storage key", v)
	}
	v.key = unescapeKey(key)
	v.share = volumeURL.Host
	if v.share == "" {
		return errors.Wrapf(errdefs.ErrParsingFailed, "volume specification %q does not include a storage file share", v)
	}
	v.name = name
	v.target = volumeURL.Path
	if v.target == "" {
		// Do not use filepath.Join, on Windows it will replace / by \
		v.target = "/run/volumes/" + v.share
	}
	return nil
}
<|endoftext|>


<commit_before>package endpoint_test

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"

	"github.com/BlueOwlOpenSource/endpoint"
	"github.com/gorilla/mux"
)

// The endpoint framework distinguishes parameters based on their types.
// All parameters of type "string" look the same, but a type that is
// defined as another type (like exampleType) is a different type.
type exampleType string
type fooParam string
type fromMiddleware string

// exampleStaticInjector will not be called until the service.Start()
// call in Example_PreRegisterServiceWithMux.  It will be called only
// once per endpoint registered.  Since it has a return value, it will
// only run if a downstream handler consumes the value it returns.
//
// The values returned by injectors and available as input parameters
// to any downstream handler.
func exampleStaticInjector() exampleType {
	return "example static value"
}

// exampleInjector will be called for each request.  We know that
// exampleInjector is a regular injector because it takes a parameter
// that is specific to the request (*http.Request).
func exampleInjector(r *http.Request) fooParam {
	return fooParam(r.FormValue("foo"))
}

type returnValue interface{}

// jsonifyResult wraps all handlers downstream of it in the call chain.
// We know that jsonifyResult is a middleware handler because its first
// argument is an function with an anonymous type (inner).   Calling inner
// invokes all handlers downstream from jsonifyResult.  The value returned
// by inner can come from the return values of the final endpoint handler
// or from values returned by any downstream middleware.  The parameters
// to inner are available as inputs to any downstream handler.
//
// Parameters are matched by their types.  Since inner returns a returnValue,
// it can come from any downstream middleware or endpoint that returns something
// of type returnValue.
func jsonifyResult(inner func(fromMiddleware) returnValue, w http.ResponseWriter) {
	v := inner("jsonify!")
	w.Header().Set("Content-Type", "application/json")
	encoded, _ := json.Marshal(v)
	w.Write(encoded)
	w.WriteHeader(200)
}

// Endpoints are grouped and started by services.  Handlers that are
// common to all endpoints are attached to the service.
var service = endpoint.PreRegisterServiceWithMux("example-service",
	exampleStaticInjector,
	jsonifyResult)

func init() {
	// The /example endpoint is bound to a handler chain
	// that combines the functions included at the service
	// level and the functions included here.  The final chain is:
	//	exampleStaticInjector, jsonifyResult, exampleInjector, exampleEndpoint.
	// ExampleStaticInjector and jsonifyResult come from the service
	// definition.  ExampleInjector and exampleEndpoint are attached when
	// the endpoint is registered.
	//
	// Handlers will execute in the order of the chain: exampleStaticInjector
	// then jsonifyResult.  When jsonifyResult calls inner(), exampleInjector
	// runs, then exampleEndpoint.   When exampleEndpoint returns, inner() returns
	// so jsonifyResult continues its work.  When jsonifyResult returns, the
	// handler chain is complete and the http server can form a reply from the
	// ResponseWriter.
	//
	// Since service is WithMux, we can use gorilla mux modifiers when
	// we register endpoints.  This allows us to trivially indicate that our
	// example endpoint supports the GET method only.
	service.RegisterEndpoint("/example", exampleInjector, exampleEndpoint).Methods("GET")
}

// This is the final endpoint handler.  The parameters it takes can
// be provided by any handler upstream from it.  It can also take the two
// values that are included by the http handler signature: http.ResponseWriter
// and *http.Request.
//
// Any values that the final endpoint handler returns must be consumed by an
// upstream middleware handler.  In this example, a "returnValue" is returned
// here and consumed by jsonifyResult.
func exampleEndpoint(sv exampleType, foo fooParam, mid fromMiddleware) returnValue {
	return map[string]string{
		"value": fmt.Sprintf("%s-%s-%s", sv, foo, mid),
	}
}

// The code below puts up a test http server, hits the /example
// endpoint, decodes the response, prints it, and exits.  This
// is just to excercise the endpoint defined above.  The interesting
// stuff happens above.
func Example() {
	muxRouter := mux.NewRouter()
	service.Start(muxRouter)
	localServer := httptest.NewServer(muxRouter)
	defer localServer.Close()
	r, err := http.Get(localServer.URL + "/example?foo=bar")
	if err != nil {
		fmt.Println("get error", err)
		return
	}
	buf, err := ioutil.ReadAll(r.Body)
	if err != nil {
		fmt.Println("read error", err)
		return
	}
	var res map[string]string
	err = json.Unmarshal(buf, &res)
	if err != nil {
		fmt.Println("unmarshal error", err)
		return
	}
	fmt.Println("Value:", res["value"])
	// Output: Value: example static value-bar-jsonify!
}
<commit_msg>doc changes<commit_after>package endpoint_test

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"

	"github.com/BlueOwlOpenSource/endpoint"
	"github.com/gorilla/mux"
)

// The endpoint framework distinguishes parameters based on their types.
// All parameters of type "string" look the same, but a type that is
// defined as another type (like exampleType) is a different type.
type exampleType string
type fooParam string
type fromMiddleware string

// exampleStaticInjector will not be called until the service.Start()
// call in Example_PreRegisterServiceWithMux.  It will be called only
// once per endpoint registered.  Since it has a return value, it will
// only run if a downstream handler consumes the value it returns.
//
// The values returned by injectors and available as input parameters
// to any downstream handler.
func exampleStaticInjector() exampleType {
	return "example static value"
}

// exampleInjector will be called for each request.  We know that
// exampleInjector is a regular injector because it takes a parameter
// that is specific to the request (*http.Request).
func exampleInjector(r *http.Request) fooParam {
	return fooParam(r.FormValue("foo"))
}

type returnValue interface{}

// jsonifyResult wraps all handlers downstream of it in the call chain.
// We know that jsonifyResult is a middleware handler because its first
// argument is an function with an anonymous type (inner).   Calling inner
// invokes all handlers downstream from jsonifyResult.  The value returned
// by inner can come from the return values of the final endpoint handler
// or from values returned by any downstream middleware.  The parameters
// to inner are available as inputs to any downstream handler.
//
// Parameters are matched by their types.  Since inner returns a returnValue,
// it can come from any downstream middleware or endpoint that returns something
// of type returnValue.
func jsonifyResult(inner func(fromMiddleware) returnValue, w http.ResponseWriter) {
	v := inner("jsonify!")
	w.Header().Set("Content-Type", "application/json")
	encoded, _ := json.Marshal(v)
	w.Write(encoded)
	w.WriteHeader(200)
}

// Endpoints are grouped and started by services.  Handlers that are
// common to all endpoints are attached to the service.
var service = endpoint.PreRegisterServiceWithMux("example-service",
	exampleStaticInjector,
	jsonifyResult)

func init() {
	// The /example endpoint is bound to a handler chain
	// that combines the functions included at the service
	// level and the functions included here.  The final chain is:
	//	exampleStaticInjector, jsonifyResult, exampleInjector, exampleEndpoint.
	// ExampleStaticInjector and jsonifyResult come from the service
	// definition.  ExampleInjector and exampleEndpoint are attached when
	// the endpoint is registered.
	//
	// Handlers will execute in the order of the chain: exampleStaticInjector
	// then jsonifyResult.  When jsonifyResult calls inner(), exampleInjector
	// runs, then exampleEndpoint.   When exampleEndpoint returns, inner() returns
	// so jsonifyResult continues its work.  When jsonifyResult returns, the
	// handler chain is complete and the http server can form a reply from the
	// ResponseWriter.
	//
	// Since service is WithMux, we can use gorilla mux modifiers when
	// we register endpoints.  This allows us to trivially indicate that our
	// example endpoint supports the GET method only.
	service.RegisterEndpoint(
		"/example", exampleInjector, exampleEndpoint).Methods("GET")
}

// This is the final endpoint handler.  The parameters it takes can
// be provided by any handler upstream from it.  It can also take the two
// values that are included by the http handler signature: http.ResponseWriter
// and *http.Request.
//
// Any values that the final endpoint handler returns must be consumed by an
// upstream middleware handler.  In this example, a "returnValue" is returned
// here and consumed by jsonifyResult.
func exampleEndpoint(sv exampleType, foo fooParam, mid fromMiddleware) returnValue {
	return map[string]string{
		"value": fmt.Sprintf("%s-%s-%s", sv, foo, mid),
	}
}

// The code below puts up a test http server, hits the /example
// endpoint, decodes the response, prints it, and exits.  This
// is just to excercise the endpoint defined above.  The interesting
// stuff happens above.
func Example() {
	muxRouter := mux.NewRouter()
	service.Start(muxRouter)
	localServer := httptest.NewServer(muxRouter)
	defer localServer.Close()
	r, err := http.Get(localServer.URL + "/example?foo=bar")
	if err != nil {
		fmt.Println("get error", err)
		return
	}
	buf, err := ioutil.ReadAll(r.Body)
	if err != nil {
		fmt.Println("read error", err)
		return
	}
	var res map[string]string
	err = json.Unmarshal(buf, &res)
	if err != nil {
		fmt.Println("unmarshal error", err)
		return
	}
	fmt.Println("Value:", res["value"])
	// Output: Value: example static value-bar-jsonify!
}
<|endoftext|>