<commit_before>// +build linux

package main

import (
	"fmt"
	"os"
	"os/exec"
	"os/user"
	"runtime"
	"strconv"
	"syscall"

	"github.com/opencontainers/runtime-spec/specs-go"
	"github.com/pkg/errors"
	"github.com/projectatomic/buildah/unshare"
	"github.com/projectatomic/buildah/util"
	"github.com/sirupsen/logrus"
	"github.com/syndtr/gocapability/capability"
	"github.com/urfave/cli"
)

const (
	// startedInUserNS is an environment variable that, if set, means that we shouldn't try
	// to create and enter a new user namespace and then re-exec ourselves.
	startedInUserNS = "_BUILDAH_STARTED_IN_USERNS"
)

var (
	unshareDescription = "Runs a command in a modified user namespace"
	unshareCommand     = cli.Command{
		Name:           "unshare",
		Usage:          "Run a command in a modified user namespace",
		Description:    unshareDescription,
		Action:         unshareCmd,
		ArgsUsage:      "[COMMAND [ARGS [...]]]",
		SkipArgReorder: true,
	}
)

type runnable interface {
	Run() error
}

func bailOnError(err error, format string, a ...interface{}) {
	if err != nil {
		if format != "" {
			logrus.Errorf("%s: %v", fmt.Sprintf(format, a...), err)
		} else {
			logrus.Errorf("%v", err)
		}
		cli.OsExiter(1)
	}
}

func maybeReexecUsingUserNamespace(c *cli.Context, evenForRoot bool) {
	// If we've already been through this once, no need to try again.
	if os.Getenv(startedInUserNS) != "" {
		return
	}

	// If this is one of the commands that doesn't need this indirection, skip it.
	switch c.Args()[0] {
	case "help", "version":
		return
	}

	// Figure out who we are.
	me, err := user.Current()
	bailOnError(err, "error determining current user")
	uidNum, err := strconv.ParseUint(me.Uid, 10, 32)
	bailOnError(err, "error parsing current UID %s", me.Uid)
	gidNum, err := strconv.ParseUint(me.Gid, 10, 32)
	bailOnError(err, "error parsing current GID %s", me.Gid)

	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	// ID mappings to use to reexec ourselves.
	var uidmap, gidmap []specs.LinuxIDMapping
	if uidNum != 0 || evenForRoot {
		// Read the set of ID mappings that we're allowed to use.  Each
		// range in /etc/subuid and /etc/subgid file is a starting host
		// ID and a range size.
		uidmap, gidmap, err = util.GetSubIDMappings(me.Username, me.Username)
		bailOnError(err, "error reading allowed ID mappings")
		if len(uidmap) == 0 {
			logrus.Warnf("Found no UID ranges set aside for user %q in /etc/subuid.", me.Username)
		}
		if len(gidmap) == 0 {
			logrus.Warnf("Found no GID ranges set aside for user %q in /etc/subgid.", me.Username)
		}
		// Map our UID and GID, then the subuid and subgid ranges,
		// consecutively, starting at 0, to get the mappings to use for
		// a copy of ourselves.
		uidmap = append([]specs.LinuxIDMapping{{HostID: uint32(uidNum), ContainerID: 0, Size: 1}}, uidmap...)
		gidmap = append([]specs.LinuxIDMapping{{HostID: uint32(gidNum), ContainerID: 0, Size: 1}}, gidmap...)
		var rangeStart uint32
		for i := range uidmap {
			uidmap[i].ContainerID = rangeStart
			rangeStart += uidmap[i].Size
		}
		rangeStart = 0
		for i := range gidmap {
			gidmap[i].ContainerID = rangeStart
			rangeStart += gidmap[i].Size
		}
	} else {
		// If we have CAP_SYS_ADMIN, then we don't need to create a new namespace in order to be able
		// to use unshare(), so don't bother creating a new user namespace at this point.
		capabilities, err := capability.NewPid(0)
		bailOnError(err, "error reading the current capabilities sets")
		if capabilities.Get(capability.EFFECTIVE, capability.CAP_SYS_ADMIN) {
			return
		}
		// Read the set of ID mappings that we're currently using.
		uidmap, gidmap, err = util.GetHostIDMappings("")
		bailOnError(err, "error reading current ID mappings")
		// Just reuse them.
		for i := range uidmap {
			uidmap[i].HostID = uidmap[i].ContainerID
		}
		for i := range gidmap {
			gidmap[i].HostID = gidmap[i].ContainerID
		}
	}

	var moreArgs []string
	// Add args to change the global defaults.
	if uidNum != 0 {
		if !c.GlobalIsSet("storage-driver") || !c.GlobalIsSet("root") || !c.GlobalIsSet("runroot") {
			logrus.Infof("Running without privileges, assuming arguments:")
			if !c.GlobalIsSet("storage-driver") {
				defaultStorageDriver := "vfs"
				logrus.Infof(" --storage-driver %q", defaultStorageDriver)
				moreArgs = append(moreArgs, "--storage-driver", defaultStorageDriver)
			}
			if !c.GlobalIsSet("root") {
				defaultRoot, err := util.UnsharedRootPath(me.HomeDir)
				bailOnError(err, "")
				logrus.Infof(" --root %q", defaultRoot)
				moreArgs = append(moreArgs, "--root", defaultRoot)
			}
			if !c.GlobalIsSet("runroot") {
				defaultRunroot, err := util.UnsharedRunrootPath(me.Uid)
				bailOnError(err, "")
				logrus.Infof(" --runroot %q", defaultRunroot)
				moreArgs = append(moreArgs, "--runroot", defaultRunroot)
			}
		}
	}

	// Unlike most uses of reexec or unshare, we're using a name that
	// _won't_ be recognized as a registered reexec handler, since we
	// _want_ to fall through reexec.Init() to the normal main().
	cmd := unshare.Command(append(append([]string{"buildah-in-a-user-namespace"}, moreArgs...), os.Args[1:]...)...)

	// If, somehow, we don't become UID 0 in our child, indicate that the child shouldn't try again.
	err = os.Setenv(startedInUserNS, "1")
	bailOnError(err, "error setting %s=1 in environment", startedInUserNS)

	// Set the default isolation type to use the "chroot" method.
	if _, ok := os.LookupEnv("BUILDAH_ISOLATION"); !ok {
		if err = os.Setenv("BUILDAH_ISOLATION", "chroot"); err != nil {
			logrus.Errorf("error setting BUILDAH_ISOLATION=chroot in environment: %v", err)
			os.Exit(1)
		}
	}

	// Reuse our stdio.
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	// Set up a new user namespace with the ID mapping.
	cmd.UnshareFlags = syscall.CLONE_NEWUSER
	cmd.UseNewuidmap = uidNum != 0
	cmd.UidMappings = uidmap
	cmd.UseNewgidmap = uidNum != 0
	cmd.GidMappings = gidmap
	cmd.GidMappingsEnableSetgroups = true

	// Finish up.
	logrus.Debugf("running %+v with environment %+v, UID map %+v, and GID map %+v", cmd.Cmd.Args, os.Environ(), cmd.UidMappings, cmd.GidMappings)
	execRunnable(cmd)
}

// execRunnable runs the specified unshare command, captures its exit status,
// and exits with the same status.
func execRunnable(cmd runnable) {
	if err := cmd.Run(); err != nil {
		if exitError, ok := errors.Cause(err).(*exec.ExitError); ok {
			if exitError.ProcessState.Exited() {
				if waitStatus, ok := exitError.ProcessState.Sys().(syscall.WaitStatus); ok {
					if waitStatus.Exited() {
						logrus.Errorf("%v", exitError)
						os.Exit(waitStatus.ExitStatus())
					}
					if waitStatus.Signaled() {
						logrus.Errorf("%v", exitError)
						os.Exit(int(waitStatus.Signal()) + 128)
					}
				}
			}
		}
		logrus.Errorf("%v", err)
		logrus.Errorf("(unable to determine exit status)")
		os.Exit(1)
	}
	os.Exit(0)
}

// unshareCmd execs whatever using the ID mappings that we want to use for ourselves
func unshareCmd(c *cli.Context) error {
	// force reexec using the configured ID mappings
	maybeReexecUsingUserNamespace(c, true)
	// exec the specified command, if there is one
	args := c.Args()
	if len(args) < 1 {
		// try to exec the shell, if one's set
		shell, shellSet := os.LookupEnv("SHELL")
		if !shellSet {
			logrus.Errorf("no command specified")
			os.Exit(1)
		}
		args = []string{shell}
	}
	cmd := exec.Command(args[0], args[1:]...)
	cmd.Env = append(os.Environ(), "USER=root", "USERNAME=root", "GROUP=root", "LOGNAME=root", "UID=0", "GID=0")
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	execRunnable(cmd)
	os.Exit(1)
	return nil
}
<commit_msg>buildah: no args is out of bounds<commit_after>// +build linux

package main

import (
	"fmt"
	"os"
	"os/exec"
	"os/user"
	"runtime"
	"strconv"
	"syscall"

	"github.com/opencontainers/runtime-spec/specs-go"
	"github.com/pkg/errors"
	"github.com/projectatomic/buildah/unshare"
	"github.com/projectatomic/buildah/util"
	"github.com/sirupsen/logrus"
	"github.com/syndtr/gocapability/capability"
	"github.com/urfave/cli"
)

const (
	// startedInUserNS is an environment variable that, if set, means that we shouldn't try
	// to create and enter a new user namespace and then re-exec ourselves.
	startedInUserNS = "_BUILDAH_STARTED_IN_USERNS"
)

var (
	unshareDescription = "Runs a command in a modified user namespace"
	unshareCommand     = cli.Command{
		Name:           "unshare",
		Usage:          "Run a command in a modified user namespace",
		Description:    unshareDescription,
		Action:         unshareCmd,
		ArgsUsage:      "[COMMAND [ARGS [...]]]",
		SkipArgReorder: true,
	}
)

type runnable interface {
	Run() error
}

func bailOnError(err error, format string, a ...interface{}) {
	if err != nil {
		if format != "" {
			logrus.Errorf("%s: %v", fmt.Sprintf(format, a...), err)
		} else {
			logrus.Errorf("%v", err)
		}
		cli.OsExiter(1)
	}
}

func maybeReexecUsingUserNamespace(c *cli.Context, evenForRoot bool) {
	// If we've already been through this once, no need to try again.
	if os.Getenv(startedInUserNS) != "" {
		return
	}

	// If this is one of the commands that doesn't need this indirection, skip it.
	if c.NArg() == 0 {
		return
	}
	switch c.Args()[0] {
	case "help", "version":
		return
	}

	// Figure out who we are.
	me, err := user.Current()
	bailOnError(err, "error determining current user")
	uidNum, err := strconv.ParseUint(me.Uid, 10, 32)
	bailOnError(err, "error parsing current UID %s", me.Uid)
	gidNum, err := strconv.ParseUint(me.Gid, 10, 32)
	bailOnError(err, "error parsing current GID %s", me.Gid)

	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	// ID mappings to use to reexec ourselves.
	var uidmap, gidmap []specs.LinuxIDMapping
	if uidNum != 0 || evenForRoot {
		// Read the set of ID mappings that we're allowed to use.  Each
		// range in /etc/subuid and /etc/subgid file is a starting host
		// ID and a range size.
		uidmap, gidmap, err = util.GetSubIDMappings(me.Username, me.Username)
		bailOnError(err, "error reading allowed ID mappings")
		if len(uidmap) == 0 {
			logrus.Warnf("Found no UID ranges set aside for user %q in /etc/subuid.", me.Username)
		}
		if len(gidmap) == 0 {
			logrus.Warnf("Found no GID ranges set aside for user %q in /etc/subgid.", me.Username)
		}
		// Map our UID and GID, then the subuid and subgid ranges,
		// consecutively, starting at 0, to get the mappings to use for
		// a copy of ourselves.
		uidmap = append([]specs.LinuxIDMapping{{HostID: uint32(uidNum), ContainerID: 0, Size: 1}}, uidmap...)
		gidmap = append([]specs.LinuxIDMapping{{HostID: uint32(gidNum), ContainerID: 0, Size: 1}}, gidmap...)
		var rangeStart uint32
		for i := range uidmap {
			uidmap[i].ContainerID = rangeStart
			rangeStart += uidmap[i].Size
		}
		rangeStart = 0
		for i := range gidmap {
			gidmap[i].ContainerID = rangeStart
			rangeStart += gidmap[i].Size
		}
	} else {
		// If we have CAP_SYS_ADMIN, then we don't need to create a new namespace in order to be able
		// to use unshare(), so don't bother creating a new user namespace at this point.
		capabilities, err := capability.NewPid(0)
		bailOnError(err, "error reading the current capabilities sets")
		if capabilities.Get(capability.EFFECTIVE, capability.CAP_SYS_ADMIN) {
			return
		}
		// Read the set of ID mappings that we're currently using.
		uidmap, gidmap, err = util.GetHostIDMappings("")
		bailOnError(err, "error reading current ID mappings")
		// Just reuse them.
		for i := range uidmap {
			uidmap[i].HostID = uidmap[i].ContainerID
		}
		for i := range gidmap {
			gidmap[i].HostID = gidmap[i].ContainerID
		}
	}

	var moreArgs []string
	// Add args to change the global defaults.
	if uidNum != 0 {
		if !c.GlobalIsSet("storage-driver") || !c.GlobalIsSet("root") || !c.GlobalIsSet("runroot") {
			logrus.Infof("Running without privileges, assuming arguments:")
			if !c.GlobalIsSet("storage-driver") {
				defaultStorageDriver := "vfs"
				logrus.Infof(" --storage-driver %q", defaultStorageDriver)
				moreArgs = append(moreArgs, "--storage-driver", defaultStorageDriver)
			}
			if !c.GlobalIsSet("root") {
				defaultRoot, err := util.UnsharedRootPath(me.HomeDir)
				bailOnError(err, "")
				logrus.Infof(" --root %q", defaultRoot)
				moreArgs = append(moreArgs, "--root", defaultRoot)
			}
			if !c.GlobalIsSet("runroot") {
				defaultRunroot, err := util.UnsharedRunrootPath(me.Uid)
				bailOnError(err, "")
				logrus.Infof(" --runroot %q", defaultRunroot)
				moreArgs = append(moreArgs, "--runroot", defaultRunroot)
			}
		}
	}

	// Unlike most uses of reexec or unshare, we're using a name that
	// _won't_ be recognized as a registered reexec handler, since we
	// _want_ to fall through reexec.Init() to the normal main().
	cmd := unshare.Command(append(append([]string{"buildah-in-a-user-namespace"}, moreArgs...), os.Args[1:]...)...)

	// If, somehow, we don't become UID 0 in our child, indicate that the child shouldn't try again.
	err = os.Setenv(startedInUserNS, "1")
	bailOnError(err, "error setting %s=1 in environment", startedInUserNS)

	// Set the default isolation type to use the "chroot" method.
	if _, ok := os.LookupEnv("BUILDAH_ISOLATION"); !ok {
		if err = os.Setenv("BUILDAH_ISOLATION", "chroot"); err != nil {
			logrus.Errorf("error setting BUILDAH_ISOLATION=chroot in environment: %v", err)
			os.Exit(1)
		}
	}

	// Reuse our stdio.
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	// Set up a new user namespace with the ID mapping.
	cmd.UnshareFlags = syscall.CLONE_NEWUSER
	cmd.UseNewuidmap = uidNum != 0
	cmd.UidMappings = uidmap
	cmd.UseNewgidmap = uidNum != 0
	cmd.GidMappings = gidmap
	cmd.GidMappingsEnableSetgroups = true

	// Finish up.
	logrus.Debugf("running %+v with environment %+v, UID map %+v, and GID map %+v", cmd.Cmd.Args, os.Environ(), cmd.UidMappings, cmd.GidMappings)
	execRunnable(cmd)
}

// execRunnable runs the specified unshare command, captures its exit status,
// and exits with the same status.
func execRunnable(cmd runnable) {
	if err := cmd.Run(); err != nil {
		if exitError, ok := errors.Cause(err).(*exec.ExitError); ok {
			if exitError.ProcessState.Exited() {
				if waitStatus, ok := exitError.ProcessState.Sys().(syscall.WaitStatus); ok {
					if waitStatus.Exited() {
						logrus.Errorf("%v", exitError)
						os.Exit(waitStatus.ExitStatus())
					}
					if waitStatus.Signaled() {
						logrus.Errorf("%v", exitError)
						os.Exit(int(waitStatus.Signal()) + 128)
					}
				}
			}
		}
		logrus.Errorf("%v", err)
		logrus.Errorf("(unable to determine exit status)")
		os.Exit(1)
	}
	os.Exit(0)
}

// unshareCmd execs whatever using the ID mappings that we want to use for ourselves
func unshareCmd(c *cli.Context) error {
	// force reexec using the configured ID mappings
	maybeReexecUsingUserNamespace(c, true)
	// exec the specified command, if there is one
	args := c.Args()
	if len(args) < 1 {
		// try to exec the shell, if one's set
		shell, shellSet := os.LookupEnv("SHELL")
		if !shellSet {
			logrus.Errorf("no command specified")
			os.Exit(1)
		}
		args = []string{shell}
	}
	cmd := exec.Command(args[0], args[1:]...)
	cmd.Env = append(os.Environ(), "USER=root", "USERNAME=root", "GROUP=root", "LOGNAME=root", "UID=0", "GID=0")
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	execRunnable(cmd)
	os.Exit(1)
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/sneakybeaky/aws-volumes/shared"
	"log"
)

func main() {
	sess, err := session.NewSession()
	if err != nil {
		log.Fatalf("failed to create session %v\n", err)
	}

	ec2Identity := shared.NewEC2Identity(sess)

	id, err := ec2Identity.GetInstanceID()

	if err != nil {
		fmt.Printf("My id is %s \n", id)
	}
}
<commit_msg>Just prints instance id and nothing else<commit_after>package main

import (
	"fmt"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/sneakybeaky/aws-volumes/shared"
	"log"
)

func main() {
	sess, err := session.NewSession()
	if err != nil {
		log.Fatalf("failed to create session %v\n", err)
	}

	ec2Identity := shared.NewEC2Identity(sess)

	id, err := ec2Identity.GetInstanceID()

	if err != nil {
		log.Fatalf("failed to get instance id %v\n", err)
	}
	fmt.Printf("%s\n", id)

}
<|endoftext|>


<commit_before>// Copyright 2015 Mikio Hara. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"net"
	"os"
	"os/signal"
	"sort"
	"syscall"
	"time"

	"github.com/mikioh/ipaddr"
	"github.com/mikioh/ipoam"
	"golang.org/x/net/icmp"
)

var rtUsageTmpl = `Usage:
	ipoam {{.Name}} [flags] destination

destination
	A hostname, DNS reg-name or IP address.

`

var (
	cmdRT = &Command{
		Func:      rtMain,
		Usage:     cmdUsage,
		UsageTmpl: rtUsageTmpl,
		CanonName: "rt",
		Aliases:   []string{"pathdisc", "traceroute"},
		Descr:     "Discover an IP-layer path",
	}

	rtPayload []byte
	rtData    = []byte("0123456789abcdefghijklmnopqrstuvwxyz")

	rtIPv4only    bool
	rtIPv6only    bool
	rtNoRevLookup bool
	rtUseICMP     bool
	rtVerbose     bool

	rtMaxHops          int
	rtTC               int
	rtPayloadLen       int
	rtPerHopProbeCount int
	rtPort             int
	rtWait             int

	rtOutboundIf string
	rtSrc        string
)

func init() {
	cmdRT.Flag.BoolVar(&rtIPv4only, "4", false, "Run IPv4 test only")
	cmdRT.Flag.BoolVar(&rtIPv6only, "6", false, "Run IPv6 test only")
	cmdRT.Flag.BoolVar(&rtNoRevLookup, "n", false, "Don't use DNS reverse lookup")
	cmdRT.Flag.BoolVar(&rtUseICMP, "m", false, "Use ICMP for probe packets instead of UDP")
	cmdRT.Flag.BoolVar(&rtVerbose, "v", false, "Show verbose information")

	cmdRT.Flag.IntVar(&rtMaxHops, "hops", 30, "Maximum IPv4 TTL or IPv6 hop-limit")
	cmdRT.Flag.IntVar(&rtTC, "tc", 0, "IPv4 TOS or IPv6 traffic-class on probe packets")
	cmdRT.Flag.IntVar(&rtPayloadLen, "pldlen", 56, "Probe packet payload length")
	cmdRT.Flag.IntVar(&rtPerHopProbeCount, "count", 3, "Per-hop probe count")
	cmdRT.Flag.IntVar(&rtPort, "port", 33434, "Base destination port, range will be [port, port+hops)")
	cmdRT.Flag.IntVar(&rtWait, "wait", 1, "Seconds between transmitting each probe")

	cmdRT.Flag.StringVar(&rtOutboundIf, "if", "", "Outbound interface name")
	cmdRT.Flag.StringVar(&rtSrc, "src", "", "Source IP address")
}

func rtMain(cmd *Command, args []string) {
	if len(args) == 0 {
		cmd.Flag.Usage()
	}

	c, ifi, err := parseDsts(args[0], rtIPv4only, rtIPv6only)
	if err != nil {
		cmd.fatal(err)
	}

	if rtMaxHops > 255 {
		rtMaxHops = 255
	}
	rtPayload = bytes.Repeat(rtData, int(rtPayloadLen)/len(rtData)+1)
	rtPayload = rtPayload[:rtPayloadLen]
	if rtWait <= 0 {
		rtWait = 1
	}
	if rtOutboundIf != "" {
		oif, err := net.InterfaceByName(rtOutboundIf)
		if err == nil {
			ifi = oif
		}
	}
	var src net.IP
	if rtSrc != "" {
		src = net.ParseIP(rtSrc)
		if src.To4() != nil {
			rtIPv4only = true
		}
		if src.To16() != nil && src.To4() == nil {
			rtIPv6only = true
		}
	}

	var ipt *ipoam.Tester
	var dst *ipaddr.Position
	for pos := c.First(); pos != nil; pos = c.Next() {
		if !rtIPv6only && pos.IP.To4() != nil {
			network := "udp4"
			address := "0.0.0.0:0"
			if src != nil {
				address = net.JoinHostPort(src.String(), "0")
			}
			if rtUseICMP {
				network = "ip4:icmp"
				address = "0.0.0.0"
				if src != nil {
					address = src.String()
				}
			}
			ipt, err = ipoam.NewTester(network, address)
			if err != nil {
				cmd.fatal(err)
			}
			defer ipt.Close()
			if rtTC >= 0 {
				ipt.IPv4PacketConn().SetTOS(rtTC)
			}
			dst = pos
			break
		}
		if !rtIPv4only && pos.IP.To16() != nil && pos.IP.To4() == nil {
			network := "udp6"
			address := "[::]:0"
			if src != nil {
				address = net.JoinHostPort(src.String(), "0")
			}
			if rtUseICMP {
				network = "ip6:ipv6-icmp"
				address = "::"
				if src != nil {
					address = src.String()
				}
			}
			ipt, err = ipoam.NewTester(network, address)
			if err != nil {
				cmd.fatal(err)
			}
			defer ipt.Close()
			if rtTC >= 0 {
				ipt.IPv6PacketConn().SetTrafficClass(rtTC)
			}
			dst = pos
			break
		}
	}
	c.Reset(nil)
	if dst == nil {
		cmd.fatal(fmt.Errorf("destination for %s not found", args[0]))
	}

	printRTBanner(args[0], c, dst)

	sig := make(chan os.Signal)
	signal.Notify(sig, os.Interrupt, syscall.SIGTERM)
	cm := ipoam.ControlMessage{ID: os.Getpid() & 0xffff, Seq: 1, Port: rtPort}
	hops := make([]rtHop, 0)
	for i := 1; i <= rtMaxHops; i++ {
		var r ipoam.Report
		hops = hops[:0]

		for j := 0; j < rtPerHopProbeCount; j++ {
			t := time.NewTimer(time.Duration(rtWait) * time.Second)
			defer t.Stop()
			begin := time.Now()
			if !rtIPv6only && dst.IP.To4() != nil {
				ipt.IPv4PacketConn().SetTTL(i)
			}
			if !rtIPv4only && dst.IP.To16() != nil && dst.IP.To4() == nil {
				ipt.IPv6PacketConn().SetHopLimit(i)
			}
			if err := ipt.Probe(rtPayload, &cm, dst.IP, ifi); err != nil {
				fmt.Fprintf(os.Stdout, "error=%q\n", err)
			}

			cm.Seq++
			if cm.Seq > 0xffff {
				cm.Seq = 1
			}
			cm.Port++
			if cm.Port > 0xffff {
				cm.Port = rtPort
			}

			select {
			case <-sig:
				os.Exit(0)
			case <-t.C:
				hops = append(hops, rtHop{rtt: time.Since(begin), r: ipoam.Report{Src: net.IPv6unspecified}})
			case r = <-ipt.Report():
				hops = append(hops, rtHop{rtt: r.Time.Sub(begin), r: r})
			}
		}

		printRTReport(i, hops)
		if hasReached(&r) {
			break
		}
	}
	os.Exit(0)
}

func printRTBanner(dsts string, c *ipaddr.Cursor, pos *ipaddr.Position) {
	bw := bufio.NewWriter(os.Stdout)
	fmt.Fprintf(bw, "Path discovery for %s: %d hops max, %d per-hop probes, %d bytes payload\n", dsts, rtMaxHops, rtPerHopProbeCount, len(rtPayload))
	if len(c.List()) > 1 {
		fmt.Fprintf(bw, "Warning: %s has multiple addresses, using %v\n", dsts, pos.IP)
	}
	bw.Flush()
}

func printRTReport(i int, hops []rtHop) {
	sort.Sort(rtHops(hops))
	bw := bufio.NewWriter(os.Stdout)
	fmt.Fprintf(bw, "% 3d  ", i)
	var prev net.IP
	for _, h := range hops {
		if h.r.Error != nil {
			continue
		}
		if h.r.Src.Equal(prev) {
			fmt.Fprintf(bw, "  %v", h.rtt)
			continue
		}
		if prev != nil {
			fmt.Fprintf(bw, "\n     ")
		}
		if h.r.Src.IsUnspecified() {
			fmt.Fprintf(bw, "*")
		} else {
			if rtNoRevLookup {
				fmt.Fprintf(bw, "%v", h.r.Src)
			} else {
				name := revLookup(h.r.Src.String())
				if name == "" {
					fmt.Fprintf(bw, "%v", h.r.Src)
				} else {
					fmt.Fprintf(bw, "%s (%v)", name, h.r.Src)
				}
			}
			if rtVerbose {
				if h.r.Dst != nil {
					fmt.Fprintf(bw, " hops=%d", h.r.Hops)
				}
				if h.r.Interface != nil {
					fmt.Fprintf(bw, " if=%s", h.r.Interface.Name)
				}
				switch body := h.r.ICMP.Body.(type) {
				case *icmp.DstUnreach:
					printICMPExtensions(bw, body.Extensions)
				case *icmp.ParamProb:
					printICMPExtensions(bw, body.Extensions)
				case *icmp.TimeExceeded:
					printICMPExtensions(bw, body.Extensions)
				}
			}
		}
		fmt.Fprintf(bw, "  %v", h.rtt)
		prev = h.r.Src
	}
	fmt.Fprintf(bw, "\n")
	bw.Flush()
}

type rtHop struct {
	rtt time.Duration
	r   ipoam.Report
}

type rtHops []rtHop

func (hops rtHops) Len() int { return len(hops) }

func (hops rtHops) Less(i, j int) bool {
	if n := bytes.Compare(hops[i].r.Src, hops[j].r.Src); n < 0 {
		return true
	}
	return false
}

func (hops rtHops) Swap(i, j int) { hops[i], hops[j] = hops[j], hops[i] }

func printICMPExtensions(w io.Writer, exts []icmp.Extension) {
	for _, ext := range exts {
		switch ext := ext.(type) {
		case *icmp.MPLSLabelStack:
			for _, l := range ext.Labels {
				fmt.Fprintf(w, " <label=%d tc=%x s=%t ttl=%d>", l.Label, l.TC, l.S, l.TTL)
			}
		case *icmp.InterfaceInfo:
			fmt.Fprintf(w, " <")
			if ext.Interface != nil {
				fmt.Fprintf(w, "if=%s", ext.Interface.Name)
			}
			if ext.Addr != nil {
				if ext.Interface != nil {
					fmt.Fprintf(w, " ")
				}
				fmt.Fprintf(w, "addr=%v", ext.Addr)
			}
			fmt.Fprintf(w, ">")
		}
	}
}
<commit_msg>ipoam/cmd/ipoam: print destination address on received packet in path discovery<commit_after>// Copyright 2015 Mikio Hara. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"net"
	"os"
	"os/signal"
	"sort"
	"syscall"
	"time"

	"github.com/mikioh/ipaddr"
	"github.com/mikioh/ipoam"
	"golang.org/x/net/icmp"
)

var rtUsageTmpl = `Usage:
	ipoam {{.Name}} [flags] destination

destination
	A hostname, DNS reg-name or IP address.

`

var (
	cmdRT = &Command{
		Func:      rtMain,
		Usage:     cmdUsage,
		UsageTmpl: rtUsageTmpl,
		CanonName: "rt",
		Aliases:   []string{"pathdisc", "traceroute"},
		Descr:     "Discover an IP-layer path",
	}

	rtPayload []byte
	rtData    = []byte("0123456789abcdefghijklmnopqrstuvwxyz")

	rtIPv4only    bool
	rtIPv6only    bool
	rtNoRevLookup bool
	rtUseICMP     bool
	rtVerbose     bool

	rtMaxHops          int
	rtTC               int
	rtPayloadLen       int
	rtPerHopProbeCount int
	rtPort             int
	rtWait             int

	rtOutboundIf string
	rtSrc        string
)

func init() {
	cmdRT.Flag.BoolVar(&rtIPv4only, "4", false, "Run IPv4 test only")
	cmdRT.Flag.BoolVar(&rtIPv6only, "6", false, "Run IPv6 test only")
	cmdRT.Flag.BoolVar(&rtNoRevLookup, "n", false, "Don't use DNS reverse lookup")
	cmdRT.Flag.BoolVar(&rtUseICMP, "m", false, "Use ICMP for probe packets instead of UDP")
	cmdRT.Flag.BoolVar(&rtVerbose, "v", false, "Show verbose information")

	cmdRT.Flag.IntVar(&rtMaxHops, "hops", 30, "Maximum IPv4 TTL or IPv6 hop-limit")
	cmdRT.Flag.IntVar(&rtTC, "tc", 0, "IPv4 TOS or IPv6 traffic-class on probe packets")
	cmdRT.Flag.IntVar(&rtPayloadLen, "pldlen", 56, "Probe packet payload length")
	cmdRT.Flag.IntVar(&rtPerHopProbeCount, "count", 3, "Per-hop probe count")
	cmdRT.Flag.IntVar(&rtPort, "port", 33434, "Base destination port, range will be [port, port+hops)")
	cmdRT.Flag.IntVar(&rtWait, "wait", 1, "Seconds between transmitting each probe")

	cmdRT.Flag.StringVar(&rtOutboundIf, "if", "", "Outbound interface name")
	cmdRT.Flag.StringVar(&rtSrc, "src", "", "Source IP address")
}

func rtMain(cmd *Command, args []string) {
	if len(args) == 0 {
		cmd.Flag.Usage()
	}

	c, ifi, err := parseDsts(args[0], rtIPv4only, rtIPv6only)
	if err != nil {
		cmd.fatal(err)
	}

	if rtMaxHops > 255 {
		rtMaxHops = 255
	}
	rtPayload = bytes.Repeat(rtData, int(rtPayloadLen)/len(rtData)+1)
	rtPayload = rtPayload[:rtPayloadLen]
	if rtWait <= 0 {
		rtWait = 1
	}
	if rtOutboundIf != "" {
		oif, err := net.InterfaceByName(rtOutboundIf)
		if err == nil {
			ifi = oif
		}
	}
	var src net.IP
	if rtSrc != "" {
		src = net.ParseIP(rtSrc)
		if src.To4() != nil {
			rtIPv4only = true
		}
		if src.To16() != nil && src.To4() == nil {
			rtIPv6only = true
		}
	}

	var ipt *ipoam.Tester
	var dst *ipaddr.Position
	for pos := c.First(); pos != nil; pos = c.Next() {
		if !rtIPv6only && pos.IP.To4() != nil {
			network := "udp4"
			address := "0.0.0.0:0"
			if src != nil {
				address = net.JoinHostPort(src.String(), "0")
			}
			if rtUseICMP {
				network = "ip4:icmp"
				address = "0.0.0.0"
				if src != nil {
					address = src.String()
				}
			}
			ipt, err = ipoam.NewTester(network, address)
			if err != nil {
				cmd.fatal(err)
			}
			defer ipt.Close()
			if rtTC >= 0 {
				ipt.IPv4PacketConn().SetTOS(rtTC)
			}
			dst = pos
			break
		}
		if !rtIPv4only && pos.IP.To16() != nil && pos.IP.To4() == nil {
			network := "udp6"
			address := "[::]:0"
			if src != nil {
				address = net.JoinHostPort(src.String(), "0")
			}
			if rtUseICMP {
				network = "ip6:ipv6-icmp"
				address = "::"
				if src != nil {
					address = src.String()
				}
			}
			ipt, err = ipoam.NewTester(network, address)
			if err != nil {
				cmd.fatal(err)
			}
			defer ipt.Close()
			if rtTC >= 0 {
				ipt.IPv6PacketConn().SetTrafficClass(rtTC)
			}
			dst = pos
			break
		}
	}
	c.Reset(nil)
	if dst == nil {
		cmd.fatal(fmt.Errorf("destination for %s not found", args[0]))
	}

	printRTBanner(args[0], c, dst)

	sig := make(chan os.Signal)
	signal.Notify(sig, os.Interrupt, syscall.SIGTERM)
	cm := ipoam.ControlMessage{ID: os.Getpid() & 0xffff, Seq: 1, Port: rtPort}
	hops := make([]rtHop, 0)
	for i := 1; i <= rtMaxHops; i++ {
		var r ipoam.Report
		hops = hops[:0]

		for j := 0; j < rtPerHopProbeCount; j++ {
			t := time.NewTimer(time.Duration(rtWait) * time.Second)
			defer t.Stop()
			begin := time.Now()
			if !rtIPv6only && dst.IP.To4() != nil {
				ipt.IPv4PacketConn().SetTTL(i)
			}
			if !rtIPv4only && dst.IP.To16() != nil && dst.IP.To4() == nil {
				ipt.IPv6PacketConn().SetHopLimit(i)
			}
			if err := ipt.Probe(rtPayload, &cm, dst.IP, ifi); err != nil {
				fmt.Fprintf(os.Stdout, "error=%q\n", err)
			}

			cm.Seq++
			if cm.Seq > 0xffff {
				cm.Seq = 1
			}
			cm.Port++
			if cm.Port > 0xffff {
				cm.Port = rtPort
			}

			select {
			case <-sig:
				os.Exit(0)
			case <-t.C:
				hops = append(hops, rtHop{rtt: time.Since(begin), r: ipoam.Report{Src: net.IPv6unspecified}})
			case r = <-ipt.Report():
				hops = append(hops, rtHop{rtt: r.Time.Sub(begin), r: r})
			}
		}

		printRTReport(i, hops)
		if hasReached(&r) {
			break
		}
	}
	os.Exit(0)
}

func printRTBanner(dsts string, c *ipaddr.Cursor, pos *ipaddr.Position) {
	bw := bufio.NewWriter(os.Stdout)
	fmt.Fprintf(bw, "Path discovery for %s: %d hops max, %d per-hop probes, %d bytes payload\n", dsts, rtMaxHops, rtPerHopProbeCount, len(rtPayload))
	if len(c.List()) > 1 {
		fmt.Fprintf(bw, "Warning: %s has multiple addresses, using %v\n", dsts, pos.IP)
	}
	bw.Flush()
}

func printRTReport(i int, hops []rtHop) {
	sort.Sort(rtHops(hops))
	bw := bufio.NewWriter(os.Stdout)
	fmt.Fprintf(bw, "% 3d  ", i)
	var prev net.IP
	for _, h := range hops {
		if h.r.Error != nil {
			continue
		}
		if h.r.Src.Equal(prev) {
			fmt.Fprintf(bw, "  %v", h.rtt)
			continue
		}
		if prev != nil {
			fmt.Fprintf(bw, "\n     ")
		}
		if h.r.Src.IsUnspecified() {
			fmt.Fprintf(bw, "*")
		} else {
			if rtNoRevLookup {
				fmt.Fprintf(bw, "%v", h.r.Src)
			} else {
				name := revLookup(h.r.Src.String())
				if name == "" {
					fmt.Fprintf(bw, "%v", h.r.Src)
				} else {
					fmt.Fprintf(bw, "%s (%v)", name, h.r.Src)
				}
			}
			if rtVerbose {
				if h.r.Dst != nil {
					fmt.Fprintf(bw, " hops=%d to=%v", h.r.Hops, h.r.Dst)
				}
				if h.r.Interface != nil {
					fmt.Fprintf(bw, " if=%s", h.r.Interface.Name)
				}
				switch body := h.r.ICMP.Body.(type) {
				case *icmp.DstUnreach:
					printICMPExtensions(bw, body.Extensions)
				case *icmp.ParamProb:
					printICMPExtensions(bw, body.Extensions)
				case *icmp.TimeExceeded:
					printICMPExtensions(bw, body.Extensions)
				}
			}
		}
		fmt.Fprintf(bw, "  %v", h.rtt)
		prev = h.r.Src
	}
	fmt.Fprintf(bw, "\n")
	bw.Flush()
}

type rtHop struct {
	rtt time.Duration
	r   ipoam.Report
}

type rtHops []rtHop

func (hops rtHops) Len() int { return len(hops) }

func (hops rtHops) Less(i, j int) bool {
	if n := bytes.Compare(hops[i].r.Src, hops[j].r.Src); n < 0 {
		return true
	}
	return false
}

func (hops rtHops) Swap(i, j int) { hops[i], hops[j] = hops[j], hops[i] }

func printICMPExtensions(w io.Writer, exts []icmp.Extension) {
	for _, ext := range exts {
		switch ext := ext.(type) {
		case *icmp.MPLSLabelStack:
			for _, l := range ext.Labels {
				fmt.Fprintf(w, " <label=%d tc=%x s=%t ttl=%d>", l.Label, l.TC, l.S, l.TTL)
			}
		case *icmp.InterfaceInfo:
			fmt.Fprintf(w, " <")
			if ext.Interface != nil {
				fmt.Fprintf(w, "if=%s", ext.Interface.Name)
			}
			if ext.Addr != nil {
				if ext.Interface != nil {
					fmt.Fprintf(w, " ")
				}
				fmt.Fprintf(w, "addr=%v", ext.Addr)
			}
			fmt.Fprintf(w, ">")
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"os"

	"github.com/aclements/go-perf/perffile"
	"github.com/aclements/go-perf/perfsession"
)

type database struct {
	// procs maps from PID to information and records for a
	// process.
	procs map[int]*proc

	// dataSrcs maps dataSrcIDs to full DataSrc information.
	// There's a lot of information in a DataSrc, but in practice
	// a given architecture will generate a small subset of the
	// possibilities. Hence, rather than storing a whole DataSrc
	// in every record, we canonicalize it to a small identifier.
	dataSrcs []perffile.DataSrc

	// maxLatency is the maximum latency value across all records
	// in this database.
	maxLatency uint32

	// metadata records metadata fields from the profile.
	metadata Metadata
}

type proc struct {
	pid     int
	comm    string
	records []record
	ipInfo  map[uint64]ipInfo
}

type record struct {
	ip      uint64
	address uint64
	latency uint32
	dataSrc dataSrcID
}

type ipInfo struct {
	funcName string
	fileName string
	line     int
}

// dataSrcID is a small integer identifying a perffile.DataSrc.
type dataSrcID uint32

type Metadata struct {
	Hostname string
	Arch     string
	CPUDesc  string   `json:"CPU"`
	CmdLine  []string `json:"Command line"`
}

// parsePerf parses a perf.data profile into a database.
func parsePerf(fileName string) *database {
	f, err := perffile.Open(fileName)
	if os.IsNotExist(err) && fileName == "perf.data" {
		// Give a friendly error for first-time users.
		fmt.Fprintf(os.Stderr, "%s.\nTo record a profile, use\n  perf mem record <command>\nor specify an alternate profile path with -i.\n", err)
		os.Exit(1)
	} else if err != nil {
		fmt.Fprintf(os.Stderr, "error loading profile: %s\n", err)
		os.Exit(1)
	}
	defer f.Close()

	db := &database{
		procs: make(map[int]*proc),
	}
	db.metadata.Hostname = f.Meta.Hostname
	db.metadata.Arch = f.Meta.Arch
	db.metadata.CPUDesc = f.Meta.CPUDesc
	db.metadata.CmdLine = f.Meta.CmdLine

	dataSrc2ID := make(map[perffile.DataSrc]dataSrcID)
	s := perfsession.New(f)

	numSamples := 0
	droppedMmaps := 0
	droppedSymbols := 0

	const requiredFormat = perffile.SampleFormatIP | perffile.SampleFormatAddr | perffile.SampleFormatWeight | perffile.SampleFormatDataSrc

	rs := f.Records(perffile.RecordsCausalOrder)
	for rs.Next() {
		r := rs.Record
		s.Update(r)

		switch r := r.(type) {
		case *perffile.RecordComm:
			// Comm events usually happen after the first
			// few samples from this PID.
			p := db.procs[r.PID]
			if p != nil {
				p.comm = r.Comm
			}

		case *perffile.RecordSample:
			if r.Format&requiredFormat != requiredFormat {
				break
			}

			numSamples++

			pidInfo := s.LookupPID(r.PID)
			mmap := pidInfo.LookupMmap(r.IP)
			if mmap == nil {
				droppedMmaps++
				break
			}

			// Find proc for r.PID.
			p, ok := db.procs[r.PID]
			if !ok {
				p = &proc{
					pid:    r.PID,
					comm:   pidInfo.Comm,
					ipInfo: make(map[uint64]ipInfo),
				}
				db.procs[r.PID] = p
			}

			// Canonicalize data source.
			dsID, ok := dataSrc2ID[r.DataSrc]
			if !ok {
				dsID = dataSrcID(len(db.dataSrcs))
				dataSrc2ID[r.DataSrc] = dsID
				db.dataSrcs = append(db.dataSrcs, r.DataSrc)
			}

			// Create the record.
			p.records = append(p.records, record{
				ip:      r.IP,
				address: r.Addr,
				latency: uint32(r.Weight),
				dataSrc: dsID,
			})

			// Update database stats.
			if uint32(r.Weight) > db.maxLatency {
				db.maxLatency = uint32(r.Weight)
			}

			// Symbolize IP.
			if _, ok := p.ipInfo[r.IP]; !ok {
				// TODO: Intern strings
				var symb perfsession.Symbolic
				if !perfsession.Symbolize(s, mmap, r.IP, &symb) {
					droppedSymbols++
				}
				if symb.FuncName == "" {
					symb.FuncName = "[unknown]"
				}
				fileName := "[unknown]"
				if symb.Line.File != nil && symb.Line.File.Name != "" {
					fileName = symb.Line.File.Name
				}
				p.ipInfo[r.IP] = ipInfo{
					funcName: symb.FuncName,
					fileName: fileName,
					line:     symb.Line.Line,
				}
			}
		}
	}

	if numSamples == 0 {
		fmt.Printf("no memory latency samples in %s (did you use \"perf mem record\"?)\n", fileName)
		os.Exit(1)
	}
	if droppedMmaps > 0 {
		fmt.Printf("warning: %d sample IPs (%d%%) occurred in unmapped memory regions\n", droppedMmaps, droppedMmaps*100/numSamples)
	}
	if droppedSymbols > 0 {
		fmt.Printf("warning: failed to symbolize %d samples (%d%%)\n", droppedSymbols, droppedSymbols*100/numSamples)
	}

	return db
}

// filter specifies a set of field values to filter records on. The
// zero value of each field means not to filter on that field.
type filter struct {
	pid      int
	funcName string
	fileName string
	line     int // Requires fileName.
	address  uint64
	dataSrc  perffile.DataSrc
}

// filter invokes cb for every record matching f.
func (db *database) filter(f *filter, cb func(*proc, *record)) {
	dsFilter := f.dataSrc != perffile.DataSrc{}
	filterProc := func(proc *proc) {
		var ds perffile.DataSrc

		// TODO: Consider creating indexes for some or all of
		// these. Then just do a list merge of the record
		// indexes.
		for i := range proc.records {
			// Avoid heap-allocating for passing rec to cb.
			rec := &proc.records[i]
			if f.address != 0 && f.address != rec.address {
				continue
			}
			ipi := proc.ipInfo[rec.ip]
			if f.funcName != "" && f.funcName != ipi.funcName {
				continue
			}
			if f.fileName != "" && f.fileName != ipi.fileName {
				continue
			}
			if f.line != 0 && f.line != ipi.line {
				continue
			}
			if !dsFilter {
				// Short-circuit dataSrc checking.
				goto good
			}

			ds = db.dataSrcs[rec.dataSrc]
			if f.dataSrc.Op != 0 && f.dataSrc.Op != ds.Op {
				continue
			}
			if f.dataSrc.Level != 0 && (f.dataSrc.Level != ds.Level || f.dataSrc.Miss != ds.Miss) {
				continue
			}
			if f.dataSrc.Snoop != 0 && f.dataSrc.Snoop != ds.Snoop {
				continue
			}
			if f.dataSrc.Locked != 0 && f.dataSrc.Locked != ds.Locked {
				continue
			}
			if f.dataSrc.TLB != 0 && f.dataSrc.TLB != ds.TLB {
				continue
			}

		good:
			cb(proc, rec)
		}
	}

	if f.pid == 0 {
		for _, proc := range db.procs {
			filterProc(proc)
		}
	} else {
		proc := db.procs[f.pid]
		if proc != nil {
			filterProc(proc)
		}
	}
}
<commit_msg>cmd/memlat: accept SampleFormatWeightStruct<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"os"

	"github.com/aclements/go-perf/perffile"
	"github.com/aclements/go-perf/perfsession"
)

type database struct {
	// procs maps from PID to information and records for a
	// process.
	procs map[int]*proc

	// dataSrcs maps dataSrcIDs to full DataSrc information.
	// There's a lot of information in a DataSrc, but in practice
	// a given architecture will generate a small subset of the
	// possibilities. Hence, rather than storing a whole DataSrc
	// in every record, we canonicalize it to a small identifier.
	dataSrcs []perffile.DataSrc

	// maxLatency is the maximum latency value across all records
	// in this database.
	maxLatency uint32

	// metadata records metadata fields from the profile.
	metadata Metadata
}

type proc struct {
	pid     int
	comm    string
	records []record
	ipInfo  map[uint64]ipInfo
}

type record struct {
	ip      uint64
	address uint64
	latency uint32
	dataSrc dataSrcID
}

type ipInfo struct {
	funcName string
	fileName string
	line     int
}

// dataSrcID is a small integer identifying a perffile.DataSrc.
type dataSrcID uint32

type Metadata struct {
	Hostname string
	Arch     string
	CPUDesc  string   `json:"CPU"`
	CmdLine  []string `json:"Command line"`
}

// parsePerf parses a perf.data profile into a database.
func parsePerf(fileName string) *database {
	f, err := perffile.Open(fileName)
	if os.IsNotExist(err) && fileName == "perf.data" {
		// Give a friendly error for first-time users.
		fmt.Fprintf(os.Stderr, "%s.\nTo record a profile, use\n  perf mem record <command>\nor specify an alternate profile path with -i.\n", err)
		os.Exit(1)
	} else if err != nil {
		fmt.Fprintf(os.Stderr, "error loading profile: %s\n", err)
		os.Exit(1)
	}
	defer f.Close()

	db := &database{
		procs: make(map[int]*proc),
	}
	db.metadata.Hostname = f.Meta.Hostname
	db.metadata.Arch = f.Meta.Arch
	db.metadata.CPUDesc = f.Meta.CPUDesc
	db.metadata.CmdLine = f.Meta.CmdLine

	dataSrc2ID := make(map[perffile.DataSrc]dataSrcID)
	s := perfsession.New(f)

	numSamples := 0
	droppedMmaps := 0
	droppedSymbols := 0

	const requiredFormat = perffile.SampleFormatIP | perffile.SampleFormatAddr | perffile.SampleFormatDataSrc

	rs := f.Records(perffile.RecordsCausalOrder)
	for rs.Next() {
		r := rs.Record
		s.Update(r)

		switch r := r.(type) {
		case *perffile.RecordComm:
			// Comm events usually happen after the first
			// few samples from this PID.
			p := db.procs[r.PID]
			if p != nil {
				p.comm = r.Comm
			}

		case *perffile.RecordSample:
			if r.Format&requiredFormat != requiredFormat {
				break
			}
			// Either Weight or WeightStruct is required.
			if r.Format&(perffile.SampleFormatWeight|perffile.SampleFormatWeightStruct) == 0 {
				break
			}

			numSamples++

			pidInfo := s.LookupPID(r.PID)
			mmap := pidInfo.LookupMmap(r.IP)
			if mmap == nil {
				droppedMmaps++
				break
			}

			// Find proc for r.PID.
			p, ok := db.procs[r.PID]
			if !ok {
				p = &proc{
					pid:    r.PID,
					comm:   pidInfo.Comm,
					ipInfo: make(map[uint64]ipInfo),
				}
				db.procs[r.PID] = p
			}

			// Canonicalize data source.
			dsID, ok := dataSrc2ID[r.DataSrc]
			if !ok {
				dsID = dataSrcID(len(db.dataSrcs))
				dataSrc2ID[r.DataSrc] = dsID
				db.dataSrcs = append(db.dataSrcs, r.DataSrc)
			}

			// Create the record.
			p.records = append(p.records, record{
				ip:      r.IP,
				address: r.Addr,
				latency: uint32(r.Weight),
				dataSrc: dsID,
			})

			// Update database stats.
			if uint32(r.Weight) > db.maxLatency {
				db.maxLatency = uint32(r.Weight)
			}

			// Symbolize IP.
			if _, ok := p.ipInfo[r.IP]; !ok {
				// TODO: Intern strings
				var symb perfsession.Symbolic
				if !perfsession.Symbolize(s, mmap, r.IP, &symb) {
					droppedSymbols++
				}
				if symb.FuncName == "" {
					symb.FuncName = "[unknown]"
				}
				fileName := "[unknown]"
				if symb.Line.File != nil && symb.Line.File.Name != "" {
					fileName = symb.Line.File.Name
				}
				p.ipInfo[r.IP] = ipInfo{
					funcName: symb.FuncName,
					fileName: fileName,
					line:     symb.Line.Line,
				}
			}
		}
	}

	if numSamples == 0 {
		fmt.Printf("no memory latency samples in %s (did you use \"perf mem record\"?)\n", fileName)
		os.Exit(1)
	}
	if droppedMmaps > 0 {
		fmt.Printf("warning: %d sample IPs (%d%%) occurred in unmapped memory regions\n", droppedMmaps, droppedMmaps*100/numSamples)
	}
	if droppedSymbols > 0 {
		fmt.Printf("warning: failed to symbolize %d samples (%d%%)\n", droppedSymbols, droppedSymbols*100/numSamples)
	}

	return db
}

// filter specifies a set of field values to filter records on. The
// zero value of each field means not to filter on that field.
type filter struct {
	pid      int
	funcName string
	fileName string
	line     int // Requires fileName.
	address  uint64
	dataSrc  perffile.DataSrc
}

// filter invokes cb for every record matching f.
func (db *database) filter(f *filter, cb func(*proc, *record)) {
	dsFilter := f.dataSrc != perffile.DataSrc{}
	filterProc := func(proc *proc) {
		var ds perffile.DataSrc

		// TODO: Consider creating indexes for some or all of
		// these. Then just do a list merge of the record
		// indexes.
		for i := range proc.records {
			// Avoid heap-allocating for passing rec to cb.
			rec := &proc.records[i]
			if f.address != 0 && f.address != rec.address {
				continue
			}
			ipi := proc.ipInfo[rec.ip]
			if f.funcName != "" && f.funcName != ipi.funcName {
				continue
			}
			if f.fileName != "" && f.fileName != ipi.fileName {
				continue
			}
			if f.line != 0 && f.line != ipi.line {
				continue
			}
			if !dsFilter {
				// Short-circuit dataSrc checking.
				goto good
			}

			ds = db.dataSrcs[rec.dataSrc]
			if f.dataSrc.Op != 0 && f.dataSrc.Op != ds.Op {
				continue
			}
			if f.dataSrc.Level != 0 && (f.dataSrc.Level != ds.Level || f.dataSrc.Miss != ds.Miss) {
				continue
			}
			if f.dataSrc.Snoop != 0 && f.dataSrc.Snoop != ds.Snoop {
				continue
			}
			if f.dataSrc.Locked != 0 && f.dataSrc.Locked != ds.Locked {
				continue
			}
			if f.dataSrc.TLB != 0 && f.dataSrc.TLB != ds.TLB {
				continue
			}

		good:
			cb(proc, rec)
		}
	}

	if f.pid == 0 {
		for _, proc := range db.procs {
			filterProc(proc)
		}
	} else {
		proc := db.procs[f.pid]
		if proc != nil {
			filterProc(proc)
		}
	}
}
<|endoftext|>


<commit_before>// SPDX-License-Identifier: MIT
//
// Copyright © 2018 Kent Gibson <warthog618@gmail.com>.

// sendsms sends an SMS using the modem.
//
// This provides an example of using the SendSMS command, as well as a test
// that the library works with the modem.
package main

import (
	"flag"
	"io"
	"log"
	"time"

	"github.com/warthog618/modem/at"
	"github.com/warthog618/modem/gsm"
	"github.com/warthog618/modem/serial"
	"github.com/warthog618/modem/trace"
	"github.com/warthog618/sms"
)

func main() {
	dev := flag.String("d", "/dev/ttyUSB0", "path to modem device")
	baud := flag.Int("b", 115200, "baud rate")
	num := flag.String("n", "+12345", "number to send to, in international format")
	msg := flag.String("m", "Zoot Zoot", "the message to send")
	timeout := flag.Duration("t", 5000*time.Millisecond, "command timeout period")
	verbose := flag.Bool("v", false, "log modem interactions")
	pdumode := flag.Bool("p", false, "send in PDU mode")
	hex := flag.Bool("x", false, "hex dump modem responses")
	flag.Parse()

	m, err := serial.New(serial.WithPort(*dev), serial.WithBaud(*baud))
	if err != nil {
		log.Fatal(err)
	}
	var mio io.ReadWriter = m
	if *hex {
		mio = trace.New(m, trace.WithReadFormat("r: %v"))
	} else if *verbose {
		mio = trace.New(m)
	}
	gopts := []gsm.Option{}
	if *pdumode {
		gopts = append(gopts, gsm.WithPDUMode)
	}
	g := gsm.New(at.New(mio, at.WithTimeout(*timeout)), gopts...)
	if err = g.Init(); err != nil {
		log.Fatal(err)
	}
	if *pdumode {
		sendPDU(g, *num, *msg)
		return
	}
	mr, err := g.SendShortMessage(*num, *msg)
	// !!! check CPIN?? on failure to determine root cause??  If ERROR 302
	log.Printf("%v %v\n", mr, err)
}

func sendPDU(g *gsm.GSM, number string, msg string) {
	pdus, err := sms.Encode([]byte(msg), sms.To(number), sms.WithAllCharsets)
	if err != nil {
		log.Fatal(err)
	}
	for i, p := range pdus {
		tp, err := p.MarshalBinary()
		if err != nil {
			log.Fatal(err)
		}
		mr, err := g.SendPDU(tp)
		if err != nil {
			// !!! check CPIN?? on failure to determine root cause??  If ERROR 302
			log.Fatal(err)
		}
		log.Printf("PDU %d: %v\n", i+1, mr)
	}
}
<commit_msg>use seconds for default timeout<commit_after>// SPDX-License-Identifier: MIT
//
// Copyright © 2018 Kent Gibson <warthog618@gmail.com>.

// sendsms sends an SMS using the modem.
//
// This provides an example of using the SendSMS command, as well as a test
// that the library works with the modem.
package main

import (
	"flag"
	"io"
	"log"
	"time"

	"github.com/warthog618/modem/at"
	"github.com/warthog618/modem/gsm"
	"github.com/warthog618/modem/serial"
	"github.com/warthog618/modem/trace"
	"github.com/warthog618/sms"
)

func main() {
	dev := flag.String("d", "/dev/ttyUSB0", "path to modem device")
	baud := flag.Int("b", 115200, "baud rate")
	num := flag.String("n", "+12345", "number to send to, in international format")
	msg := flag.String("m", "Zoot Zoot", "the message to send")
	timeout := flag.Duration("t", 5*time.Second, "command timeout period")
	verbose := flag.Bool("v", false, "log modem interactions")
	pdumode := flag.Bool("p", false, "send in PDU mode")
	hex := flag.Bool("x", false, "hex dump modem responses")
	flag.Parse()

	m, err := serial.New(serial.WithPort(*dev), serial.WithBaud(*baud))
	if err != nil {
		log.Fatal(err)
	}
	var mio io.ReadWriter = m
	if *hex {
		mio = trace.New(m, trace.WithReadFormat("r: %v"))
	} else if *verbose {
		mio = trace.New(m)
	}
	gopts := []gsm.Option{}
	if *pdumode {
		gopts = append(gopts, gsm.WithPDUMode)
	}
	g := gsm.New(at.New(mio, at.WithTimeout(*timeout)), gopts...)
	if err = g.Init(); err != nil {
		log.Fatal(err)
	}
	if *pdumode {
		sendPDU(g, *num, *msg)
		return
	}
	mr, err := g.SendShortMessage(*num, *msg)
	// !!! check CPIN?? on failure to determine root cause??  If ERROR 302
	log.Printf("%v %v\n", mr, err)
}

func sendPDU(g *gsm.GSM, number string, msg string) {
	pdus, err := sms.Encode([]byte(msg), sms.To(number), sms.WithAllCharsets)
	if err != nil {
		log.Fatal(err)
	}
	for i, p := range pdus {
		tp, err := p.MarshalBinary()
		if err != nil {
			log.Fatal(err)
		}
		mr, err := g.SendPDU(tp)
		if err != nil {
			// !!! check CPIN?? on failure to determine root cause??  If ERROR 302
			log.Fatal(err)
		}
		log.Printf("PDU %d: %v\n", i+1, mr)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"strings"

	"github.com/nlopes/slack"
)

const (
	iconPath = "/static/icon.png"
	memePath = "/static/spongemock.jpg"
)

var (
	atk     = os.Getenv("AUTHENTICATION_TOKEN")
	vtk     = os.Getenv("VERIFICATION_TOKEN")
	appURL  = os.Getenv("APP_URL")
	iconURL string
	memeURL string
	api     = slack.New(atk)
)

func transformText(m string) string {
	var buffer bytes.Buffer
	for i := 0; i < len(m); i++ {
		ch := m[i : i+1]
		if rand.Int()%2 == 0 {
			ch = strings.ToUpper(ch)
		} else {
			ch = strings.ToLower(ch)
		}
		buffer.WriteString(ch)
	}
	return buffer.String()
}

func isValidSlackRequest(r *http.Request) bool {
	if r.Method != "POST" {
		log.Printf("want method POST, got %s\n", r.Method)
		return false
	}
	err := r.ParseForm()
	if err != nil {
		log.Printf("invalid form data: %s\n", err)
		return false
	}
	if cmd := r.PostFormValue("command"); cmd != "/spongemock" {
		log.Printf("want command /spongemock, got %s\n", cmd)
		return false
	}
	if tk := r.PostFormValue("token"); tk != vtk {
		log.Printf("received invalid token %s\n", tk)
		return false
	}
	if url := r.PostFormValue("response_url"); url == "" {
		log.Println("did not receive response url")
		return false
	}
	return true
}

func getLastSlackMessage(c string) (string, error) {
	h, err := api.GetChannelHistory(c, slack.NewHistoryParameters())
	if err != nil {
		log.Printf("history API request error: %s", err)
		return "", err
	}

	for _, msg := range h.Messages {
		if msg.SubType != "" || msg.Text == "" {
			continue
		}
		return msg.Text, nil
	}

	err = errors.New("no last message found")
	log.Println(err)
	return "", err
}

func handleSlack(w http.ResponseWriter, r *http.Request) {
	status := http.StatusOK
	defer func() {
		w.WriteHeader(status)
	}()
	if !isValidSlackRequest(r) {
		status = http.StatusBadRequest
		return
	}
	channel := r.PostFormValue("channel_id")
	lastMessage, err := getLastSlackMessage(channel)
	if err != nil {
		status = http.StatusInternalServerError
		return
	}
	params := slack.NewPostMessageParameters()
	params.Username = "Spongebob"
	params.Attachments = []slack.Attachment{{
		Text:     transformText(lastMessage),
		Fallback: "*Spongebob mocking meme*",
		ImageURL: memeURL,
	}}
	params.IconURL = iconURL
	_, _, err = api.PostMessage(channel, "", params)
	if err != nil {
		status = http.StatusInternalServerError
	}
}

func main() {
	port := os.Getenv("PORT")
	if port == "" {
		log.Fatal("$PORT must be set!")
	}
	if atk == "" {
		log.Fatal("$AUTHENTICATION_TOKEN must be set!")
	}
	if vtk == "" {
		log.Fatal("$VERIFICATION_TOKEN must be set!")
	}
	if appURL == "" {
		log.Fatal("$APP_URL must be set!")
	}
	u, err := url.Parse(appURL)
	if err != nil {
		log.Fatal("invalid $APP_URL %s", appURL)
	}
	icon, _ := url.Parse(iconPath)
	iconURL = u.ResolveReference(icon).String()
	meme, _ := url.Parse(memePath)
	memeURL = u.ResolveReference(meme).String()

	fs := http.FileServer(http.Dir("static"))
	http.Handle("/static/", http.StripPrefix("/static/", fs))
	http.HandleFunc("/slack", handleSlack)

	log.Fatal(http.ListenAndServe(":"+port, nil))
}
<commit_msg>Properly escape HTML characters<commit_after>package main

import (
	"bytes"
	"errors"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"strings"

	"github.com/nlopes/slack"
)

const (
	iconPath = "/static/icon.png"
	memePath = "/static/spongemock.jpg"
)

var (
	atk     = os.Getenv("AUTHENTICATION_TOKEN")
	vtk     = os.Getenv("VERIFICATION_TOKEN")
	appURL  = os.Getenv("APP_URL")
	iconURL string
	memeURL string
	api     = slack.New(atk)

	textRegexp = regexp.MustCompile("&|<|>|.?")
)

func transformText(m string) string {
	var buffer bytes.Buffer
	letters := textRegexp.FindAllString(m, -1)
	for _, ch := range letters {
		// ignore html escaped entities
		if len(ch) > 1 {
			buffer.WriteString(ch)
			continue
		}
		if rand.Int()%2 == 0 {
			ch = strings.ToUpper(ch)
		} else {
			ch = strings.ToLower(ch)
		}
		buffer.WriteString(ch)
	}
	return buffer.String()
}

func isValidSlackRequest(r *http.Request) bool {
	if r.Method != "POST" {
		log.Printf("want method POST, got %s\n", r.Method)
		return false
	}
	err := r.ParseForm()
	if err != nil {
		log.Printf("invalid form data: %s\n", err)
		return false
	}
	if cmd := r.PostFormValue("command"); cmd != "/spongemock" {
		log.Printf("want command /spongemock, got %s\n", cmd)
		return false
	}
	if tk := r.PostFormValue("token"); tk != vtk {
		log.Printf("received invalid token %s\n", tk)
		return false
	}
	if url := r.PostFormValue("response_url"); url == "" {
		log.Println("did not receive response url")
		return false
	}
	return true
}

func getLastSlackMessage(c string) (string, error) {
	h, err := api.GetChannelHistory(c, slack.NewHistoryParameters())
	if err != nil {
		log.Printf("history API request error: %s", err)
		return "", err
	}

	for _, msg := range h.Messages {
		if msg.SubType != "" || msg.Text == "" {
			continue
		}
		return msg.Text, nil
	}

	err = errors.New("no last message found")
	log.Println(err)
	return "", err
}

func handleSlack(w http.ResponseWriter, r *http.Request) {
	status := http.StatusOK
	defer func() {
		w.WriteHeader(status)
	}()
	if !isValidSlackRequest(r) {
		status = http.StatusBadRequest
		return
	}
	channel := r.PostFormValue("channel_id")
	lastMessage, err := getLastSlackMessage(channel)
	if err != nil {
		status = http.StatusInternalServerError
		return
	}
	params := slack.NewPostMessageParameters()
	params.Username = "Spongebob"
	params.Attachments = []slack.Attachment{{
		Text:     transformText(lastMessage),
		Fallback: "*Spongebob mocking meme*",
		ImageURL: memeURL,
	}}
	params.IconURL = iconURL
	_, _, err = api.PostMessage(channel, "", params)
	if err != nil {
		status = http.StatusInternalServerError
	}
}

func main() {
	port := os.Getenv("PORT")
	if port == "" {
		log.Fatal("$PORT must be set!")
	}
	if atk == "" {
		log.Fatal("$AUTHENTICATION_TOKEN must be set!")
	}
	if vtk == "" {
		log.Fatal("$VERIFICATION_TOKEN must be set!")
	}
	if appURL == "" {
		log.Fatal("$APP_URL must be set!")
	}
	u, err := url.Parse(appURL)
	if err != nil {
		log.Fatal("invalid $APP_URL %s", appURL)
	}
	icon, _ := url.Parse(iconPath)
	iconURL = u.ResolveReference(icon).String()
	meme, _ := url.Parse(memePath)
	memeURL = u.ResolveReference(meme).String()

	fs := http.FileServer(http.Dir("static"))
	http.Handle("/static/", http.StripPrefix("/static/", fs))
	http.HandleFunc("/slack", handleSlack)

	log.Fatal(http.ListenAndServe(":"+port, nil))
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
)

// Installer is used to figure out what is being requested and how to install
type Installer struct {
	Request string
	Spec    *Specification
}

/* Requests can be a number of things:
a URL, e.g. https://example.com/stowage/
a reference to a repo, e.g. myrepo\some-binary
a local file, e.g. ./somedir/some-binary.json
a docker hub reference, e.g. ealexhudson/stowage

If it doesn't look like a URL or repo reference, we try it as a local
file. If that doesn't work, we assume it's a Docker hub reference.

*/
func (i *Installer) setup() bool {
	name := i.Request

	if strings.Index(name, "://") > -1 {
		// this is a URL
		return i.loadSpecFromURL(name)
	}

	repoSep := strings.Index(name, ":")
	if repoSep > -1 {
		// this could be a repo reference
		repo := name[0:repoSep]
		name = name[repoSep+1:]

		if strings.Index(repo, "/") == -1 {
			// repo names cannot have slashes in them; this must be a
			// docker hub reference!
			return i.loadSpecFromRepo(repo, name)
		}
	}

	_ = i.loadSpecFromFie(name)

	if i.Spec == nil {
		spec := Specification{
			Name:    name,
			Image:   name,
			Command: "",
		}
		i.Spec = &spec
	}

	return true
}

func (i *Installer) loadSpecFromFie(path string) bool {
	store := createStorage()
	spec, err := store.loadSpecification(path)
	fmt.Println("Loading from file")
	if err != nil {
		fmt.Println("Failure!")
		return false
	}

	i.Spec = &spec
	fmt.Println("Success!")
	return true
}

func (i *Installer) loadSpecFromURL(url string) bool {
	var spec Specification

	response, err := http.Get(url)
	if err != nil {
		fmt.Println("Specfile missing from repository!")
		return false
	}
	fmt.Println(response.Body)
	buf, _ := ioutil.ReadAll(response.Body)
	json.Unmarshal(buf, &spec)

	i.Spec = &spec
	return true
}

func (i *Installer) loadSpecFromRepo(repoName string, name string) bool {
	store := createStorage()

	repo, err := store.loadRepositoryByName(repoName)
	if err != nil {
		fmt.Println("No such repository.")
		return false
	}

	urlForSpec := repo.getURLForSpec(name)

	return i.loadSpecFromURL(urlForSpec)
}

func (i *Installer) run() bool {
	store := createStorage()
	store.saveSpecification(i.Spec)

	binary := Binary{name: i.Spec.Name, spec: i.Spec}
	binary.install()

	fmt.Printf("%s installed\n", i.Spec.Name)
	return true
}
<commit_msg>Make install less verbose when using files.<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
)

// Installer is used to figure out what is being requested and how to install
type Installer struct {
	Request string
	Spec    *Specification
}

/* Requests can be a number of things:
a URL, e.g. https://example.com/stowage/
a reference to a repo, e.g. myrepo\some-binary
a local file, e.g. ./somedir/some-binary.json
a docker hub reference, e.g. ealexhudson/stowage

If it doesn't look like a URL or repo reference, we try it as a local
file. If that doesn't work, we assume it's a Docker hub reference.

*/
func (i *Installer) setup() bool {
	name := i.Request

	if strings.Index(name, "://") > -1 {
		// this is a URL
		return i.loadSpecFromURL(name)
	}

	repoSep := strings.Index(name, ":")
	if repoSep > -1 {
		// this could be a repo reference
		repo := name[0:repoSep]
		name = name[repoSep+1:]

		if strings.Index(repo, "/") == -1 {
			// repo names cannot have slashes in them; this must be a
			// docker hub reference!
			return i.loadSpecFromRepo(repo, name)
		}
	}

	_ = i.loadSpecFromFie(name)

	if i.Spec == nil {
		spec := Specification{
			Name:    name,
			Image:   name,
			Command: "",
		}
		i.Spec = &spec
	}

	return true
}

func (i *Installer) loadSpecFromFie(path string) bool {
	store := createStorage()
	spec, err := store.loadSpecification(path)
	if err != nil {
		return false
	}

	i.Spec = &spec
	return true
}

func (i *Installer) loadSpecFromURL(url string) bool {
	var spec Specification

	response, err := http.Get(url)
	if err != nil {
		fmt.Println("Specfile missing from repository!")
		return false
	}
	fmt.Println(response.Body)
	buf, _ := ioutil.ReadAll(response.Body)
	json.Unmarshal(buf, &spec)

	i.Spec = &spec
	return true
}

func (i *Installer) loadSpecFromRepo(repoName string, name string) bool {
	store := createStorage()

	repo, err := store.loadRepositoryByName(repoName)
	if err != nil {
		fmt.Println("No such repository.")
		return false
	}

	urlForSpec := repo.getURLForSpec(name)

	return i.loadSpecFromURL(urlForSpec)
}

func (i *Installer) run() bool {
	store := createStorage()
	store.saveSpecification(i.Spec)

	binary := Binary{name: i.Spec.Name, spec: i.Spec}
	binary.install()

	fmt.Printf("%s installed\n", i.Spec.Name)
	return true
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	fmtlog "log"
	"net/http"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"time"

	"github.com/cenk/backoff"
	"github.com/containous/flaeg"
	"github.com/containous/staert"
	"github.com/containous/traefik/acme"
	"github.com/containous/traefik/collector"
	"github.com/containous/traefik/configuration"
	"github.com/containous/traefik/job"
	"github.com/containous/traefik/log"
	"github.com/containous/traefik/provider/ecs"
	"github.com/containous/traefik/provider/kubernetes"
	"github.com/containous/traefik/safe"
	"github.com/containous/traefik/server"
	"github.com/containous/traefik/server/uuid"
	traefikTls "github.com/containous/traefik/tls"
	"github.com/containous/traefik/types"
	"github.com/containous/traefik/version"
	"github.com/coreos/go-systemd/daemon"
	"github.com/ogier/pflag"
	"github.com/sirupsen/logrus"
)

func main() {
	// traefik config inits
	traefikConfiguration := NewTraefikConfiguration()
	traefikPointersConfiguration := NewTraefikDefaultPointersConfiguration()

	// traefik Command init
	traefikCmd := &flaeg.Command{
		Name: "traefik",
		Description: `traefik is a modern HTTP reverse proxy and load balancer made to deploy microservices with ease.
Complete documentation is available at https://traefik.io`,
		Config:                traefikConfiguration,
		DefaultPointersConfig: traefikPointersConfiguration,
		Run: func() error {
			run(&traefikConfiguration.GlobalConfiguration, traefikConfiguration.ConfigFile)
			return nil
		},
	}

	// storeconfig Command init
	storeConfigCmd := newStoreConfigCmd(traefikConfiguration, traefikPointersConfiguration)

	// init flaeg source
	f := flaeg.New(traefikCmd, os.Args[1:])
	// add custom parsers
	f.AddParser(reflect.TypeOf(configuration.EntryPoints{}), &configuration.EntryPoints{})
	f.AddParser(reflect.TypeOf(configuration.DefaultEntryPoints{}), &configuration.DefaultEntryPoints{})
	f.AddParser(reflect.TypeOf(traefikTls.RootCAs{}), &traefikTls.RootCAs{})
	f.AddParser(reflect.TypeOf(types.Constraints{}), &types.Constraints{})
	f.AddParser(reflect.TypeOf(kubernetes.Namespaces{}), &kubernetes.Namespaces{})
	f.AddParser(reflect.TypeOf(ecs.Clusters{}), &ecs.Clusters{})
	f.AddParser(reflect.TypeOf([]acme.Domain{}), &acme.Domains{})
	f.AddParser(reflect.TypeOf(types.Buckets{}), &types.Buckets{})

	// add commands
	f.AddCommand(newVersionCmd())
	f.AddCommand(newBugCmd(traefikConfiguration, traefikPointersConfiguration))
	f.AddCommand(storeConfigCmd)
	f.AddCommand(newHealthCheckCmd(traefikConfiguration, traefikPointersConfiguration))

	usedCmd, err := f.GetCommand()
	if err != nil {
		fmtlog.Println(err)
		os.Exit(-1)
	}

	if _, err := f.Parse(usedCmd); err != nil {
		if err == pflag.ErrHelp {
			os.Exit(0)
		}
		fmtlog.Printf("Error parsing command: %s\n", err)
		os.Exit(-1)
	}

	// staert init
	s := staert.NewStaert(traefikCmd)
	// init TOML source
	toml := staert.NewTomlSource("traefik", []string{traefikConfiguration.ConfigFile, "/etc/traefik/", "$HOME/.traefik/", "."})

	// add sources to staert
	s.AddSource(toml)
	s.AddSource(f)
	if _, err := s.LoadConfig(); err != nil {
		fmtlog.Printf("Error reading TOML config file %s : %s\n", toml.ConfigFileUsed(), err)
		os.Exit(-1)
	}

	traefikConfiguration.ConfigFile = toml.ConfigFileUsed()

	kv, err := createKvSource(traefikConfiguration)
	if err != nil {
		fmtlog.Printf("Error creating kv store: %s\n", err)
		os.Exit(-1)
	}
	storeConfigCmd.Run = runStoreConfig(kv, traefikConfiguration)

	// if a KV Store is enable and no sub-command called in args
	if kv != nil && usedCmd == traefikCmd {
		if traefikConfiguration.Cluster == nil {
			traefikConfiguration.Cluster = &types.Cluster{Node: uuid.Get()}
		}
		if traefikConfiguration.Cluster.Store == nil {
			traefikConfiguration.Cluster.Store = &types.Store{Prefix: kv.Prefix, Store: kv.Store}
		}
		s.AddSource(kv)
		operation := func() error {
			_, err := s.LoadConfig()
			return err
		}
		notify := func(err error, time time.Duration) {
			log.Errorf("Load config error: %+v, retrying in %s", err, time)
		}
		err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify)
		if err != nil {
			fmtlog.Printf("Error loading configuration: %s\n", err)
			os.Exit(-1)
		}
	}

	if err := s.Run(); err != nil {
		fmtlog.Printf("Error running traefik: %s\n", err)
		os.Exit(-1)
	}

	os.Exit(0)
}

func run(globalConfiguration *configuration.GlobalConfiguration, configFile string) {
	configureLogging(globalConfiguration)

	if len(configFile) > 0 {
		log.Infof("Using TOML configuration file %s", configFile)
	}

	http.DefaultTransport.(*http.Transport).Proxy = http.ProxyFromEnvironment

	globalConfiguration.SetEffectiveConfiguration(configFile)
	globalConfiguration.ValidateConfiguration()

	jsonConf, _ := json.Marshal(globalConfiguration)
	log.Infof("Traefik version %s built on %s", version.Version, version.BuildDate)

	if globalConfiguration.CheckNewVersion {
		checkNewVersion()
	}

	stats(globalConfiguration)

	log.Debugf("Global configuration loaded %s", string(jsonConf))
	svr := server.NewServer(*globalConfiguration, configuration.NewProviderAggregator(globalConfiguration))
	svr.Start()
	defer svr.Close()

	sent, err := daemon.SdNotify(false, "READY=1")
	if !sent && err != nil {
		log.Error("Fail to notify", err)
	}

	t, err := daemon.SdWatchdogEnabled(false)
	if err != nil {
		log.Error("Problem with watchdog", err)
	} else if t != 0 {
		// Send a ping each half time given
		t = t / 2
		log.Info("Watchdog activated with timer each ", t)
		safe.Go(func() {
			tick := time.Tick(t)
			for range tick {
				_, errHealthCheck := healthCheck(*globalConfiguration)
				if globalConfiguration.Ping == nil || errHealthCheck == nil {
					if ok, _ := daemon.SdNotify(false, "WATCHDOG=1"); !ok {
						log.Error("Fail to tick watchdog")
					}
				} else {
					log.Error(errHealthCheck)
				}
			}
		})
	}

	svr.Wait()
	log.Info("Shutting down")
	logrus.Exit(0)
}

func configureLogging(globalConfiguration *configuration.GlobalConfiguration) {
	// configure default log flags
	fmtlog.SetFlags(fmtlog.Lshortfile | fmtlog.LstdFlags)

	if globalConfiguration.Debug {
		globalConfiguration.LogLevel = "DEBUG"
	}

	// configure log level
	level, err := logrus.ParseLevel(strings.ToLower(globalConfiguration.LogLevel))
	if err != nil {
		log.Error("Error getting level", err)
	}
	log.SetLevel(level)

	// configure log output file
	logFile := globalConfiguration.TraefikLogsFile
	if len(logFile) > 0 {
		log.Warn("top-level traefikLogsFile has been deprecated -- please use traefiklog.filepath")
	}
	if globalConfiguration.TraefikLog != nil && len(globalConfiguration.TraefikLog.FilePath) > 0 {
		logFile = globalConfiguration.TraefikLog.FilePath
	}

	// configure log format
	var formatter logrus.Formatter
	if globalConfiguration.TraefikLog != nil && globalConfiguration.TraefikLog.Format == "json" {
		formatter = &logrus.JSONFormatter{}
	} else {
		disableColors := len(logFile) > 0
		formatter = &logrus.TextFormatter{DisableColors: disableColors, FullTimestamp: true, DisableSorting: true}
	}
	log.SetFormatter(formatter)

	if len(logFile) > 0 {
		dir := filepath.Dir(logFile)

		if err := os.MkdirAll(dir, 0755); err != nil {
			log.Errorf("Failed to create log path %s: %s", dir, err)
		}

		err = log.OpenFile(logFile)
		logrus.RegisterExitHandler(func() {
			if err := log.CloseFile(); err != nil {
				log.Error("Error closing log", err)
			}
		})
		if err != nil {
			log.Error("Error opening file", err)
		}
	}
}

func checkNewVersion() {
	ticker := time.Tick(24 * time.Hour)
	safe.Go(func() {
		for time.Sleep(10 * time.Minute); ; <-ticker {
			version.CheckNewVersion()
		}
	})
}

func stats(globalConfiguration *configuration.GlobalConfiguration) {
	if globalConfiguration.SendAnonymousUsage {
		log.Info(`
Stats collection is enabled.
Many thanks for contributing to Traefik's improvement by allowing us to receive anonymous information from your configuration.
Help us improve Traefik by leaving this feature on :)
More details on: https://docs.traefik.io/basics/#collected-data
`)
		collect(globalConfiguration)
	} else {
		log.Info(`
Stats collection is disabled.
Help us improve Traefik by turning this feature on :)
More details on: https://docs.traefik.io/basics/#collected-data
`)
	}
}

func collect(globalConfiguration *configuration.GlobalConfiguration) {
	ticker := time.Tick(24 * time.Hour)
	safe.Go(func() {
		for time.Sleep(10 * time.Minute); ; <-ticker {
			if err := collector.Collect(globalConfiguration); err != nil {
				log.Debug(err)
			}
		}
	})
}
<commit_msg>refactor: use positive error code.<commit_after>package main

import (
	"encoding/json"
	fmtlog "log"
	"net/http"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"time"

	"github.com/cenk/backoff"
	"github.com/containous/flaeg"
	"github.com/containous/staert"
	"github.com/containous/traefik/acme"
	"github.com/containous/traefik/collector"
	"github.com/containous/traefik/configuration"
	"github.com/containous/traefik/job"
	"github.com/containous/traefik/log"
	"github.com/containous/traefik/provider/ecs"
	"github.com/containous/traefik/provider/kubernetes"
	"github.com/containous/traefik/safe"
	"github.com/containous/traefik/server"
	"github.com/containous/traefik/server/uuid"
	traefikTls "github.com/containous/traefik/tls"
	"github.com/containous/traefik/types"
	"github.com/containous/traefik/version"
	"github.com/coreos/go-systemd/daemon"
	"github.com/ogier/pflag"
	"github.com/sirupsen/logrus"
)

func main() {
	// traefik config inits
	traefikConfiguration := NewTraefikConfiguration()
	traefikPointersConfiguration := NewTraefikDefaultPointersConfiguration()

	// traefik Command init
	traefikCmd := &flaeg.Command{
		Name: "traefik",
		Description: `traefik is a modern HTTP reverse proxy and load balancer made to deploy microservices with ease.
Complete documentation is available at https://traefik.io`,
		Config:                traefikConfiguration,
		DefaultPointersConfig: traefikPointersConfiguration,
		Run: func() error {
			run(&traefikConfiguration.GlobalConfiguration, traefikConfiguration.ConfigFile)
			return nil
		},
	}

	// storeconfig Command init
	storeConfigCmd := newStoreConfigCmd(traefikConfiguration, traefikPointersConfiguration)

	// init flaeg source
	f := flaeg.New(traefikCmd, os.Args[1:])
	// add custom parsers
	f.AddParser(reflect.TypeOf(configuration.EntryPoints{}), &configuration.EntryPoints{})
	f.AddParser(reflect.TypeOf(configuration.DefaultEntryPoints{}), &configuration.DefaultEntryPoints{})
	f.AddParser(reflect.TypeOf(traefikTls.RootCAs{}), &traefikTls.RootCAs{})
	f.AddParser(reflect.TypeOf(types.Constraints{}), &types.Constraints{})
	f.AddParser(reflect.TypeOf(kubernetes.Namespaces{}), &kubernetes.Namespaces{})
	f.AddParser(reflect.TypeOf(ecs.Clusters{}), &ecs.Clusters{})
	f.AddParser(reflect.TypeOf([]acme.Domain{}), &acme.Domains{})
	f.AddParser(reflect.TypeOf(types.Buckets{}), &types.Buckets{})

	// add commands
	f.AddCommand(newVersionCmd())
	f.AddCommand(newBugCmd(traefikConfiguration, traefikPointersConfiguration))
	f.AddCommand(storeConfigCmd)
	f.AddCommand(newHealthCheckCmd(traefikConfiguration, traefikPointersConfiguration))

	usedCmd, err := f.GetCommand()
	if err != nil {
		fmtlog.Println(err)
		os.Exit(1)
	}

	if _, err := f.Parse(usedCmd); err != nil {
		if err == pflag.ErrHelp {
			os.Exit(0)
		}
		fmtlog.Printf("Error parsing command: %s\n", err)
		os.Exit(1)
	}

	// staert init
	s := staert.NewStaert(traefikCmd)
	// init TOML source
	toml := staert.NewTomlSource("traefik", []string{traefikConfiguration.ConfigFile, "/etc/traefik/", "$HOME/.traefik/", "."})

	// add sources to staert
	s.AddSource(toml)
	s.AddSource(f)
	if _, err := s.LoadConfig(); err != nil {
		fmtlog.Printf("Error reading TOML config file %s : %s\n", toml.ConfigFileUsed(), err)
		os.Exit(1)
	}

	traefikConfiguration.ConfigFile = toml.ConfigFileUsed()

	kv, err := createKvSource(traefikConfiguration)
	if err != nil {
		fmtlog.Printf("Error creating kv store: %s\n", err)
		os.Exit(1)
	}
	storeConfigCmd.Run = runStoreConfig(kv, traefikConfiguration)

	// if a KV Store is enable and no sub-command called in args
	if kv != nil && usedCmd == traefikCmd {
		if traefikConfiguration.Cluster == nil {
			traefikConfiguration.Cluster = &types.Cluster{Node: uuid.Get()}
		}
		if traefikConfiguration.Cluster.Store == nil {
			traefikConfiguration.Cluster.Store = &types.Store{Prefix: kv.Prefix, Store: kv.Store}
		}
		s.AddSource(kv)
		operation := func() error {
			_, err := s.LoadConfig()
			return err
		}
		notify := func(err error, time time.Duration) {
			log.Errorf("Load config error: %+v, retrying in %s", err, time)
		}
		err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify)
		if err != nil {
			fmtlog.Printf("Error loading configuration: %s\n", err)
			os.Exit(1)
		}
	}

	if err := s.Run(); err != nil {
		fmtlog.Printf("Error running traefik: %s\n", err)
		os.Exit(1)
	}

	os.Exit(0)
}

func run(globalConfiguration *configuration.GlobalConfiguration, configFile string) {
	configureLogging(globalConfiguration)

	if len(configFile) > 0 {
		log.Infof("Using TOML configuration file %s", configFile)
	}

	http.DefaultTransport.(*http.Transport).Proxy = http.ProxyFromEnvironment

	globalConfiguration.SetEffectiveConfiguration(configFile)
	globalConfiguration.ValidateConfiguration()

	jsonConf, _ := json.Marshal(globalConfiguration)
	log.Infof("Traefik version %s built on %s", version.Version, version.BuildDate)

	if globalConfiguration.CheckNewVersion {
		checkNewVersion()
	}

	stats(globalConfiguration)

	log.Debugf("Global configuration loaded %s", string(jsonConf))
	svr := server.NewServer(*globalConfiguration, configuration.NewProviderAggregator(globalConfiguration))
	svr.Start()
	defer svr.Close()

	sent, err := daemon.SdNotify(false, "READY=1")
	if !sent && err != nil {
		log.Error("Fail to notify", err)
	}

	t, err := daemon.SdWatchdogEnabled(false)
	if err != nil {
		log.Error("Problem with watchdog", err)
	} else if t != 0 {
		// Send a ping each half time given
		t = t / 2
		log.Info("Watchdog activated with timer each ", t)
		safe.Go(func() {
			tick := time.Tick(t)
			for range tick {
				_, errHealthCheck := healthCheck(*globalConfiguration)
				if globalConfiguration.Ping == nil || errHealthCheck == nil {
					if ok, _ := daemon.SdNotify(false, "WATCHDOG=1"); !ok {
						log.Error("Fail to tick watchdog")
					}
				} else {
					log.Error(errHealthCheck)
				}
			}
		})
	}

	svr.Wait()
	log.Info("Shutting down")
	logrus.Exit(0)
}

func configureLogging(globalConfiguration *configuration.GlobalConfiguration) {
	// configure default log flags
	fmtlog.SetFlags(fmtlog.Lshortfile | fmtlog.LstdFlags)

	if globalConfiguration.Debug {
		globalConfiguration.LogLevel = "DEBUG"
	}

	// configure log level
	level, err := logrus.ParseLevel(strings.ToLower(globalConfiguration.LogLevel))
	if err != nil {
		log.Error("Error getting level", err)
	}
	log.SetLevel(level)

	// configure log output file
	logFile := globalConfiguration.TraefikLogsFile
	if len(logFile) > 0 {
		log.Warn("top-level traefikLogsFile has been deprecated -- please use traefiklog.filepath")
	}
	if globalConfiguration.TraefikLog != nil && len(globalConfiguration.TraefikLog.FilePath) > 0 {
		logFile = globalConfiguration.TraefikLog.FilePath
	}

	// configure log format
	var formatter logrus.Formatter
	if globalConfiguration.TraefikLog != nil && globalConfiguration.TraefikLog.Format == "json" {
		formatter = &logrus.JSONFormatter{}
	} else {
		disableColors := len(logFile) > 0
		formatter = &logrus.TextFormatter{DisableColors: disableColors, FullTimestamp: true, DisableSorting: true}
	}
	log.SetFormatter(formatter)

	if len(logFile) > 0 {
		dir := filepath.Dir(logFile)

		if err := os.MkdirAll(dir, 0755); err != nil {
			log.Errorf("Failed to create log path %s: %s", dir, err)
		}

		err = log.OpenFile(logFile)
		logrus.RegisterExitHandler(func() {
			if err := log.CloseFile(); err != nil {
				log.Error("Error closing log", err)
			}
		})
		if err != nil {
			log.Error("Error opening file", err)
		}
	}
}

func checkNewVersion() {
	ticker := time.Tick(24 * time.Hour)
	safe.Go(func() {
		for time.Sleep(10 * time.Minute); ; <-ticker {
			version.CheckNewVersion()
		}
	})
}

func stats(globalConfiguration *configuration.GlobalConfiguration) {
	if globalConfiguration.SendAnonymousUsage {
		log.Info(`
Stats collection is enabled.
Many thanks for contributing to Traefik's improvement by allowing us to receive anonymous information from your configuration.
Help us improve Traefik by leaving this feature on :)
More details on: https://docs.traefik.io/basics/#collected-data
`)
		collect(globalConfiguration)
	} else {
		log.Info(`
Stats collection is disabled.
Help us improve Traefik by turning this feature on :)
More details on: https://docs.traefik.io/basics/#collected-data
`)
	}
}

func collect(globalConfiguration *configuration.GlobalConfiguration) {
	ticker := time.Tick(24 * time.Hour)
	safe.Go(func() {
		for time.Sleep(10 * time.Minute); ; <-ticker {
			if err := collector.Collect(globalConfiguration); err != nil {
				log.Debug(err)
			}
		}
	})
}
<|endoftext|>


<commit_before>// transcribe is a tool for transcribing audio files using Google Speech API. It
// is intended for bulk processing of large (> 1 min) audio files and automates
// GCS upload (and removal).
package main

import (
	"context"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"cloud.google.com/go/speech/apiv1"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/storage/v1"

	speechpb "google.golang.org/genproto/googleapis/cloud/speech/v1"
	"os/exec"
	"path"
	"sync"
)

var (
	project = flag.String("project", "", "GCP project to use. The project must have the Speech API enabled.")
	output  = flag.String("out", ".", "Directory to place output text files.")
	bucket  = flag.String("bucket", "", "Temporary GCS bucket to hold the audio files. If not provided, a new transient bucket will be created.")
	mono    = flag.Bool("mono", false, "Convert stereo audio file to mono (required if stereo).")
)

func init() {
	flag.Usage = func() {
		fmt.Fprint(os.Stderr, `
usage: transcribe [options] file [...]

Transcribe transcribes audio files using Google Speech API. It is intended
for bulk processing of large (> 1 min) audio files and automates GCS upload
(and removal). Supported format: wav 44.1kHz (stereo or mono).
Options:
`)
		flag.PrintDefaults()
	}
}

func main() {
	flag.Parse()

	files := flag.Args()
	if len(files) == 0 {
		fmt.Fprintln(os.Stderr, "transcribe: no files provided.")
		flag.Usage()
		os.Exit(1)
	}
	if *project == "" {
		fmt.Fprintln(os.Stderr, "transcribe: no project provided.")
		flag.Usage()
		os.Exit(2)
	}

	// (2) Create tmp location, if needed.

	cl, err := newStorageClient(context.Background())
	if err != nil {
		log.Fatalf("Failed to create GCS client: %v", err)
	}

	if *bucket == "" {
		*bucket = fmt.Sprintf("transcribe-%v", time.Now().UnixNano())

		if _, err := cl.Buckets.Insert(*project, &storage.Bucket{Name: *bucket}).Do(); err != nil {
			log.Fatalf("Failed to create tmp bucket %v: %v", *bucket, err)
		}
		log.Printf("Using temporary GCS bucket '%v'", *bucket)

		defer func() {
			if err := cl.Buckets.Delete(*bucket).Do(); err != nil {
				log.Printf("Failed to delete tmp bucket %v: %v", *bucket, err)
			}
		}()
	}

	scl, err := speech.NewClient(context.Background())
	if err != nil {
		log.Fatalf("Failed to create speech client: %v", err)
	}

	log.Printf("Transcribing %v files in parallel", len(files))

	// (3) Upload and transcribe the files in parallel

	var wg sync.WaitGroup
	for _, name := range files {
		wg.Add(1)
		go func(name string) {
			defer wg.Done()

			if !strings.HasSuffix(strings.ToLower(name), ".wav") {
				log.Printf("File %v is not a supported format. Ignoring.", name)
				return
			}

			out := filepath.Join(*output, filepath.Base(name)+".txt")
			if _, err := os.Stat(out); err == nil || !os.IsNotExist(err) {
				log.Printf("File %v already transcribed. Ignoring.", name)
				return
			}

			log.Printf("Transcribing %v ...", filepath.Base(name))

			// (a) If stereo, convert first to mono

			if *mono {
				tmp := filepath.Join(os.TempDir(), filepath.Base(name))

				out, err := exec.Command("sox", name, tmp, "remix", "1-2").CombinedOutput()
				if err != nil {
					log.Printf("Failed to convert %v to mono (err=%v): %v. Do you have sox installed?", name, err, string(out))
					return
				}
				defer os.Remove(tmp)

				name = tmp
			}

			// (b) Upload

			obj := path.Join("tmp/audio", strings.ToLower(filepath.Base(name)))
			fd, err := os.Open(name)
			if err != nil {
				log.Printf("Failed to read %v: %v", name, err)
				return
			}
			defer fd.Close()

			if _, err := cl.Objects.Insert(*bucket, &storage.Object{Name: obj}).Media(fd).Do(); err != nil {
				log.Printf("Failed to upload %v: %v", name, err)
				return
			}
			defer cl.Objects.Delete(*bucket, obj).Do()

			// (c) Transcribe

			req := &speechpb.LongRunningRecognizeRequest{
				Config: &speechpb.RecognitionConfig{
					Encoding:        speechpb.RecognitionConfig_LINEAR16,
					SampleRateHertz: 44100,
					LanguageCode:    "en-US",
				},
				Audio: &speechpb.RecognitionAudio{
					AudioSource: &speechpb.RecognitionAudio_Uri{Uri: fmt.Sprintf("gs://%v/%v", *bucket, obj)},
				},
			}

			op, err := scl.LongRunningRecognize(context.Background(), req)
			if err != nil {
				log.Printf("Failed to transcribe %v: %v", name, err)
				return
			}
			resp, err := op.Wait(context.Background())
			if err != nil {
				log.Printf("Failed to transcribe %v: %v", name, err)
				return
			}

			// TODO(herohde) 6/11/2017: Add simple post-processing.

			var phrases []string
			for _, result := range resp.Results {
				for _, alt := range result.Alternatives {
					// Add text, if low confidence?
					phrases = append(phrases, alt.Transcript)
				}
			}
			data := strings.Join(phrases, " ")

			// (d) Write output

			// if err := ioutil.WriteFile(base+".raw.txt", []byte(data), 0644); err != nil {
			//    log.Printf("Failed to write raw output: %v", err)
			// }

			// data = strings.Replace(data, "paragraph", "\n", -1)
			// data = strings.Replace(data, "Paragraph", "\n", -1)
			data = strings.Replace(data, "  ", " ", -1)

			if err := ioutil.WriteFile(out, []byte(data), 0644); err != nil {
				log.Printf("Failed to write output: %v", err)
			}

			log.Printf("Transcribed %v", filepath.Base(name))
		}(name)
	}
	wg.Wait()

	log.Print("Done")
}

func newStorageClient(ctx context.Context) (*storage.Service, error) {
	httpClient, err := google.DefaultClient(context.Background(), storage.CloudPlatformScope)
	if err != nil {
		return nil, err
	}
	return storage.New(httpClient)
}
<commit_msg>Remove space after newline<commit_after>// transcribe is a tool for transcribing audio files using Google Speech API. It
// is intended for bulk processing of large (> 1 min) audio files and automates
// GCS upload (and removal).
package main

import (
	"context"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"cloud.google.com/go/speech/apiv1"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/storage/v1"

	speechpb "google.golang.org/genproto/googleapis/cloud/speech/v1"
	"os/exec"
	"path"
	"sync"
)

var (
	project = flag.String("project", "", "GCP project to use. The project must have the Speech API enabled.")
	output  = flag.String("out", ".", "Directory to place output text files.")
	bucket  = flag.String("bucket", "", "Temporary GCS bucket to hold the audio files. If not provided, a new transient bucket will be created.")
	mono    = flag.Bool("mono", false, "Convert stereo audio file to mono (required if stereo).")
)

func init() {
	flag.Usage = func() {
		fmt.Fprint(os.Stderr, `
usage: transcribe [options] file [...]

Transcribe transcribes audio files using Google Speech API. It is intended
for bulk processing of large (> 1 min) audio files and automates GCS upload
(and removal). Supported format: wav 44.1kHz (stereo or mono).
Options:
`)
		flag.PrintDefaults()
	}
}

func main() {
	flag.Parse()

	files := flag.Args()
	if len(files) == 0 {
		fmt.Fprintln(os.Stderr, "transcribe: no files provided.")
		flag.Usage()
		os.Exit(1)
	}
	if *project == "" {
		fmt.Fprintln(os.Stderr, "transcribe: no project provided.")
		flag.Usage()
		os.Exit(2)
	}

	// (2) Create tmp location, if needed.

	cl, err := newStorageClient(context.Background())
	if err != nil {
		log.Fatalf("Failed to create GCS client: %v", err)
	}

	if *bucket == "" {
		*bucket = fmt.Sprintf("transcribe-%v", time.Now().UnixNano())

		if _, err := cl.Buckets.Insert(*project, &storage.Bucket{Name: *bucket}).Do(); err != nil {
			log.Fatalf("Failed to create tmp bucket %v: %v", *bucket, err)
		}
		log.Printf("Using temporary GCS bucket '%v'", *bucket)

		defer func() {
			if err := cl.Buckets.Delete(*bucket).Do(); err != nil {
				log.Printf("Failed to delete tmp bucket %v: %v", *bucket, err)
			}
		}()
	}

	scl, err := speech.NewClient(context.Background())
	if err != nil {
		log.Fatalf("Failed to create speech client: %v", err)
	}

	log.Printf("Transcribing %v files in parallel", len(files))

	// (3) Upload and transcribe the files in parallel

	var wg sync.WaitGroup
	for _, name := range files {
		wg.Add(1)
		go func(name string) {
			defer wg.Done()

			if !strings.HasSuffix(strings.ToLower(name), ".wav") {
				log.Printf("File %v is not a supported format. Ignoring.", name)
				return
			}

			out := filepath.Join(*output, filepath.Base(name)+".txt")
			if _, err := os.Stat(out); err == nil || !os.IsNotExist(err) {
				log.Printf("File %v already transcribed. Ignoring.", name)
				return
			}

			log.Printf("Transcribing %v ...", filepath.Base(name))

			// (a) If stereo, convert first to mono

			if *mono {
				tmp := filepath.Join(os.TempDir(), filepath.Base(name))

				out, err := exec.Command("sox", name, tmp, "remix", "1-2").CombinedOutput()
				if err != nil {
					log.Printf("Failed to convert %v to mono (err=%v): %v. Do you have sox installed?", name, err, string(out))
					return
				}
				defer os.Remove(tmp)

				name = tmp
			}

			// (b) Upload

			obj := path.Join("tmp/audio", strings.ToLower(filepath.Base(name)))
			fd, err := os.Open(name)
			if err != nil {
				log.Printf("Failed to read %v: %v", name, err)
				return
			}
			defer fd.Close()

			if _, err := cl.Objects.Insert(*bucket, &storage.Object{Name: obj}).Media(fd).Do(); err != nil {
				log.Printf("Failed to upload %v: %v", name, err)
				return
			}
			defer cl.Objects.Delete(*bucket, obj).Do()

			// (c) Transcribe

			req := &speechpb.LongRunningRecognizeRequest{
				Config: &speechpb.RecognitionConfig{
					Encoding:        speechpb.RecognitionConfig_LINEAR16,
					SampleRateHertz: 44100,
					LanguageCode:    "en-US",
				},
				Audio: &speechpb.RecognitionAudio{
					AudioSource: &speechpb.RecognitionAudio_Uri{Uri: fmt.Sprintf("gs://%v/%v", *bucket, obj)},
				},
			}

			op, err := scl.LongRunningRecognize(context.Background(), req)
			if err != nil {
				log.Printf("Failed to transcribe %v: %v", name, err)
				return
			}
			resp, err := op.Wait(context.Background())
			if err != nil {
				log.Printf("Failed to transcribe %v: %v", name, err)
				return
			}

			var phrases []string
			for _, result := range resp.Results {
				for _, alt := range result.Alternatives {
					// Add text, if low confidence?
					phrases = append(phrases, alt.Transcript)
				}
			}
			data := strings.Join(phrases, " ")

			// (d) Write output

			// TODO(herohde) 6/11/2017: Add simple post-processing.

			data = strings.Replace(data, "  ", " ", -1)
			data = strings.Replace(data, "\n ", "\n", -1)

			if err := ioutil.WriteFile(out, []byte(data), 0644); err != nil {
				log.Printf("Failed to write output: %v", err)
			}

			log.Printf("Transcribed %v", filepath.Base(name))
		}(name)
	}
	wg.Wait()

	log.Print("Done")
}

func newStorageClient(ctx context.Context) (*storage.Service, error) {
	httpClient, err := google.DefaultClient(context.Background(), storage.CloudPlatformScope)
	if err != nil {
		return nil, err
	}
	return storage.New(httpClient)
}
<|endoftext|>