<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sync

import (
	"sync/atomic"
	"unsafe"
)

// A WaitGroup waits for a collection of goroutines to finish.
// The main goroutine calls Add to set the number of
// goroutines to wait for.  Then each of the goroutines
// runs and calls Done when finished.  At the same time,
// Wait can be used to block until all goroutines have finished.
type WaitGroup struct {
	m       Mutex
	counter int32
	waiters int32
	sema    *uint32
}

// WaitGroup creates a new semaphore each time the old semaphore
// is released. This is to avoid the following race:
//
// G1: Add(1)
// G1: go G2()
// G1: Wait() // Context switch after Unlock() and before Semacquire().
// G2: Done() // Release semaphore: sema == 1, waiters == 0. G1 doesn't run yet.
// G3: Wait() // Finds counter == 0, waiters == 0, doesn't block.
// G3: Add(1) // Makes counter == 1, waiters == 0.
// G3: go G4()
// G3: Wait() // G1 still hasn't run, G3 finds sema == 1, unblocked! Bug.

// Add adds delta, which may be negative, to the WaitGroup counter.
// If the counter becomes zero, all goroutines blocked on Wait are released.
// If the counter goes negative, Add panics.
//
// Note that calls with positive delta must happen before the call to Wait,
// or else Wait may wait for too small a group. Typically this means the calls
// to Add should execute before the statement creating the goroutine or
// other event to be waited for. See the WaitGroup example.
func (wg *WaitGroup) Add(delta int) {
	if raceenabled {
		_ = wg.m.state // trigger nil deref early
		if delta < 0 {
			// Synchronize decrements with Wait.
			raceReleaseMerge(unsafe.Pointer(wg))
		}
		raceDisable()
		defer raceEnable()
	}
	v := atomic.AddInt32(&wg.counter, int32(delta))
	if raceenabled {
		if delta > 0 && v == int32(delta) {
			// The first increment must be synchronized with Wait.
			// Need to model this as a read, because there can be
			// several concurrent wg.counter transitions from 0.
			raceRead(unsafe.Pointer(&wg.sema))
		}
	}
	if v < 0 {
		panic("sync: negative WaitGroup counter")
	}
	if v > 0 || atomic.LoadInt32(&wg.waiters) == 0 {
		return
	}
	wg.m.Lock()
	if atomic.LoadInt32(&wg.counter) == 0 {
		for i := int32(0); i < wg.waiters; i++ {
			runtime_Semrelease(wg.sema)
		}
		wg.waiters = 0
		wg.sema = nil
	}
	wg.m.Unlock()
}

// Done decrements the WaitGroup counter.
func (wg *WaitGroup) Done() {
	wg.Add(-1)
}

// Wait blocks until the WaitGroup counter is zero.
func (wg *WaitGroup) Wait() {
	if raceenabled {
		_ = wg.m.state // trigger nil deref early
		raceDisable()
	}
	if atomic.LoadInt32(&wg.counter) == 0 {
		if raceenabled {
			raceEnable()
			raceAcquire(unsafe.Pointer(wg))
		}
		return
	}
	wg.m.Lock()
	w := atomic.AddInt32(&wg.waiters, 1)
	// This code is racing with the unlocked path in Add above.
	// The code above modifies counter and then reads waiters.
	// We must modify waiters and then read counter (the opposite order)
	// to avoid missing an Add.
	if atomic.LoadInt32(&wg.counter) == 0 {
		atomic.AddInt32(&wg.waiters, -1)
		if raceenabled {
			raceEnable()
			raceAcquire(unsafe.Pointer(wg))
			raceDisable()
		}
		wg.m.Unlock()
		if raceenabled {
			raceEnable()
		}
		return
	}
	if raceenabled && w == 1 {
		// Wait must be synchronized with the first Add.
		// Need to model this is as a write to race with the read in Add.
		// As a consequence, can do the write only for the first waiter,
		// otherwise concurrent Waits will race with each other.
		raceWrite(unsafe.Pointer(&wg.sema))
	}
	if wg.sema == nil {
		wg.sema = new(uint32)
	}
	s := wg.sema
	wg.m.Unlock()
	runtime_Semacquire(s)
	if raceenabled {
		raceEnable()
		raceAcquire(unsafe.Pointer(wg))
	}
}
<commit_msg>sync: be more explicit in WaitGroup docs<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sync

import (
	"sync/atomic"
	"unsafe"
)

// A WaitGroup waits for a collection of goroutines to finish.
// The main goroutine calls Add to set the number of
// goroutines to wait for.  Then each of the goroutines
// runs and calls Done when finished.  At the same time,
// Wait can be used to block until all goroutines have finished.
type WaitGroup struct {
	m       Mutex
	counter int32
	waiters int32
	sema    *uint32
}

// WaitGroup creates a new semaphore each time the old semaphore
// is released. This is to avoid the following race:
//
// G1: Add(1)
// G1: go G2()
// G1: Wait() // Context switch after Unlock() and before Semacquire().
// G2: Done() // Release semaphore: sema == 1, waiters == 0. G1 doesn't run yet.
// G3: Wait() // Finds counter == 0, waiters == 0, doesn't block.
// G3: Add(1) // Makes counter == 1, waiters == 0.
// G3: go G4()
// G3: Wait() // G1 still hasn't run, G3 finds sema == 1, unblocked! Bug.

// Add adds delta, which may be negative, to the WaitGroup counter.
// If the counter becomes zero, all goroutines blocked on Wait are released.
// If the counter goes negative, Add panics.
//
// Note that calls with a positive delta that occur when the counter is zero
// must happen before a Wait. Calls with a negative delta, or calls with a
// positive delta that start when the counter is greater than zero, may happen
// at any time.
// Typically this means the calls to Add should execute before the statement
// creating the goroutine or other event to be waited for.
// See the WaitGroup example.
func (wg *WaitGroup) Add(delta int) {
	if raceenabled {
		_ = wg.m.state // trigger nil deref early
		if delta < 0 {
			// Synchronize decrements with Wait.
			raceReleaseMerge(unsafe.Pointer(wg))
		}
		raceDisable()
		defer raceEnable()
	}
	v := atomic.AddInt32(&wg.counter, int32(delta))
	if raceenabled {
		if delta > 0 && v == int32(delta) {
			// The first increment must be synchronized with Wait.
			// Need to model this as a read, because there can be
			// several concurrent wg.counter transitions from 0.
			raceRead(unsafe.Pointer(&wg.sema))
		}
	}
	if v < 0 {
		panic("sync: negative WaitGroup counter")
	}
	if v > 0 || atomic.LoadInt32(&wg.waiters) == 0 {
		return
	}
	wg.m.Lock()
	if atomic.LoadInt32(&wg.counter) == 0 {
		for i := int32(0); i < wg.waiters; i++ {
			runtime_Semrelease(wg.sema)
		}
		wg.waiters = 0
		wg.sema = nil
	}
	wg.m.Unlock()
}

// Done decrements the WaitGroup counter.
func (wg *WaitGroup) Done() {
	wg.Add(-1)
}

// Wait blocks until the WaitGroup counter is zero.
func (wg *WaitGroup) Wait() {
	if raceenabled {
		_ = wg.m.state // trigger nil deref early
		raceDisable()
	}
	if atomic.LoadInt32(&wg.counter) == 0 {
		if raceenabled {
			raceEnable()
			raceAcquire(unsafe.Pointer(wg))
		}
		return
	}
	wg.m.Lock()
	w := atomic.AddInt32(&wg.waiters, 1)
	// This code is racing with the unlocked path in Add above.
	// The code above modifies counter and then reads waiters.
	// We must modify waiters and then read counter (the opposite order)
	// to avoid missing an Add.
	if atomic.LoadInt32(&wg.counter) == 0 {
		atomic.AddInt32(&wg.waiters, -1)
		if raceenabled {
			raceEnable()
			raceAcquire(unsafe.Pointer(wg))
			raceDisable()
		}
		wg.m.Unlock()
		if raceenabled {
			raceEnable()
		}
		return
	}
	if raceenabled && w == 1 {
		// Wait must be synchronized with the first Add.
		// Need to model this is as a write to race with the read in Add.
		// As a consequence, can do the write only for the first waiter,
		// otherwise concurrent Waits will race with each other.
		raceWrite(unsafe.Pointer(&wg.sema))
	}
	if wg.sema == nil {
		wg.sema = new(uint32)
	}
	s := wg.sema
	wg.m.Unlock()
	runtime_Semacquire(s)
	if raceenabled {
		raceEnable()
		raceAcquire(unsafe.Pointer(wg))
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package testing

import (
	"runtime"
)

// AllocsPerRun returns the average number of allocations during calls to f.
//
// To compute the number of allocations, the function will first be run once as
// a warm-up.  The average number of allocations over the specified number of
// runs will then be measured and returned.
//
// AllocsPerRun sets GOMAXPROCS to 1 during its measurement and will restore
// it before returning.
func AllocsPerRun(runs int, f func()) (avg float64) {
	defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(1))

	// Warm up the function
	f()

	// Measure the starting statistics
	var memstats runtime.MemStats
	runtime.ReadMemStats(&memstats)
	mallocs := 0 - memstats.Mallocs

	// Run the function the specified number of times
	for i := 0; i < runs; i++ {
		f()
	}

	// Read the final statistics
	runtime.ReadMemStats(&memstats)
	mallocs += memstats.Mallocs

	// Average the mallocs over the runs (not counting the warm-up)
	return float64(mallocs) / float64(runs)
}
<commit_msg>testing: quantize AllocsPerRun As the code now says:<commit_after>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package testing

import (
	"runtime"
)

// AllocsPerRun returns the average number of allocations during calls to f.
// Although the return value has type float64, it will always be an integral value.
//
// To compute the number of allocations, the function will first be run once as
// a warm-up.  The average number of allocations over the specified number of
// runs will then be measured and returned.
//
// AllocsPerRun sets GOMAXPROCS to 1 during its measurement and will restore
// it before returning.
func AllocsPerRun(runs int, f func()) (avg float64) {
	defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(1))

	// Warm up the function
	f()

	// Measure the starting statistics
	var memstats runtime.MemStats
	runtime.ReadMemStats(&memstats)
	mallocs := 0 - memstats.Mallocs

	// Run the function the specified number of times
	for i := 0; i < runs; i++ {
		f()
	}

	// Read the final statistics
	runtime.ReadMemStats(&memstats)
	mallocs += memstats.Mallocs

	// Average the mallocs over the runs (not counting the warm-up).
	// We are forced to return a float64 because the API is silly, but do
	// the division as integers so we can ask if AllocsPerRun()==1
	// instead of AllocsPerRun()<2.
	return float64(mallocs / uint64(runs))
}
<|endoftext|>


<commit_before>// +build linux

package main

import (
	"fmt"
	"os"
	"os/exec"
	"os/user"
	"runtime"
	"strconv"
	"syscall"

	"github.com/containers/buildah/unshare"
	"github.com/containers/buildah/util"
	"github.com/opencontainers/runtime-spec/specs-go"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"github.com/syndtr/gocapability/capability"
	"github.com/urfave/cli"
)

const (
	// startedInUserNS is an environment variable that, if set, means that we shouldn't try
	// to create and enter a new user namespace and then re-exec ourselves.
	startedInUserNS = "_BUILDAH_STARTED_IN_USERNS"
)

var (
	unshareDescription = "Runs a command in a modified user namespace"
	unshareCommand     = cli.Command{
		Name:           "unshare",
		Usage:          "Run a command in a modified user namespace",
		Description:    unshareDescription,
		Action:         unshareCmd,
		ArgsUsage:      "[COMMAND [ARGS [...]]]",
		SkipArgReorder: true,
	}
)

type runnable interface {
	Run() error
}

func bailOnError(err error, format string, a ...interface{}) {
	if err != nil {
		if format != "" {
			logrus.Errorf("%s: %v", fmt.Sprintf(format, a...), err)
		} else {
			logrus.Errorf("%v", err)
		}
		cli.OsExiter(1)
	}
}

func maybeReexecUsingUserNamespace(c *cli.Context, evenForRoot bool) {
	// If we've already been through this once, no need to try again.
	if os.Getenv(startedInUserNS) != "" {
		return
	}

	// If this is one of the commands that doesn't need this indirection, skip it.
	if c.NArg() == 0 {
		return
	}
	switch c.Args()[0] {
	case "help", "version":
		return
	}

	// Figure out who we are.
	me, err := user.Current()
	bailOnError(err, "error determining current user")
	uidNum, err := strconv.ParseUint(me.Uid, 10, 32)
	bailOnError(err, "error parsing current UID %s", me.Uid)
	gidNum, err := strconv.ParseUint(me.Gid, 10, 32)
	bailOnError(err, "error parsing current GID %s", me.Gid)

	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	// ID mappings to use to reexec ourselves.
	var uidmap, gidmap []specs.LinuxIDMapping
	if uidNum != 0 || evenForRoot {
		// Read the set of ID mappings that we're allowed to use.  Each
		// range in /etc/subuid and /etc/subgid file is a starting host
		// ID and a range size.
		uidmap, gidmap, err = util.GetSubIDMappings(me.Username, me.Username)
		bailOnError(err, "error reading allowed ID mappings")
		if len(uidmap) == 0 {
			logrus.Warnf("Found no UID ranges set aside for user %q in /etc/subuid.", me.Username)
		}
		if len(gidmap) == 0 {
			logrus.Warnf("Found no GID ranges set aside for user %q in /etc/subgid.", me.Username)
		}
		// Map our UID and GID, then the subuid and subgid ranges,
		// consecutively, starting at 0, to get the mappings to use for
		// a copy of ourselves.
		uidmap = append([]specs.LinuxIDMapping{{HostID: uint32(uidNum), ContainerID: 0, Size: 1}}, uidmap...)
		gidmap = append([]specs.LinuxIDMapping{{HostID: uint32(gidNum), ContainerID: 0, Size: 1}}, gidmap...)
		var rangeStart uint32
		for i := range uidmap {
			uidmap[i].ContainerID = rangeStart
			rangeStart += uidmap[i].Size
		}
		rangeStart = 0
		for i := range gidmap {
			gidmap[i].ContainerID = rangeStart
			rangeStart += gidmap[i].Size
		}
	} else {
		// If we have CAP_SYS_ADMIN, then we don't need to create a new namespace in order to be able
		// to use unshare(), so don't bother creating a new user namespace at this point.
		capabilities, err := capability.NewPid(0)
		bailOnError(err, "error reading the current capabilities sets")
		if capabilities.Get(capability.EFFECTIVE, capability.CAP_SYS_ADMIN) {
			return
		}
		// Read the set of ID mappings that we're currently using.
		uidmap, gidmap, err = util.GetHostIDMappings("")
		bailOnError(err, "error reading current ID mappings")
		// Just reuse them.
		for i := range uidmap {
			uidmap[i].HostID = uidmap[i].ContainerID
		}
		for i := range gidmap {
			gidmap[i].HostID = gidmap[i].ContainerID
		}
	}

	var moreArgs []string
	// Add args to change the global defaults.
	if uidNum != 0 {
		if !c.GlobalIsSet("storage-driver") || !c.GlobalIsSet("root") || !c.GlobalIsSet("runroot") {
			logrus.Infof("Running without privileges, assuming arguments:")
			if !c.GlobalIsSet("storage-driver") {
				defaultStorageDriver := "vfs"
				logrus.Infof(" --storage-driver %q", defaultStorageDriver)
				moreArgs = append(moreArgs, "--storage-driver", defaultStorageDriver)
			}
			if !c.GlobalIsSet("root") {
				defaultRoot, err := util.UnsharedRootPath(me.HomeDir)
				bailOnError(err, "")
				logrus.Infof(" --root %q", defaultRoot)
				moreArgs = append(moreArgs, "--root", defaultRoot)
			}
			if !c.GlobalIsSet("runroot") {
				defaultRunroot, err := util.UnsharedRunrootPath(me.Uid)
				bailOnError(err, "")
				logrus.Infof(" --runroot %q", defaultRunroot)
				moreArgs = append(moreArgs, "--runroot", defaultRunroot)
			}
		}
	}

	// Unlike most uses of reexec or unshare, we're using a name that
	// _won't_ be recognized as a registered reexec handler, since we
	// _want_ to fall through reexec.Init() to the normal main().
	cmd := unshare.Command(append(append([]string{"buildah-in-a-user-namespace"}, moreArgs...), os.Args[1:]...)...)

	// If, somehow, we don't become UID 0 in our child, indicate that the child shouldn't try again.
	err = os.Setenv(startedInUserNS, "1")
	bailOnError(err, "error setting %s=1 in environment", startedInUserNS)

	// Set the default isolation type to use the "rootless" method.
	if _, present := os.LookupEnv("BUILDAH_ISOLATION"); !present {
		if err = os.Setenv("BUILDAH_ISOLATION", "rootless"); err != nil {
			logrus.Errorf("error setting BUILDAH_ISOLATION=rootless in environment: %v", err)
			os.Exit(1)
		}
	}

	// Reuse our stdio.
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	// Set up a new user namespace with the ID mapping.
	cmd.UnshareFlags = syscall.CLONE_NEWUSER
	cmd.UseNewuidmap = uidNum != 0
	cmd.UidMappings = uidmap
	cmd.UseNewgidmap = uidNum != 0
	cmd.GidMappings = gidmap
	cmd.GidMappingsEnableSetgroups = true

	// Finish up.
	logrus.Debugf("running %+v with environment %+v, UID map %+v, and GID map %+v", cmd.Cmd.Args, os.Environ(), cmd.UidMappings, cmd.GidMappings)
	execRunnable(cmd)
}

// execRunnable runs the specified unshare command, captures its exit status,
// and exits with the same status.
func execRunnable(cmd runnable) {
	if err := cmd.Run(); err != nil {
		if exitError, ok := errors.Cause(err).(*exec.ExitError); ok {
			if exitError.ProcessState.Exited() {
				if waitStatus, ok := exitError.ProcessState.Sys().(syscall.WaitStatus); ok {
					if waitStatus.Exited() {
						logrus.Errorf("%v", exitError)
						os.Exit(waitStatus.ExitStatus())
					}
					if waitStatus.Signaled() {
						logrus.Errorf("%v", exitError)
						os.Exit(int(waitStatus.Signal()) + 128)
					}
				}
			}
		}
		logrus.Errorf("%v", err)
		logrus.Errorf("(unable to determine exit status)")
		os.Exit(1)
	}
	os.Exit(0)
}

// unshareCmd execs whatever using the ID mappings that we want to use for ourselves
func unshareCmd(c *cli.Context) error {
	// force reexec using the configured ID mappings
	maybeReexecUsingUserNamespace(c, true)
	// exec the specified command, if there is one
	args := c.Args()
	if len(args) < 1 {
		// try to exec the shell, if one's set
		shell, shellSet := os.LookupEnv("SHELL")
		if !shellSet {
			logrus.Errorf("no command specified")
			os.Exit(1)
		}
		args = []string{shell}
	}
	cmd := exec.Command(args[0], args[1:]...)
	cmd.Env = append(os.Environ(), "USER=root", "USERNAME=root", "GROUP=root", "LOGNAME=root", "UID=0", "GID=0")
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	execRunnable(cmd)
	os.Exit(1)
	return nil
}
<commit_msg>unshare: create a new mount namespace<commit_after>// +build linux

package main

import (
	"fmt"
	"os"
	"os/exec"
	"os/user"
	"runtime"
	"strconv"
	"syscall"

	"github.com/containers/buildah/unshare"
	"github.com/containers/buildah/util"
	"github.com/opencontainers/runtime-spec/specs-go"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"github.com/syndtr/gocapability/capability"
	"github.com/urfave/cli"
)

const (
	// startedInUserNS is an environment variable that, if set, means that we shouldn't try
	// to create and enter a new user namespace and then re-exec ourselves.
	startedInUserNS = "_BUILDAH_STARTED_IN_USERNS"
)

var (
	unshareDescription = "Runs a command in a modified user namespace"
	unshareCommand     = cli.Command{
		Name:           "unshare",
		Usage:          "Run a command in a modified user namespace",
		Description:    unshareDescription,
		Action:         unshareCmd,
		ArgsUsage:      "[COMMAND [ARGS [...]]]",
		SkipArgReorder: true,
	}
)

type runnable interface {
	Run() error
}

func bailOnError(err error, format string, a ...interface{}) {
	if err != nil {
		if format != "" {
			logrus.Errorf("%s: %v", fmt.Sprintf(format, a...), err)
		} else {
			logrus.Errorf("%v", err)
		}
		cli.OsExiter(1)
	}
}

func maybeReexecUsingUserNamespace(c *cli.Context, evenForRoot bool) {
	// If we've already been through this once, no need to try again.
	if os.Getenv(startedInUserNS) != "" {
		return
	}

	// If this is one of the commands that doesn't need this indirection, skip it.
	if c.NArg() == 0 {
		return
	}
	switch c.Args()[0] {
	case "help", "version":
		return
	}

	// Figure out who we are.
	me, err := user.Current()
	bailOnError(err, "error determining current user")
	uidNum, err := strconv.ParseUint(me.Uid, 10, 32)
	bailOnError(err, "error parsing current UID %s", me.Uid)
	gidNum, err := strconv.ParseUint(me.Gid, 10, 32)
	bailOnError(err, "error parsing current GID %s", me.Gid)

	runtime.LockOSThread()
	defer runtime.UnlockOSThread()

	// ID mappings to use to reexec ourselves.
	var uidmap, gidmap []specs.LinuxIDMapping
	if uidNum != 0 || evenForRoot {
		// Read the set of ID mappings that we're allowed to use.  Each
		// range in /etc/subuid and /etc/subgid file is a starting host
		// ID and a range size.
		uidmap, gidmap, err = util.GetSubIDMappings(me.Username, me.Username)
		bailOnError(err, "error reading allowed ID mappings")
		if len(uidmap) == 0 {
			logrus.Warnf("Found no UID ranges set aside for user %q in /etc/subuid.", me.Username)
		}
		if len(gidmap) == 0 {
			logrus.Warnf("Found no GID ranges set aside for user %q in /etc/subgid.", me.Username)
		}
		// Map our UID and GID, then the subuid and subgid ranges,
		// consecutively, starting at 0, to get the mappings to use for
		// a copy of ourselves.
		uidmap = append([]specs.LinuxIDMapping{{HostID: uint32(uidNum), ContainerID: 0, Size: 1}}, uidmap...)
		gidmap = append([]specs.LinuxIDMapping{{HostID: uint32(gidNum), ContainerID: 0, Size: 1}}, gidmap...)
		var rangeStart uint32
		for i := range uidmap {
			uidmap[i].ContainerID = rangeStart
			rangeStart += uidmap[i].Size
		}
		rangeStart = 0
		for i := range gidmap {
			gidmap[i].ContainerID = rangeStart
			rangeStart += gidmap[i].Size
		}
	} else {
		// If we have CAP_SYS_ADMIN, then we don't need to create a new namespace in order to be able
		// to use unshare(), so don't bother creating a new user namespace at this point.
		capabilities, err := capability.NewPid(0)
		bailOnError(err, "error reading the current capabilities sets")
		if capabilities.Get(capability.EFFECTIVE, capability.CAP_SYS_ADMIN) {
			return
		}
		// Read the set of ID mappings that we're currently using.
		uidmap, gidmap, err = util.GetHostIDMappings("")
		bailOnError(err, "error reading current ID mappings")
		// Just reuse them.
		for i := range uidmap {
			uidmap[i].HostID = uidmap[i].ContainerID
		}
		for i := range gidmap {
			gidmap[i].HostID = gidmap[i].ContainerID
		}
	}

	var moreArgs []string
	// Add args to change the global defaults.
	if uidNum != 0 {
		if !c.GlobalIsSet("storage-driver") || !c.GlobalIsSet("root") || !c.GlobalIsSet("runroot") {
			logrus.Infof("Running without privileges, assuming arguments:")
			if !c.GlobalIsSet("storage-driver") {
				defaultStorageDriver := "vfs"
				logrus.Infof(" --storage-driver %q", defaultStorageDriver)
				moreArgs = append(moreArgs, "--storage-driver", defaultStorageDriver)
			}
			if !c.GlobalIsSet("root") {
				defaultRoot, err := util.UnsharedRootPath(me.HomeDir)
				bailOnError(err, "")
				logrus.Infof(" --root %q", defaultRoot)
				moreArgs = append(moreArgs, "--root", defaultRoot)
			}
			if !c.GlobalIsSet("runroot") {
				defaultRunroot, err := util.UnsharedRunrootPath(me.Uid)
				bailOnError(err, "")
				logrus.Infof(" --runroot %q", defaultRunroot)
				moreArgs = append(moreArgs, "--runroot", defaultRunroot)
			}
		}
	}

	// Unlike most uses of reexec or unshare, we're using a name that
	// _won't_ be recognized as a registered reexec handler, since we
	// _want_ to fall through reexec.Init() to the normal main().
	cmd := unshare.Command(append(append([]string{"buildah-in-a-user-namespace"}, moreArgs...), os.Args[1:]...)...)

	// If, somehow, we don't become UID 0 in our child, indicate that the child shouldn't try again.
	err = os.Setenv(startedInUserNS, "1")
	bailOnError(err, "error setting %s=1 in environment", startedInUserNS)

	// Set the default isolation type to use the "rootless" method.
	if _, present := os.LookupEnv("BUILDAH_ISOLATION"); !present {
		if err = os.Setenv("BUILDAH_ISOLATION", "rootless"); err != nil {
			logrus.Errorf("error setting BUILDAH_ISOLATION=rootless in environment: %v", err)
			os.Exit(1)
		}
	}

	// Reuse our stdio.
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	// Set up a new user namespace with the ID mapping.
	cmd.UnshareFlags = syscall.CLONE_NEWUSER|syscall.CLONE_NEWNS
	cmd.UseNewuidmap = uidNum != 0
	cmd.UidMappings = uidmap
	cmd.UseNewgidmap = uidNum != 0
	cmd.GidMappings = gidmap
	cmd.GidMappingsEnableSetgroups = true

	// Finish up.
	logrus.Debugf("running %+v with environment %+v, UID map %+v, and GID map %+v", cmd.Cmd.Args, os.Environ(), cmd.UidMappings, cmd.GidMappings)
	execRunnable(cmd)
}

// execRunnable runs the specified unshare command, captures its exit status,
// and exits with the same status.
func execRunnable(cmd runnable) {
	if err := cmd.Run(); err != nil {
		if exitError, ok := errors.Cause(err).(*exec.ExitError); ok {
			if exitError.ProcessState.Exited() {
				if waitStatus, ok := exitError.ProcessState.Sys().(syscall.WaitStatus); ok {
					if waitStatus.Exited() {
						logrus.Errorf("%v", exitError)
						os.Exit(waitStatus.ExitStatus())
					}
					if waitStatus.Signaled() {
						logrus.Errorf("%v", exitError)
						os.Exit(int(waitStatus.Signal()) + 128)
					}
				}
			}
		}
		logrus.Errorf("%v", err)
		logrus.Errorf("(unable to determine exit status)")
		os.Exit(1)
	}
	os.Exit(0)
}

// unshareCmd execs whatever using the ID mappings that we want to use for ourselves
func unshareCmd(c *cli.Context) error {
	// force reexec using the configured ID mappings
	maybeReexecUsingUserNamespace(c, true)
	// exec the specified command, if there is one
	args := c.Args()
	if len(args) < 1 {
		// try to exec the shell, if one's set
		shell, shellSet := os.LookupEnv("SHELL")
		if !shellSet {
			logrus.Errorf("no command specified")
			os.Exit(1)
		}
		args = []string{shell}
	}
	cmd := exec.Command(args[0], args[1:]...)
	cmd.Env = append(os.Environ(), "USER=root", "USERNAME=root", "GROUP=root", "LOGNAME=root", "UID=0", "GID=0")
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	execRunnable(cmd)
	os.Exit(1)
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bitbucket.org/anacrolix/go.torrent/dht"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"os/signal"
)

type pingResponse struct {
	addr string
	krpc dht.Msg
}

var (
	tableFileName = flag.String("tableFile", "", "name of file for storing node info")
	serveAddr     = flag.String("serveAddr", ":0", "local UDP address")

	s dht.Server
)

func loadTable() error {
	if *tableFileName == "" {
		return nil
	}
	f, err := os.Open(*tableFileName)
	if os.IsNotExist(err) {
		return nil
	}
	if err != nil {
		return fmt.Errorf("error opening table file: %s", err)
	}
	defer f.Close()
	added := 0
	for {
		b := make([]byte, dht.CompactNodeInfoLen)
		_, err := io.ReadFull(f, b)
		if err == io.EOF {
			break
		}
		if err != nil {
			return fmt.Errorf("error reading table file: %s", err)
		}
		var ni dht.NodeInfo
		err = ni.UnmarshalCompact(b)
		if err != nil {
			return fmt.Errorf("error unmarshaling compact node info: %s", err)
		}
		s.AddNode(ni)
		added++
	}
	log.Printf("loaded %d nodes from table file", added)
	return nil
}

func init() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
	flag.Parse()
	err := loadTable()
	if err != nil {
		log.Fatalf("error loading table: %s", err)
	}
	s.Socket, err = net.ListenUDP("udp4", func() *net.UDPAddr {
		addr, err := net.ResolveUDPAddr("udp4", *serveAddr)
		if err != nil {
			log.Fatalf("error resolving serve addr: %s", err)
		}
		return addr
	}())
	if err != nil {
		log.Fatal(err)
	}
	log.Printf("dht server on %s", s.Socket.LocalAddr())
	s.Init()
	setupSignals()
}

func saveTable() error {
	goodNodes := s.GoodNodes()
	if *tableFileName == "" {
		if len(goodNodes) != 0 {
			log.Printf("discarding %d good nodes!", len(goodNodes))
		}
		return nil
	}
	f, err := os.OpenFile(*tableFileName, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
	if err != nil {
		return fmt.Errorf("error opening table file: %s", err)
	}
	defer f.Close()
	for _, nodeInfo := range goodNodes {
		var b [dht.CompactNodeInfoLen]byte
		err := nodeInfo.PutCompact(b[:])
		if err != nil {
			return fmt.Errorf("error compacting node info: %s", err)
		}
		_, err = f.Write(b[:])
		if err != nil {
			return fmt.Errorf("error writing compact node info: %s", err)
		}
	}
	log.Printf("saved %d nodes to table file", len(goodNodes))
	return nil
}

func setupSignals() {
	ch := make(chan os.Signal)
	signal.Notify(ch)
	go func() {
		<-ch
		s.StopServing()
	}()
}

func main() {
	go func() {
		err := s.Bootstrap()
		if err != nil {
			log.Printf("error bootstrapping: %s", err)
			s.StopServing()
		}
	}()
	err := s.Serve()
	if err := saveTable(); err != nil {
		log.Printf("error saving node table: %s", err)
	}
	if err != nil {
		log.Fatalf("error serving dht: %s", err)
	}
}
<commit_msg>DHT node table should be loaded after the server is initialized<commit_after>package main

import (
	"bitbucket.org/anacrolix/go.torrent/dht"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"os/signal"
)

type pingResponse struct {
	addr string
	krpc dht.Msg
}

var (
	tableFileName = flag.String("tableFile", "", "name of file for storing node info")
	serveAddr     = flag.String("serveAddr", ":0", "local UDP address")

	s dht.Server
)

func loadTable() error {
	if *tableFileName == "" {
		return nil
	}
	f, err := os.Open(*tableFileName)
	if os.IsNotExist(err) {
		return nil
	}
	if err != nil {
		return fmt.Errorf("error opening table file: %s", err)
	}
	defer f.Close()
	added := 0
	for {
		b := make([]byte, dht.CompactNodeInfoLen)
		_, err := io.ReadFull(f, b)
		if err == io.EOF {
			break
		}
		if err != nil {
			return fmt.Errorf("error reading table file: %s", err)
		}
		var ni dht.NodeInfo
		err = ni.UnmarshalCompact(b)
		if err != nil {
			return fmt.Errorf("error unmarshaling compact node info: %s", err)
		}
		s.AddNode(ni)
		added++
	}
	log.Printf("loaded %d nodes from table file", added)
	return nil
}

func init() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
	flag.Parse()
	var err error
	s.Socket, err = net.ListenUDP("udp4", func() *net.UDPAddr {
		addr, err := net.ResolveUDPAddr("udp4", *serveAddr)
		if err != nil {
			log.Fatalf("error resolving serve addr: %s", err)
		}
		return addr
	}())
	if err != nil {
		log.Fatal(err)
	}
	s.Init()
	err = loadTable()
	if err != nil {
		log.Fatalf("error loading table: %s", err)
	}
	log.Printf("dht server on %s, ID is %q", s.Socket.LocalAddr(), s.IDString())
	setupSignals()
}

func saveTable() error {
	goodNodes := s.GoodNodes()
	if *tableFileName == "" {
		if len(goodNodes) != 0 {
			log.Printf("discarding %d good nodes!", len(goodNodes))
		}
		return nil
	}
	f, err := os.OpenFile(*tableFileName, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
	if err != nil {
		return fmt.Errorf("error opening table file: %s", err)
	}
	defer f.Close()
	for _, nodeInfo := range goodNodes {
		var b [dht.CompactNodeInfoLen]byte
		err := nodeInfo.PutCompact(b[:])
		if err != nil {
			return fmt.Errorf("error compacting node info: %s", err)
		}
		_, err = f.Write(b[:])
		if err != nil {
			return fmt.Errorf("error writing compact node info: %s", err)
		}
	}
	log.Printf("saved %d nodes to table file", len(goodNodes))
	return nil
}

func setupSignals() {
	ch := make(chan os.Signal)
	signal.Notify(ch)
	go func() {
		<-ch
		s.StopServing()
	}()
}

func main() {
	go func() {
		err := s.Bootstrap()
		if err != nil {
			log.Printf("error bootstrapping: %s", err)
			s.StopServing()
		}
	}()
	err := s.Serve()
	if err := saveTable(); err != nil {
		log.Printf("error saving node table: %s", err)
	}
	if err != nil {
		log.Fatalf("error serving dht: %s", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strings"
	"time"

	"github.com/gocolly/colly"
	"github.com/gocolly/colly/extensions"
)

func main() {
	c := colly.NewCollector()
	extensions.RandomUserAgent(c)
	c.Limit(&colly.LimitRule{
		Delay: 5 * time.Second,
	})

	c.OnHTML("td.collection_objectname", func(e *colly.HTMLElement) {
		linkElem := e.DOM.Find("a[href]")
		link, _ := linkElem.Attr("href")
		num := strings.Split(link, "/")[2]
		name := linkElem.Text()
		fmt.Printf("%s: %s\n", num, name)
	})

	c.OnRequest(func(r *colly.Request) {
		fmt.Println("Visiting", r.URL)
	})

	c.Visit(`https://boardgamegeek.com/browse/boardgame/page/1`)
}
<commit_msg>scrape all pages<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/gocolly/colly"
	"github.com/gocolly/colly/extensions"
)

func main() {
	var (
		waitSeconds int
		limitFlag   int
	)
	flag.IntVar(&waitSeconds, "wait", 5, "wait sets the default wait between web requests")
	flag.IntVar(&limitFlag, "limit", 0, "limit sets the max number of pages to check (0 is no limit)")
	flag.Parse()

	var totalPages int
	var wg *sync.WaitGroup = new(sync.WaitGroup)
	const geekURL string = "https://boardgamegeek.com/browse/boardgame/page/%v"
	c := colly.NewCollector()
	extensions.RandomUserAgent(c)
	if waitSeconds > 0 {
		c.Limit(&colly.LimitRule{
			Delay: time.Duration(waitSeconds) * time.Second,
		})
	}

	c.OnHTML("td.collection_objectname", func(e *colly.HTMLElement) {
		linkElem := e.DOM.Find("a[href]")
		link, _ := linkElem.Attr("href")
		num := strings.Split(link, "/")[2]
		name := linkElem.Text()
		wg.Add(1)
		game := fmt.Sprintf("%s: %s\n", num, name)
		f, err := os.OpenFile("games.txt", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666)
		if err != nil {
			log.Fatalf("Couldn't open file: %s", err)
		}
		go WriteToFile(game, f, wg)
	})

	c.OnHTML("#main_content", func(e *colly.HTMLElement) {
		if totalPages == 0 {
			lastPageText := e.DOM.Find(`div.infobox a[title="last page"]`).Text()
			totalPages, err := strconv.Atoi(strings.Trim(lastPageText, "[]"))
			if err != nil {
				log.Fatalf("Couldn't parse page total: %s", err)
			}
			for i := 2; i <= coalesceInt(limitFlag, totalPages); i++ {
				c.Visit(fmt.Sprintf(geekURL, i))
			}
		}
	})

	c.OnRequest(func(r *colly.Request) {
		log.Println("Visiting", r.URL)
	})

	c.OnResponse(func(r *colly.Response) {
		if r.StatusCode != 200 {
			e := HttpResponseError{
				Code:     r.StatusCode,
				Filename: r.FileName(),
				Body:     string(r.Body),
			}
			log.Println("Error", e.Code, e.Body)
			AddError(e)
		}
	})

	log.Println("First visit", fmt.Sprintf(geekURL, 1))
	c.Visit(fmt.Sprintf(geekURL, 1))

	wg.Wait()
	log.Println("Complete,", totalPages, "games saved.")
	errorData, err := json.Marshal(errorList)
	if err != nil {
		log.Fatalf("marshalling errorList: %s", err)
	}
	err = ioutil.WriteFile("errors.json", errorData, 0666)
	if err != nil {
		log.Fatalf("writing error.json: %s", err)
	}
	log.Println("Errors logged:", len(errorList))
}

var fileLock sync.Mutex

func WriteToFile(game string, f *os.File, w *sync.WaitGroup) {
	fileLock.Lock()
	defer fileLock.Unlock()
	f.Write([]byte(game))
}

var errorList []HttpResponseError

func AddError(e HttpResponseError) {
	errorLock.Lock()
	defer errorLock.Unlock()
	errorList = append(errorList, e)
}

var errorLock sync.Mutex

type HttpResponseError struct {
	Code     int
	Filename string
	Body     string
}

func coalesceInt(nums ...int) int {
	for _, num := range nums {
		if num != 0 {
			return num
		}
	}
	return 0
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"time"

	"github.com/go-ggz/ggz/config"
	"github.com/go-ggz/ggz/version"

	"github.com/joho/godotenv"
	_ "github.com/joho/godotenv/autoload"
	"github.com/rs/zerolog"
	"github.com/rs/zerolog/log"
	"gopkg.in/urfave/cli.v2"
)

func setupLogging() {
	zerolog.SetGlobalLevel(zerolog.InfoLevel)
	if config.Server.Debug {
		zerolog.SetGlobalLevel(zerolog.DebugLevel)
	}

	if config.Logs.Pretty {
		log.Logger = log.Output(
			zerolog.ConsoleWriter{
				Out:     os.Stderr,
				NoColor: !config.Logs.Color,
			},
		)
	}
}

func main() {
	if env := os.Getenv("GGZ_ENV_FILE"); env != "" {
		if err := godotenv.Load(env); err != nil {
			log.Fatal().Err(err).Msg("Cannot start load config from env")
		}
	}

	app := &cli.App{
		Name:      "gzz server",
		Usage:     "shorten url service",
		Copyright: "Copyright (c) 2018 Bo-Yi Wu",
		Version:   version.PrintCLIVersion(),
		Compiled:  time.Now(),
		Authors: []*cli.Author{
			{
				Name:  "Bo-Yi Wu",
				Email: "appleboy.tw@gmail.com",
			},
		},

		Flags: []cli.Flag{
			&cli.BoolFlag{
				Name:        "debug",
				Value:       true,
				Usage:       "Activate debug information",
				EnvVars:     []string{"GGZ_SERVER_DEBUG"},
				Destination: &config.Server.Debug,
			},
			&cli.BoolFlag{
				Name:        "color",
				Value:       true,
				Usage:       "Enable pprof debugging server",
				EnvVars:     []string{"GGZ_LOGS_COLOR"},
				Destination: &config.Logs.Color,
			},
			&cli.BoolFlag{
				Name:        "pretty",
				Value:       true,
				Usage:       "Enable pprof debugging server",
				EnvVars:     []string{"GGZ_LOGS_PRETTY"},
				Destination: &config.Logs.Pretty,
			},
		},

		Before: func(c *cli.Context) error {
			setupLogging()

			return nil
		},

		Commands: []*cli.Command{
			Server(),
			Ping(),
			Mail(),
		},
	}

	cli.HelpFlag = &cli.BoolFlag{
		Name:    "help",
		Aliases: []string{"h"},
		Usage:   "Show the help, so what you see now",
	}

	cli.VersionFlag = &cli.BoolFlag{
		Name:    "version",
		Aliases: []string{"v"},
		Usage:   "Print the current version of that tool",
	}

	if err := app.Run(os.Args); err != nil {
		os.Exit(1)
	}
}
<commit_msg>chore: add debug<commit_after>package main

import (
	"os"
	"time"

	"github.com/go-ggz/ggz/config"
	"github.com/go-ggz/ggz/version"

	"github.com/joho/godotenv"
	_ "github.com/joho/godotenv/autoload"
	"github.com/rs/zerolog"
	"github.com/rs/zerolog/log"
	"gopkg.in/urfave/cli.v2"
)

func setupLogging() {
	zerolog.SetGlobalLevel(zerolog.InfoLevel)
	if config.Server.Debug {
		zerolog.SetGlobalLevel(zerolog.DebugLevel)
	}

	if config.Logs.Pretty {
		log.Logger = log.Output(
			zerolog.ConsoleWriter{
				Out:     os.Stderr,
				NoColor: !config.Logs.Color,
			},
		)
	}
}

func main() {
	if env := os.Getenv("GGZ_ENV_FILE"); env != "" {
		if err := godotenv.Load(env); err != nil {
			log.Fatal().Err(err).Msg("Cannot start load config from env")
		}
	}

	app := &cli.App{
		Name:      "gzz server",
		Usage:     "shorten url service",
		Copyright: "Copyright (c) 2018 Bo-Yi Wu",
		Version:   version.PrintCLIVersion(),
		Compiled:  time.Now(),
		Authors: []*cli.Author{
			{
				Name:  "Bo-Yi Wu",
				Email: "appleboy.tw@gmail.com",
			},
		},

		Flags: []cli.Flag{
			&cli.BoolFlag{
				Name:        "debug",
				Value:       true,
				Usage:       "Activate debug information",
				EnvVars:     []string{"GGZ_SERVER_DEBUG"},
				Destination: &config.Server.Debug,
			},
			&cli.BoolFlag{
				Name:        "color",
				Value:       true,
				Usage:       "Enable pprof debugging server",
				EnvVars:     []string{"GGZ_LOGS_COLOR"},
				Destination: &config.Logs.Color,
			},
			&cli.BoolFlag{
				Name:        "pretty",
				Value:       true,
				Usage:       "Enable pprof debugging server",
				EnvVars:     []string{"GGZ_LOGS_PRETTY"},
				Destination: &config.Logs.Pretty,
			},
		},

		Before: func(c *cli.Context) error {
			setupLogging()

			return nil
		},

		Commands: []*cli.Command{
			Server(),
			Ping(),
			Mail(),
		},
	}

	cli.HelpFlag = &cli.BoolFlag{
		Name:    "help",
		Aliases: []string{"h"},
		Usage:   "Show the help, so what you see now",
	}

	cli.VersionFlag = &cli.BoolFlag{
		Name:    "version",
		Aliases: []string{"v"},
		Usage:   "Print the current version of that tool",
	}

	if err := app.Run(os.Args); err != nil {
		log.Fatal().Err(err).Msg("can't run app")
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
	"time"
)

var (
	dir   = flag.String("dir", "", "Directory to put data files")
	since = flag.Duration("since", 1*time.Hour, "Fetch things modified in this past duration.")
)

const (
	maxItemsPerPage = 1000

	rvTimeFormat = "2006-01-02 15:04:05"

	queryTmpl = "https://codereview.appspot.com/search?closed=3&{{CC_OR_REVIEWER}}=golang-dev%40googlegroups.com&private=1&modified_before={{DATE_BEFORE}}&modifed_after={{DATE_AFTER}}&order=-modified&keys_only=False&with_messages=False&cursor={{CURSOR}}&limit={{LIMIT}}&format=json"

	// JSON with the text of messages. e.g.
	// https://codereview.appspot.com/api/6454085?messages=true
	reviewTmpl = "https://codereview.appspot.com/api/{{ISSUE_NUMBER}}?messages=true"
)

// itemsPerPage is the number of items to fetch for a single page.
// Changed by tests.
var itemsPerPage = 100 // maxItemsPerPage

var updatewg = new(sync.WaitGroup)

func main() {
	flag.Parse()
	if *dir == "" {
		log.Fatalf("--dir flag is required.")
	}
	if fi, err := os.Stat(*dir); err != nil || !fi.IsDir() {
		log.Fatalf("Directory %q needs to exist.", *dir)
	}

	for _, to := range []string{"reviewer", "cc"} {
		updatewg.Add(1)
		go loadReviews(to, updatewg)
	}
	updatewg.Wait()
}

func loadReviews(to string, wg *sync.WaitGroup) {
	defer wg.Done()
	cursor := ""
	for {
		url := urlWithParams(queryTmpl, map[string]string{
			"CC_OR_REVIEWER": to,
			"DATE_AFTER":     time.Now().Add(-*since).Format(rvTimeFormat),
			"DATE_BEFORE":    "2020-01-01 00:00:00",
			"CURSOR":         cursor,
			"LIMIT":          fmt.Sprint(itemsPerPage),
		})
		log.Printf("Fetching %s", url)
		res, err := http.Get(url)
		if err != nil {
			log.Fatal(err)
		}
		var reviews []*Review
		reviews, cursor, err = ParseReviews(res.Body)
		if err != nil {
			log.Fatal(err)
		}
		log.Printf("for cursor %q, Got %d reviews", cursor, len(reviews))
		for _, r := range reviews {
			wg.Add(1)
			go updateReview(r, wg)
		}
		res.Body.Close()
		if cursor == "" || len(reviews) == 0 {
			break
		}
	}
}

// updateReview checks to see if r (which lacks comments) has a higher
// modification time than the version we have on disk and if necessary
// fetches the full (with comments) version of r and puts it on disk.
func updateReview(r *Review, wg *sync.WaitGroup) {
	defer wg.Done()
	dr, err := loadDiskFullReview(r.Issue)
	if err == nil && dr.Modified == r.Modified {
		// Nothing to do.
		return
	}
	if err != nil && !os.IsNotExist(err) {
		log.Fatalf("Error loading issue %d: %v", r.Issue, err)
	}
	url := urlWithParams(reviewTmpl, map[string]string{
		"ISSUE_NUMBER": fmt.Sprint(r.Issue),
	})
	log.Printf("Fetching %s", url)
	res, err := http.Get(url)
	if err != nil || res.StatusCode != 200 {
		log.Fatalf("Error fetching %s: %+v, %v", url, res, err)
	}
	defer res.Body.Close()
	all, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Fatal(err)
	}
	var buf bytes.Buffer
	if err := json.Indent(&buf, all, "", "\t"); err != nil {
		log.Fatal(err)
	}

	dstFile := issueDiskPath(r.Issue)
	dir := filepath.Dir(dstFile)
	if err := os.MkdirAll(dir, 0755); err != nil {
		log.Fatal(err)
	}
	tf, err := ioutil.TempFile(dir, "issue")
	if err != nil {
		log.Fatal(err)
	}
	_, err = tf.Write(buf.Bytes())
	if err != nil {
		log.Fatal(err)
	}
	if err := tf.Close(); err != nil {
		log.Fatal(err)
	}
	if err := os.Rename(tf.Name(), dstFile); err != nil {
		log.Fatal(err)
	}
}

func issueDiskPath(id int) string {
	return filepath.Join(*dir, "reviews", fmt.Sprintf("%d.json", id))
}

func loadDiskFullReview(id int) (*Review, error) {
	f, err := os.Open(issueDiskPath(id))
	if err != nil {
		return nil, err
	}
	defer f.Close()
	r := new(Review)
	return r, json.NewDecoder(f).Decode(&r)
}

var urlParam = regexp.MustCompile(`{{\w+}}`)

func urlWithParams(urlTempl string, m map[string]string) string {
	return urlParam.ReplaceAllStringFunc(urlTempl, func(param string) string {
		return url.QueryEscape(m[strings.Trim(param, "{}")])
	})
}

type issueResult struct {
	Closed   bool   `json:"closed"`
	Modified string `json:"modified"`
}

type Message struct {
	Sender string `json:"sender"`
	Text   string `json:"text"`
	Date   string `json:"date"` // "2012-04-07 00:51:58.602055"
}

type byMessageDate []*Message

func (s byMessageDate) Len() int           { return len(s) }
func (s byMessageDate) Less(i, j int) bool { return s[i].Date < s[j].Date }
func (s byMessageDate) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }

type monthQueryResult struct {
	Cursor  string    `json:"cursor"`
	Results []*Review `json:"results"`
}

// Time unmarshals a time in rietveld's format.
type Time time.Time

func (t *Time) UnmarshalJSON(b []byte) error {
	if len(b) < 2 || b[0] != '"' || b[len(b)-1] != '"' {
		return fmt.Errorf("types: failed to unmarshal non-string value %q as an RFC 3339 time")
	}
	// TODO: pic
	tm, err := time.Parse("2006-01-02 15:04:05", string(b[1:len(b)-1]))
	if err != nil {
		return err
	}
	*t = Time(tm)
	return nil
}

func (t Time) String() string { return time.Time(t).String() }

type Review struct {
	Issue      int        `json:"issue"`
	Desc       string     `json:"description"`
	OwnerEmail string     `json:"owner_email"`
	Owner      string     `json:"owner"`
	Created    Time       `json:"created"`
	Modified   string     `json:"modified"` // just a string; more reliable to do string equality tests on it
	Messages   []*Message `json:"messages"`
	Reviewers  []string   `json:"reviewers"`
	CC         []string   `json:"cc"`
}

func ParseReviews(r io.Reader) (reviews []*Review, cursor string, err error) {
	var d monthQueryResult
	err = json.NewDecoder(r).Decode(&d)
	return d.Results, d.Cursor, err
}

// Reviewer returns the email address of an explicit reviewer, if any, else
// returns the empty string.
func (r *Review) Reviewer() string {
	for _, who := range r.Reviewers {
		if strings.HasSuffix(who, "@googlegroups.com") {
			continue
		}
		return who
	}
	return ""
}
<commit_msg>better query<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
	"time"
)

var (
	dir   = flag.String("dir", "", "Directory to put data files")
	since = flag.Duration("since", 1*time.Hour, "Fetch things modified in this past duration.")
)

const (
	maxItemsPerPage = 1000

	rvTimeFormat = "2006-01-02 15:04:05"

	// queryTmpl = "https://codereview.appspot.com/search?closed=3&{{CC_OR_REVIEWER}}=golang-dev%40googlegroups.com&private=1&modified_before={{DATE_BEFORE}}&modifed_after={{DATE_AFTER}}&order=-modified&keys_only=False&with_messages=False&cursor={{CURSOR}}&limit={{LIMIT}}&format=json"

	// closed=1 means "unknown"
	queryTmpl = "https://codereview.appspot.com/search?closed=1&owner=&{{CC_OR_REVIEWER}}=golang-dev&cc=&repo_guid=&base=&private=1&created_before=&created_after=&modified_before=&modified_after=&order=-modified&format=json&keys_only=False&with_messages=False&cursor={{CURSOR}}&limit={{LIMIT}}"

	// JSON with the text of messages. e.g.
	// https://codereview.appspot.com/api/6454085?messages=true
	reviewTmpl = "https://codereview.appspot.com/api/{{ISSUE_NUMBER}}?messages=true"
)

// itemsPerPage is the number of items to fetch for a single page.
// Changed by tests.
var itemsPerPage = 100 // maxItemsPerPage

var updatewg = new(sync.WaitGroup)

func main() {
	flag.Parse()
	if *dir == "" {
		log.Fatalf("--dir flag is required.")
	}
	if fi, err := os.Stat(*dir); err != nil || !fi.IsDir() {
		log.Fatalf("Directory %q needs to exist.", *dir)
	}

	for _, to := range []string{"reviewer"} { // needs index: , "cc"
		updatewg.Add(1)
		go loadReviews(to, updatewg)
	}
	updatewg.Wait()
}

func loadReviews(to string, wg *sync.WaitGroup) {
	defer wg.Done()
	cursor := ""
	oldestWant := time.Now().Add(-*since).Format(rvTimeFormat)
	for {
		url := urlWithParams(queryTmpl, map[string]string{
			"CC_OR_REVIEWER": to,
			"CURSOR":         cursor,
			"LIMIT":          fmt.Sprint(itemsPerPage),
		})
		log.Printf("Fetching %s", url)
		res, err := http.Get(url)
		if err != nil {
			log.Fatal(err)
		}
		var reviews []*Review
		reviews, cursor, err = ParseReviews(res.Body)
		if err != nil {
			log.Fatal(err)
		}
		ngood := 0 // where good means "new enough"
		for _, r := range reviews {
			if r.Modified >= oldestWant {
				ngood++
				wg.Add(1)
				go updateReview(r, wg)
			}
		}
		log.Printf("for cursor %q, Got %d reviews (%d in time window)", cursor, len(reviews), ngood)
		res.Body.Close()
		if cursor == "" || ngood == 0 {
			break
		}
	}
}

var httpGate = make(chan bool, 25)

// updateReview checks to see if r (which lacks comments) has a higher
// modification time than the version we have on disk and if necessary
// fetches the full (with comments) version of r and puts it on disk.
func updateReview(r *Review, wg *sync.WaitGroup) {
	defer wg.Done()
	dr, err := loadDiskFullReview(r.Issue)
	if err == nil && dr.Modified == r.Modified {
		// Nothing to do.
		return
	}
	if err != nil && !os.IsNotExist(err) {
		log.Fatalf("Error loading issue %d: %v", r.Issue, err)
	}

	httpGate <- true
	defer func() { <-httpGate }()

	url := urlWithParams(reviewTmpl, map[string]string{
		"ISSUE_NUMBER": fmt.Sprint(r.Issue),
	})
	log.Printf("Fetching %s", url)
	res, err := http.Get(url)
	if err != nil || res.StatusCode != 200 {
		log.Fatalf("Error fetching %s: %+v, %v", url, res, err)
	}
	defer res.Body.Close()
	all, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Fatal(err)
	}
	var buf bytes.Buffer
	if err := json.Indent(&buf, all, "", "\t"); err != nil {
		log.Fatal(err)
	}

	dstFile := issueDiskPath(r.Issue)
	dir := filepath.Dir(dstFile)
	if err := os.MkdirAll(dir, 0755); err != nil {
		log.Fatal(err)
	}
	tf, err := ioutil.TempFile(dir, "issue")
	if err != nil {
		log.Fatal(err)
	}
	_, err = tf.Write(buf.Bytes())
	if err != nil {
		log.Fatal(err)
	}
	if err := tf.Close(); err != nil {
		log.Fatal(err)
	}
	if err := os.Rename(tf.Name(), dstFile); err != nil {
		log.Fatal(err)
	}
}

func issueDiskPath(id int) string {
	return filepath.Join(*dir, "reviews", fmt.Sprintf("%d.json", id))
}

func loadDiskFullReview(id int) (*Review, error) {
	f, err := os.Open(issueDiskPath(id))
	if err != nil {
		return nil, err
	}
	defer f.Close()
	r := new(Review)
	return r, json.NewDecoder(f).Decode(&r)
}

var urlParam = regexp.MustCompile(`{{\w+}}`)

func urlWithParams(urlTempl string, m map[string]string) string {
	return urlParam.ReplaceAllStringFunc(urlTempl, func(param string) string {
		return url.QueryEscape(m[strings.Trim(param, "{}")])
	})
}

type issueResult struct {
	Closed   bool   `json:"closed"`
	Modified string `json:"modified"`
}

type Message struct {
	Sender string `json:"sender"`
	Text   string `json:"text"`
	Date   string `json:"date"` // "2012-04-07 00:51:58.602055"
}

type byMessageDate []*Message

func (s byMessageDate) Len() int           { return len(s) }
func (s byMessageDate) Less(i, j int) bool { return s[i].Date < s[j].Date }
func (s byMessageDate) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }

type monthQueryResult struct {
	Cursor  string    `json:"cursor"`
	Results []*Review `json:"results"`
}

// Time unmarshals a time in rietveld's format.
type Time time.Time

func (t *Time) UnmarshalJSON(b []byte) error {
	if len(b) < 2 || b[0] != '"' || b[len(b)-1] != '"' {
		return fmt.Errorf("types: failed to unmarshal non-string value %q as an RFC 3339 time")
	}
	// TODO: pic
	tm, err := time.Parse("2006-01-02 15:04:05", string(b[1:len(b)-1]))
	if err != nil {
		return err
	}
	*t = Time(tm)
	return nil
}

func (t Time) String() string { return time.Time(t).String() }

type Review struct {
	Issue      int        `json:"issue"`
	Desc       string     `json:"description"`
	OwnerEmail string     `json:"owner_email"`
	Owner      string     `json:"owner"`
	Created    Time       `json:"created"`
	Modified   string     `json:"modified"` // just a string; more reliable to do string equality tests on it
	Messages   []*Message `json:"messages"`
	Reviewers  []string   `json:"reviewers"`
	CC         []string   `json:"cc"`
}

func ParseReviews(r io.Reader) (reviews []*Review, cursor string, err error) {
	var d monthQueryResult
	err = json.NewDecoder(r).Decode(&d)
	return d.Results, d.Cursor, err
}

// Reviewer returns the email address of an explicit reviewer, if any, else
// returns the empty string.
func (r *Review) Reviewer() string {
	for _, who := range r.Reviewers {
		if strings.HasSuffix(who, "@googlegroups.com") {
			continue
		}
		return who
	}
	return ""
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"syscall"

	"github.com/Symantec/Dominator/hypervisor/dhcpd"
	"github.com/Symantec/Dominator/hypervisor/httpd"
	"github.com/Symantec/Dominator/hypervisor/manager"
	"github.com/Symantec/Dominator/hypervisor/metadatad"
	"github.com/Symantec/Dominator/hypervisor/rpcd"
	"github.com/Symantec/Dominator/hypervisor/tftpbootd"
	"github.com/Symantec/Dominator/lib/constants"
	"github.com/Symantec/Dominator/lib/flags/loadflags"
	"github.com/Symantec/Dominator/lib/flagutil"
	"github.com/Symantec/Dominator/lib/log/serverlogger"
	"github.com/Symantec/Dominator/lib/net"
	"github.com/Symantec/Dominator/lib/srpc/setupserver"
	"github.com/Symantec/tricorder/go/tricorder"
)

const (
	dirPerms = syscall.S_IRWXU | syscall.S_IRGRP | syscall.S_IXGRP |
		syscall.S_IROTH | syscall.S_IXOTH
)

var (
	imageServerHostname = flag.String("imageServerHostname", "localhost",
		"Hostname of image server")
	imageServerPortNum = flag.Uint("imageServerPortNum",
		constants.ImageServerPortNumber,
		"Port number of image server")
	networkBootImage = flag.String("networkBootImage", "pxelinux.0",
		"Name of boot image passed via DHCP option")
	portNum = flag.Uint("portNum", constants.HypervisorPortNumber,
		"Port number to allocate and listen on for HTTP/RPC")
	showVGA  = flag.Bool("showVGA", false, "If true, show VGA console")
	stateDir = flag.String("stateDir", "/var/lib/hypervisor",
		"Name of state directory")
	testMemoryAvailable = flag.Uint64("testMemoryAvailable", 0,
		"test if memory is allocatable and exit (units of MiB)")
	tftpbootImageStream = flag.String("tftpbootImageStream", "",
		"Name of default image stream for network booting")
	username = flag.String("username", "nobody",
		"Name of user to run VMs")
	volumeDirectories flagutil.StringList
)

func init() {
	flag.Var(&volumeDirectories, "volumeDirectories",
		"Comma separated list of volume directories. If empty, scan for space")
}

func main() {
	if err := loadflags.LoadForDaemon("hypervisor"); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	flag.Parse()
	if *testMemoryAvailable > 0 {
		nBytes := *testMemoryAvailable << 20
		mem := make([]byte, nBytes)
		for pos := uint64(0); pos < nBytes; pos += 4096 {
			mem[pos] = 0
		}
		os.Exit(0)
	}
	tricorder.RegisterFlags()
	if os.Geteuid() != 0 {
		fmt.Fprintln(os.Stderr, "Must run the Hypervisor as root")
		os.Exit(1)
	}
	logger := serverlogger.New("")
	if err := setupserver.SetupTls(); err != nil {
		logger.Fatalln(err)
	}
	if err := os.MkdirAll(*stateDir, dirPerms); err != nil {
		logger.Fatalf("Cannot create state directory: %s\n", err)
	}
	bridges, err := net.ListBridges()
	if err != nil {
		logger.Fatalf("Cannot list bridges: %s\n", err)
	}
	bridgeNames := make([]string, 0, len(bridges))
	vlanIdToBridge := make(map[uint]string)
	for _, bridge := range bridges {
		if vlanId, err := net.GetBridgeVlanId(bridge.Name); err != nil {
			logger.Fatalf("Cannot get VLAN Id for bridge: %s: %s\n",
				bridge.Name, err)
		} else if vlanId < 0 {
			logger.Printf("Bridge: %s has no EtherNet port, ignoring\n",
				bridge.Name)
		} else {
			bridgeNames = append(bridgeNames, bridge.Name)
			vlanIdToBridge[uint(vlanId)] = bridge.Name
			logger.Printf("Bridge: %s, VLAN Id: %d\n", bridge.Name, vlanId)
		}
	}
	dhcpServer, err := dhcpd.New(bridgeNames, logger)
	if err != nil {
		logger.Fatalf("Cannot start DHCP server: %s\n", err)
	}
	if err := dhcpServer.SetNetworkBootImage(*networkBootImage); err != nil {
		logger.Fatalf("Cannot set NetworkBootImage name: %s\n", err)
	}
	imageServerAddress := fmt.Sprintf("%s:%d",
		*imageServerHostname, *imageServerPortNum)
	tftpbootServer, err := tftpbootd.New(imageServerAddress,
		*tftpbootImageStream, logger)
	if err != nil {
		logger.Fatalf("Cannot start tftpboot server: %s\n", err)
	}
	managerObj, err := manager.New(manager.StartOptions{
		ImageServerAddress: imageServerAddress,
		DhcpServer:         dhcpServer,
		Logger:             logger,
		ShowVgaConsole:     *showVGA,
		StateDir:           *stateDir,
		Username:           *username,
		VlanIdToBridge:     vlanIdToBridge,
		VolumeDirectories:  volumeDirectories,
	})
	if err != nil {
		logger.Fatalf("Cannot start hypervisor: %s\n", err)
	}
	httpd.AddHtmlWriter(managerObj)
	if len(bridges) < 1 {
		logger.Println("No bridges found: entering log-only mode")
	} else {
		rpcHtmlWriter, err := rpcd.Setup(managerObj, dhcpServer, tftpbootServer,
			logger)
		if err != nil {
			logger.Fatalf("Cannot start rpcd: %s\n", err)
		}
		httpd.AddHtmlWriter(rpcHtmlWriter)
	}
	httpd.AddHtmlWriter(logger)
	err = metadatad.StartServer(*portNum, bridges, managerObj, logger)
	if err != nil {
		logger.Fatalf("Cannot start metadata server: %s\n", err)
	}
	if err := httpd.StartServer(*portNum, managerObj, false); err != nil {
		logger.Fatalf("Unable to create http server: %s\n", err)
	}
}
<commit_msg>Add -dhcpServerOnBridgesOnly option to hypervisor.<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"syscall"

	"github.com/Symantec/Dominator/hypervisor/dhcpd"
	"github.com/Symantec/Dominator/hypervisor/httpd"
	"github.com/Symantec/Dominator/hypervisor/manager"
	"github.com/Symantec/Dominator/hypervisor/metadatad"
	"github.com/Symantec/Dominator/hypervisor/rpcd"
	"github.com/Symantec/Dominator/hypervisor/tftpbootd"
	"github.com/Symantec/Dominator/lib/constants"
	"github.com/Symantec/Dominator/lib/flags/loadflags"
	"github.com/Symantec/Dominator/lib/flagutil"
	"github.com/Symantec/Dominator/lib/log/serverlogger"
	"github.com/Symantec/Dominator/lib/net"
	"github.com/Symantec/Dominator/lib/srpc/setupserver"
	"github.com/Symantec/tricorder/go/tricorder"
)

const (
	dirPerms = syscall.S_IRWXU | syscall.S_IRGRP | syscall.S_IXGRP |
		syscall.S_IROTH | syscall.S_IXOTH
)

var (
	dhcpServerOnBridgesOnly = flag.Bool("dhcpServerOnBridgesOnly", false,
		"If true, run the DHCP server on bridge interfaces only")
	imageServerHostname = flag.String("imageServerHostname", "localhost",
		"Hostname of image server")
	imageServerPortNum = flag.Uint("imageServerPortNum",
		constants.ImageServerPortNumber,
		"Port number of image server")
	networkBootImage = flag.String("networkBootImage", "pxelinux.0",
		"Name of boot image passed via DHCP option")
	portNum = flag.Uint("portNum", constants.HypervisorPortNumber,
		"Port number to allocate and listen on for HTTP/RPC")
	showVGA  = flag.Bool("showVGA", false, "If true, show VGA console")
	stateDir = flag.String("stateDir", "/var/lib/hypervisor",
		"Name of state directory")
	testMemoryAvailable = flag.Uint64("testMemoryAvailable", 0,
		"test if memory is allocatable and exit (units of MiB)")
	tftpbootImageStream = flag.String("tftpbootImageStream", "",
		"Name of default image stream for network booting")
	username = flag.String("username", "nobody",
		"Name of user to run VMs")
	volumeDirectories flagutil.StringList
)

func init() {
	flag.Var(&volumeDirectories, "volumeDirectories",
		"Comma separated list of volume directories. If empty, scan for space")
}

func main() {
	if err := loadflags.LoadForDaemon("hypervisor"); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	flag.Parse()
	if *testMemoryAvailable > 0 {
		nBytes := *testMemoryAvailable << 20
		mem := make([]byte, nBytes)
		for pos := uint64(0); pos < nBytes; pos += 4096 {
			mem[pos] = 0
		}
		os.Exit(0)
	}
	tricorder.RegisterFlags()
	if os.Geteuid() != 0 {
		fmt.Fprintln(os.Stderr, "Must run the Hypervisor as root")
		os.Exit(1)
	}
	logger := serverlogger.New("")
	if err := setupserver.SetupTls(); err != nil {
		logger.Fatalln(err)
	}
	if err := os.MkdirAll(*stateDir, dirPerms); err != nil {
		logger.Fatalf("Cannot create state directory: %s\n", err)
	}
	bridges, err := net.ListBridges()
	if err != nil {
		logger.Fatalf("Cannot list bridges: %s\n", err)
	}
	bridgeNames := make([]string, 0, len(bridges))
	vlanIdToBridge := make(map[uint]string)
	for _, bridge := range bridges {
		if vlanId, err := net.GetBridgeVlanId(bridge.Name); err != nil {
			logger.Fatalf("Cannot get VLAN Id for bridge: %s: %s\n",
				bridge.Name, err)
		} else if vlanId < 0 {
			logger.Printf("Bridge: %s has no EtherNet port, ignoring\n",
				bridge.Name)
		} else {
			if *dhcpServerOnBridgesOnly {
				bridgeNames = append(bridgeNames, bridge.Name)
			}
			vlanIdToBridge[uint(vlanId)] = bridge.Name
			logger.Printf("Bridge: %s, VLAN Id: %d\n", bridge.Name, vlanId)
		}
	}
	dhcpServer, err := dhcpd.New(bridgeNames, logger)
	if err != nil {
		logger.Fatalf("Cannot start DHCP server: %s\n", err)
	}
	if err := dhcpServer.SetNetworkBootImage(*networkBootImage); err != nil {
		logger.Fatalf("Cannot set NetworkBootImage name: %s\n", err)
	}
	imageServerAddress := fmt.Sprintf("%s:%d",
		*imageServerHostname, *imageServerPortNum)
	tftpbootServer, err := tftpbootd.New(imageServerAddress,
		*tftpbootImageStream, logger)
	if err != nil {
		logger.Fatalf("Cannot start tftpboot server: %s\n", err)
	}
	managerObj, err := manager.New(manager.StartOptions{
		ImageServerAddress: imageServerAddress,
		DhcpServer:         dhcpServer,
		Logger:             logger,
		ShowVgaConsole:     *showVGA,
		StateDir:           *stateDir,
		Username:           *username,
		VlanIdToBridge:     vlanIdToBridge,
		VolumeDirectories:  volumeDirectories,
	})
	if err != nil {
		logger.Fatalf("Cannot start hypervisor: %s\n", err)
	}
	httpd.AddHtmlWriter(managerObj)
	if len(bridges) < 1 {
		logger.Println("No bridges found: entering log-only mode")
	} else {
		rpcHtmlWriter, err := rpcd.Setup(managerObj, dhcpServer, tftpbootServer,
			logger)
		if err != nil {
			logger.Fatalf("Cannot start rpcd: %s\n", err)
		}
		httpd.AddHtmlWriter(rpcHtmlWriter)
	}
	httpd.AddHtmlWriter(logger)
	err = metadatad.StartServer(*portNum, bridges, managerObj, logger)
	if err != nil {
		logger.Fatalf("Cannot start metadata server: %s\n", err)
	}
	if err := httpd.StartServer(*portNum, managerObj, false); err != nil {
		logger.Fatalf("Unable to create http server: %s\n", err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"errors"
	"fmt"
	"io"
	"os"

	"istio.io/manager/cmd"
	"istio.io/manager/cmd/version"
	"istio.io/manager/platform/kube/inject"

	"github.com/spf13/cobra"
)

const (
	// DefaultHubEnvVar is the environment variable that defaults the value of the --hub flag to istioctl kube-inject
	DefaultHubEnvVar = "MANAGER_HUB"
	// DefaultTagEnvVar is the environment variable that defaults the value of the --tag flag to istioctl kube-inject
	DefaultTagEnvVar = "MANAGER_TAG"
)

var (
	hub             string
	tag             string
	sidecarProxyUID int64
	verbosity       int
	versionStr      string // override build version
	enableCoreDump  bool
	meshConfig      string

	inFilename  string
	outFilename string
)

var (
	injectCmd = &cobra.Command{
		Use:   "kube-inject",
		Short: "Inject istio sidecar proxy into kubernetes resources",
		Long: `
Use kube-inject to manually inject istio sidecar proxy into kubernetes
resource files. Unsupported resources are left unmodified so it is
safe to run kube-inject over a single file that contains multiple
Service, ConfigMap, Deployment, etc. definitions for a complex
application. Its best to do this when the resource is initially
created.

Example usage:

	kubectl apply -f <(istioctl kube-inject -f <resource.yaml>)
`,
		RunE: func(_ *cobra.Command, _ []string) (err error) {
			if inFilename == "" {
				return errors.New("filename not specified (see --filename or -f)")
			}
			if hub == "" {
				return fmt.Errorf("specify --hub or define %v", DefaultHubEnvVar)
			}
			if tag == "" {
				return fmt.Errorf("specify --tag or define %v", DefaultTagEnvVar)
			}
			var reader io.Reader
			if inFilename == "-" {
				reader = os.Stdin
			} else {
				if reader, err = os.Open(inFilename); err != nil {
					return err
				}
			}

			var writer io.Writer
			if outFilename == "" {
				writer = os.Stdout
			} else {
				var file *os.File
				if file, err = os.Create(outFilename); err != nil {
					return err
				}
				writer = file
				defer func() { err = file.Close() }()
			}

			if versionStr == "" {
				versionStr = version.Line()
			}

			mesh, err := cmd.GetMeshConfig(client.GetKubernetesClient(), namespace, meshConfig)
			if err != nil {
				return err
			}
			params := &inject.Params{
				InitImage:       inject.InitImageName(hub, tag),
				ProxyImage:      inject.ProxyImageName(hub, tag),
				Verbosity:       verbosity,
				SidecarProxyUID: sidecarProxyUID,
				Version:         versionStr,
				EnableCoreDump:  enableCoreDump,
				Mesh:            mesh,
			}
			if meshConfig != cmd.DefaultConfigMapName {
				params.MeshConfigMapName = meshConfig
			}
			return inject.IntoResourceFile(params, reader, writer)
		},
	}
)

func init() {
	injectCmd.PersistentFlags().StringVar(&hub, "hub",
		os.Getenv(DefaultHubEnvVar), "Docker hub")
	injectCmd.PersistentFlags().StringVar(&tag, "tag",
		os.Getenv(DefaultTagEnvVar), "Docker tag")
	injectCmd.PersistentFlags().StringVarP(&inFilename, "filename", "f",
		"", "Input kubernetes resource filename")
	injectCmd.PersistentFlags().StringVarP(&outFilename, "output", "o",
		"", "Modified output kubernetes resource filename")
	injectCmd.PersistentFlags().IntVar(&verbosity, "verbosity",
		inject.DefaultVerbosity, "Runtime verbosity")
	injectCmd.PersistentFlags().Int64Var(&sidecarProxyUID, "sidecarProxyUID",
		inject.DefaultSidecarProxyUID, "Sidecar proxy UID")
	injectCmd.PersistentFlags().StringVar(&versionStr, "setVersionString",
		"", "Override version info injected into resource")
	injectCmd.PersistentFlags().StringVar(&meshConfig, "meshConfig", cmd.DefaultConfigMapName,
		fmt.Sprintf("ConfigMap name for Istio mesh configuration, key should be %q", cmd.ConfigMapKey))

	// Default --coreDump=true for pre-alpha development. Core dump
	// settings (i.e. sysctl kernel.*) affect all pods in a node and
	// require privileges. This option should only be used by the cluster
	// admin (see https://kubernetes.io/docs/concepts/cluster-administration/sysctl-cluster/)
	injectCmd.PersistentFlags().BoolVar(&enableCoreDump, "coreDump",
		true, "Enable/Disable core dumps in injected proxy (--coreDump=true affects "+
			"all pods in a node and should only be used the cluster admin)")
}
<commit_msg>istioctl: improve error message when istio ConfigMap isn't found (#580)<commit_after>// Copyright 2017 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"errors"
	"fmt"
	"io"
	"os"

	"istio.io/manager/cmd"
	"istio.io/manager/cmd/version"
	"istio.io/manager/platform/kube/inject"

	"github.com/spf13/cobra"
)

const (
	// DefaultHubEnvVar is the environment variable that defaults the value of the --hub flag to istioctl kube-inject
	DefaultHubEnvVar = "MANAGER_HUB"
	// DefaultTagEnvVar is the environment variable that defaults the value of the --tag flag to istioctl kube-inject
	DefaultTagEnvVar = "MANAGER_TAG"
)

var (
	hub             string
	tag             string
	sidecarProxyUID int64
	verbosity       int
	versionStr      string // override build version
	enableCoreDump  bool
	meshConfig      string

	inFilename  string
	outFilename string
)

var (
	injectCmd = &cobra.Command{
		Use:   "kube-inject",
		Short: "Inject istio sidecar proxy into kubernetes resources",
		Long: `
Use kube-inject to manually inject istio sidecar proxy into kubernetes
resource files. Unsupported resources are left unmodified so it is
safe to run kube-inject over a single file that contains multiple
Service, ConfigMap, Deployment, etc. definitions for a complex
application. Its best to do this when the resource is initially
created.

Example usage:

	kubectl apply -f <(istioctl kube-inject -f <resource.yaml>)
`,
		RunE: func(_ *cobra.Command, _ []string) (err error) {
			if inFilename == "" {
				return errors.New("filename not specified (see --filename or -f)")
			}
			if hub == "" {
				return fmt.Errorf("specify --hub or define %v", DefaultHubEnvVar)
			}
			if tag == "" {
				return fmt.Errorf("specify --tag or define %v", DefaultTagEnvVar)
			}
			var reader io.Reader
			if inFilename == "-" {
				reader = os.Stdin
			} else {
				if reader, err = os.Open(inFilename); err != nil {
					return err
				}
			}

			var writer io.Writer
			if outFilename == "" {
				writer = os.Stdout
			} else {
				var file *os.File
				if file, err = os.Create(outFilename); err != nil {
					return err
				}
				writer = file
				defer func() { err = file.Close() }()
			}

			if versionStr == "" {
				versionStr = version.Line()
			}

			mesh, err := cmd.GetMeshConfig(client.GetKubernetesClient(), namespace, meshConfig)
			if err != nil {
				return fmt.Errorf("Istio configuration not found. Verify istio configmap is "+
					"installed in namespace %q with `kubectl get -n %s configmap istio`",
					namespace, namespace)
			}
			params := &inject.Params{
				InitImage:       inject.InitImageName(hub, tag),
				ProxyImage:      inject.ProxyImageName(hub, tag),
				Verbosity:       verbosity,
				SidecarProxyUID: sidecarProxyUID,
				Version:         versionStr,
				EnableCoreDump:  enableCoreDump,
				Mesh:            mesh,
			}
			if meshConfig != cmd.DefaultConfigMapName {
				params.MeshConfigMapName = meshConfig
			}
			return inject.IntoResourceFile(params, reader, writer)
		},
	}
)

func init() {
	injectCmd.PersistentFlags().StringVar(&hub, "hub",
		os.Getenv(DefaultHubEnvVar), "Docker hub")
	injectCmd.PersistentFlags().StringVar(&tag, "tag",
		os.Getenv(DefaultTagEnvVar), "Docker tag")
	injectCmd.PersistentFlags().StringVarP(&inFilename, "filename", "f",
		"", "Input kubernetes resource filename")
	injectCmd.PersistentFlags().StringVarP(&outFilename, "output", "o",
		"", "Modified output kubernetes resource filename")
	injectCmd.PersistentFlags().IntVar(&verbosity, "verbosity",
		inject.DefaultVerbosity, "Runtime verbosity")
	injectCmd.PersistentFlags().Int64Var(&sidecarProxyUID, "sidecarProxyUID",
		inject.DefaultSidecarProxyUID, "Sidecar proxy UID")
	injectCmd.PersistentFlags().StringVar(&versionStr, "setVersionString",
		"", "Override version info injected into resource")
	injectCmd.PersistentFlags().StringVar(&meshConfig, "meshConfig", cmd.DefaultConfigMapName,
		fmt.Sprintf("ConfigMap name for Istio mesh configuration, key should be %q", cmd.ConfigMapKey))

	// Default --coreDump=true for pre-alpha development. Core dump
	// settings (i.e. sysctl kernel.*) affect all pods in a node and
	// require privileges. This option should only be used by the cluster
	// admin (see https://kubernetes.io/docs/concepts/cluster-administration/sysctl-cluster/)
	injectCmd.PersistentFlags().BoolVar(&enableCoreDump, "coreDump",
		true, "Enable/Disable core dumps in injected proxy (--coreDump=true affects "+
			"all pods in a node and should only be used the cluster admin)")
}
<|endoftext|>


<commit_before>/*
Copyright 2016 Mirantis

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"os"

	"github.com/Mirantis/virtlet/pkg/manager"

	"github.com/golang/glog"
)

var (
	libvirtUri = flag.String("libvirt-uri", "qemu:///system",
		"Libvirt connection URI")
	pool = flag.String("pool", "default",
		"Storage pool in which the images should be stored")
	storageBackend = flag.String("storage-backend", "dir",
		"Libvirt storage pool type/backend")
	boltPath = flag.String("bolt-path", "/var/data/virtlet/virtlet.db",
		"Path to the bolt database file")
	listen = flag.String("listen", "/run/virtlet.sock",
		"The unix socket to listen on, e.g. /run/virtlet.sock")
	cniPluginsDir = flag.String("cni-bin-dir", "/opt/cni/bin",
		"Path to CNI plugin binaries")
	cniConfigsDir = flag.String("cni-conf-dir", "/etc/cni/net.d",
		"Location of CNI configurations (first file name in lexicographic order will be chosen)")
)

func main() {
	flag.Parse()

	server, err := manager.NewVirtletManager(*libvirtUri, *pool, *storageBackend, *boltPath, *cniPluginsDir, *cniConfigsDir)
	if err != nil {
		glog.Errorf("Initializing server failed: %v", err)
		os.Exit(1)
	}
	glog.V(1).Infof("Starting server on socket %s", *listen)
	if err = server.Serve(*listen); err != nil {
		glog.Errorf("Serving failed: %v", err)
	}
}
<commit_msg>Fix virtlet exit status on Serve() failures<commit_after>/*
Copyright 2016 Mirantis

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"os"

	"github.com/Mirantis/virtlet/pkg/manager"

	"github.com/golang/glog"
)

var (
	libvirtUri = flag.String("libvirt-uri", "qemu:///system",
		"Libvirt connection URI")
	pool = flag.String("pool", "default",
		"Storage pool in which the images should be stored")
	storageBackend = flag.String("storage-backend", "dir",
		"Libvirt storage pool type/backend")
	boltPath = flag.String("bolt-path", "/var/data/virtlet/virtlet.db",
		"Path to the bolt database file")
	listen = flag.String("listen", "/run/virtlet.sock",
		"The unix socket to listen on, e.g. /run/virtlet.sock")
	cniPluginsDir = flag.String("cni-bin-dir", "/opt/cni/bin",
		"Path to CNI plugin binaries")
	cniConfigsDir = flag.String("cni-conf-dir", "/etc/cni/net.d",
		"Location of CNI configurations (first file name in lexicographic order will be chosen)")
)

func main() {
	flag.Parse()

	server, err := manager.NewVirtletManager(*libvirtUri, *pool, *storageBackend, *boltPath, *cniPluginsDir, *cniConfigsDir)
	if err != nil {
		glog.Errorf("Initializing server failed: %v", err)
		os.Exit(1)
	}
	glog.V(1).Infof("Starting server on socket %s", *listen)
	if err = server.Serve(*listen); err != nil {
		glog.Errorf("Serving failed: %v", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// SPDX-License-Identifier: MIT
//
// Copyright © 2018 Kent Gibson <warthog618@gmail.com>.

// waitsms waits for SMSs to be received by the modem, and dumps them to
// stdout.
//
// This provides an example of using indications, as well as a test that the
// library works with the modem.
//
// The modem device provided must support nofications, or no SMSs will be seen.
// (the notification port is typically USB2, hence the default)
package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"time"

	"github.com/warthog618/modem/at"
	"github.com/warthog618/modem/gsm"
	"github.com/warthog618/modem/serial"
	"github.com/warthog618/modem/trace"
)

var version = "undefined"

func main() {
	dev := flag.String("d", "/dev/ttyUSB2", "path to modem device")
	baud := flag.Int("b", 115200, "baud rate")
	period := flag.Duration("p", 10*time.Minute, "period to wait")
	timeout := flag.Duration("t", 400*time.Millisecond, "command timeout period")
	verbose := flag.Bool("v", false, "log modem interactions")
	hex := flag.Bool("x", false, "hex dump modem responses")
	vsn := flag.Bool("version", false, "report version and exit")
	flag.Parse()
	if *vsn {
		fmt.Printf("%s %s\n", os.Args[0], version)
		os.Exit(0)
	}
	m, err := serial.New(serial.WithPort(*dev), serial.WithBaud(*baud))
	if err != nil {
		log.Println(err)
		return
	}
	defer m.Close()
	var mio io.ReadWriter = m
	if *hex {
		mio = trace.New(m, trace.WithReadFormat("r: %v"))
	} else if *verbose {
		mio = trace.New(m)
	}
	g := gsm.New(at.New(mio, at.WithTimeout(*timeout)))
	err = g.Init()
	if err != nil {
		log.Println(err)
		return
	}

	go pollSignalQuality(g, timeout)

	err = g.StartMessageRx(
		func(msg gsm.Message) {
			log.Printf("%s: %s\n", msg.Number, msg.Message)
		},
		func(err error) {
			log.Printf("err: %v\n", err)
		})
	if err != nil {
		log.Println(err)
		return
	}
	defer g.StopMessageRx()

	for {
		select {
		case <-time.After(*period):
			log.Println("exiting...")
			return
		case <-g.Closed():
			log.Fatal("modem closed, exiting...")
		}
	}
}

// pollSignalQuality polls the modem to read signal quality every minute.
//
// This is run in parallel to SMS reception to demonstrate separate goroutines
// interacting with the modem.
func pollSignalQuality(g *gsm.GSM, timeout *time.Duration) {
	for {
		select {
		case <-time.After(time.Minute):
			i, err := g.Command("+CSQ")
			if err != nil {
				log.Println(err)
			} else {
				log.Printf("Signal quality: %v\n", i)
			}
		case <-g.Closed():
			return
		}
	}
}
<commit_msg>fix typo<commit_after>// SPDX-License-Identifier: MIT
//
// Copyright © 2018 Kent Gibson <warthog618@gmail.com>.

// waitsms waits for SMSs to be received by the modem, and dumps them to
// stdout.
//
// This provides an example of using indications, as well as a test that the
// library works with the modem.
//
// The modem device provided must support notifications or no SMSs will be seen.
// (the notification port is typically USB2, hence the default)
package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"time"

	"github.com/warthog618/modem/at"
	"github.com/warthog618/modem/gsm"
	"github.com/warthog618/modem/serial"
	"github.com/warthog618/modem/trace"
)

var version = "undefined"

func main() {
	dev := flag.String("d", "/dev/ttyUSB2", "path to modem device")
	baud := flag.Int("b", 115200, "baud rate")
	period := flag.Duration("p", 10*time.Minute, "period to wait")
	timeout := flag.Duration("t", 400*time.Millisecond, "command timeout period")
	verbose := flag.Bool("v", false, "log modem interactions")
	hex := flag.Bool("x", false, "hex dump modem responses")
	vsn := flag.Bool("version", false, "report version and exit")
	flag.Parse()
	if *vsn {
		fmt.Printf("%s %s\n", os.Args[0], version)
		os.Exit(0)
	}
	m, err := serial.New(serial.WithPort(*dev), serial.WithBaud(*baud))
	if err != nil {
		log.Println(err)
		return
	}
	defer m.Close()
	var mio io.ReadWriter = m
	if *hex {
		mio = trace.New(m, trace.WithReadFormat("r: %v"))
	} else if *verbose {
		mio = trace.New(m)
	}
	g := gsm.New(at.New(mio, at.WithTimeout(*timeout)))
	err = g.Init()
	if err != nil {
		log.Println(err)
		return
	}

	go pollSignalQuality(g, timeout)

	err = g.StartMessageRx(
		func(msg gsm.Message) {
			log.Printf("%s: %s\n", msg.Number, msg.Message)
		},
		func(err error) {
			log.Printf("err: %v\n", err)
		})
	if err != nil {
		log.Println(err)
		return
	}
	defer g.StopMessageRx()

	for {
		select {
		case <-time.After(*period):
			log.Println("exiting...")
			return
		case <-g.Closed():
			log.Fatal("modem closed, exiting...")
		}
	}
}

// pollSignalQuality polls the modem to read signal quality every minute.
//
// This is run in parallel to SMS reception to demonstrate separate goroutines
// interacting with the modem.
func pollSignalQuality(g *gsm.GSM, timeout *time.Duration) {
	for {
		select {
		case <-time.After(time.Minute):
			i, err := g.Command("+CSQ")
			if err != nil {
				log.Println(err)
			} else {
				log.Printf("Signal quality: %v\n", i)
			}
		case <-g.Closed():
			return
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"os/exec"
)

const version = "0.1.3"

func main() {
	up := flag.Bool("update", false, "update command")
	flag.Parse()

	fmt.Printf("version %s\n", version)

	var code int
	if *up {
		if out, err := update(srcPath); err != nil {
			fmt.Printf("Update fail: %v\n", err)
			fmt.Println(string(out))
			os.Exit(1)
		}
		fmt.Println("Update done.")
		os.Exit(0)
	}

	fmt.Println("Command done.")

	os.Exit(code)
}

const srcPath = "github.com/dvrkps/dojo/cmdupdate"

func update(srcPath string) ([]byte, error) {
	fmt.Println("Update start.")
	cmd := exec.Command("go", "get", "-u", srcPath)
	return cmd.CombinedOutput()
}
<commit_msg>cmdupdate: clean update func<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"os/exec"
)

const version = "0.1.3"

func main() {
	up := flag.Bool("update", false, "update command")
	flag.Parse()

	fmt.Printf("version %s\n", version)

	var code int
	if *up {
		if out, err := update(srcPath); err != nil {
			fmt.Printf("Update fail: %v\n", err)
			fmt.Println(string(out))
			os.Exit(1)
		}
		fmt.Println("Update done.")
		os.Exit(0)
	}

	fmt.Println("Command done.")

	os.Exit(code)
}

const srcPath = "github.com/dvrkps/dojo/cmdupdate"

func update(srcPath string) ([]byte, error) {
	cmd := exec.Command("go", "get", "-u", srcPath)
	return cmd.CombinedOutput()
}
<|endoftext|>


<commit_before>// +build !nonative

package collector

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"os"
	"regexp"
	"strconv"
	"strings"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/prometheus"
)

const (
	procDiskStats = "/proc/diskstats"
)

type diskStat struct {
	name          string
	metric        prometheus.Metric
	documentation string
}

var (
	ignoredDevices = flag.String("diskstatsIgnoredDevices", "^(ram|loop)\\d+$", "Regexp of devices to ignore for diskstats.")

	// Docs from https://www.kernel.org/doc/Documentation/iostats.txt
	diskStatsMetrics = []diskStat{
		{"reads_completed", prometheus.NewCounter(), "The total number of reads completed successfully."},
		{"reads_merged", prometheus.NewCounter(), "The number of reads merged. See https://www.kernel.org/doc/Documentation/iostats.txt"},
		{"sectors_read", prometheus.NewCounter(), "The total number of sectors read successfully."},
		{"read_time_ms", prometheus.NewCounter(), "the total number of milliseconds spent by all reads."},
		{"writes_completed", prometheus.NewCounter(), "The total number of writes completed successfully."},
		{"writes_merged", prometheus.NewCounter(), "The number of writes merged. See https://www.kernel.org/doc/Documentation/iostats.txt"},
		{"sectors_written", prometheus.NewCounter(), "The total number of sectors written successfully."},
		{"write_time_ms", prometheus.NewCounter(), "This is the total number of milliseconds spent by all writes."},
		{"io_now", prometheus.NewGauge(), "The number of I/Os currently in progress."},
		{"io_time_ms", prometheus.NewCounter(), "Milliseconds spent doing I/Os."},
		{"io_time_weighted", prometheus.NewCounter(), "The weighted # of milliseconds spent doing I/Os. See https://www.kernel.org/doc/Documentation/iostats.txt"},
	}
)

type diskstatsCollector struct {
	registry              prometheus.Registry
	config                Config
	ignoredDevicesPattern *regexp.Regexp
}

func init() {
	Factories["diskstats"] = NewDiskstatsCollector
}

// Takes a config struct and prometheus registry and returns a new Collector exposing
// disk device stats.
func NewDiskstatsCollector(config Config, registry prometheus.Registry) (Collector, error) {
	c := diskstatsCollector{
		config:                config,
		registry:              registry,
		ignoredDevicesPattern: regexp.MustCompile(*ignoredDevices),
	}

	for _, v := range diskStatsMetrics {
		registry.Register(
			"node_disk_"+v.name,
			v.documentation,
			prometheus.NilLabels,
			v.metric,
		)
	}
	return &c, nil
}

func (c *diskstatsCollector) Update() (updates int, err error) {
	diskStats, err := getDiskStats()
	if err != nil {
		return updates, fmt.Errorf("Couldn't get diskstats: %s", err)
	}
	for dev, stats := range diskStats {
		if c.ignoredDevicesPattern.MatchString(dev) {
			glog.V(1).Infof("Ignoring device: %s", dev)
			continue
		}
		for k, value := range stats {
			updates++
			v, err := strconv.ParseFloat(value, 64)
			if err != nil {
				return updates, fmt.Errorf("Invalid value %s in diskstats: %s", value, err)
			}
			labels := map[string]string{"device": dev}
			counter, ok := diskStatsMetrics[k].metric.(prometheus.Counter)
			if ok {
				counter.Set(labels, v)
			} else {
				var gauge = diskStatsMetrics[k].metric.(prometheus.Gauge)
				gauge.Set(labels, v)
			}
		}
	}
	return updates, err
}

func getDiskStats() (map[string]map[int]string, error) {
	file, err := os.Open(procDiskStats)
	if err != nil {
		return nil, err
	}
	return parseDiskStats(file)
}

func parseDiskStats(r io.ReadCloser) (map[string]map[int]string, error) {
	defer r.Close()
	diskStats := map[string]map[int]string{}
	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		parts := strings.Fields(string(scanner.Text()))
		if len(parts) != len(diskStatsMetrics)+3 { // we strip major, minor and dev
			return nil, fmt.Errorf("Invalid line in %s: %s", procDiskStats, scanner.Text())
		}
		dev := parts[2]
		diskStats[dev] = map[int]string{}
		for i, v := range parts[3:] {
			diskStats[dev][i] = v
		}
	}
	return diskStats, nil
}
<commit_msg>Ignore disk partitions by default, it's the disks themselves you usually want.<commit_after>// +build !nonative

package collector

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"os"
	"regexp"
	"strconv"
	"strings"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/prometheus"
)

const (
	procDiskStats = "/proc/diskstats"
)

type diskStat struct {
	name          string
	metric        prometheus.Metric
	documentation string
}

var (
	ignoredDevices = flag.String("diskstatsIgnoredDevices", "^(ram|loop|[hs]d[a-z])\\d+$", "Regexp of devices to ignore for diskstats.")

	// Docs from https://www.kernel.org/doc/Documentation/iostats.txt
	diskStatsMetrics = []diskStat{
		{"reads_completed", prometheus.NewCounter(), "The total number of reads completed successfully."},
		{"reads_merged", prometheus.NewCounter(), "The number of reads merged. See https://www.kernel.org/doc/Documentation/iostats.txt"},
		{"sectors_read", prometheus.NewCounter(), "The total number of sectors read successfully."},
		{"read_time_ms", prometheus.NewCounter(), "the total number of milliseconds spent by all reads."},
		{"writes_completed", prometheus.NewCounter(), "The total number of writes completed successfully."},
		{"writes_merged", prometheus.NewCounter(), "The number of writes merged. See https://www.kernel.org/doc/Documentation/iostats.txt"},
		{"sectors_written", prometheus.NewCounter(), "The total number of sectors written successfully."},
		{"write_time_ms", prometheus.NewCounter(), "This is the total number of milliseconds spent by all writes."},
		{"io_now", prometheus.NewGauge(), "The number of I/Os currently in progress."},
		{"io_time_ms", prometheus.NewCounter(), "Milliseconds spent doing I/Os."},
		{"io_time_weighted", prometheus.NewCounter(), "The weighted # of milliseconds spent doing I/Os. See https://www.kernel.org/doc/Documentation/iostats.txt"},
	}
)

type diskstatsCollector struct {
	registry              prometheus.Registry
	config                Config
	ignoredDevicesPattern *regexp.Regexp
}

func init() {
	Factories["diskstats"] = NewDiskstatsCollector
}

// Takes a config struct and prometheus registry and returns a new Collector exposing
// disk device stats.
func NewDiskstatsCollector(config Config, registry prometheus.Registry) (Collector, error) {
	c := diskstatsCollector{
		config:                config,
		registry:              registry,
		ignoredDevicesPattern: regexp.MustCompile(*ignoredDevices),
	}

	for _, v := range diskStatsMetrics {
		registry.Register(
			"node_disk_"+v.name,
			v.documentation,
			prometheus.NilLabels,
			v.metric,
		)
	}
	return &c, nil
}

func (c *diskstatsCollector) Update() (updates int, err error) {
	diskStats, err := getDiskStats()
	if err != nil {
		return updates, fmt.Errorf("Couldn't get diskstats: %s", err)
	}
	for dev, stats := range diskStats {
		if c.ignoredDevicesPattern.MatchString(dev) {
			glog.V(1).Infof("Ignoring device: %s", dev)
			continue
		}
		for k, value := range stats {
			updates++
			v, err := strconv.ParseFloat(value, 64)
			if err != nil {
				return updates, fmt.Errorf("Invalid value %s in diskstats: %s", value, err)
			}
			labels := map[string]string{"device": dev}
			counter, ok := diskStatsMetrics[k].metric.(prometheus.Counter)
			if ok {
				counter.Set(labels, v)
			} else {
				var gauge = diskStatsMetrics[k].metric.(prometheus.Gauge)
				gauge.Set(labels, v)
			}
		}
	}
	return updates, err
}

func getDiskStats() (map[string]map[int]string, error) {
	file, err := os.Open(procDiskStats)
	if err != nil {
		return nil, err
	}
	return parseDiskStats(file)
}

func parseDiskStats(r io.ReadCloser) (map[string]map[int]string, error) {
	defer r.Close()
	diskStats := map[string]map[int]string{}
	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		parts := strings.Fields(string(scanner.Text()))
		if len(parts) != len(diskStatsMetrics)+3 { // we strip major, minor and dev
			return nil, fmt.Errorf("Invalid line in %s: %s", procDiskStats, scanner.Text())
		}
		dev := parts[2]
		diskStats[dev] = map[int]string{}
		for i, v := range parts[3:] {
			diskStats[dev][i] = v
		}
	}
	return diskStats, nil
}
<|endoftext|>