<commit_before>package auth

import (
	"bytes"
	"encoding/json"
	"github.com/guregu/kami"
	"github.com/the-information/ori/account"
	"github.com/the-information/ori/config"
	"github.com/the-information/ori/rest"
	"golang.org/x/net/context"
	"google.golang.org/appengine/log"
	"io/ioutil"
	"net/http"
)

var authKey = "__auth_ctx"

// Middleware sets up the request context so account information can be
// retrieved with auth.GetAccount(ctx). It panics if config.Get(ctx) fails.
func Middleware(ctx context.Context, w http.ResponseWriter, r *http.Request) context.Context {

	var conf config.Global
	if err := config.Get(ctx, &conf); err != nil {
		panic("Could not get AuthSecret: " + err.Error())
	}
	claimSet, err := Decode([]byte(r.Header.Get("Authorization")), []byte(conf.AuthSecret))
	if err != nil {
		return context.WithValue(ctx, authKey, err)
	} else if claimSet == SuperClaimSet {
		return context.WithValue(ctx, authKey, &account.Super)
	} else if claimSet == NobodyClaimSet {
		return context.WithValue(ctx, authKey, &account.Nobody)
	}

	var acct account.Account
	if err := account.Get(ctx, claimSet.Sub, &acct); err != nil {
		rest.WriteJSON(w, &rest.Response{
			Code: http.StatusUnauthorized,
			Body: &rest.Message{"Could not retrieve account with key " + claimSet.Sub + ": " + err.Error()},
		})
		return nil
	} else {
		return context.WithValue(ctx, authKey, &acct)
	}

}

var forbiddenMessage = "You do not have permission to access this resource."

type key int

// An AuthCheck is a handler-like function that determines whether a given HTTP request ought to be
// allowed to continue executing based on the request's authentication status. They can be
// used with Check to check credentials on a route without having to do so in the
// core HTTP handler for the route, which is very convenient.
type AuthCheck func(ctx context.Context, w http.ResponseWriter, r *http.Request) bool

// Super is an AuthCheck that grants access to the superuser (that is, the user
// identified by `Authorization: <secret>`).
func Super(ctx context.Context, w http.ResponseWriter, r *http.Request) bool {

	var acct account.Account
	GetAccount(ctx, &acct)
	return acct.Super()

}

// HasRole returns an AuthCheck that grants access if `account.HasRole(role)` is true.
func HasRole(role string) AuthCheck {

	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) bool {

		var acct account.Account
		GetAccount(ctx, &acct)
		return acct.HasRole(role)

	}

}

// AccountMatchesParam returns an AuthCheck that grants access if paramName is the same
// as the account's ID; so, for instance, on a route to /accounts/:accountId, with
// a request to /accounts/asdf, the AuthCheck will return true if the account's ID is asdf.
func AccountMatchesParam(paramName string) AuthCheck {

	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) bool {

		var acct account.Account
		err := GetAccount(ctx, &acct)
		return err == nil && acct.Key(ctx).Encode() == kami.Param(ctx, paramName)

	}

}

// AccountOwnsObject returns an AuthCheck that reads the request body and grants
// access if the request body is an object that has the property `property` and its
// value is the same as the account's encoded datastore key. So, for instance,
// on a request with a body shaped like `{"owner": "asdf"}`, AccountOwnsObject("owner")
// will return true if the authorized account's encoded datastore key is `asdf`.
func AccountOwnsObject(property string) AuthCheck {

	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) bool {

		if r.Body == nil {
			return false
		}

		var acct account.Account
		err := GetAccount(ctx, &acct)
		if err != nil {
			return false
		} else if acct.Super() {
			return true
		}

		// buffer the http body so we can look at it
		buf := bytes.Buffer{}
		buf.ReadFrom(r.Body)

		r.Body = ioutil.NopCloser(&buf)
		data := buf.Bytes()

		props := make(map[string]interface{}, 8)

		if err := json.Unmarshal(data, props); err != nil {
			return false
		}

		val, ok := props[property]
		if !ok {
			return false
		}

		return val.(string) == acct.Key(ctx).Encode()
	}

}

// Checker is an object returned by auth.Check.
type Checker struct {
	checks []AuthCheck
}

// Check produces a Checker for a set of AuthChecks. If all of the AuthChecks
// fail, Checker.Then will return a 403 to the user and prevent the underlying
// handler from being called.
func Check(checks ...AuthCheck) *Checker {
	return &Checker{checks: checks}
}

// Then produces a kami.Handler that wraps another handler with authentication magic.
// If every AuthCheck associated with the Checker fails, Then will reject the HTTP request
// with a 403. If even one of them passes, Then will call h.
// This produces a very pleasant syntax for authenticating routes:
// 	func listAccounts(ctx context.Context, w http.ResponseWriter, r *http.Request) {
//		w.Write([]byte("[]"))
//	}
//
// 	kami.Get("/accounts", auth.Check(auth.Super, auth.HasRole("admin")).Then(listAccounts))
func (c *Checker) Then(h kami.HandlerFunc) kami.HandlerFunc {

	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) {

		var acct account.Account
		GetAccount(ctx, &acct)

		// run the checks
		passed := false
		for _, check := range c.checks {
			if check(ctx, w, r) {
				passed = true
			}
		}

		// did one of them pass?
		if passed {
			log.Infof(ctx, "%s: access granted", acct.Email)
			// mark this as a context which passed authentication
			ctx = context.WithValue(ctx, "__auth_check_ctx", "passed")
			h(ctx, w, r)
		} else {

			log.Warningf(ctx, "%s: access denied", acct.Email)
			rest.WriteJSON(w, &rest.Response{
				Code: http.StatusForbidden,
				Body: &rest.Message{forbiddenMessage},
			})

		}

	}

}

// GetAccount retrieves the authorized account for ctx and copies it into account.
// It returns the error if one was encountered.
// Note that for special user types (account.Nobody and account.Super) both
// the returned key and error will be nil.
func GetAccount(ctx context.Context, acct *account.Account) error {

	switch t := ctx.Value(authKey).(type) {
	case error:
		return t
	case *account.Account:
		*acct = *t
		return nil
	default:
		panic("Type assertion failed, we should never get here")
	}

}
<commit_msg>Keep calm, return ErrNotInAuthContext<commit_after>package auth

import (
	"bytes"
	"encoding/json"
	"errors"
	"github.com/guregu/kami"
	"github.com/the-information/ori/account"
	"github.com/the-information/ori/config"
	"github.com/the-information/ori/rest"
	"golang.org/x/net/context"
	"google.golang.org/appengine/log"
	"io/ioutil"
	"net/http"
)

var authKey = "__auth_ctx"

var ErrNotInAuthContext = errors.New("That context object was not run through auth.Middleware!")

// Middleware sets up the request context so account information can be
// retrieved with auth.GetAccount(ctx). It panics if config.Get(ctx) fails.
func Middleware(ctx context.Context, w http.ResponseWriter, r *http.Request) context.Context {

	var conf config.Global
	if err := config.Get(ctx, &conf); err != nil {
		panic("Could not get AuthSecret: " + err.Error())
	}
	claimSet, err := Decode([]byte(r.Header.Get("Authorization")), []byte(conf.AuthSecret))
	if err != nil {
		return context.WithValue(ctx, authKey, err)
	} else if claimSet == SuperClaimSet {
		return context.WithValue(ctx, authKey, &account.Super)
	} else if claimSet == NobodyClaimSet {
		return context.WithValue(ctx, authKey, &account.Nobody)
	}

	var acct account.Account
	if err := account.Get(ctx, claimSet.Sub, &acct); err != nil {
		rest.WriteJSON(w, &rest.Response{
			Code: http.StatusUnauthorized,
			Body: &rest.Message{"Could not retrieve account with key " + claimSet.Sub + ": " + err.Error()},
		})
		return nil
	} else {
		return context.WithValue(ctx, authKey, &acct)
	}

}

var forbiddenMessage = "You do not have permission to access this resource."

type key int

// An AuthCheck is a handler-like function that determines whether a given HTTP request ought to be
// allowed to continue executing based on the request's authentication status. They can be
// used with Check to check credentials on a route without having to do so in the
// core HTTP handler for the route, which is very convenient.
type AuthCheck func(ctx context.Context, w http.ResponseWriter, r *http.Request) bool

// Super is an AuthCheck that grants access to the superuser (that is, the user
// identified by `Authorization: <secret>`).
func Super(ctx context.Context, w http.ResponseWriter, r *http.Request) bool {

	var acct account.Account
	GetAccount(ctx, &acct)
	return acct.Super()

}

// HasRole returns an AuthCheck that grants access if `account.HasRole(role)` is true.
func HasRole(role string) AuthCheck {

	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) bool {

		var acct account.Account
		GetAccount(ctx, &acct)
		return acct.HasRole(role)

	}

}

// AccountMatchesParam returns an AuthCheck that grants access if paramName is the same
// as the account's ID; so, for instance, on a route to /accounts/:accountId, with
// a request to /accounts/asdf, the AuthCheck will return true if the account's ID is asdf.
func AccountMatchesParam(paramName string) AuthCheck {

	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) bool {

		var acct account.Account
		err := GetAccount(ctx, &acct)
		return err == nil && acct.Key(ctx).Encode() == kami.Param(ctx, paramName)

	}

}

// AccountOwnsObject returns an AuthCheck that reads the request body and grants
// access if the request body is an object that has the property `property` and its
// value is the same as the account's encoded datastore key. So, for instance,
// on a request with a body shaped like `{"owner": "asdf"}`, AccountOwnsObject("owner")
// will return true if the authorized account's encoded datastore key is `asdf`.
func AccountOwnsObject(property string) AuthCheck {

	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) bool {

		if r.Body == nil {
			return false
		}

		var acct account.Account
		err := GetAccount(ctx, &acct)
		if err != nil {
			return false
		} else if acct.Super() {
			return true
		}

		// buffer the http body so we can look at it
		buf := bytes.Buffer{}
		buf.ReadFrom(r.Body)

		r.Body = ioutil.NopCloser(&buf)
		data := buf.Bytes()

		props := make(map[string]interface{}, 8)

		if err := json.Unmarshal(data, props); err != nil {
			return false
		}

		val, ok := props[property]
		if !ok {
			return false
		}

		return val.(string) == acct.Key(ctx).Encode()
	}

}

// Checker is an object returned by auth.Check.
type Checker struct {
	checks []AuthCheck
}

// Check produces a Checker for a set of AuthChecks. If all of the AuthChecks
// fail, Checker.Then will return a 403 to the user and prevent the underlying
// handler from being called.
func Check(checks ...AuthCheck) *Checker {
	return &Checker{checks: checks}
}

// Then produces a kami.Handler that wraps another handler with authentication magic.
// If every AuthCheck associated with the Checker fails, Then will reject the HTTP request
// with a 403. If even one of them passes, Then will call h.
// This produces a very pleasant syntax for authenticating routes:
// 	func listAccounts(ctx context.Context, w http.ResponseWriter, r *http.Request) {
//		w.Write([]byte("[]"))
//	}
//
// 	kami.Get("/accounts", auth.Check(auth.Super, auth.HasRole("admin")).Then(listAccounts))
func (c *Checker) Then(h kami.HandlerFunc) kami.HandlerFunc {

	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) {

		var acct account.Account
		GetAccount(ctx, &acct)

		// run the checks
		passed := false
		for _, check := range c.checks {
			if check(ctx, w, r) {
				passed = true
			}
		}

		// did one of them pass?
		if passed {
			log.Infof(ctx, "%s: access granted", acct.Email)
			// mark this as a context which passed authentication
			ctx = context.WithValue(ctx, "__auth_check_ctx", "passed")
			h(ctx, w, r)
		} else {

			log.Warningf(ctx, "%s: access denied", acct.Email)
			rest.WriteJSON(w, &rest.Response{
				Code: http.StatusForbidden,
				Body: &rest.Message{forbiddenMessage},
			})

		}

	}

}

// GetAccount retrieves the authorized account for ctx and copies it into account.
// It returns the error if one was encountered.
// Note that for special user types (account.Nobody and account.Super) both
// the returned key and error will be nil.
func GetAccount(ctx context.Context, acct *account.Account) error {

	switch t := ctx.Value(authKey).(type) {
	case error:
		return t
	case *account.Account:
		*acct = *t
		return nil
	default:
		return ErrNotInAuthContext
	}

}
<|endoftext|>


<commit_before>//go:build linux && cgo

package device

import (
	"fmt"
	"github.com/lxc/lxd/shared/logger"
	"strings"

	udev "github.com/jochenvg/go-udev"

	deviceConfig "github.com/lxc/lxd/lxd/device/config"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/validate"
)

// unixHotplugIsOurDevice indicates whether the unixHotplug device event qualifies as part of our device.
// This function is not defined against the unixHotplug struct type so that it can be used in event
// callbacks without needing to keep a reference to the unixHotplug device struct.
func unixHotplugIsOurDevice(config deviceConfig.Device, unixHotplug *UnixHotplugEvent) bool {
	// Check if event matches criteria for this device, if not return.
	if (config["vendorid"] != "" && config["vendorid"] != unixHotplug.Vendor) || (config["productid"] != "" && config["productid"] != unixHotplug.Product) {
		return false
	}

	return true
}

type unixHotplug struct {
	deviceCommon
}

// isRequired indicates whether the device config requires this device to start OK.
func (d *unixHotplug) isRequired() bool {
	// Defaults to not required.
	return shared.IsTrue(d.config["required"])
}

// validateConfig checks the supplied config for correctness.
func (d *unixHotplug) validateConfig(instConf instance.ConfigReader) error {
	if !instanceSupported(instConf.Type(), instancetype.Container) {
		return ErrUnsupportedDevType
	}

	rules := map[string]func(string) error{
		"vendorid":  validate.Optional(validate.IsDeviceID),
		"productid": validate.Optional(validate.IsDeviceID),
		"uid":       unixValidUserID,
		"gid":       unixValidUserID,
		"mode":      unixValidOctalFileMode,
		"required":  validate.Optional(validate.IsBool),
	}

	err := d.config.Validate(rules)
	if err != nil {
		return err
	}

	if d.config["vendorid"] == "" && d.config["productid"] == "" {
		return fmt.Errorf("Unix hotplug devices require a vendorid or a productid")
	}

	return nil
}

// Register is run after the device is started or when LXD starts.
func (d *unixHotplug) Register() error {
	// Extract variables needed to run the event hook so that the reference to this device
	// struct is not needed to be kept in memory.
	devicesPath := d.inst.DevicesPath()
	devConfig := d.config
	deviceName := d.name
	state := d.state

	// Handler for when a UnixHotplug event occurs.
	f := func(e UnixHotplugEvent) (*deviceConfig.RunConfig, error) {
		runConf := deviceConfig.RunConfig{}

		if e.Action == "add" {
			if !unixHotplugIsOurDevice(devConfig, &e) {
				return nil, nil
			}

			if e.Subsystem == "block" {
				err := unixDeviceSetupBlockNum(state, devicesPath, "unix", deviceName, devConfig, e.Major, e.Minor, e.Path, false, &runConf)
				if err != nil {
					return nil, err
				}
			} else {
				err := unixDeviceSetupCharNum(state, devicesPath, "unix", deviceName, devConfig, e.Major, e.Minor, e.Path, false, &runConf)
				if err != nil {
					return nil, err
				}
			}
		} else if e.Action == "remove" {
			relativeTargetPath := strings.TrimPrefix(e.Path, "/")
			err := unixDeviceRemove(devicesPath, "unix", deviceName, relativeTargetPath, &runConf)
			if err != nil {
				return nil, err
			}

			// Add a post hook function to remove the specific unix hotplug device file after unmount.
			runConf.PostHooks = []func() error{func() error {
				err := unixDeviceDeleteFiles(state, devicesPath, "unix", deviceName, relativeTargetPath)
				if err != nil {
					return fmt.Errorf("Failed to delete files for device '%s': %w", deviceName, err)
				}

				return nil
			}}
		}

		runConf.Uevents = append(runConf.Uevents, e.UeventParts)

		return &runConf, nil
	}

	unixHotplugRegisterHandler(d.inst, d.name, f)

	return nil
}

// Start is run when the device is added to the instance.
func (d *unixHotplug) Start() (*deviceConfig.RunConfig, error) {
	runConf := deviceConfig.RunConfig{}
	runConf.PostHooks = []func() error{d.Register}

	device := d.loadUnixDevice()
	if d.isRequired() && device == nil {
		return nil, fmt.Errorf("Required Unix Hotplug device not found")
	}
	if device == nil {
		return &runConf, nil
	}

	devnum := device.Devnum()
	major := uint32(devnum.Major())
	minor := uint32(devnum.Minor())

	// setup device
	var err error
	if device.Subsystem() == "block" {
		err = unixDeviceSetupBlockNum(d.state, d.inst.DevicesPath(), "unix", d.name, d.config, major, minor, device.Devnode(), false, &runConf)
	} else {
		err = unixDeviceSetupCharNum(d.state, d.inst.DevicesPath(), "unix", d.name, d.config, major, minor, device.Devnode(), false, &runConf)
	}

	if err != nil {
		return nil, err
	}

	return &runConf, nil
}

// Stop is run when the device is removed from the instance.
func (d *unixHotplug) Stop() (*deviceConfig.RunConfig, error) {
	unixHotplugUnregisterHandler(d.inst, d.name)

	runConf := deviceConfig.RunConfig{
		PostHooks: []func() error{d.postStop},
	}

	err := unixDeviceRemove(d.inst.DevicesPath(), "unix", d.name, "", &runConf)
	if err != nil {
		return nil, err
	}

	return &runConf, nil
}

// postStop is run after the device is removed from the instance.
func (d *unixHotplug) postStop() error {
	err := unixDeviceDeleteFiles(d.state, d.inst.DevicesPath(), "unix", d.name, "")
	if err != nil {
		return fmt.Errorf("Failed to delete files for device '%s': %w", d.name, err)
	}

	return nil
}

// loadUnixDevice scans the host machine for unix devices with matching product/vendor ids
// and returns the first matching device with the subsystem type char or block.
func (d *unixHotplug) loadUnixDevice() *udev.Device {
	// Find device if exists
	u := udev.Udev{}
	e := u.NewEnumerate()

	if d.config["vendorid"] != "" {
		err := e.AddMatchProperty("ID_VENDOR_ID", d.config["vendorid"])
		if err != nil {
			logger.Warn("Failed to add property to device", logger.Ctx{"property_name": "ID_VENDOR_ID", "property_value": d.config["vendorid"], "err": err})
		}
	}

	if d.config["productid"] != "" {
		err := e.AddMatchProperty("ID_MODEL_ID", d.config["productid"])
		if err != nil {
			logger.Warn("Failed to add property to device", logger.Ctx{"property_name": "ID_MODEL_ID", "property_value": d.config["productid"], "err": err})
		}
	}

	err := e.AddMatchIsInitialized()
	if err != nil {
		logger.Warn("Failed to add initialised property to device", logger.Ctx{"err": err})
	}

	devices, _ := e.Devices()
	var device *udev.Device
	for i := range devices {
		device = devices[i]

		devnum := device.Devnum()
		if devnum.Major() == 0 || devnum.Minor() == 0 {
			continue
		}

		if device.Devnode() == "" {
			continue
		}

		if !strings.HasPrefix(device.Subsystem(), "usb") {
			return device
		}
	}

	return nil
}
<commit_msg>lxd/device: Fixes imports.<commit_after>//go:build linux && cgo

package device

import (
	"fmt"
	"strings"

	"github.com/jochenvg/go-udev"

	deviceConfig "github.com/lxc/lxd/lxd/device/config"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/validate"
)

// unixHotplugIsOurDevice indicates whether the unixHotplug device event qualifies as part of our device.
// This function is not defined against the unixHotplug struct type so that it can be used in event
// callbacks without needing to keep a reference to the unixHotplug device struct.
func unixHotplugIsOurDevice(config deviceConfig.Device, unixHotplug *UnixHotplugEvent) bool {
	// Check if event matches criteria for this device, if not return.
	if (config["vendorid"] != "" && config["vendorid"] != unixHotplug.Vendor) || (config["productid"] != "" && config["productid"] != unixHotplug.Product) {
		return false
	}

	return true
}

type unixHotplug struct {
	deviceCommon
}

// isRequired indicates whether the device config requires this device to start OK.
func (d *unixHotplug) isRequired() bool {
	// Defaults to not required.
	return shared.IsTrue(d.config["required"])
}

// validateConfig checks the supplied config for correctness.
func (d *unixHotplug) validateConfig(instConf instance.ConfigReader) error {
	if !instanceSupported(instConf.Type(), instancetype.Container) {
		return ErrUnsupportedDevType
	}

	rules := map[string]func(string) error{
		"vendorid":  validate.Optional(validate.IsDeviceID),
		"productid": validate.Optional(validate.IsDeviceID),
		"uid":       unixValidUserID,
		"gid":       unixValidUserID,
		"mode":      unixValidOctalFileMode,
		"required":  validate.Optional(validate.IsBool),
	}

	err := d.config.Validate(rules)
	if err != nil {
		return err
	}

	if d.config["vendorid"] == "" && d.config["productid"] == "" {
		return fmt.Errorf("Unix hotplug devices require a vendorid or a productid")
	}

	return nil
}

// Register is run after the device is started or when LXD starts.
func (d *unixHotplug) Register() error {
	// Extract variables needed to run the event hook so that the reference to this device
	// struct is not needed to be kept in memory.
	devicesPath := d.inst.DevicesPath()
	devConfig := d.config
	deviceName := d.name
	state := d.state

	// Handler for when a UnixHotplug event occurs.
	f := func(e UnixHotplugEvent) (*deviceConfig.RunConfig, error) {
		runConf := deviceConfig.RunConfig{}

		if e.Action == "add" {
			if !unixHotplugIsOurDevice(devConfig, &e) {
				return nil, nil
			}

			if e.Subsystem == "block" {
				err := unixDeviceSetupBlockNum(state, devicesPath, "unix", deviceName, devConfig, e.Major, e.Minor, e.Path, false, &runConf)
				if err != nil {
					return nil, err
				}
			} else {
				err := unixDeviceSetupCharNum(state, devicesPath, "unix", deviceName, devConfig, e.Major, e.Minor, e.Path, false, &runConf)
				if err != nil {
					return nil, err
				}
			}
		} else if e.Action == "remove" {
			relativeTargetPath := strings.TrimPrefix(e.Path, "/")
			err := unixDeviceRemove(devicesPath, "unix", deviceName, relativeTargetPath, &runConf)
			if err != nil {
				return nil, err
			}

			// Add a post hook function to remove the specific unix hotplug device file after unmount.
			runConf.PostHooks = []func() error{func() error {
				err := unixDeviceDeleteFiles(state, devicesPath, "unix", deviceName, relativeTargetPath)
				if err != nil {
					return fmt.Errorf("Failed to delete files for device '%s': %w", deviceName, err)
				}

				return nil
			}}
		}

		runConf.Uevents = append(runConf.Uevents, e.UeventParts)

		return &runConf, nil
	}

	unixHotplugRegisterHandler(d.inst, d.name, f)

	return nil
}

// Start is run when the device is added to the instance.
func (d *unixHotplug) Start() (*deviceConfig.RunConfig, error) {
	runConf := deviceConfig.RunConfig{}
	runConf.PostHooks = []func() error{d.Register}

	device := d.loadUnixDevice()
	if d.isRequired() && device == nil {
		return nil, fmt.Errorf("Required Unix Hotplug device not found")
	}
	if device == nil {
		return &runConf, nil
	}

	devnum := device.Devnum()
	major := uint32(devnum.Major())
	minor := uint32(devnum.Minor())

	// setup device
	var err error
	if device.Subsystem() == "block" {
		err = unixDeviceSetupBlockNum(d.state, d.inst.DevicesPath(), "unix", d.name, d.config, major, minor, device.Devnode(), false, &runConf)
	} else {
		err = unixDeviceSetupCharNum(d.state, d.inst.DevicesPath(), "unix", d.name, d.config, major, minor, device.Devnode(), false, &runConf)
	}

	if err != nil {
		return nil, err
	}

	return &runConf, nil
}

// Stop is run when the device is removed from the instance.
func (d *unixHotplug) Stop() (*deviceConfig.RunConfig, error) {
	unixHotplugUnregisterHandler(d.inst, d.name)

	runConf := deviceConfig.RunConfig{
		PostHooks: []func() error{d.postStop},
	}

	err := unixDeviceRemove(d.inst.DevicesPath(), "unix", d.name, "", &runConf)
	if err != nil {
		return nil, err
	}

	return &runConf, nil
}

// postStop is run after the device is removed from the instance.
func (d *unixHotplug) postStop() error {
	err := unixDeviceDeleteFiles(d.state, d.inst.DevicesPath(), "unix", d.name, "")
	if err != nil {
		return fmt.Errorf("Failed to delete files for device '%s': %w", d.name, err)
	}

	return nil
}

// loadUnixDevice scans the host machine for unix devices with matching product/vendor ids
// and returns the first matching device with the subsystem type char or block.
func (d *unixHotplug) loadUnixDevice() *udev.Device {
	// Find device if exists
	u := udev.Udev{}
	e := u.NewEnumerate()

	if d.config["vendorid"] != "" {
		err := e.AddMatchProperty("ID_VENDOR_ID", d.config["vendorid"])
		if err != nil {
			logger.Warn("Failed to add property to device", logger.Ctx{"property_name": "ID_VENDOR_ID", "property_value": d.config["vendorid"], "err": err})
		}
	}

	if d.config["productid"] != "" {
		err := e.AddMatchProperty("ID_MODEL_ID", d.config["productid"])
		if err != nil {
			logger.Warn("Failed to add property to device", logger.Ctx{"property_name": "ID_MODEL_ID", "property_value": d.config["productid"], "err": err})
		}
	}

	err := e.AddMatchIsInitialized()
	if err != nil {
		logger.Warn("Failed to add initialised property to device", logger.Ctx{"err": err})
	}

	devices, _ := e.Devices()
	var device *udev.Device
	for i := range devices {
		device = devices[i]

		devnum := device.Devnum()
		if devnum.Major() == 0 || devnum.Minor() == 0 {
			continue
		}

		if device.Devnode() == "" {
			continue
		}

		if !strings.HasPrefix(device.Subsystem(), "usb") {
			return device
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"github.com/spf13/cobra"

	// Used by cgo
	_ "github.com/lxc/lxd/lxd/include"
)

/*
#include "config.h"

#include <ctype.h>
#include <errno.h>
#include <fcntl.h>
#include <grp.h>
#include <sched.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/fsuid.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/vfs.h>
#include <unistd.h>

#include "lxd.h"
#include "error_utils.h"
#include "file_utils.h"
#include "macro.h"
#include "memory_utils.h"
#include "mount_utils.h"
#include "process_utils.h"
#include "syscall_numbers.h"
#include "syscall_wrappers.h"

#ifndef PIPEFS_MAGIC
#define PIPEFS_MAGIC 0x50495045
#endif

#define FD_PIPE_UIDMAP 3
#define FD_PIPE_GIDMAP 4

#define MAP_SIZE ((ssize_t)4096)

typedef __typeof__(((struct statfs *)NULL)->f_type) fs_type_magic;
static bool is_fs_type(const struct statfs *fs, fs_type_magic magic_val)
{
	return (fs->f_type == (fs_type_magic)magic_val);
}

static bool fhas_fs_type(int fd, fs_type_magic magic_val)
{
	int ret;
	struct statfs sb;

	ret = fstatfs(fd, &sb);
	if (ret < 0)
		return false;

	return is_fs_type(&sb, magic_val);
}

static inline bool drop_groups(void)
{
	return setgroups(0, NULL) == 0;
}

static inline bool switch_resids(uid_t uid, gid_t gid)
{
	if (setresgid(gid, gid, gid))
		return false;

	if (setresuid(uid, uid, uid))
		return false;

	if (setfsgid(-1) != gid)
		return false;

	if (setfsuid(-1) != uid)
		return false;

	return true;
}

static int write_id_mapping(bool is_uid)
{
	__do_close int fd_idmap = -EBADF;
	ssize_t bytes_read, ret;
	char buf[PROC_PID_IDMAP_LEN], idmap[LXC_IDMAPLEN];

	if (is_uid)
		ret = strnprintf(buf, sizeof(buf), "/proc/%d/uid_map", getppid());
	else
		ret = strnprintf(buf, sizeof(buf), "/proc/%d/gid_map", getppid());
	if (ret < 0)
		return -errno;

	fd_idmap = open(buf, O_WRONLY | O_CLOEXEC | O_NOCTTY | O_NOFOLLOW);
	if (fd_idmap < 0)
		return -1;

	if (is_uid)
		bytes_read = read_nointr(FD_PIPE_UIDMAP, idmap, sizeof(idmap));
	else
		bytes_read = read_nointr(FD_PIPE_GIDMAP, idmap, sizeof(idmap));
	if (bytes_read < 0 || bytes_read >= sizeof(idmap))
		return -errno;

	ret = write_nointr(fd_idmap, idmap, bytes_read);
	if (ret < 0 || ret != bytes_read)
		return -errno;

	return 0;
}

static int write_id_mappings(void)
{
	int ret;

	ret = write_id_mapping(true);
	if (!ret)
		ret = write_id_mapping(false);
	return ret;
}

static int safe_uint(const char *numstr, unsigned int *converted)
{
	char *err = NULL;
	unsigned long int uli;

	while (isspace(*numstr))
		numstr++;

	if (*numstr == '-')
		return -EINVAL;

	errno = 0;
	uli = strtoul(numstr, &err, 0);
	if (errno == ERANGE && uli == ULONG_MAX)
		return -ERANGE;

	if (err == numstr || *err != '\0')
		return -EINVAL;

	if (uli > UINT_MAX)
		return -ERANGE;

	*converted = (unsigned int)uli;
	return 0;
}

__attribute__ ((noinline)) int __forkusernsexec(void)
{
	__do_free_string_list char **argvp = NULL;
	pid_t pid;
	ssize_t ret;
	int fd_socket[2];
	char c;
	char *cur;
	unsigned int keep_fd_up_to = STDERR_FILENO;

	if (geteuid() != 0)
		return log_error(EXIT_FAILURE, "Error: forkexec requires root privileges");

	cur = advance_arg(false);
	if (!cur || (strcmp(cur, "--help") == 0 ||
	     strcmp(cur, "--version") == 0 || strcmp(cur, "-h") == 0))
		return EXIT_SUCCESS;

	if (strncmp(cur, "--keep-fd-up-to=", STRLITERALLEN("--keep-fd-up-to=")) == 0) {
		cur += STRLITERALLEN("--keep-fd-up-to=");
		ret = safe_uint(cur, &keep_fd_up_to);
		if (ret)
			return log_error(EXIT_FAILURE, "Invalid fd number %s specified", cur);

		cur = advance_arg(true);
	}

	if (!fhas_fs_type(FD_PIPE_UIDMAP, PIPEFS_MAGIC))
		return log_error(EXIT_FAILURE, "Error: Missing UID map FD");

	if (!fhas_fs_type(FD_PIPE_GIDMAP, PIPEFS_MAGIC))
		return log_error(EXIT_FAILURE, "Error: Missing GID map FD");

	ret = socketpair(AF_LOCAL, SOCK_STREAM | SOCK_CLOEXEC, 0, fd_socket);
	if (ret < 0)
		return EXIT_FAILURE;

	pid = fork();
	if (pid < 0)
		return EXIT_FAILURE;

	if (pid == 0) {
		close_prot_errno_disarm(fd_socket[0]);

		ret = read_nointr(fd_socket[1], &c, 1);
		if (ret != 1)
			_exit(EXIT_FAILURE);

		ret = write_id_mappings();
		if (ret < 0)
			_exit(EXIT_FAILURE);

		ret = write_nointr(fd_socket[1], &c, 1);
		if (ret != 1)
			_exit(EXIT_FAILURE);

		_exit(EXIT_SUCCESS);
	}

	close_prot_errno_disarm(fd_socket[1]);

	ret = unshare(CLONE_NEWUSER);
	if (ret) {
		kill(pid, SIGKILL);
		goto out_reap;
	}

	ret = write_nointr(fd_socket[0], &c, 1);
	if (ret == 1)
		ret = read_nointr(fd_socket[0], &c, 1);
	if (ret != 1) {
		kill(pid, SIGKILL);
		goto out_reap;
	}

	if (!drop_groups()) {
		kill(pid, SIGKILL);
		goto out_reap;
	}

	if (!switch_resids(0, 0)) {
		kill(pid, SIGKILL);
		goto out_reap;
	}

	close_prot_errno_disarm(fd_socket[0]);

out_reap:
	ret = wait_for_pid(pid);
	if (ret)
		return EXIT_FAILURE;

	for (; cur; cur = advance_arg(false)) {
		ret = push_vargs(&argvp, cur);
		if (ret < 0)
			return log_error(EXIT_FAILURE, "Failed to add %s to arg array", cur);
	}

	if (!argvp || !*argvp)
		return log_error(EXIT_FAILURE, "No command specified");

	ret = lxd_close_range(keep_fd_up_to + 1, UINT_MAX, CLOSE_RANGE_CLOEXEC);
	if (ret && !IN_SET(errno, ENOSYS, EINVAL))
		return log_error(EXIT_FAILURE, "Aborting forkusernsexec to prevent leaking file descriptors");

	execvp(argvp[0], argvp);
	return EXIT_FAILURE;
}

void forkusernsexec(void)
{
	_exit(__forkusernsexec());
}
*/
import "C"

type cmdForkusernsexec struct {
	global     *cmdGlobal
	keepFdUpTo int
}

func (c *cmdForkusernsexec) Command() *cobra.Command {
	// Main subcommand
	cmd := &cobra.Command{}
	cmd.Use = "forkusernsexec <cmd> <args>"
	cmd.Short = "Run command in user namespace"
	cmd.Flags().IntVar(&c.keepFdUpTo, "keep-fd-up-to", (1 << 31), fmt.Sprintf("Keep all fds below and including this one and close all the ones above it"))
	cmd.Long = `Description:
  Run command in user namespace

  This command is used to execute a program in a new user namespace.
`
	cmd.RunE = c.Run
	cmd.Hidden = true

	return cmd
}

func (c *cmdForkusernsexec) Run(cmd *cobra.Command, args []string) error {
	return fmt.Errorf("This command should have been intercepted in cgo")
}
<commit_msg>forkusernsexec: log verbosely<commit_after>package main

import (
	"fmt"

	"github.com/spf13/cobra"

	// Used by cgo
	_ "github.com/lxc/lxd/lxd/include"
)

/*
#include "config.h"

#include <ctype.h>
#include <errno.h>
#include <fcntl.h>
#include <grp.h>
#include <sched.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/fsuid.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/vfs.h>
#include <unistd.h>

#include "lxd.h"
#include "error_utils.h"
#include "file_utils.h"
#include "macro.h"
#include "memory_utils.h"
#include "mount_utils.h"
#include "process_utils.h"
#include "syscall_numbers.h"
#include "syscall_wrappers.h"

#ifndef PIPEFS_MAGIC
#define PIPEFS_MAGIC 0x50495045
#endif

#define FD_PIPE_UIDMAP 3
#define FD_PIPE_GIDMAP 4

#define MAP_SIZE ((ssize_t)4096)

typedef __typeof__(((struct statfs *)NULL)->f_type) fs_type_magic;
static bool is_fs_type(const struct statfs *fs, fs_type_magic magic_val)
{
	return (fs->f_type == (fs_type_magic)magic_val);
}

static bool fhas_fs_type(int fd, fs_type_magic magic_val)
{
	int ret;
	struct statfs sb;

	ret = fstatfs(fd, &sb);
	if (ret < 0)
		return false;

	return is_fs_type(&sb, magic_val);
}

static inline bool drop_groups(void)
{
	return setgroups(0, NULL) == 0;
}

static inline bool switch_resids(uid_t uid, gid_t gid)
{
	if (setresgid(gid, gid, gid))
		return false;

	if (setresuid(uid, uid, uid))
		return false;

	if (setfsgid(-1) != gid)
		return false;

	if (setfsuid(-1) != uid)
		return false;

	return true;
}

static int write_id_mapping(bool is_uid)
{
	__do_close int fd_idmap = -EBADF;
	ssize_t bytes_read, ret;
	char buf[PROC_PID_IDMAP_LEN], idmap[LXC_IDMAPLEN];

	if (is_uid)
		ret = strnprintf(buf, sizeof(buf), "/proc/%d/uid_map", getppid());
	else
		ret = strnprintf(buf, sizeof(buf), "/proc/%d/gid_map", getppid());
	if (ret < 0)
		return -errno;

	fd_idmap = open(buf, O_WRONLY | O_CLOEXEC | O_NOCTTY | O_NOFOLLOW);
	if (fd_idmap < 0)
		return -1;

	if (is_uid)
		bytes_read = read_nointr(FD_PIPE_UIDMAP, idmap, sizeof(idmap));
	else
		bytes_read = read_nointr(FD_PIPE_GIDMAP, idmap, sizeof(idmap));
	if (bytes_read < 0 || bytes_read >= sizeof(idmap))
		return -errno;

	ret = write_nointr(fd_idmap, idmap, bytes_read);
	if (ret < 0 || ret != bytes_read)
		return -errno;

	return 0;
}

static int write_id_mappings(void)
{
	int ret;

	ret = write_id_mapping(true);
	if (!ret)
		ret = write_id_mapping(false);
	return ret;
}

static int safe_uint(const char *numstr, unsigned int *converted)
{
	char *err = NULL;
	unsigned long int uli;

	while (isspace(*numstr))
		numstr++;

	if (*numstr == '-')
		return -EINVAL;

	errno = 0;
	uli = strtoul(numstr, &err, 0);
	if (errno == ERANGE && uli == ULONG_MAX)
		return -ERANGE;

	if (err == numstr || *err != '\0')
		return -EINVAL;

	if (uli > UINT_MAX)
		return -ERANGE;

	*converted = (unsigned int)uli;
	return 0;
}

__attribute__ ((noinline)) int __forkusernsexec(void)
{
	__do_free_string_list char **argvp = NULL;
	pid_t pid;
	ssize_t ret;
	int fd_socket[2];
	char c;
	char *cur;
	unsigned int keep_fd_up_to = STDERR_FILENO;

	if (geteuid() != 0)
		return log_error(EXIT_FAILURE, "Error: forkexec requires root privileges");

	cur = advance_arg(false);
	if (!cur || (strcmp(cur, "--help") == 0 ||
	     strcmp(cur, "--version") == 0 || strcmp(cur, "-h") == 0))
		return EXIT_SUCCESS;

	if (strncmp(cur, "--keep-fd-up-to=", STRLITERALLEN("--keep-fd-up-to=")) == 0) {
		cur += STRLITERALLEN("--keep-fd-up-to=");
		ret = safe_uint(cur, &keep_fd_up_to);
		if (ret)
			return log_error(EXIT_FAILURE, "Invalid fd number %s specified", cur);

		cur = advance_arg(true);
	}

	if (!fhas_fs_type(FD_PIPE_UIDMAP, PIPEFS_MAGIC))
		return log_error(EXIT_FAILURE, "Error: Missing UID map FD");

	if (!fhas_fs_type(FD_PIPE_GIDMAP, PIPEFS_MAGIC))
		return log_error(EXIT_FAILURE, "Error: Missing GID map FD");

	ret = socketpair(AF_LOCAL, SOCK_STREAM | SOCK_CLOEXEC, 0, fd_socket);
	if (ret < 0)
		return log_error(EXIT_FAILURE, "Failed to create socketpair");

	pid = fork();
	if (pid < 0)
		return EXIT_FAILURE;

	if (pid == 0) {
		close_prot_errno_disarm(fd_socket[0]);

		ret = read_nointr(fd_socket[1], &c, 1);
		if (ret != 1)
			_exit(EXIT_FAILURE);

		ret = write_id_mappings();
		if (ret < 0)
			_exit(EXIT_FAILURE);

		ret = write_nointr(fd_socket[1], &c, 1);
		if (ret != 1)
			_exit(EXIT_FAILURE);

		_exit(EXIT_SUCCESS);
	}

	close_prot_errno_disarm(fd_socket[1]);

	ret = unshare(CLONE_NEWUSER);
	if (ret) {
		kill(pid, SIGKILL);
		fprintf(stderr, "%m - Failed to create user namespace");
		goto out_reap;
	}

	ret = write_nointr(fd_socket[0], &c, 1);
	if (ret == 1)
		ret = read_nointr(fd_socket[0], &c, 1);
	if (ret != 1) {
		kill(pid, SIGKILL);
		fprintf(stderr, "%m - Failed to read or write to socketpair fd");
		goto out_reap;
	}

	if (!drop_groups()) {
		kill(pid, SIGKILL);
		fprintf(stderr, "%m - Failed to drop supplimentary groups");
		goto out_reap;
	}

	if (!switch_resids(0, 0)) {
		kill(pid, SIGKILL);
		fprintf(stderr, "%m - Failed to switch uid and gid to 0");
		goto out_reap;
	}

	close_prot_errno_disarm(fd_socket[0]);

out_reap:
	ret = wait_for_pid(pid);
	if (ret)
		return log_error(EXIT_FAILURE, "Child process failed");

	for (; cur; cur = advance_arg(false)) {
		ret = push_vargs(&argvp, cur);
		if (ret < 0)
			return log_error(EXIT_FAILURE, "Failed to add %s to arg array", cur);
	}

	if (!argvp || !*argvp)
		return log_error(EXIT_FAILURE, "No command specified");

	ret = lxd_close_range(keep_fd_up_to + 1, UINT_MAX, CLOSE_RANGE_CLOEXEC);
	if (ret && !IN_SET(errno, ENOSYS, EINVAL))
		return log_error(EXIT_FAILURE, "Aborting forkusernsexec to prevent leaking file descriptors");

	execvp(argvp[0], argvp);
	return log_error(EXIT_FAILURE, "Failed to execute \"%s\"", argvp[0]);
}

void forkusernsexec(void)
{
	_exit(__forkusernsexec());
}
*/
import "C"

type cmdForkusernsexec struct {
	global     *cmdGlobal
	keepFdUpTo int
}

func (c *cmdForkusernsexec) Command() *cobra.Command {
	// Main subcommand
	cmd := &cobra.Command{}
	cmd.Use = "forkusernsexec <cmd> <args>"
	cmd.Short = "Run command in user namespace"
	cmd.Flags().IntVar(&c.keepFdUpTo, "keep-fd-up-to", (1 << 31), fmt.Sprintf("Keep all fds below and including this one and close all the ones above it"))
	cmd.Long = `Description:
  Run command in user namespace

  This command is used to execute a program in a new user namespace.
`
	cmd.RunE = c.Run
	cmd.Hidden = true

	return cmd
}

func (c *cmdForkusernsexec) Run(cmd *cobra.Command, args []string) error {
	return fmt.Errorf("This command should have been intercepted in cgo")
}
<|endoftext|>


<commit_before>package nettypes

import (
	"fmt"
	"net"
)

type EthType [2]byte

var (
	All                 = EthType{0x00, 0x03}
	IPv4                = EthType{0x08, 0x00}
	ARP                 = EthType{0x08, 0x06}
	WakeOnLAN           = EthType{0x08, 0x42}
	TRILL               = EthType{0x22, 0xF3}
	DECnetPhase4        = EthType{0x60, 0x03}
	RARP                = EthType{0x80, 0x35}
	AppleTalk           = EthType{0x80, 0x9B}
	AARP                = EthType{0x80, 0xF3}
	IPX1                = EthType{0x81, 0x37}
	IPX2                = EthType{0x81, 0x38}
	QNXQnet             = EthType{0x82, 0x04}
	IPv6                = EthType{0x86, 0xDD}
	EthernetFlowControl = EthType{0x88, 0x08}
	IEEE802_3           = EthType{0x88, 0x09}
	CobraNet            = EthType{0x88, 0x19}
	MPLSUnicast         = EthType{0x88, 0x47}
	MPLSMulticast       = EthType{0x88, 0x48}
	PPPoEDiscovery      = EthType{0x88, 0x63}
	PPPoESession        = EthType{0x88, 0x64}
	JumboFrames         = EthType{0x88, 0x70}
	HomePlug1_0MME      = EthType{0x88, 0x7B}
	IEEE802_1X          = EthType{0x88, 0x8E}
	PROFINET            = EthType{0x88, 0x92}
	HyperSCSI           = EthType{0x88, 0x9A}
	AoE                 = EthType{0x88, 0xA2}
	EtherCAT            = EthType{0x88, 0xA4}
	EthernetPowerlink   = EthType{0x88, 0xAB}
	LLDP                = EthType{0x88, 0xCC}
	SERCOS3             = EthType{0x88, 0xCD}
	HomePlugAVMME       = EthType{0x88, 0xE1}
	MRP                 = EthType{0x88, 0xE3}
	IEEE802_1AE         = EthType{0x88, 0xE5}
	IEEE1588            = EthType{0x88, 0xF7}
	IEEE802_1ag         = EthType{0x89, 0x02}
	FCoE                = EthType{0x89, 0x06}
	FCoEInit            = EthType{0x89, 0x14}
	RoCE                = EthType{0x89, 0x15}
	CTP                 = EthType{0x90, 0x00}
	VeritasLLT          = EthType{0xCA, 0xFE}
)

func (e EthType) String() string {
	switch e {
	case All:
		return "All"
	case IPv4:
		return "IPv4"
	case ARP:
		return "ARP"
	case WakeOnLAN:
		return "WakeOnLAN"
	case TRILL:
		return "TRILL"
	case DECnetPhase4:
		return "DECnetPhase4"
	case RARP:
		return "RARP"
	case AppleTalk:
		return "AppleTalk"
	case AARP:
		return "AARP"
	case IPX1:
		return "IPX1"
	case IPX2:
		return "IPX2"
	case QNXQnet:
		return "QNXQnet"
	case IPv6:
		return "IPv6"
	case EthernetFlowControl:
		return "EthernetFlowControl"
	case IEEE802_3:
		return "IEEE802_3"
	case CobraNet:
		return "CobraNet"
	case MPLSUnicast:
		return "MPLSUnicast"
	case MPLSMulticast:
		return "MPLSMulticast"
	case PPPoEDiscovery:
		return "PPPoEDiscovery"
	case PPPoESession:
		return "PPPoESession"
	case JumboFrames:
		return "JumboFrames"
	case HomePlug1_0MME:
		return "HomePlug1_0MME"
	case IEEE802_1X:
		return "IEEE802_1X"
	case PROFINET:
		return "PROFINET"
	case HyperSCSI:
		return "HyperSCSI"
	case AoE:
		return "AoE"
	case EtherCAT:
		return "EtherCAT"
	case EthernetPowerlink:
		return "EthernetPowerlink"
	case LLDP:
		return "LLDP"
	case SERCOS3:
		return "SERCOS3"
	case HomePlugAVMME:
		return "HomePlugAVMME"
	case MRP:
		return "MRP"
	case IEEE802_1AE:
		return "IEEE802_1AE"
	case IEEE1588:
		return "IEEE1588"
	case IEEE802_1ag:
		return "IEEE802_1ag"
	case FCoE:
		return "FCoE"
	case FCoEInit:
		return "FCoEInit"
	case RoCE:
		return "RoCE"
	case CTP:
		return "CTP"
	case VeritasLLT:
		return "VeritasLLT"
	default:
		return fmt.Sprintf("unknown type:%02x", e[0:2])
	}
}

type Tagging uint32

const (
	NotTagged    Tagging = 0
	Tagged       Tagging = 4
	DoubleTagged Tagging = 8
)

type Frame []byte

func (f *Frame) String(l uint32, indent int) string {
	return fmt.Sprintf(padLeft("Mac Len    : %d\n", "\t", indent), l) +
		fmt.Sprintf(padLeft("MAC Source : %s\n", "\t", indent), f.MACSource()) +
		fmt.Sprintf(padLeft("MAC Dest   : %s\n", "\t", indent), f.MACDestination()) +
		fmt.Sprintf(padLeft("MAC Type   : %s\n", "\t", indent), f.MACEthertype()) +
		f.GetPayString(l, indent)
}

func (f *Frame) MACSource() net.HardwareAddr {
	return net.HardwareAddr((*f)[6:12])
}

func (f *Frame) MACDestination() net.HardwareAddr {
	return net.HardwareAddr((*f)[:6])
}

func (f *Frame) MACTagging() Tagging {
	b1, b2 := (*f)[12], (*f)[13]
	if b1 == 0x81 && b2 == 0x00 {
		b3, b4 := (*f)[16], (*f)[17]
		if b3 == 0x81 && b4 == 0x00 {
			return DoubleTagged
		}
		return Tagged
	}
	if (b1 == 0x91 || b1 == 0x92) && b2 == 0x00 {
		return DoubleTagged
	}
	if b1 == 0x88 && b2 == 0xA8 {
		return DoubleTagged
	}
	return NotTagged
}

func (f *Frame) MACEthertype() EthType {
	pos := 12 + f.MACTagging()
	return EthType{(*f)[pos], (*f)[pos+1]}
}

func (f *Frame) MACPayload() ([]byte, uint32) {
	off := 12 + uint32(f.MACTagging()) + 2
	return (*f)[off:], off
}

func (f *Frame) GetPayString(frameLen uint32, indent int) string {
	p, off := f.MACPayload()
	frameLen -= off
	indent++
	switch f.MACEthertype() {
	case ARP:
		return ARP_P(p).String(indent)
	case IPv4:
		return IPv4_P(p).String(frameLen, indent)
	default:
		return "unknown eth payload...\n"
	}
}

func IsMACBroadcast(addr net.HardwareAddr) bool {
	return addr[0] == 0xFF && addr[1] == 0xFF && addr[2] == 0xFF && addr[3] == 0xFF && addr[4] == 0xFF && addr[5] == 0xFF
}

func IsMACMulticastIPv4(addr net.HardwareAddr) bool {
	return addr[0] == 0x01 && addr[1] == 0x00 && addr[2] == 0x5E
}

func IsMACMulticastIPv6(addr net.HardwareAddr) bool {
	return addr[0] == 0x33 && addr[1] == 0x33
}
<commit_msg>proper tagging<commit_after>package nettypes

import (
	"fmt"
	"net"

	"github.com/nathanjsweet/zsocket/inet"
)

type EthType [2]byte

var (
	All                 = EthType{0x00, 0x03}
	IPv4                = EthType{0x08, 0x00}
	ARP                 = EthType{0x08, 0x06}
	WakeOnLAN           = EthType{0x08, 0x42}
	TRILL               = EthType{0x22, 0xF3}
	DECnetPhase4        = EthType{0x60, 0x03}
	RARP                = EthType{0x80, 0x35}
	AppleTalk           = EthType{0x80, 0x9B}
	AARP                = EthType{0x80, 0xF3}
	IPX1                = EthType{0x81, 0x37}
	IPX2                = EthType{0x81, 0x38}
	QNXQnet             = EthType{0x82, 0x04}
	IPv6                = EthType{0x86, 0xDD}
	EthernetFlowControl = EthType{0x88, 0x08}
	IEEE802_3           = EthType{0x88, 0x09}
	CobraNet            = EthType{0x88, 0x19}
	MPLSUnicast         = EthType{0x88, 0x47}
	MPLSMulticast       = EthType{0x88, 0x48}
	PPPoEDiscovery      = EthType{0x88, 0x63}
	PPPoESession        = EthType{0x88, 0x64}
	JumboFrames         = EthType{0x88, 0x70}
	HomePlug1_0MME      = EthType{0x88, 0x7B}
	IEEE802_1X          = EthType{0x88, 0x8E}
	PROFINET            = EthType{0x88, 0x92}
	HyperSCSI           = EthType{0x88, 0x9A}
	AoE                 = EthType{0x88, 0xA2}
	EtherCAT            = EthType{0x88, 0xA4}
	EthernetPowerlink   = EthType{0x88, 0xAB}
	LLDP                = EthType{0x88, 0xCC}
	SERCOS3             = EthType{0x88, 0xCD}
	HomePlugAVMME       = EthType{0x88, 0xE1}
	MRP                 = EthType{0x88, 0xE3}
	IEEE802_1AE         = EthType{0x88, 0xE5}
	IEEE1588            = EthType{0x88, 0xF7}
	IEEE802_1ag         = EthType{0x89, 0x02}
	FCoE                = EthType{0x89, 0x06}
	FCoEInit            = EthType{0x89, 0x14}
	RoCE                = EthType{0x89, 0x15}
	CTP                 = EthType{0x90, 0x00}
	VeritasLLT          = EthType{0xCA, 0xFE}
)

func (e EthType) String() string {
	switch e {
	case All:
		return "All"
	case IPv4:
		return "IPv4"
	case ARP:
		return "ARP"
	case WakeOnLAN:
		return "WakeOnLAN"
	case TRILL:
		return "TRILL"
	case DECnetPhase4:
		return "DECnetPhase4"
	case RARP:
		return "RARP"
	case AppleTalk:
		return "AppleTalk"
	case AARP:
		return "AARP"
	case IPX1:
		return "IPX1"
	case IPX2:
		return "IPX2"
	case QNXQnet:
		return "QNXQnet"
	case IPv6:
		return "IPv6"
	case EthernetFlowControl:
		return "EthernetFlowControl"
	case IEEE802_3:
		return "IEEE802_3"
	case CobraNet:
		return "CobraNet"
	case MPLSUnicast:
		return "MPLSUnicast"
	case MPLSMulticast:
		return "MPLSMulticast"
	case PPPoEDiscovery:
		return "PPPoEDiscovery"
	case PPPoESession:
		return "PPPoESession"
	case JumboFrames:
		return "JumboFrames"
	case HomePlug1_0MME:
		return "HomePlug1_0MME"
	case IEEE802_1X:
		return "IEEE802_1X"
	case PROFINET:
		return "PROFINET"
	case HyperSCSI:
		return "HyperSCSI"
	case AoE:
		return "AoE"
	case EtherCAT:
		return "EtherCAT"
	case EthernetPowerlink:
		return "EthernetPowerlink"
	case LLDP:
		return "LLDP"
	case SERCOS3:
		return "SERCOS3"
	case HomePlugAVMME:
		return "HomePlugAVMME"
	case MRP:
		return "MRP"
	case IEEE802_1AE:
		return "IEEE802_1AE"
	case IEEE1588:
		return "IEEE1588"
	case IEEE802_1ag:
		return "IEEE802_1ag"
	case FCoE:
		return "FCoE"
	case FCoEInit:
		return "FCoEInit"
	case RoCE:
		return "RoCE"
	case CTP:
		return "CTP"
	case VeritasLLT:
		return "VeritasLLT"
	default:
		return fmt.Sprintf("unknown type:%02x", e[0:2])
	}
}

type VLANTag uint32

const (
	NotTagged VLANTag = 0
	Tagged    VLANTag = 4
)

type PCP uint8

const (
	BK = PCP(0x01)
	BE = PCP(0x00)
	EE = PCP(0x02)
	CA = PCP(0x03)
	VI = PCP(0x04)
	VO = PCP(0x05)
	IC = PCP(0x06)
	NC = PCP(0x07)
)

func (pcp PCP) String() string {
	switch pcp {
	case BK:
		return "BK - Background"
	case BE:
		return "BE - Best Effort"
	case EE:
		return "EE - Excellent Effort"
	case CA:
		return "CA - Critical Applications"
	case VI:
		return "VI - Video"
	case VO:
		return "VO - Voice"
	case IC:
		return "IC - Internetwork Control"
	case NC:
		return "NC - Network Control"
	}
	return fmt.Sprintf("corrupt type: %v", uint8(pcp))
}

type Frame []byte

func (f *Frame) String(l uint32, indent int) string {
	s := fmt.Sprintf(padLeft("Mac Len    : %d\n", "\t", indent), l) +
		fmt.Sprintf(padLeft("MAC Source : %s\n", "\t", indent), f.MACSource()) +
		fmt.Sprintf(padLeft("MAC Dest   : %s\n", "\t", indent), f.MACDestination())
	mT := f.VLANTag()
	if mT == Tagged {
		s += fmt.Sprint(padLeft("VLAN Info  : \n", "\t", indent))
		s += fmt.Sprintf(padLeft("PCP        : %s\n", "\t", indent), f.VLANPCP())
		s += fmt.Sprintf(padLeft("DEI        : %s\n", "\t", indent), f.VLANDEI())
		s += fmt.Sprintf(padLeft("ID         : %s\n", "\t", indent), f.VLANID())
	}
	s += fmt.Sprintf(padLeft("MAC Type   : %s\n", "\t", indent), f.MACEthertype(mT)) +
		f.GetPayString(l, indent, mT)
}

func (f *Frame) MACSource() net.HardwareAddr {
	return net.HardwareAddr((*f)[6:12])
}

func (f *Frame) MACDestination() net.HardwareAddr {
	return net.HardwareAddr((*f)[:6])
}

func (f *Frame) VLANTag() VLANTag {
	if (*f)[12] == 0x81 && (*f)[13] == 0x00 {
		return Tagged
	}
	return NotTagged
}

func (f *Frame) VLANPCP() PCP {
	return PCP((*f)[14] & 0xE0)
}

func (f *Frame) VLANDEI() bool {
	return (*f)[14]&0x20 == 0x20
}

func (f *Frame) VLANID() uint16 {
	return inet.NToHS([]byte{(*f)[14] & 0x0f, (*f)[15]})
}

func (f *Frame) MACEthertype(tag VLANTag) EthType {
	pos := 12 + tag
	return EthType{(*f)[pos], (*f)[pos+1]}
}

func (f *Frame) MACPayload(tag VLANTag) ([]byte, uint32) {
	off := uint32(14 + tag)
	return (*f)[off:], off
}

func (f *Frame) GetPayString(frameLen uint32, indent int, tag VLANTag) string {
	p, off := f.MACPayload(tag)
	frameLen -= off
	indent++
	switch f.MACEthertype(tag) {
	case ARP:
		return ARP_P(p).String(indent)
	case IPv4:
		return IPv4_P(p).String(frameLen, indent)
	default:
		return "unknown eth payload...\n"
	}
}

func IsMACBroadcast(addr net.HardwareAddr) bool {
	return addr[0] == 0xFF && addr[1] == 0xFF && addr[2] == 0xFF && addr[3] == 0xFF && addr[4] == 0xFF && addr[5] == 0xFF
}

func IsMACMulticastIPv4(addr net.HardwareAddr) bool {
	return addr[0] == 0x01 && addr[1] == 0x00 && addr[2] == 0x5E
}

func IsMACMulticastIPv6(addr net.HardwareAddr) bool {
	return addr[0] == 0x33 && addr[1] == 0x33
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"flag"
	"encoding/json"
	"os"
	"regexp"
	"sync/atomic"
	"log"
	"time"
	"path/filepath"
	"github.com/APTrust/bagman"
	"launchpad.net/goamz/aws"
	"launchpad.net/goamz/s3"
	"github.com/bitly/go-nsq"
)

type Channels struct {
	FetchChannel     chan bagman.ProcessResult
	UnpackChannel    chan bagman.ProcessResult
	CleanUpChannel   chan bagman.ProcessResult
	ResultsChannel   chan bagman.ProcessResult
}

// Global vars.
var channels *Channels
var config bagman.Config
var jsonLog *log.Logger
var messageLog *log.Logger
var taskCounter = int64(0)
var volume *bagman.Volume
var succeeded = int64(0)
var failed = int64(0)
var bytesInS3 = int64(0)
var bytesProcessed = int64(0)

func main() {

	loadConfig()
	initVolume()
	initChannels()
	initGoRoutines(channels)

	// ----------------------------------------------------------------
	// START HERE
	// Bind message handler here!!!
	// See https://github.com/APTrust/bagman/blob/develop/nsqtest/nsqtest.go
	// ----------------------------------------------------------------

	waitForAllTasks()
}

func loadConfig() {
	// Load the config or die.
	requestedConfig := flag.String("config", "", "configuration to run")
	flag.Parse()
	config = bagman.LoadRequestedConfig(requestedConfig)
	jsonLog, messageLog = bagman.InitLoggers(config.LogDirectory, "bagman_cli")
}

// Set up the volume to keep track of how much disk space is
// available. We want to avoid downloading large files when
// we know ahead of time that the volume containing the tar
// directory doesn't have enough space to accomodate them.
func initVolume() {
	var err error
	volume, err = bagman.NewVolume(config.TarDirectory)
	if err != nil {
		panic(err.Error())
	}
}

// Set up the channels. It's essential that that the fetchChannel
// be limited to a relatively low number. If we are downloading
// 1GB tar files, we need space to store the tar file AND the
// untarred version. That's about 2 x 1GB. We do not want to pull
// down 1000 files at once, or we'll run out of disk space!
// If config sets fetchers to 10, we can pull down 10 files at a
// time. The fetch queue could hold 10 * 4 = 40 items, so we'd
// have max 40 tar files + untarred directories on disk at once.
// The number of workers should be close to the number of CPU
// cores.
func initChannels() {
	fetcherBufferSize := config.Fetchers * 4
	workerBufferSize := config.Workers * 10

	channels = &Channels{}
	channels.FetchChannel = make(chan bagman.ProcessResult, fetcherBufferSize)
	channels.UnpackChannel = make(chan bagman.ProcessResult, workerBufferSize)
	channels.CleanUpChannel = make(chan bagman.ProcessResult, workerBufferSize)
	channels.ResultsChannel = make(chan bagman.ProcessResult, workerBufferSize)
}

// Set up our go routines. We do NOT want one go routine per
// S3 file. If we do that, the system will run out of file handles,
// as we'll have tens of thousands of open connections to S3
// trying to write data into tens of thousands of local files.
func initGoRoutines(channels *Channels) {
	for i := 0; i < config.Fetchers; i++ {
		go doFetch()
	}

	for i := 0; i < config.Workers; i++ {
		go doUnpack()
		go logResult()
		go doCleanUp()
	}
}

func HandleMessage(message *nsq.Message, outputChannel chan *nsq.FinishedMessage) {
	fmt.Println(string(message.Body))
	message.Attempts++
	atomic.AddInt64(&taskCounter, 1)

	var s3File bagman.S3File
	err := json.Unmarshal(message.Body, &s3File)
	if err != nil {
		messageLog.Println("[ERROR] Could not unmarshal JSON data from nsq")
		finishedMessage := &nsq.FinishedMessage{message.Id, 1000, false}
		outputChannel <- finishedMessage
	}

	// Create the result struct and pass it down the pipeline
	result := bagman.ProcessResult{
		message,
		outputChannel,
		&s3File,
		nil,
		nil,
		nil,
		nil,
		true}
	channels.FetchChannel <- result
	messageLog.Println("[INFO]", "Put", s3File.Key.Key, "into fetch queue")
}

// This function blocks until all tasks are complete.
// sync.WaitGroup is the standard way of doing this, but
// WaitGroup is really for counting go routines. We have only a handful
// of go routines, and a huge number of tasks. We're counting the tasks.
func waitForAllTasks() {
	for {
		if atomic.LoadInt64(&taskCounter) == 0 {
			break
		}
		time.Sleep(5 * time.Second)
	}
}

// This runs as a go routine to fetch files from S3.
func doFetch() {
	for result := range channels.FetchChannel {
		s3Key := result.S3File.Key
		// Disk needs filesize * 2 disk space to accomodate tar file & untarred files
		err := volume.Reserve(uint64(s3Key.Size * 2))
		if err != nil {
			// Not enough room on disk
			messageLog.Println("[WARNING]", "Requeueing",
				s3Key.Key, "- not enough disk space")
			result.Error = err
			result.Retry = true
			channels.ResultsChannel <- result
		} else {
			messageLog.Println("[INFO]", "Fetching", s3Key)
			fetchResult := Fetch(result.S3File.BucketName, s3Key)
			result.FetchResult = fetchResult
			result.Retry = fetchResult.Retry
			if fetchResult.Error != nil {
				// Fetch from S3 failed. Requeue.
				result.Error = fetchResult.Error
				channels.ResultsChannel <- result
			} else {
				// Got S3 file. Untar it.
				// And touch the message, so nsqd knows we're making progress.
				result.NsqMessage.Touch()
				channels.UnpackChannel <- result
			}
		}
	}
}

// This runs as a go routine to untar files downloaded from S3.
func doUnpack() {
	for result := range channels.UnpackChannel {
		if result.Error != nil {
			// Unpack failed. Go to end.
			messageLog.Println("[INFO]", "Nothing to unpack for",
				result.S3File.Key.Key)
			channels.ResultsChannel <- result
		} else {
			// Unpacked! Now process the bag and touch message
			// so nsqd knows we're making progress.
			messageLog.Println("[INFO]", "Unpacking", result.S3File.Key.Key)
			result.NsqMessage.Touch()
			ProcessBagFile(&result)
			channels.ResultsChannel <- result
		}
	}
}

// This runs as a go routine to remove the files we downloaded
// and untarred.
func doCleanUp() {
	for result := range channels.CleanUpChannel {
		messageLog.Println("[INFO]", "Cleaning up", result.S3File.Key.Key)
		if result.FetchResult.LocalTarFile != "" {
			// Clean up any files we downloaded and unpacked
			errors := CleanUp(result.FetchResult.LocalTarFile)
			if errors != nil && len(errors) > 0 {
				messageLog.Println("[WARNING]", "Errors cleaning up",
					result.FetchResult.LocalTarFile)
				for _, e := range errors {
					messageLog.Println("[ERROR]", e)
				}
			}
		}
		// Let our volumn tracker know we just freed up some disk space.
		volume.Release(uint64(result.S3File.Key.Size * 2))
		// Reduce outstanding task count.
		atomic.AddInt64(&taskCounter, -1)

		// Build and send message back to NSQ, indicating whether
		// processing succeeded.
		succeeded := true
		requeueDelayMs := 0
		if result.Error != nil {
			succeeded = false
			requeueDelayMs = 60000
		}
		finishedMessage := &nsq.FinishedMessage{
			result.NsqMessage.Id,
			requeueDelayMs,
			succeeded}
		result.NsqOutputChannel <- finishedMessage
	}
}

// This prints to the log the result of the program's attempt to fetch,
// untar, unbag and verify an individual S3 tar file.
func logResult() {
	for result := range channels.ResultsChannel {
		json, err := json.Marshal(result)
		if err != nil {
			messageLog.Println("[ERROR]", err)
		}
		jsonLog.Println(string(json))
		atomic.AddInt64(&bytesInS3, int64(result.S3File.Key.Size))
		if(result.Error != nil) {
			atomic.AddInt64(&failed, 1)
			messageLog.Println("[ERROR]",
				result.S3File.BucketName,
				result.S3File.Key.Key,
				"->", result.Error)
		} else {
			atomic.AddInt64(&succeeded, 1)
			atomic.AddInt64(&bytesProcessed, int64(result.S3File.Key.Size))
			messageLog.Println("[INFO]", result.S3File.Key.Key, "-> finished OK")
		}
		channels.CleanUpChannel <- result
	}
}

// This fetches a file from S3 and stores it locally.
func Fetch(bucketName string, key s3.Key) (result *bagman.FetchResult) {
	client, err := bagman.GetClient(aws.USEast)
	if err != nil {
		fetchResult := new(bagman.FetchResult)
		fetchResult.Error = err
		return fetchResult
	}
	// TODO: We fetched this bucket before. Do we need to fetch it again?
	bucket := client.Bucket(bucketName)
	tarFilePath := filepath.Join(config.TarDirectory, key.Key)
	return bagman.FetchToFile(bucket, key, tarFilePath)
}

// This deletes the tar file and all of the files that were
// unpacked from it. Param file is the path the tar file.
func CleanUp(file string) (errors []error) {
	errors = make([]error, 0)
	err := os.Remove(file)
	if err != nil {
		errors = append(errors, err)
	}
	// TODO: Don't assume the untarred dir name is the same as
	// the tar file. We have its actual name in the TarResult.
	re := regexp.MustCompile("\\.tar$")
	untarredDir := re.ReplaceAllString(file, "")
	err = os.RemoveAll(untarredDir)
	if err != nil {
		messageLog.Printf("Error deleting dir %s: %s\n", untarredDir, err.Error())
		errors = append(errors, err)
	}
	return errors
}

// Runs tests on the bag file at path and returns information about
// whether it was successfully unpacked, valid and complete.
func ProcessBagFile(result *bagman.ProcessResult) {
	result.TarResult = bagman.Untar(result.FetchResult.LocalTarFile)
	if result.TarResult.Error != nil {
		result.Error = result.TarResult.Error
		// If we can't untar this, there's no reason to retry...
		// but we'll have to revisit this. There may be cases
		// where we do want to retry, such as if disk was full.
		result.Retry = false
	} else {
		result.BagReadResult = bagman.ReadBag(result.TarResult.OutputDir)
		if result.BagReadResult.Error != nil {
			result.Error = result.BagReadResult.Error
			// Something was wrong with this bag. Bad checksum,
			// missing file, etc. Don't reprocess it.
			result.Retry = false
		}
	}
}
<commit_msg>Interim commit.<commit_after>package main

import (
	"fmt"
	"flag"
	"encoding/json"
	"os"
	"regexp"
	"sync/atomic"
	"log"
	"time"
	"path/filepath"
	"github.com/APTrust/bagman"
	"launchpad.net/goamz/aws"
	"launchpad.net/goamz/s3"
	"github.com/bitly/go-nsq"
)

type Channels struct {
	FetchChannel     chan bagman.ProcessResult
	UnpackChannel    chan bagman.ProcessResult
	CleanUpChannel   chan bagman.ProcessResult
	ResultsChannel   chan bagman.ProcessResult
}

// Global vars.
var channels *Channels
var config bagman.Config
var jsonLog *log.Logger
var messageLog *log.Logger
var taskCounter = int64(0)
var volume *bagman.Volume
var succeeded = int64(0)
var failed = int64(0)
var bytesInS3 = int64(0)
var bytesProcessed = int64(0)

// bag_processor receives messages from nsqd describing
// items in the S3 receiving buckets. Each item/message
// follows this flow:
//
// 1. Fetch channel: fetches the file from S3.
// 2. Unpack channel: untars the bag files, parses and validates
//    the bag, reads tags, generates checksums and generic file
//    UUIDs.
// 3. **** TODO **** Copy files to S3 permanent storage.
// 4. Results channel: tells the queue whether processing
//    succeeded, and if not, whether the item should be requeued.
//    Also logs results to json and message logs.
// 5. Cleanup channe: cleans up the files after processing
//    completes.
//
// If a failure occurs anywhere in the first three steps,
// processing goes directly to the Results Channel, which
// records the error and the disposition (retry/give up).
//
// As long as the message from nsq contains valid JSON,
// steps 4 and 5 ALWAYS run.
//
// The bag processor has so many responsibilities because
// downloading, untarring and running checksums on
// multi-gigabyte files takes a lot of time. We want to
// avoid having separate processes repeatedly download and
// untar the files, so bag_processor performs all operations
// that require local access to the raw contents of the bags.
func main() {

	loadConfig()
	initVolume()
	initChannels()
	initGoRoutines(channels)

	reader, err := nsq.NewReader(config.BagProcessorTopic, config.BagProcessorChannel)
	if err != nil {
		messageLog.Fatalf(err.Error())
	}

	handler := &BagProcessor{}
	reader.SetMaxInFlight(100)
	reader.AddAsyncHandler(handler)
	reader.ConnectToLookupd(config.NsqLookupd)

	// This reader blocks until we get an interrupt, so our program does not exit.
	<-reader.ExitChan

	//waitForAllTasks()
}

func loadConfig() {
	// Load the config or die.
	requestedConfig := flag.String("config", "", "configuration to run")
	flag.Parse()
	config = bagman.LoadRequestedConfig(requestedConfig)
	jsonLog, messageLog = bagman.InitLoggers(config.LogDirectory, "bag_processor")
}

// Set up the volume to keep track of how much disk space is
// available. We want to avoid downloading large files when
// we know ahead of time that the volume containing the tar
// directory doesn't have enough space to accomodate them.
func initVolume() {
	var err error
	volume, err = bagman.NewVolume(config.TarDirectory)
	if err != nil {
		panic(err.Error())
	}
}

// Set up the channels. It's essential that that the fetchChannel
// be limited to a relatively low number. If we are downloading
// 1GB tar files, we need space to store the tar file AND the
// untarred version. That's about 2 x 1GB. We do not want to pull
// down 1000 files at once, or we'll run out of disk space!
// If config sets fetchers to 10, we can pull down 10 files at a
// time. The fetch queue could hold 10 * 4 = 40 items, so we'd
// have max 40 tar files + untarred directories on disk at once.
// The number of workers should be close to the number of CPU
// cores.
func initChannels() {
	fetcherBufferSize := config.Fetchers * 4
	workerBufferSize := config.Workers * 10

	channels = &Channels{}
	channels.FetchChannel = make(chan bagman.ProcessResult, fetcherBufferSize)
	channels.UnpackChannel = make(chan bagman.ProcessResult, workerBufferSize)
	channels.CleanUpChannel = make(chan bagman.ProcessResult, workerBufferSize)
	channels.ResultsChannel = make(chan bagman.ProcessResult, workerBufferSize)
}

// Set up our go routines. We do NOT want one go routine per
// S3 file. If we do that, the system will run out of file handles,
// as we'll have tens of thousands of open connections to S3
// trying to write data into tens of thousands of local files.
func initGoRoutines(channels *Channels) {
	for i := 0; i < config.Fetchers; i++ {
		go doFetch()
	}

	for i := 0; i < config.Workers; i++ {
		go doUnpack()
		go logResult()
		go doCleanUp()
	}
}

type BagProcessor struct {

}

// MessageHandler handles messages from the queue, putting each
// item into the pipleline.
func (*BagProcessor) HandleMessage(message *nsq.Message, outputChannel chan *nsq.FinishedMessage) {

	fmt.Println(string(message.Body))
	message.Attempts++
	atomic.AddInt64(&taskCounter, 1)

	var s3File bagman.S3File
	err := json.Unmarshal(message.Body, &s3File)
	if err != nil {
		messageLog.Println("[ERROR] Could not unmarshal JSON data from nsq:",
			string(message.Body))
		finishedMessage := &nsq.FinishedMessage{message.Id, 0, true}
		outputChannel <- finishedMessage
		return
	}

	// Create the result struct and pass it down the pipeline
	result := bagman.ProcessResult{
		message,
		outputChannel,
		&s3File,
		nil,
		nil,
		nil,
		nil,
		true}
	channels.FetchChannel <- result
	messageLog.Println("[INFO]", "Put", s3File.Key.Key, "into fetch queue")
}

// This function blocks until all tasks are complete.
// sync.WaitGroup is the standard way of doing this, but
// WaitGroup is really for counting go routines. We have only a handful
// of go routines, and a huge number of tasks. We're counting the tasks.
func waitForAllTasks() {
	for {
		if atomic.LoadInt64(&taskCounter) == 0 {
			break
		}
		time.Sleep(5 * time.Second)
	}
}

// This runs as a go routine to fetch files from S3.
func doFetch() {
	for result := range channels.FetchChannel {
		s3Key := result.S3File.Key
		// Disk needs filesize * 2 disk space to accomodate tar file & untarred files
		err := volume.Reserve(uint64(s3Key.Size * 2))
		if err != nil {
			// Not enough room on disk
			messageLog.Println("[WARNING]", "Requeueing",
				s3Key.Key, "- not enough disk space")
			result.Error = err
			result.Retry = true
			channels.ResultsChannel <- result
		} else {
			messageLog.Println("[INFO]", "Fetching", s3Key)
			fetchResult := Fetch(result.S3File.BucketName, s3Key)
			result.FetchResult = fetchResult
			result.Retry = fetchResult.Retry
			if fetchResult.Error != nil {
				// Fetch from S3 failed. Requeue.
				result.Error = fetchResult.Error
				channels.ResultsChannel <- result
			} else {
				// Got S3 file. Untar it.
				// And touch the message, so nsqd knows we're making progress.
				result.NsqMessage.Touch()
				channels.UnpackChannel <- result
			}
		}
	}
}

// This runs as a go routine to untar files downloaded from S3.
func doUnpack() {
	for result := range channels.UnpackChannel {
		if result.Error != nil {
			// Unpack failed. Go to end.
			messageLog.Println("[INFO]", "Nothing to unpack for",
				result.S3File.Key.Key)
			channels.ResultsChannel <- result
		} else {
			// Unpacked! Now process the bag and touch message
			// so nsqd knows we're making progress.
			messageLog.Println("[INFO]", "Unpacking", result.S3File.Key.Key)
			result.NsqMessage.Touch()
			ProcessBagFile(&result)
			channels.ResultsChannel <- result
		}
	}
}

// This runs as a go routine to remove the files we downloaded
// and untarred.
func doCleanUp() {
	for result := range channels.CleanUpChannel {
		messageLog.Println("[INFO]", "Cleaning up", result.S3File.Key.Key)
		if result.S3File.Key.Key != "" && result.FetchResult.LocalTarFile != "" {
			// Clean up any files we downloaded and unpacked
			errors := CleanUp(result.FetchResult.LocalTarFile)
			if errors != nil && len(errors) > 0 {
				messageLog.Println("[WARNING]", "Errors cleaning up",
					result.FetchResult.LocalTarFile)
				for _, e := range errors {
					messageLog.Println("[ERROR]", e)
				}
			}
		}
		// Let our volumn tracker know we just freed up some disk space.
		volume.Release(uint64(result.S3File.Key.Size * 2))
		// Reduce outstanding task count.
		atomic.AddInt64(&taskCounter, -1)

		// Build and send message back to NSQ, indicating whether
		// processing succeeded.
		succeeded := true
		requeueDelayMs := 0
		if result.Error != nil {
			succeeded = false
			requeueDelayMs = 60000
		}
		finishedMessage := &nsq.FinishedMessage{
			result.NsqMessage.Id,
			requeueDelayMs,
			succeeded}
		result.NsqOutputChannel <- finishedMessage
		//cmd := nsq.Finish(result.NsqMessage.Id)
	}
}

// This prints to the log the result of the program's attempt to fetch,
// untar, unbag and verify an individual S3 tar file.
func logResult() {
	for result := range channels.ResultsChannel {
		json, err := json.Marshal(result)
		if err != nil {
			messageLog.Println("[ERROR]", err)
		}
		jsonLog.Println(string(json))
		atomic.AddInt64(&bytesInS3, int64(result.S3File.Key.Size))
		if(result.Error != nil) {
			atomic.AddInt64(&failed, 1)
			messageLog.Println("[ERROR]",
				result.S3File.BucketName,
				result.S3File.Key.Key,
				"->", result.Error)
		} else {
			atomic.AddInt64(&succeeded, 1)
			atomic.AddInt64(&bytesProcessed, int64(result.S3File.Key.Size))
			messageLog.Println("[INFO]", result.S3File.Key.Key, "-> finished OK")
		}
		channels.CleanUpChannel <- result
	}
}

// This fetches a file from S3 and stores it locally.
func Fetch(bucketName string, key s3.Key) (result *bagman.FetchResult) {
	client, err := bagman.GetClient(aws.USEast)
	if err != nil {
		fetchResult := new(bagman.FetchResult)
		fetchResult.Error = err
		return fetchResult
	}
	// TODO: We fetched this bucket before. Do we need to fetch it again?
	bucket := client.Bucket(bucketName)
	tarFilePath := filepath.Join(config.TarDirectory, key.Key)
	return bagman.FetchToFile(bucket, key, tarFilePath)
}

// This deletes the tar file and all of the files that were
// unpacked from it. Param file is the path the tar file.
func CleanUp(file string) (errors []error) {
	errors = make([]error, 0)
	err := os.Remove(file)
	if err != nil {
		errors = append(errors, err)
	}
	// TODO: Don't assume the untarred dir name is the same as
	// the tar file. We have its actual name in the TarResult.
	re := regexp.MustCompile("\\.tar$")
	untarredDir := re.ReplaceAllString(file, "")
	err = os.RemoveAll(untarredDir)
	if err != nil {
		messageLog.Printf("Error deleting dir %s: %s\n", untarredDir, err.Error())
		errors = append(errors, err)
	}
	return errors
}

// Runs tests on the bag file at path and returns information about
// whether it was successfully unpacked, valid and complete.
func ProcessBagFile(result *bagman.ProcessResult) {
	result.TarResult = bagman.Untar(result.FetchResult.LocalTarFile)
	if result.TarResult.Error != nil {
		result.Error = result.TarResult.Error
		// If we can't untar this, there's no reason to retry...
		// but we'll have to revisit this. There may be cases
		// where we do want to retry, such as if disk was full.
		result.Retry = false
	} else {
		result.BagReadResult = bagman.ReadBag(result.TarResult.OutputDir)
		if result.BagReadResult.Error != nil {
			result.Error = result.BagReadResult.Error
			// Something was wrong with this bag. Bad checksum,
			// missing file, etc. Don't reprocess it.
			result.Retry = false
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 tsuru-autoscale authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package action

import (
	"errors"
	"net/http"
	"strings"

	"github.com/tsuru/tsuru-autoscale/db"
)

// Action represents an AutoScale action to increase or decrease the
// number of the units.
type Action struct {
	Name    string
	URL     string
	Method  string
	Body    string
	Headers map[string]string
}

func New(a *Action) error {
	if a.URL == "" {
		return errors.New("action: url required")
	}
	if a.Method == "" {
		return errors.New("action: method required")
	}
	conn, err := db.Conn()
	if err != nil {
		return err
	}
	defer conn.Close()
	return conn.DataSources().Insert(&a)
}

func (a *Action) Do() error {
	req, err := http.NewRequest(a.Method, a.URL, strings.NewReader(a.Body))
	if err != nil {
		return err
	}
	for key, value := range a.Headers {
		req.Header.Add(key, value)
	}
	client := &http.Client{}
	_, err = client.Do(req)
	return err
}
<commit_msg>action: add comments.<commit_after>// Copyright 2015 tsuru-autoscale authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package action

import (
	"errors"
	"net/http"
	"strings"

	"github.com/tsuru/tsuru-autoscale/db"
)

// Action represents an AutoScale action to increase or decrease the
// number of the units.
type Action struct {
	Name    string
	URL     string
	Method  string
	Body    string
	Headers map[string]string
}

// New creates a new action.
func New(a *Action) error {
	if a.URL == "" {
		return errors.New("action: url required")
	}
	if a.Method == "" {
		return errors.New("action: method required")
	}
	conn, err := db.Conn()
	if err != nil {
		return err
	}
	defer conn.Close()
	return conn.DataSources().Insert(&a)
}

func (a *Action) Do() error {
	req, err := http.NewRequest(a.Method, a.URL, strings.NewReader(a.Body))
	if err != nil {
		return err
	}
	for key, value := range a.Headers {
		req.Header.Add(key, value)
	}
	client := &http.Client{}
	_, err = client.Do(req)
	return err
}
<|endoftext|>


<commit_before>// +build windows

package adapter

import (
	"net"
	"os/exec"

	"github.com/songgao/water"
)

func setAdapterAddress(name string, ipNet net.IPNet, gateway net.IP) error {
	return exec.Command("netsh", "interface", "ip", "set", "address",
		"name=", name, "source=static", "address="+ipNet.String(),
		"mask="+net.IP(ipNet.Mask).String(), "gateway="+gateway.String()).Run()
}

func stopDevice(name string) error {
	return nil
}

func getConfig() water.Config {
	cfg := water.Config{
		DeviceType: water.TAP,
	}

	cfg.ComponentID = "tap0901"
	return cfg
}
<commit_msg>Fixed gateway on windows<commit_after>// +build windows

package adapter

import (
	"net"
	"os/exec"

	"github.com/songgao/water"
)

func setAdapterAddress(name string, ipNet net.IPNet, gateway net.IP) error {
	return exec.Command("netsh", "interface", "ip", "set", "address",
		"name=", name, "source=static", "address="+ipNet.String(),
		"mask="+net.IP(ipNet.Mask).String()).Run()
}

func stopDevice(name string) error {
	return nil
}

func getConfig() water.Config {
	cfg := water.Config{
		DeviceType: water.TAP,
	}

	cfg.ComponentID = "tap0901"
	return cfg
}
<|endoftext|>


<commit_before>package certUtil

import (
	"crypto/x509"
    "testing"
	"os"
	"io/ioutil"
	"github.com/stretchr/testify/assert"
)

func TestFetchCertificateChain(t *testing.T) {
	file, err := os.Open("../tests/comodo.der.crt")
	if err != nil {
		t.Error(err)
	}

	data, err := ioutil.ReadAll(file)
	if err != nil {
		t.Error(err)
	}

	cert, err := x509.ParseCertificate(data)
	if err != nil {
		t.Error(err)
	}

	certs, err := FetchCertificateChain(cert)
	if err != nil {
		t.Error(err)
	}

	assert.Equal(t, 3, len(certs))
}
<commit_msg>all: gofmt<commit_after>package certUtil

import (
	"crypto/x509"
	"github.com/stretchr/testify/assert"
	"io/ioutil"
	"os"
	"testing"
)

func TestFetchCertificateChain(t *testing.T) {
	file, err := os.Open("../tests/comodo.der.crt")
	if err != nil {
		t.Error(err)
	}

	data, err := ioutil.ReadAll(file)
	if err != nil {
		t.Error(err)
	}

	cert, err := x509.ParseCertificate(data)
	if err != nil {
		t.Error(err)
	}

	certs, err := FetchCertificateChain(cert)
	if err != nil {
		t.Error(err)
	}

	assert.Equal(t, 3, len(certs))
}
<|endoftext|>


<commit_before>package action

import (
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/Masterminds/glide/cfg"
	"github.com/Masterminds/glide/dependency"
	"github.com/Masterminds/glide/gb"
	"github.com/Masterminds/glide/godep"
	"github.com/Masterminds/glide/gpm"
	"github.com/Masterminds/glide/msg"
	gpath "github.com/Masterminds/glide/path"
	"github.com/Masterminds/glide/util"
)

// Create creates/initializes a new Glide repository.
//
// This will fail if a glide.yaml already exists.
//
// By default, this will scan the present source code directory for dependencies.
//
// If skipImport is set to true, this will not attempt to import from an existing
// GPM, Godep, or GB project if one should exist. However, it will still attempt
// to read the local source to determine required packages.
func Create(base string, skipImport bool) {
	glidefile := gpath.GlideFile
	// Guard against overwrites.
	guardYAML(glidefile)

	// Guess deps
	conf := guessDeps(base, skipImport)
	// Write YAML
	if err := conf.WriteFile(glidefile); err != nil {
		msg.Die("Could not save %s: %s", glidefile, err)
	}
}

// guardYAML fails if the given file already exists.
//
// This prevents an important file from being overwritten.
func guardYAML(filename string) {
	if _, err := os.Stat(filename); err == nil {
		msg.Die("Cowardly refusing to overwrite existing YAML.")
	}
}

// guessDeps attempts to resolve all of the dependencies for a given project.
//
// base is the directory to start with.
// skipImport will skip running the automatic imports.
//
// FIXME: This function is likely a one-off that has a more standard alternative.
// It's also long and could use a refactor.
func guessDeps(base string, skipImport bool) *cfg.Config {
	buildContext, err := util.GetBuildContext()
	if err != nil {
		msg.Die("Failed to build an import context: %s", err)
	}
	name := buildContext.PackageName(base)

	msg.Info("Generating a YAML configuration file and guessing the dependencies")

	config := new(cfg.Config)

	// Get the name of the top level package
	config.Name = name

	// Import by looking at other package managers and looking over the
	// entire directory structure.

	// Attempt to import from other package managers.
	if !skipImport {
		guessImportDeps(base, config)
	}

	// Resolve dependencies by looking at the tree.
	r, err := dependency.NewResolver(base)
	if err != nil {
		msg.Die("Error creating a dependency resolver: %s", err)
	}

	h := &dependency.DefaultMissingPackageHandler{Missing: []string{}, Gopath: []string{}}
	r.Handler = h

	sortable, err := r.ResolveLocal(false)
	if err != nil {
		msg.Die("Error resolving local dependencies: %s", err)
	}

	sort.Strings(sortable)

	vpath := r.VendorDir
	if !strings.HasSuffix(vpath, "/") {
		vpath = vpath + string(os.PathSeparator)
	}

	for _, pa := range sortable {
		n := strings.TrimPrefix(pa, vpath)
		root, subpkg := util.NormalizeName(n)

		if !config.HasDependency(root) {
			msg.Info("Found reference to %s\n", n)
			d := &cfg.Dependency{
				Name: root,
			}
			if len(subpkg) > 0 {
				d.Subpackages = []string{subpkg}
			}
			config.Imports = append(config.Imports, d)
		} else {
			if len(subpkg) > 0 {
				subpkg = strings.TrimPrefix(subpkg, "/")
				d := config.Imports.Get(root)
				if !d.HasSubpackage(subpkg) {
					msg.Info("Adding sub-package %s to %s\n", subpkg, root)
					d.Subpackages = append(d.Subpackages, subpkg)
				}
			}
		}
	}

	return config
}

func guessImportDeps(base string, config *cfg.Config) {
	msg.Info("Attempting to import from other package managers (use --skip-import to skip)")
	deps := []*cfg.Dependency{}
	absBase, err := filepath.Abs(base)
	if err != nil {
		msg.Die("Failed to resolve location of %s: %s", base, err)
	}

	if d, ok := guessImportGodep(absBase); ok {
		msg.Info("Importing Godep configuration")
		msg.Warn("Godep uses commit id versions. Consider using Semantic Versions with Glide")
		deps = d
	} else if d, ok := guessImportGPM(absBase); ok {
		msg.Info("Importing GPM configuration")
		deps = d
	} else if d, ok := guessImportGB(absBase); ok {
		msg.Info("Importing GB configuration")
		deps = d
	}

	for _, i := range deps {
		msg.Info("Found imported reference to %s\n", i.Name)
		config.Imports = append(config.Imports, i)
	}
}

func guessImportGodep(dir string) ([]*cfg.Dependency, bool) {
	d, err := godep.Parse(dir)
	if err != nil || len(d) == 0 {
		return []*cfg.Dependency{}, false
	}

	return d, true
}

func guessImportGPM(dir string) ([]*cfg.Dependency, bool) {
	d, err := gpm.Parse(dir)
	if err != nil || len(d) == 0 {
		return []*cfg.Dependency{}, false
	}

	return d, true
}

func guessImportGB(dir string) ([]*cfg.Dependency, bool) {
	d, err := gb.Parse(dir)
	if err != nil || len(d) == 0 {
		return []*cfg.Dependency{}, false
	}

	return d, true
}
<commit_msg>Fixed bug where Gom was not being imported at init<commit_after>package action

import (
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/Masterminds/glide/cfg"
	"github.com/Masterminds/glide/dependency"
	"github.com/Masterminds/glide/gb"
	"github.com/Masterminds/glide/godep"
	"github.com/Masterminds/glide/gom"
	"github.com/Masterminds/glide/gpm"
	"github.com/Masterminds/glide/msg"
	gpath "github.com/Masterminds/glide/path"
	"github.com/Masterminds/glide/util"
)

// Create creates/initializes a new Glide repository.
//
// This will fail if a glide.yaml already exists.
//
// By default, this will scan the present source code directory for dependencies.
//
// If skipImport is set to true, this will not attempt to import from an existing
// GPM, Godep, or GB project if one should exist. However, it will still attempt
// to read the local source to determine required packages.
func Create(base string, skipImport bool) {
	glidefile := gpath.GlideFile
	// Guard against overwrites.
	guardYAML(glidefile)

	// Guess deps
	conf := guessDeps(base, skipImport)
	// Write YAML
	if err := conf.WriteFile(glidefile); err != nil {
		msg.Die("Could not save %s: %s", glidefile, err)
	}
}

// guardYAML fails if the given file already exists.
//
// This prevents an important file from being overwritten.
func guardYAML(filename string) {
	if _, err := os.Stat(filename); err == nil {
		msg.Die("Cowardly refusing to overwrite existing YAML.")
	}
}

// guessDeps attempts to resolve all of the dependencies for a given project.
//
// base is the directory to start with.
// skipImport will skip running the automatic imports.
//
// FIXME: This function is likely a one-off that has a more standard alternative.
// It's also long and could use a refactor.
func guessDeps(base string, skipImport bool) *cfg.Config {
	buildContext, err := util.GetBuildContext()
	if err != nil {
		msg.Die("Failed to build an import context: %s", err)
	}
	name := buildContext.PackageName(base)

	msg.Info("Generating a YAML configuration file and guessing the dependencies")

	config := new(cfg.Config)

	// Get the name of the top level package
	config.Name = name

	// Import by looking at other package managers and looking over the
	// entire directory structure.

	// Attempt to import from other package managers.
	if !skipImport {
		guessImportDeps(base, config)
	}

	// Resolve dependencies by looking at the tree.
	r, err := dependency.NewResolver(base)
	if err != nil {
		msg.Die("Error creating a dependency resolver: %s", err)
	}

	h := &dependency.DefaultMissingPackageHandler{Missing: []string{}, Gopath: []string{}}
	r.Handler = h

	sortable, err := r.ResolveLocal(false)
	if err != nil {
		msg.Die("Error resolving local dependencies: %s", err)
	}

	sort.Strings(sortable)

	vpath := r.VendorDir
	if !strings.HasSuffix(vpath, "/") {
		vpath = vpath + string(os.PathSeparator)
	}

	for _, pa := range sortable {
		n := strings.TrimPrefix(pa, vpath)
		root, subpkg := util.NormalizeName(n)

		if !config.HasDependency(root) {
			msg.Info("Found reference to %s\n", n)
			d := &cfg.Dependency{
				Name: root,
			}
			if len(subpkg) > 0 {
				d.Subpackages = []string{subpkg}
			}
			config.Imports = append(config.Imports, d)
		} else {
			if len(subpkg) > 0 {
				subpkg = strings.TrimPrefix(subpkg, "/")
				d := config.Imports.Get(root)
				if !d.HasSubpackage(subpkg) {
					msg.Info("Adding sub-package %s to %s\n", subpkg, root)
					d.Subpackages = append(d.Subpackages, subpkg)
				}
			}
		}
	}

	return config
}

func guessImportDeps(base string, config *cfg.Config) {
	msg.Info("Attempting to import from other package managers (use --skip-import to skip)")
	deps := []*cfg.Dependency{}
	absBase, err := filepath.Abs(base)
	if err != nil {
		msg.Die("Failed to resolve location of %s: %s", base, err)
	}

	if d, ok := guessImportGodep(absBase); ok {
		msg.Info("Importing Godep configuration")
		msg.Warn("Godep uses commit id versions. Consider using Semantic Versions with Glide")
		deps = d
	} else if d, ok := guessImportGPM(absBase); ok {
		msg.Info("Importing GPM configuration")
		deps = d
	} else if d, ok := guessImportGB(absBase); ok {
		msg.Info("Importing GB configuration")
		deps = d
	} else if d, ok := guessImportGom(absBase); ok {
		msg.Info("Importing GB configuration")
		deps = d
	}

	for _, i := range deps {
		msg.Info("Found imported reference to %s\n", i.Name)
		config.Imports = append(config.Imports, i)
	}
}

func guessImportGodep(dir string) ([]*cfg.Dependency, bool) {
	d, err := godep.Parse(dir)
	if err != nil || len(d) == 0 {
		return []*cfg.Dependency{}, false
	}

	return d, true
}

func guessImportGPM(dir string) ([]*cfg.Dependency, bool) {
	d, err := gpm.Parse(dir)
	if err != nil || len(d) == 0 {
		return []*cfg.Dependency{}, false
	}

	return d, true
}

func guessImportGB(dir string) ([]*cfg.Dependency, bool) {
	d, err := gb.Parse(dir)
	if err != nil || len(d) == 0 {
		return []*cfg.Dependency{}, false
	}

	return d, true
}

func guessImportGom(dir string) ([]*cfg.Dependency, bool) {
	d, err := gom.Parse(dir)
	if err != nil || len(d) == 0 {
		return []*cfg.Dependency{}, false
	}

	return d, true
}
<|endoftext|>


<commit_before>package client

import (
	"fmt"

	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/pinentry"
	keybase1 "github.com/keybase/client/protocol/go"
)

type SecretEntry struct {
	pinentry *pinentry.Pinentry
	terminal *Terminal
	initRes  *error
}

type Printer interface {
	Printf(format string, a ...interface{}) (n int, err error)
}

func NewSecretEntry(t *Terminal) *SecretEntry {
	return &SecretEntry{terminal: t}
}

func (se *SecretEntry) Init() (err error) {

	G.Log.Debug("+ SecretEntry.Init()")

	if se.initRes != nil {
		G.Log.Debug("- SecretEntry.Init() -> cached %s", libkb.ErrToOk(*se.initRes))
		return *se.initRes
	}

	if G.Env.GetNoPinentry() {
		G.Log.Debug("| Pinentry skipped due to config")
	} else {
		pe := pinentry.New(G.Env.GetPinentry(), G.Log)
		if e2, fatalerr := pe.Init(); fatalerr != nil {
			err = fatalerr
		} else if e2 != nil {
			G.Log.Debug("| Pinentry initialization failed: %s", e2)
		} else {
			se.pinentry = pe
			G.Log.Debug("| Pinentry initialized")
		}
	}

	if err == nil {
		if se.pinentry == nil && se.terminal == nil {
			err = fmt.Errorf("No terminal and pinentry init; cannot input secrets")
		}
	}

	se.initRes = &err

	G.Log.Debug("- SecretEntry.Init() -> %s", libkb.ErrToOk(err))
	return err
}

func (se *SecretEntry) Get(arg keybase1.SecretEntryArg, termArg *keybase1.SecretEntryArg, printer Printer) (res *keybase1.SecretEntryRes, err error) {

	if err = se.Init(); err != nil {
		return
	}

	if pe := se.pinentry; pe != nil {
		if len(arg.Reason) > 0 {
			printer.Printf("Collecting your passphrase via pinentry.  Why?  %s.\n", arg.Reason)
		}
		res, err = pe.Get(arg)
	} else if se.terminal == nil {
		err = NoTerminalError{}
	} else {
		if termArg == nil {
			termArg = &arg
		}
		res, err = se.terminal.GetSecret(termArg)
	}

	return
}
<commit_msg>PR feedback<commit_after>package client

import (
	"fmt"

	"github.com/keybase/client/go/libkb"
	"github.com/keybase/client/go/pinentry"
	keybase1 "github.com/keybase/client/protocol/go"
)

type SecretEntry struct {
	pinentry *pinentry.Pinentry
	terminal *Terminal
	initRes  *error
}

type Printer interface {
	Printf(format string, a ...interface{}) (n int, err error)
}

func NewSecretEntry(t *Terminal) *SecretEntry {
	return &SecretEntry{terminal: t}
}

func (se *SecretEntry) Init() (err error) {

	G.Log.Debug("+ SecretEntry.Init()")

	if se.initRes != nil {
		G.Log.Debug("- SecretEntry.Init() -> cached %s", libkb.ErrToOk(*se.initRes))
		return *se.initRes
	}

	if G.Env.GetNoPinentry() {
		G.Log.Debug("| Pinentry skipped due to config")
	} else {
		pe := pinentry.New(G.Env.GetPinentry(), G.Log)
		if e2, fatalerr := pe.Init(); fatalerr != nil {
			err = fatalerr
		} else if e2 != nil {
			G.Log.Debug("| Pinentry initialization failed: %s", e2)
		} else {
			se.pinentry = pe
			G.Log.Debug("| Pinentry initialized")
		}
	}

	if err == nil {
		if se.pinentry == nil && se.terminal == nil {
			err = fmt.Errorf("No terminal and pinentry init; cannot input secrets")
		}
	}

	se.initRes = &err

	G.Log.Debug("- SecretEntry.Init() -> %s", libkb.ErrToOk(err))
	return err
}

func (se *SecretEntry) Get(arg keybase1.SecretEntryArg, termArg *keybase1.SecretEntryArg, printer Printer) (res *keybase1.SecretEntryRes, err error) {

	if err = se.Init(); err != nil {
		return
	}

	if pe := se.pinentry; pe != nil {
		if len(arg.Reason) > 0 {
			printer.Printf("Collecting your passphrase.  Why?  %s.\n", arg.Reason)
		}
		res, err = pe.Get(arg)
	} else if se.terminal == nil {
		err = NoTerminalError{}
	} else {
		if termArg == nil {
			termArg = &arg
		}
		res, err = se.terminal.GetSecret(termArg)
	}

	return
}
<|endoftext|>