<commit_before>package util

import (
	"fmt"
	"io"
	"net/url"
	"os"
	"path"
	"strings"
	"syscall"

	"github.com/containers/image/docker/reference"
	"github.com/containers/image/pkg/sysregistriesv2"
	"github.com/containers/image/signature"
	is "github.com/containers/image/storage"
	"github.com/containers/image/transports"
	"github.com/containers/image/types"
	"github.com/containers/storage"
	"github.com/docker/distribution/registry/api/errcode"
	specs "github.com/opencontainers/runtime-spec/specs-go"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const (
	minimumTruncatedIDLength = 3
	// DefaultTransport is a prefix that we apply to an image name if we
	// can't find one in the local Store, in order to generate a source
	// reference for the image that we can then copy to the local Store.
	DefaultTransport = "docker://"
)

var (
	// RegistryDefaultPathPrefix contains a per-registry listing of default prefixes
	// to prepend to image names that only contain a single path component.
	RegistryDefaultPathPrefix = map[string]string{
		"index.docker.io": "library",
		"docker.io":       "library",
	}
)

// ResolveName checks if name is a valid image name, and if that name doesn't
// include a domain portion, returns a list of the names which it might
// correspond to in the set of configured registries, the transport used to
// pull the image, and a boolean which is true iff
// 1) the list of search registries was used, and 2) it was empty.
//
// The returned image names never include a transport: prefix, and if transport != "",
// (transport, image) should be a valid input to alltransports.ParseImageName.
// transport == "" indicates that image that already exists in a local storage,
// and the name is valid for store.Image() / storage.Transport.ParseStoreReference().
//
// NOTE: The "list of search registries is empty" check does not count blocked registries,
// and neither the implied "localhost" nor a possible firstRegistry are counted
func ResolveName(name string, firstRegistry string, sc *types.SystemContext, store storage.Store) ([]string, string, bool, error) {
	if name == "" {
		return nil, "", false, nil
	}

	// Maybe it's a truncated image ID.  Don't prepend a registry name, then.
	if len(name) >= minimumTruncatedIDLength {
		if img, err := store.Image(name); err == nil && img != nil && strings.HasPrefix(img.ID, name) {
			// It's a truncated version of the ID of an image that's present in local storage;
			// we need only expand the ID.
			return []string{img.ID}, "", false, nil
		}
	}

	// If the image includes a transport's name as a prefix, use it as-is.
	if strings.HasPrefix(name, DefaultTransport) {
		return []string{strings.TrimPrefix(name, DefaultTransport)}, DefaultTransport, false, nil
	}
	split := strings.SplitN(name, ":", 2)
	if len(split) == 2 {
		if trans := transports.Get(split[0]); trans != nil {
			return []string{split[1]}, trans.Name(), false, nil
		}
	}
	// If the image name already included a domain component, we're done.
	named, err := reference.ParseNormalizedNamed(name)
	if err != nil {
		return nil, "", false, errors.Wrapf(err, "error parsing image name %q", name)
	}
	if named.String() == name {
		// Parsing produced the same result, so there was a domain name in there to begin with.
		return []string{name}, DefaultTransport, false, nil
	}
	if reference.Domain(named) != "" && RegistryDefaultPathPrefix[reference.Domain(named)] != "" {
		// If this domain can cause us to insert something in the middle, check if that happened.
		repoPath := reference.Path(named)
		domain := reference.Domain(named)
		tag := ""
		if tagged, ok := named.(reference.Tagged); ok {
			tag = ":" + tagged.Tag()
		}
		digest := ""
		if digested, ok := named.(reference.Digested); ok {
			digest = "@" + digested.Digest().String()
		}
		defaultPrefix := RegistryDefaultPathPrefix[reference.Domain(named)] + "/"
		if strings.HasPrefix(repoPath, defaultPrefix) && path.Join(domain, repoPath[len(defaultPrefix):])+tag+digest == name {
			// Yup, parsing just inserted a bit in the middle, so there was a domain name there to begin with.
			return []string{name}, DefaultTransport, false, nil
		}
	}

	// Figure out the list of registries.
	var registries []string
	searchRegistries, err := sysregistriesv2.UnqualifiedSearchRegistries(sc)
	if err != nil {
		logrus.Debugf("unable to read configured registries to complete %q: %v", name, err)
		searchRegistries = nil
	}
	for _, registry := range searchRegistries {
		reg, err := sysregistriesv2.FindRegistry(sc, registry)
		if err != nil {
			logrus.Debugf("unable to read registry configuraitno for %#v: %v", registry, err)
			continue
		}
		if reg == nil || !reg.Blocked {
			registries = append(registries, registry)
		}
	}
	searchRegistriesAreEmpty := len(registries) == 0

	// Create all of the combinations.  Some registries need an additional component added, so
	// use our lookaside map to keep track of them.  If there are no configured registries, we'll
	// return a name using "localhost" as the registry name.
	candidates := []string{}
	initRegistries := []string{"localhost"}
	if firstRegistry != "" && firstRegistry != "localhost" {
		initRegistries = append([]string{firstRegistry}, initRegistries...)
	}
	for _, registry := range append(initRegistries, registries...) {
		if registry == "" {
			continue
		}
		middle := ""
		if prefix, ok := RegistryDefaultPathPrefix[registry]; ok && strings.IndexRune(name, '/') == -1 {
			middle = prefix
		}
		candidate := path.Join(registry, middle, name)
		candidates = append(candidates, candidate)
	}
	return candidates, DefaultTransport, searchRegistriesAreEmpty, nil
}

// ExpandNames takes unqualified names, parses them as image names, and returns
// the fully expanded result, including a tag.  Names which don't include a registry
// name will be marked for the most-preferred registry (i.e., the first one in our
// configuration).
func ExpandNames(names []string, firstRegistry string, systemContext *types.SystemContext, store storage.Store) ([]string, error) {
	expanded := make([]string, 0, len(names))
	for _, n := range names {
		var name reference.Named
		nameList, _, _, err := ResolveName(n, firstRegistry, systemContext, store)
		if err != nil {
			return nil, errors.Wrapf(err, "error parsing name %q", n)
		}
		if len(nameList) == 0 {
			named, err := reference.ParseNormalizedNamed(n)
			if err != nil {
				return nil, errors.Wrapf(err, "error parsing name %q", n)
			}
			name = named
		} else {
			named, err := reference.ParseNormalizedNamed(nameList[0])
			if err != nil {
				return nil, errors.Wrapf(err, "error parsing name %q", nameList[0])
			}
			name = named
		}
		name = reference.TagNameOnly(name)
		expanded = append(expanded, name.String())
	}
	return expanded, nil
}

// FindImage locates the locally-stored image which corresponds to a given name.
func FindImage(store storage.Store, firstRegistry string, systemContext *types.SystemContext, image string) (types.ImageReference, *storage.Image, error) {
	var ref types.ImageReference
	var img *storage.Image
	var err error
	names, _, _, err := ResolveName(image, firstRegistry, systemContext, store)
	if err != nil {
		return nil, nil, errors.Wrapf(err, "error parsing name %q", image)
	}
	for _, name := range names {
		ref, err = is.Transport.ParseStoreReference(store, name)
		if err != nil {
			logrus.Debugf("error parsing reference to image %q: %v", name, err)
			continue
		}
		img, err = is.Transport.GetStoreImage(store, ref)
		if err != nil {
			img2, err2 := store.Image(name)
			if err2 != nil {
				logrus.Debugf("error locating image %q: %v", name, err2)
				continue
			}
			img = img2
		}
		break
	}
	if ref == nil || img == nil {
		return nil, nil, errors.Wrapf(err, "error locating image with name %q (%v)", image, names)
	}
	return ref, img, nil
}

// AddImageNames adds the specified names to the specified image.
func AddImageNames(store storage.Store, firstRegistry string, systemContext *types.SystemContext, image *storage.Image, addNames []string) error {
	names, err := ExpandNames(addNames, firstRegistry, systemContext, store)
	if err != nil {
		return err
	}
	err = store.SetNames(image.ID, append(image.Names, names...))
	if err != nil {
		return errors.Wrapf(err, "error adding names (%v) to image %q", names, image.ID)
	}
	return nil
}

// GetFailureCause checks the type of the error "err" and returns a new
// error message that reflects the reason of the failure.
// In case err type is not a familiar one the error "defaultError" is returned.
func GetFailureCause(err, defaultError error) error {
	switch nErr := errors.Cause(err).(type) {
	case errcode.Errors:
		return err
	case errcode.Error, *url.Error:
		return nErr
	default:
		return defaultError
	}
}

// WriteError writes `lastError` into `w` if not nil and return the next error `err`
func WriteError(w io.Writer, err error, lastError error) error {
	if lastError != nil {
		fmt.Fprintln(w, lastError)
	}
	return err
}

// Runtime is the default command to use to run the container.
func Runtime() string {
	runtime := os.Getenv("BUILDAH_RUNTIME")
	if runtime != "" {
		return runtime
	}
	return DefaultRuntime
}

// StringInSlice returns a boolean indicating if the exact value s is present
// in the slice slice.
func StringInSlice(s string, slice []string) bool {
	for _, v := range slice {
		if v == s {
			return true
		}
	}
	return false
}

// GetContainerIDs uses ID mappings to compute the container-level IDs that will
// correspond to a UID/GID pair on the host.
func GetContainerIDs(uidmap, gidmap []specs.LinuxIDMapping, uid, gid uint32) (uint32, uint32, error) {
	uidMapped := true
	for _, m := range uidmap {
		uidMapped = false
		if uid >= m.HostID && uid < m.HostID+m.Size {
			uid = (uid - m.HostID) + m.ContainerID
			uidMapped = true
			break
		}
	}
	if !uidMapped {
		return 0, 0, errors.Errorf("container uses ID mappings (%#v), but doesn't map UID %d", uidmap, uid)
	}
	gidMapped := true
	for _, m := range gidmap {
		gidMapped = false
		if gid >= m.HostID && gid < m.HostID+m.Size {
			gid = (gid - m.HostID) + m.ContainerID
			gidMapped = true
			break
		}
	}
	if !gidMapped {
		return 0, 0, errors.Errorf("container uses ID mappings (%#v), but doesn't map GID %d", gidmap, gid)
	}
	return uid, gid, nil
}

// GetHostIDs uses ID mappings to compute the host-level IDs that will
// correspond to a UID/GID pair in the container.
func GetHostIDs(uidmap, gidmap []specs.LinuxIDMapping, uid, gid uint32) (uint32, uint32, error) {
	uidMapped := true
	for _, m := range uidmap {
		uidMapped = false
		if uid >= m.ContainerID && uid < m.ContainerID+m.Size {
			uid = (uid - m.ContainerID) + m.HostID
			uidMapped = true
			break
		}
	}
	if !uidMapped {
		return 0, 0, errors.Errorf("container uses ID mappings (%#v), but doesn't map UID %d", uidmap, uid)
	}
	gidMapped := true
	for _, m := range gidmap {
		gidMapped = false
		if gid >= m.ContainerID && gid < m.ContainerID+m.Size {
			gid = (gid - m.ContainerID) + m.HostID
			gidMapped = true
			break
		}
	}
	if !gidMapped {
		return 0, 0, errors.Errorf("container uses ID mappings (%#v), but doesn't map GID %d", gidmap, gid)
	}
	return uid, gid, nil
}

// GetHostRootIDs uses ID mappings in spec to compute the host-level IDs that will
// correspond to UID/GID 0/0 in the container.
func GetHostRootIDs(spec *specs.Spec) (uint32, uint32, error) {
	if spec.Linux == nil {
		return 0, 0, nil
	}
	return GetHostIDs(spec.Linux.UIDMappings, spec.Linux.GIDMappings, 0, 0)
}

// GetPolicyContext sets up, initializes and returns a new context for the specified policy
func GetPolicyContext(ctx *types.SystemContext) (*signature.PolicyContext, error) {
	policy, err := signature.DefaultPolicy(ctx)
	if err != nil {
		return nil, err
	}

	policyContext, err := signature.NewPolicyContext(policy)
	if err != nil {
		return nil, err
	}
	return policyContext, nil
}

// logIfNotErrno logs the error message unless err is either nil or one of the
// listed syscall.Errno values.  It returns true if it logged an error.
func logIfNotErrno(err error, what string, ignores ...syscall.Errno) (logged bool) {
	if err == nil {
		return false
	}
	if errno, isErrno := err.(syscall.Errno); isErrno {
		for _, ignore := range ignores {
			if errno == ignore {
				return false
			}
		}
	}
	logrus.Error(what)
	return true
}

// LogIfNotRetryable logs "what" if err is set and is not an EINTR or EAGAIN
// syscall.Errno.  Returns "true" if we can continue.
func LogIfNotRetryable(err error, what string) (retry bool) {
	return !logIfNotErrno(err, what, syscall.EINTR, syscall.EAGAIN)
}

// LogIfUnexpectedWhileDraining logs "what" if err is set and is not an EINTR
// or EAGAIN or EIO syscall.Errno.
func LogIfUnexpectedWhileDraining(err error, what string) {
	logIfNotErrno(err, what, syscall.EINTR, syscall.EAGAIN, syscall.EIO)
}
<commit_msg>util: use strings.ContainsRune instead of index check<commit_after>package util

import (
	"fmt"
	"io"
	"net/url"
	"os"
	"path"
	"strings"
	"syscall"

	"github.com/containers/image/docker/reference"
	"github.com/containers/image/pkg/sysregistriesv2"
	"github.com/containers/image/signature"
	is "github.com/containers/image/storage"
	"github.com/containers/image/transports"
	"github.com/containers/image/types"
	"github.com/containers/storage"
	"github.com/docker/distribution/registry/api/errcode"
	specs "github.com/opencontainers/runtime-spec/specs-go"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const (
	minimumTruncatedIDLength = 3
	// DefaultTransport is a prefix that we apply to an image name if we
	// can't find one in the local Store, in order to generate a source
	// reference for the image that we can then copy to the local Store.
	DefaultTransport = "docker://"
)

var (
	// RegistryDefaultPathPrefix contains a per-registry listing of default prefixes
	// to prepend to image names that only contain a single path component.
	RegistryDefaultPathPrefix = map[string]string{
		"index.docker.io": "library",
		"docker.io":       "library",
	}
)

// ResolveName checks if name is a valid image name, and if that name doesn't
// include a domain portion, returns a list of the names which it might
// correspond to in the set of configured registries, the transport used to
// pull the image, and a boolean which is true iff
// 1) the list of search registries was used, and 2) it was empty.
//
// The returned image names never include a transport: prefix, and if transport != "",
// (transport, image) should be a valid input to alltransports.ParseImageName.
// transport == "" indicates that image that already exists in a local storage,
// and the name is valid for store.Image() / storage.Transport.ParseStoreReference().
//
// NOTE: The "list of search registries is empty" check does not count blocked registries,
// and neither the implied "localhost" nor a possible firstRegistry are counted
func ResolveName(name string, firstRegistry string, sc *types.SystemContext, store storage.Store) ([]string, string, bool, error) {
	if name == "" {
		return nil, "", false, nil
	}

	// Maybe it's a truncated image ID.  Don't prepend a registry name, then.
	if len(name) >= minimumTruncatedIDLength {
		if img, err := store.Image(name); err == nil && img != nil && strings.HasPrefix(img.ID, name) {
			// It's a truncated version of the ID of an image that's present in local storage;
			// we need only expand the ID.
			return []string{img.ID}, "", false, nil
		}
	}

	// If the image includes a transport's name as a prefix, use it as-is.
	if strings.HasPrefix(name, DefaultTransport) {
		return []string{strings.TrimPrefix(name, DefaultTransport)}, DefaultTransport, false, nil
	}
	split := strings.SplitN(name, ":", 2)
	if len(split) == 2 {
		if trans := transports.Get(split[0]); trans != nil {
			return []string{split[1]}, trans.Name(), false, nil
		}
	}
	// If the image name already included a domain component, we're done.
	named, err := reference.ParseNormalizedNamed(name)
	if err != nil {
		return nil, "", false, errors.Wrapf(err, "error parsing image name %q", name)
	}
	if named.String() == name {
		// Parsing produced the same result, so there was a domain name in there to begin with.
		return []string{name}, DefaultTransport, false, nil
	}
	if reference.Domain(named) != "" && RegistryDefaultPathPrefix[reference.Domain(named)] != "" {
		// If this domain can cause us to insert something in the middle, check if that happened.
		repoPath := reference.Path(named)
		domain := reference.Domain(named)
		tag := ""
		if tagged, ok := named.(reference.Tagged); ok {
			tag = ":" + tagged.Tag()
		}
		digest := ""
		if digested, ok := named.(reference.Digested); ok {
			digest = "@" + digested.Digest().String()
		}
		defaultPrefix := RegistryDefaultPathPrefix[reference.Domain(named)] + "/"
		if strings.HasPrefix(repoPath, defaultPrefix) && path.Join(domain, repoPath[len(defaultPrefix):])+tag+digest == name {
			// Yup, parsing just inserted a bit in the middle, so there was a domain name there to begin with.
			return []string{name}, DefaultTransport, false, nil
		}
	}

	// Figure out the list of registries.
	var registries []string
	searchRegistries, err := sysregistriesv2.UnqualifiedSearchRegistries(sc)
	if err != nil {
		logrus.Debugf("unable to read configured registries to complete %q: %v", name, err)
		searchRegistries = nil
	}
	for _, registry := range searchRegistries {
		reg, err := sysregistriesv2.FindRegistry(sc, registry)
		if err != nil {
			logrus.Debugf("unable to read registry configuraitno for %#v: %v", registry, err)
			continue
		}
		if reg == nil || !reg.Blocked {
			registries = append(registries, registry)
		}
	}
	searchRegistriesAreEmpty := len(registries) == 0

	// Create all of the combinations.  Some registries need an additional component added, so
	// use our lookaside map to keep track of them.  If there are no configured registries, we'll
	// return a name using "localhost" as the registry name.
	candidates := []string{}
	initRegistries := []string{"localhost"}
	if firstRegistry != "" && firstRegistry != "localhost" {
		initRegistries = append([]string{firstRegistry}, initRegistries...)
	}
	for _, registry := range append(initRegistries, registries...) {
		if registry == "" {
			continue
		}
		middle := ""
		if prefix, ok := RegistryDefaultPathPrefix[registry]; ok && !strings.ContainsRune(name, '/') {
			middle = prefix
		}
		candidate := path.Join(registry, middle, name)
		candidates = append(candidates, candidate)
	}
	return candidates, DefaultTransport, searchRegistriesAreEmpty, nil
}

// ExpandNames takes unqualified names, parses them as image names, and returns
// the fully expanded result, including a tag.  Names which don't include a registry
// name will be marked for the most-preferred registry (i.e., the first one in our
// configuration).
func ExpandNames(names []string, firstRegistry string, systemContext *types.SystemContext, store storage.Store) ([]string, error) {
	expanded := make([]string, 0, len(names))
	for _, n := range names {
		var name reference.Named
		nameList, _, _, err := ResolveName(n, firstRegistry, systemContext, store)
		if err != nil {
			return nil, errors.Wrapf(err, "error parsing name %q", n)
		}
		if len(nameList) == 0 {
			named, err := reference.ParseNormalizedNamed(n)
			if err != nil {
				return nil, errors.Wrapf(err, "error parsing name %q", n)
			}
			name = named
		} else {
			named, err := reference.ParseNormalizedNamed(nameList[0])
			if err != nil {
				return nil, errors.Wrapf(err, "error parsing name %q", nameList[0])
			}
			name = named
		}
		name = reference.TagNameOnly(name)
		expanded = append(expanded, name.String())
	}
	return expanded, nil
}

// FindImage locates the locally-stored image which corresponds to a given name.
func FindImage(store storage.Store, firstRegistry string, systemContext *types.SystemContext, image string) (types.ImageReference, *storage.Image, error) {
	var ref types.ImageReference
	var img *storage.Image
	var err error
	names, _, _, err := ResolveName(image, firstRegistry, systemContext, store)
	if err != nil {
		return nil, nil, errors.Wrapf(err, "error parsing name %q", image)
	}
	for _, name := range names {
		ref, err = is.Transport.ParseStoreReference(store, name)
		if err != nil {
			logrus.Debugf("error parsing reference to image %q: %v", name, err)
			continue
		}
		img, err = is.Transport.GetStoreImage(store, ref)
		if err != nil {
			img2, err2 := store.Image(name)
			if err2 != nil {
				logrus.Debugf("error locating image %q: %v", name, err2)
				continue
			}
			img = img2
		}
		break
	}
	if ref == nil || img == nil {
		return nil, nil, errors.Wrapf(err, "error locating image with name %q (%v)", image, names)
	}
	return ref, img, nil
}

// AddImageNames adds the specified names to the specified image.
func AddImageNames(store storage.Store, firstRegistry string, systemContext *types.SystemContext, image *storage.Image, addNames []string) error {
	names, err := ExpandNames(addNames, firstRegistry, systemContext, store)
	if err != nil {
		return err
	}
	err = store.SetNames(image.ID, append(image.Names, names...))
	if err != nil {
		return errors.Wrapf(err, "error adding names (%v) to image %q", names, image.ID)
	}
	return nil
}

// GetFailureCause checks the type of the error "err" and returns a new
// error message that reflects the reason of the failure.
// In case err type is not a familiar one the error "defaultError" is returned.
func GetFailureCause(err, defaultError error) error {
	switch nErr := errors.Cause(err).(type) {
	case errcode.Errors:
		return err
	case errcode.Error, *url.Error:
		return nErr
	default:
		return defaultError
	}
}

// WriteError writes `lastError` into `w` if not nil and return the next error `err`
func WriteError(w io.Writer, err error, lastError error) error {
	if lastError != nil {
		fmt.Fprintln(w, lastError)
	}
	return err
}

// Runtime is the default command to use to run the container.
func Runtime() string {
	runtime := os.Getenv("BUILDAH_RUNTIME")
	if runtime != "" {
		return runtime
	}
	return DefaultRuntime
}

// StringInSlice returns a boolean indicating if the exact value s is present
// in the slice slice.
func StringInSlice(s string, slice []string) bool {
	for _, v := range slice {
		if v == s {
			return true
		}
	}
	return false
}

// GetContainerIDs uses ID mappings to compute the container-level IDs that will
// correspond to a UID/GID pair on the host.
func GetContainerIDs(uidmap, gidmap []specs.LinuxIDMapping, uid, gid uint32) (uint32, uint32, error) {
	uidMapped := true
	for _, m := range uidmap {
		uidMapped = false
		if uid >= m.HostID && uid < m.HostID+m.Size {
			uid = (uid - m.HostID) + m.ContainerID
			uidMapped = true
			break
		}
	}
	if !uidMapped {
		return 0, 0, errors.Errorf("container uses ID mappings (%#v), but doesn't map UID %d", uidmap, uid)
	}
	gidMapped := true
	for _, m := range gidmap {
		gidMapped = false
		if gid >= m.HostID && gid < m.HostID+m.Size {
			gid = (gid - m.HostID) + m.ContainerID
			gidMapped = true
			break
		}
	}
	if !gidMapped {
		return 0, 0, errors.Errorf("container uses ID mappings (%#v), but doesn't map GID %d", gidmap, gid)
	}
	return uid, gid, nil
}

// GetHostIDs uses ID mappings to compute the host-level IDs that will
// correspond to a UID/GID pair in the container.
func GetHostIDs(uidmap, gidmap []specs.LinuxIDMapping, uid, gid uint32) (uint32, uint32, error) {
	uidMapped := true
	for _, m := range uidmap {
		uidMapped = false
		if uid >= m.ContainerID && uid < m.ContainerID+m.Size {
			uid = (uid - m.ContainerID) + m.HostID
			uidMapped = true
			break
		}
	}
	if !uidMapped {
		return 0, 0, errors.Errorf("container uses ID mappings (%#v), but doesn't map UID %d", uidmap, uid)
	}
	gidMapped := true
	for _, m := range gidmap {
		gidMapped = false
		if gid >= m.ContainerID && gid < m.ContainerID+m.Size {
			gid = (gid - m.ContainerID) + m.HostID
			gidMapped = true
			break
		}
	}
	if !gidMapped {
		return 0, 0, errors.Errorf("container uses ID mappings (%#v), but doesn't map GID %d", gidmap, gid)
	}
	return uid, gid, nil
}

// GetHostRootIDs uses ID mappings in spec to compute the host-level IDs that will
// correspond to UID/GID 0/0 in the container.
func GetHostRootIDs(spec *specs.Spec) (uint32, uint32, error) {
	if spec.Linux == nil {
		return 0, 0, nil
	}
	return GetHostIDs(spec.Linux.UIDMappings, spec.Linux.GIDMappings, 0, 0)
}

// GetPolicyContext sets up, initializes and returns a new context for the specified policy
func GetPolicyContext(ctx *types.SystemContext) (*signature.PolicyContext, error) {
	policy, err := signature.DefaultPolicy(ctx)
	if err != nil {
		return nil, err
	}

	policyContext, err := signature.NewPolicyContext(policy)
	if err != nil {
		return nil, err
	}
	return policyContext, nil
}

// logIfNotErrno logs the error message unless err is either nil or one of the
// listed syscall.Errno values.  It returns true if it logged an error.
func logIfNotErrno(err error, what string, ignores ...syscall.Errno) (logged bool) {
	if err == nil {
		return false
	}
	if errno, isErrno := err.(syscall.Errno); isErrno {
		for _, ignore := range ignores {
			if errno == ignore {
				return false
			}
		}
	}
	logrus.Error(what)
	return true
}

// LogIfNotRetryable logs "what" if err is set and is not an EINTR or EAGAIN
// syscall.Errno.  Returns "true" if we can continue.
func LogIfNotRetryable(err error, what string) (retry bool) {
	return !logIfNotErrno(err, what, syscall.EINTR, syscall.EAGAIN)
}

// LogIfUnexpectedWhileDraining logs "what" if err is set and is not an EINTR
// or EAGAIN or EIO syscall.Errno.
func LogIfUnexpectedWhileDraining(err error, what string) {
	logIfNotErrno(err, what, syscall.EINTR, syscall.EAGAIN, syscall.EIO)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package server

import (
	"net"
	"testing"
)

func TestLoopbackHostPort(t *testing.T) {
	host, port, err := LoopbackHostPort("1.2.3.4:443")
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if host != "1.2.3.4" {
		t.Fatalf("expected 1.2.3.4 as host, got %q", host)
	}
	if port != "443" {
		t.Fatalf("expected 443 as port, got %q", port)
	}

	host, port, err = LoopbackHostPort("0.0.0.0:443")
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if ip := net.ParseIP(host); ip == nil || !ip.IsLoopback() {
		t.Fatalf("expected host to be loopback, got %q", host)
	}
	if port != "443" {
		t.Fatalf("expected 443 as port, got %q", port)
	}

	host, port, err = LoopbackHostPort("[ff06:0:0:0:0:0:0:c3]:443")
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if host != "ff06:0:0:0:0:0:0:c3" {
		t.Fatalf("expected ff06:0:0:0:0:0:0:c3 as host, got %q", host)
	}
	if port != "443" {
		t.Fatalf("expected 443 as port, got %q", port)
	}

	host, port, err = LoopbackHostPort("[::]:443")
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if ip := net.ParseIP(host); ip == nil || !ip.IsLoopback() || ip.To4() != nil {
		t.Fatalf("expected IPv6 host to be loopback, got %q", host)
	}

	if port != "443" {
		t.Fatalf("expected 443 as port, got %q", port)
	}
}
<commit_msg>Split TestLoopbackHostPort into 2 tests<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package server

import (
	"net"
	"testing"
)

func TestLoopbackHostPortIPv4(t *testing.T) {
	_, ipv6only, err := isIPv6LoopbackSupported()
	if err != nil {
		t.Fatalf("fail to enumerate network interface, %s", err)
	}
	if ipv6only {
		t.Fatalf("no ipv4 loopback interface")
	}

	host, port, err := LoopbackHostPort("1.2.3.4:443")
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if host != "1.2.3.4" {
		t.Fatalf("expected 1.2.3.4 as host, got %q", host)
	}
	if port != "443" {
		t.Fatalf("expected 443 as port, got %q", port)
	}

	host, port, err = LoopbackHostPort("0.0.0.0:443")
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if ip := net.ParseIP(host); ip == nil || !ip.IsLoopback() {
		t.Fatalf("expected host to be loopback, got %q", host)
	}
	if port != "443" {
		t.Fatalf("expected 443 as port, got %q", port)
	}
}
func TestLoopbackHostPortIPv6(t *testing.T) {
	ipv6, _, err := isIPv6LoopbackSupported()
	if err != nil {
		t.Fatalf("fail to enumerate network interface, %s", err)
	}
	if !ipv6 {
		t.Fatalf("no ipv6 loopback interface")
	}

	host, port, err := LoopbackHostPort("[ff06:0:0:0:0:0:0:c3]:443")
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if host != "ff06:0:0:0:0:0:0:c3" {
		t.Fatalf("expected ff06:0:0:0:0:0:0:c3 as host, got %q", host)
	}
	if port != "443" {
		t.Fatalf("expected 443 as port, got %q", port)
	}

	host, port, err = LoopbackHostPort("[::]:443")
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if ip := net.ParseIP(host); ip == nil || !ip.IsLoopback() || ip.To4() != nil {
		t.Fatalf("expected IPv6 host to be loopback, got %q", host)
	}
	if port != "443" {
		t.Fatalf("expected 443 as port, got %q", port)
	}
}

func isIPv6LoopbackSupported() (ipv6 bool, ipv6only bool, err error) {
	addrs, err := net.InterfaceAddrs()
	if err != nil {
		return false, false, err
	}
	ipv4 := false
	for _, address := range addrs {
		ipnet, ok := address.(*net.IPNet)
		if !ok || !ipnet.IP.IsLoopback() {
			continue
		}
		if ipnet.IP.To4() == nil {
			ipv6 = true
			continue
		}
		ipv4 = true
	}
	ipv6only = ipv6 && !ipv4
	return ipv6, ipv6only, nil
}
<|endoftext|>


<commit_before>package hello

import (
	"net/http"
	"github.com/gin-gonic/gin"
)

// This function's name is a must. App Engine uses it to drive the requests properly.
func init() {
	// Starts a new Gin instance with no middle-ware
	r := gin.New()

	// Define your handlers
	r.GET("/", func(c *gin.Context){
		c.String(200, "Hello World!")
	})
	r.GET("/ping", func(c *gin.Context){
		c.String(200, "pong")
	})

	// Handle all requests using net/http
	http.Handle("/", r)
}<commit_msg>fix format<commit_after>package hello

import (
	"github.com/gin-gonic/gin"
	"net/http"
)

// This function's name is a must. App Engine uses it to drive the requests properly.
func init() {
	// Starts a new Gin instance with no middle-ware
	r := gin.New()

	// Define your handlers
	r.GET("/", func(c *gin.Context) {
		c.String(200, "Hello World!")
	})
	r.GET("/ping", func(c *gin.Context) {
		c.String(200, "pong")
	})

	// Handle all requests using net/http
	http.Handle("/", r)
}
<|endoftext|>


<commit_before><commit_msg>Another plcae to check in CPUsets<commit_after><|endoftext|>


<commit_before>package main

import (
    "bytes"
    "fmt"
)

type RingBuffer struct {
    buffer []interface{}
    beginPos, endPos int
}

type InvalidIndexError struct {
    index int
}

func (iie *InvalidIndexError) Error() string {
    return fmt.Sprintf("%d is not a valid index for an element in this buffer", iie.index)
}

func (rb *RingBuffer) Length() int {
    if rb.beginPos < rb.endPos {
        return rb.endPos - rb.beginPos + 1
    } else {
        return len(rb.buffer)
    }
}

func (rb *RingBuffer) Item(index int) (interface{}, error) {
    if index < 0 || index >= len(rb.buffer) || beginPos == 0 && index > endPos {
        return nil, &InvalidIndexError{index}
    }

    pos := rb.endPos - index

    if pos < 0 {
        pos += len(rb.buffer)
    }

    return rb.buffer[pos], nil
}

func (rb *RingBuffer) Append(value interface{}) {
    if rb.beginPos == -1 || rb.endPos == -1 {
        rb.beginPos = 0
        rb.endPos = 0
        rb.buffer[0] = value
        return
    }

    if rb.endPos == len(rb.buffer) - 1 {
        rb.endPos = 0
    } else {
        rb.endPos++
    }

    if rb.beginPos == len(rb.buffer) - 1 {
        rb.beginPos = 0
    } else if rb.beginPos == rb.endPos {
        rb.beginPos++
    }

    rb.buffer[rb.endPos] = value
}

func (rb *RingBuffer) String() string {
    var buf bytes.Buffer
    buf.WriteString("[ ")

    for i := 0; i < len(rb.buffer) - 1; i++ {
        rbi, _ := rb.Item(i)
        buf.WriteString(fmt.Sprintf("%v ", rbi))
    }

    rbi, _ := rb.Item(len(rb.buffer) - 1)
    buf.WriteString(fmt.Sprintf("%v]", rbi))
    return buf.String()
}

func NewRingBuffer(size int) *RingBuffer {
    return &RingBuffer{
        buffer: make([]interface{}, size),
        beginPos: -1,
        endPos: -1,
    }
}
<commit_msg>fix: forgot to prefix names of beginPos and endPos with receiver name<commit_after>package main

import (
    "bytes"
    "fmt"
)

type RingBuffer struct {
    buffer []interface{}
    beginPos, endPos int
}

type InvalidIndexError struct {
    index int
}

func (iie *InvalidIndexError) Error() string {
    return fmt.Sprintf("%d is not a valid index for an element in this buffer", iie.index)
}

func (rb *RingBuffer) Length() int {
    if rb.beginPos < rb.endPos {
        return rb.endPos - rb.beginPos + 1
    } else {
        return len(rb.buffer)
    }
}

func (rb *RingBuffer) Item(index int) (interface{}, error) {
    if index < 0 || index >= len(rb.buffer) || rb.beginPos == 0 && index > rb.endPos {
        return nil, &InvalidIndexError{index}
    }

    pos := rb.endPos - index

    if pos < 0 {
        pos += len(rb.buffer)
    }

    return rb.buffer[pos], nil
}

func (rb *RingBuffer) Append(value interface{}) {
    if rb.beginPos == -1 || rb.endPos == -1 {
        rb.beginPos = 0
        rb.endPos = 0
        rb.buffer[0] = value
        return
    }

    if rb.endPos == len(rb.buffer) - 1 {
        rb.endPos = 0
    } else {
        rb.endPos++
    }

    if rb.beginPos == len(rb.buffer) - 1 {
        rb.beginPos = 0
    } else if rb.beginPos == rb.endPos {
        rb.beginPos++
    }

    rb.buffer[rb.endPos] = value
}

func (rb *RingBuffer) String() string {
    var buf bytes.Buffer
    buf.WriteString("[ ")

    for i := 0; i < len(rb.buffer) - 1; i++ {
        rbi, _ := rb.Item(i)
        buf.WriteString(fmt.Sprintf("%v ", rbi))
    }

    rbi, _ := rb.Item(len(rb.buffer) - 1)
    buf.WriteString(fmt.Sprintf("%v]", rbi))
    return buf.String()
}

func NewRingBuffer(size int) *RingBuffer {
    return &RingBuffer{
        buffer: make([]interface{}, size),
        beginPos: -1,
        endPos: -1,
    }
}
<|endoftext|>


<commit_before>package openapi

import (
	"testing"
)

const (
	exampleCom  = "https://example.com"
	exampleMail = "foo@example.com"
)

type candidate struct {
	label  string
	in     validater
	hasErr bool
}

func testValidater(t *testing.T, candidates []candidate) {
	t.Helper()
	for _, c := range candidates {
		if err := c.in.Validate(); (err != nil) != c.hasErr {
			if c.hasErr {
				t.Error("error should be occurred, but not")
				continue
			}
			t.Errorf("error is occurred: %s", err)
		}
	}
}

func TestHasDuplicatedParameter(t *testing.T) {
	t.Run("no duplicated param", testHasDuplicatedParameterFalse)
	t.Run("there's duplicated param", testHasDuplicatedParameterTrue)
}

func testHasDuplicatedParameterFalse(t *testing.T) {
	params := []*Parameter{
		&Parameter{Name: "foo", In: "header"},
		&Parameter{Name: "foo", In: "path", Required: true},
		&Parameter{Name: "bar", In: "path", Required: true},
	}
	if hasDuplicatedParameter(params) {
		t.Error("should return false")
	}
}

func testHasDuplicatedParameterTrue(t *testing.T) {
	params := []*Parameter{
		&Parameter{Name: "foo", In: "header"},
		&Parameter{Name: "foo", In: "header"},
	}
	if !hasDuplicatedParameter(params) {
		t.Error("should return true")
	}
}

func TestMustURL(t *testing.T) {
	candidates := []struct {
		label  string
		in     string
		hasErr bool
	}{
		{"empty", "", true},
		{"valid HTTP url", "http://example.com", false},
		{"allowed relative path", "foo/bar/baz", true},
		{"absolute path", "/foo/bar/baz", false},
		{"plain string", "foobarbaz", true},
	}
	for _, c := range candidates {
		if err := mustURL(c.label, c.in); (err != nil) != c.hasErr {
			t.Logf("error occured at %s", c.label)
			if c.hasErr {
				t.Error("error should occured, but not")
				return
			}
			t.Error("error should not occurred, but occurred")
			return
		}
	}
}

func TestContactValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", Contact{}, true},
		{"withURL", Contact{URL: exampleCom}, false},
		{"invalidURL", Contact{URL: "foobar"}, true},
		{"withEmail", Contact{Email: exampleMail}, true},
		{"valid", Contact{URL: exampleCom, Email: exampleMail}, false},
		{"invalidEmail", Contact{URL: exampleCom, Email: "foobar"}, true},
	}

	testValidater(t, candidates)
}

func TestLicenseValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", License{}, true},
		{"withName", License{Name: "foobar"}, true},
		{"withURL", License{URL: exampleCom}, true},
		{"invalidURL", License{Name: "foobar", URL: "foobar"}, true},
		{"valid", License{Name: "foobar", URL: exampleCom}, false},
	}
	testValidater(t, candidates)
}

func TestServerValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", Server{}, true},
		{"invalidURL", Server{URL: "foobar%"}, true},
		{"withURL", Server{URL: exampleCom}, false},
	}
	testValidater(t, candidates)
}

func TestServerVariableValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", ServerVariable{}, true},
		{"withDefault", ServerVariable{Default: "default"}, false},
	}
	testValidater(t, candidates)
}

func TestPathsValidate(t *testing.T) {
	t.Run("duplicate pathItem", testPathItemDuplicate)
}

func getPaths(id1, id2 string) Paths {
	return Paths{
		"/foo/bar": &PathItem{
			Get:  &Operation{OperationID: id1, Responses: Responses{"200": &Response{Description: "foo"}}},
			Post: &Operation{OperationID: id2, Responses: Responses{"200": &Response{Description: "foo"}}},
		},
	}
}

func testPathItemDuplicate(t *testing.T) {
	candidates := []candidate{
		{"invalid", getPaths("foobar", "foobar"), true},
		{"valid", getPaths("foo", "bar"), false},
	}
	testValidater(t, candidates)
}

func TestExternalDocumentationValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", ExternalDocumentation{}, true},
		{"invalidURL", ExternalDocumentation{URL: "foobar"}, true},
		{"valid", ExternalDocumentation{URL: exampleCom}, false},
	}
	testValidater(t, candidates)
}

func TestTagValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", Tag{}, true},
		{"withEmptyExternalDocs", Tag{ExternalDocs: &ExternalDocumentation{}}, true},
		{"withValidExternalDocs", Tag{ExternalDocs: &ExternalDocumentation{URL: exampleCom}}, true},

		{"withName", Tag{Name: "foo"}, false},
	}
	testValidater(t, candidates)
}

func TestDiscriminatorValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", Discriminator{}, true},
		{"withPropertyName", Discriminator{PropertyName: "foobar"}, false},
	}
	testValidater(t, candidates)
}

func TestXMLValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", XML{}, true},
		{"invalidURLNamespace", XML{Namespace: "foobar"}, true},
		{"withNamespace", XML{Namespace: exampleCom}, false},
	}
	testValidater(t, candidates)
}

func TestOAuthFlowValidate(t *testing.T) {
	mockScopes := map[string]string{"foo": "bar"}

	empty := OAuthFlow{}
	aURL := OAuthFlow{AuthorizationURL: exampleCom}
	tURL := OAuthFlow{TokenURL: exampleCom}
	rURL := OAuthFlow{RefreshURL: exampleCom}
	scopes := OAuthFlow{Scopes: mockScopes}
	atURL := OAuthFlow{AuthorizationURL: exampleCom, TokenURL: exampleCom}
	arURL := OAuthFlow{AuthorizationURL: exampleCom, RefreshURL: exampleCom}
	aURLscopes := OAuthFlow{AuthorizationURL: exampleCom, Scopes: mockScopes}
	trURL := OAuthFlow{TokenURL: exampleCom, RefreshURL: exampleCom}
	tURLscopes := OAuthFlow{TokenURL: exampleCom, Scopes: mockScopes}
	rURLscopes := OAuthFlow{RefreshURL: exampleCom, Scopes: mockScopes}
	atrURL := OAuthFlow{AuthorizationURL: exampleCom, TokenURL: exampleCom, RefreshURL: exampleCom}
	atURLscopes := OAuthFlow{AuthorizationURL: exampleCom, TokenURL: exampleCom, Scopes: mockScopes}
	arURLscopes := OAuthFlow{AuthorizationURL: exampleCom, RefreshURL: exampleCom, Scopes: mockScopes}
	trURLscopes := OAuthFlow{TokenURL: exampleCom, RefreshURL: exampleCom, Scopes: mockScopes}
	atrURLscopes := OAuthFlow{AuthorizationURL: exampleCom, TokenURL: exampleCom, RefreshURL: exampleCom, Scopes: mockScopes}
	invalidURL := OAuthFlow{AuthorizationURL: "foobar", TokenURL: "foobar", RefreshURL: "foobar", Scopes: mockScopes}
	zeroMap := OAuthFlow{AuthorizationURL: exampleCom, TokenURL: exampleCom, RefreshURL: exampleCom, Scopes: map[string]string{}}

	candidates := []struct {
		label   string
		in      OAuthFlow
		haveErr [4]bool
	}{
		{"empty", empty, [4]bool{true, true, true, true}},
		{"aURL", aURL, [4]bool{true, true, true, true}},
		{"tURL", tURL, [4]bool{true, true, true, true}},
		{"rURL", rURL, [4]bool{true, true, true, true}},
		{"scopes", scopes, [4]bool{true, true, true, true}},
		{"aURL/tURL", atURL, [4]bool{true, true, true, true}},
		{"aURL/rURL", arURL, [4]bool{true, true, true, true}},
		{"aURL/scopes", aURLscopes, [4]bool{false, true, true, true}},
		{"tURL/rURL", trURL, [4]bool{true, true, true, true}},
		{"tURL/scopes", tURLscopes, [4]bool{true, false, false, true}},
		{"rURL/scopes", rURLscopes, [4]bool{true, true, true, true}},
		{"aURL/tURL/rURL", atrURL, [4]bool{true, true, true, true}},
		{"aURL/tURL/scopes", atURLscopes, [4]bool{false, false, false, false}},
		{"aURL/rURL/scopes", arURLscopes, [4]bool{false, true, true, true}},
		{"tURL/rURL/scopes", trURLscopes, [4]bool{true, false, false, true}},
		{"aURL/tURL/rURL/scopes", atrURLscopes, [4]bool{false, false, false, false}},

		{"invalidURL", invalidURL, [4]bool{true, true, true, true}},
		{"zero length map", zeroMap, [4]bool{true, true, true, true}},
	}
	for _, c := range candidates {
		testOAuthFlowValidate(t, c.label, c.in, c.haveErr)
	}
}

var flowTypes = []string{"implicit", "password", "clientCredentials", "authorizationCode"}

func testOAuthFlowValidate(t *testing.T, label string, oauthFlow OAuthFlow, haveErr [4]bool) {
	if err := oauthFlow.Validate(""); err == nil {
		t.Logf("%s-empty", label)
		t.Error("error should be occurred, but not")
	}
	if err := oauthFlow.Validate("foobar"); err == nil {
		t.Logf("%s-wrongtype", label)
		t.Error("error should be occurred, but not")
	}
	for i, flowType := range flowTypes {
		if err := oauthFlow.Validate(flowType); (err != nil) != haveErr[i] {
			t.Logf("%s-%s", label, flowType)
			if haveErr[i] {
				t.Error("error should be occurred, but not")
				continue
			}
			t.Error("error should not be occurred, but occurred")
			t.Log(err)
		}
	}
}
<commit_msg>add test for OASVersion<commit_after>package openapi

import (
	"testing"
)

const (
	exampleCom  = "https://example.com"
	exampleMail = "foo@example.com"
)

type candidate struct {
	label  string
	in     validater
	hasErr bool
}

func testValidater(t *testing.T, candidates []candidate) {
	t.Helper()
	for _, c := range candidates {
		if err := c.in.Validate(); (err != nil) != c.hasErr {
			if c.hasErr {
				t.Error("error should be occurred, but not")
				continue
			}
			t.Errorf("error is occurred: %s", err)
		}
	}
}

func TestHasDuplicatedParameter(t *testing.T) {
	t.Run("no duplicated param", testHasDuplicatedParameterFalse)
	t.Run("there's duplicated param", testHasDuplicatedParameterTrue)
}

func testHasDuplicatedParameterFalse(t *testing.T) {
	params := []*Parameter{
		&Parameter{Name: "foo", In: "header"},
		&Parameter{Name: "foo", In: "path", Required: true},
		&Parameter{Name: "bar", In: "path", Required: true},
	}
	if hasDuplicatedParameter(params) {
		t.Error("should return false")
	}
}

func testHasDuplicatedParameterTrue(t *testing.T) {
	params := []*Parameter{
		&Parameter{Name: "foo", In: "header"},
		&Parameter{Name: "foo", In: "header"},
	}
	if !hasDuplicatedParameter(params) {
		t.Error("should return true")
	}
}

func TestMustURL(t *testing.T) {
	candidates := []struct {
		label  string
		in     string
		hasErr bool
	}{
		{"empty", "", true},
		{"valid HTTP url", "http://example.com", false},
		{"allowed relative path", "foo/bar/baz", true},
		{"absolute path", "/foo/bar/baz", false},
		{"plain string", "foobarbaz", true},
	}
	for _, c := range candidates {
		if err := mustURL(c.label, c.in); (err != nil) != c.hasErr {
			t.Logf("error occured at %s", c.label)
			if c.hasErr {
				t.Error("error should occured, but not")
				return
			}
			t.Error("error should not occurred, but occurred")
			return
		}
	}
}

func TestValidateOASVersion(t *testing.T) {
	candidates := []struct {
		label  string
		in     string
		hasErr bool
	}{
		{"empty", "", true},
		{"invalidVersion", "foobar", true},
		{"swagger", "2.0", true},
		{"valid", "3.0.0", false},
	}
	for _, c := range candidates {
		if err := validateOASVersion(c.in); (err != nil) != c.hasErr {
			if c.hasErr {
				t.Error("error should be occurred, but not")
				continue
			}
			t.Errorf("error should not be occured: %s", err)
		}
	}
}

func TestContactValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", Contact{}, true},
		{"withURL", Contact{URL: exampleCom}, false},
		{"invalidURL", Contact{URL: "foobar"}, true},
		{"withEmail", Contact{Email: exampleMail}, true},
		{"valid", Contact{URL: exampleCom, Email: exampleMail}, false},
		{"invalidEmail", Contact{URL: exampleCom, Email: "foobar"}, true},
	}

	testValidater(t, candidates)
}

func TestLicenseValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", License{}, true},
		{"withName", License{Name: "foobar"}, true},
		{"withURL", License{URL: exampleCom}, true},
		{"invalidURL", License{Name: "foobar", URL: "foobar"}, true},
		{"valid", License{Name: "foobar", URL: exampleCom}, false},
	}
	testValidater(t, candidates)
}

func TestServerValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", Server{}, true},
		{"invalidURL", Server{URL: "foobar%"}, true},
		{"withURL", Server{URL: exampleCom}, false},
	}
	testValidater(t, candidates)
}

func TestServerVariableValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", ServerVariable{}, true},
		{"withDefault", ServerVariable{Default: "default"}, false},
	}
	testValidater(t, candidates)
}

func TestPathsValidate(t *testing.T) {
	t.Run("duplicate pathItem", testPathItemDuplicate)
}

func getPaths(id1, id2 string) Paths {
	return Paths{
		"/foo/bar": &PathItem{
			Get:  &Operation{OperationID: id1, Responses: Responses{"200": &Response{Description: "foo"}}},
			Post: &Operation{OperationID: id2, Responses: Responses{"200": &Response{Description: "foo"}}},
		},
	}
}

func testPathItemDuplicate(t *testing.T) {
	candidates := []candidate{
		{"invalid", getPaths("foobar", "foobar"), true},
		{"valid", getPaths("foo", "bar"), false},
	}
	testValidater(t, candidates)
}

func TestExternalDocumentationValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", ExternalDocumentation{}, true},
		{"invalidURL", ExternalDocumentation{URL: "foobar"}, true},
		{"valid", ExternalDocumentation{URL: exampleCom}, false},
	}
	testValidater(t, candidates)
}

func TestTagValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", Tag{}, true},
		{"withEmptyExternalDocs", Tag{ExternalDocs: &ExternalDocumentation{}}, true},
		{"withValidExternalDocs", Tag{ExternalDocs: &ExternalDocumentation{URL: exampleCom}}, true},

		{"withName", Tag{Name: "foo"}, false},
	}
	testValidater(t, candidates)
}

func TestDiscriminatorValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", Discriminator{}, true},
		{"withPropertyName", Discriminator{PropertyName: "foobar"}, false},
	}
	testValidater(t, candidates)
}

func TestXMLValidate(t *testing.T) {
	candidates := []candidate{
		{"empty", XML{}, true},
		{"invalidURLNamespace", XML{Namespace: "foobar"}, true},
		{"withNamespace", XML{Namespace: exampleCom}, false},
	}
	testValidater(t, candidates)
}

func TestOAuthFlowValidate(t *testing.T) {
	mockScopes := map[string]string{"foo": "bar"}

	empty := OAuthFlow{}
	aURL := OAuthFlow{AuthorizationURL: exampleCom}
	tURL := OAuthFlow{TokenURL: exampleCom}
	rURL := OAuthFlow{RefreshURL: exampleCom}
	scopes := OAuthFlow{Scopes: mockScopes}
	atURL := OAuthFlow{AuthorizationURL: exampleCom, TokenURL: exampleCom}
	arURL := OAuthFlow{AuthorizationURL: exampleCom, RefreshURL: exampleCom}
	aURLscopes := OAuthFlow{AuthorizationURL: exampleCom, Scopes: mockScopes}
	trURL := OAuthFlow{TokenURL: exampleCom, RefreshURL: exampleCom}
	tURLscopes := OAuthFlow{TokenURL: exampleCom, Scopes: mockScopes}
	rURLscopes := OAuthFlow{RefreshURL: exampleCom, Scopes: mockScopes}
	atrURL := OAuthFlow{AuthorizationURL: exampleCom, TokenURL: exampleCom, RefreshURL: exampleCom}
	atURLscopes := OAuthFlow{AuthorizationURL: exampleCom, TokenURL: exampleCom, Scopes: mockScopes}
	arURLscopes := OAuthFlow{AuthorizationURL: exampleCom, RefreshURL: exampleCom, Scopes: mockScopes}
	trURLscopes := OAuthFlow{TokenURL: exampleCom, RefreshURL: exampleCom, Scopes: mockScopes}
	atrURLscopes := OAuthFlow{AuthorizationURL: exampleCom, TokenURL: exampleCom, RefreshURL: exampleCom, Scopes: mockScopes}
	invalidURL := OAuthFlow{AuthorizationURL: "foobar", TokenURL: "foobar", RefreshURL: "foobar", Scopes: mockScopes}
	zeroMap := OAuthFlow{AuthorizationURL: exampleCom, TokenURL: exampleCom, RefreshURL: exampleCom, Scopes: map[string]string{}}

	candidates := []struct {
		label   string
		in      OAuthFlow
		haveErr [4]bool
	}{
		{"empty", empty, [4]bool{true, true, true, true}},
		{"aURL", aURL, [4]bool{true, true, true, true}},
		{"tURL", tURL, [4]bool{true, true, true, true}},
		{"rURL", rURL, [4]bool{true, true, true, true}},
		{"scopes", scopes, [4]bool{true, true, true, true}},
		{"aURL/tURL", atURL, [4]bool{true, true, true, true}},
		{"aURL/rURL", arURL, [4]bool{true, true, true, true}},
		{"aURL/scopes", aURLscopes, [4]bool{false, true, true, true}},
		{"tURL/rURL", trURL, [4]bool{true, true, true, true}},
		{"tURL/scopes", tURLscopes, [4]bool{true, false, false, true}},
		{"rURL/scopes", rURLscopes, [4]bool{true, true, true, true}},
		{"aURL/tURL/rURL", atrURL, [4]bool{true, true, true, true}},
		{"aURL/tURL/scopes", atURLscopes, [4]bool{false, false, false, false}},
		{"aURL/rURL/scopes", arURLscopes, [4]bool{false, true, true, true}},
		{"tURL/rURL/scopes", trURLscopes, [4]bool{true, false, false, true}},
		{"aURL/tURL/rURL/scopes", atrURLscopes, [4]bool{false, false, false, false}},

		{"invalidURL", invalidURL, [4]bool{true, true, true, true}},
		{"zero length map", zeroMap, [4]bool{true, true, true, true}},
	}
	for _, c := range candidates {
		testOAuthFlowValidate(t, c.label, c.in, c.haveErr)
	}
}

var flowTypes = []string{"implicit", "password", "clientCredentials", "authorizationCode"}

func testOAuthFlowValidate(t *testing.T, label string, oauthFlow OAuthFlow, haveErr [4]bool) {
	if err := oauthFlow.Validate(""); err == nil {
		t.Logf("%s-empty", label)
		t.Error("error should be occurred, but not")
	}
	if err := oauthFlow.Validate("foobar"); err == nil {
		t.Logf("%s-wrongtype", label)
		t.Error("error should be occurred, but not")
	}
	for i, flowType := range flowTypes {
		if err := oauthFlow.Validate(flowType); (err != nil) != haveErr[i] {
			t.Logf("%s-%s", label, flowType)
			if haveErr[i] {
				t.Error("error should be occurred, but not")
				continue
			}
			t.Error("error should not be occurred, but occurred")
			t.Log(err)
		}
	}
}
<|endoftext|>


<commit_before>package cli

import (
	"encoding/json"
	"fmt"
	"github.com/gruntwork-io/terragrunt/errors"
	"github.com/gruntwork-io/terragrunt/options"
	"github.com/gruntwork-io/terragrunt/util"
	"github.com/hashicorp/hcl2/hcl"
	"github.com/hashicorp/hcl2/hcl/hclsyntax"
	"github.com/hashicorp/hcl2/hclwrite"
	"github.com/mattn/go-zglob"
	"github.com/zclconf/go-cty/cty"
	"io/ioutil"
	"path/filepath"
)

func applyAwsProviderPatch(terragruntOptions *options.TerragruntOptions) error {
	if len(terragruntOptions.AwsProviderPatchOverrides) == 0 {
		return errors.WithStackTrace(MissingOverrides(OPT_TERRAGRUNT_OVERRIDE_ATTR))
	}

	terraformFilesInModules, err := findAllTerraformFilesInModules(terragruntOptions)
	if err != nil {
		return err
	}

	for _, terraformFile := range terraformFilesInModules {
		terragruntOptions.Logger.Printf("Looking at file %s...", terraformFile)
		originalTerraformFileContents, err := util.ReadFileAsString(terraformFile)
		if err != nil {
			return err
		}

		updatedTerraformFileContents, codeWasUpdated, err := patchAwsProviderInTerraformCode(originalTerraformFileContents, terraformFile, terragruntOptions.AwsProviderPatchOverrides)
		if err != nil {
			return err
		}

		if codeWasUpdated {
			terragruntOptions.Logger.Printf("Patching AWS provider in %s", terraformFile)
			if err := util.WriteFileWithSamePermissions(terraformFile, terraformFile, []byte(updatedTerraformFileContents)); err != nil {
				return err
			}
		}
	}

	return nil
}

// The format we expect in the .terraform/modules/modules.json file
type TerraformModulesJson struct {
	Modules []TerraformModule
}

type TerraformModule struct {
	Key    string
	Source string
	Dir    string
}

// findAllTerraformFiles returns all Terraform source files within the modules being used by this Terragrunt
// configuration. To be more specific, it only returns the source files downloaded for module "xxx" { ... } blocks into
// the .terraform/modules folder; it does NOT return Terraform files for the top-level (AKA "root") module.
//
// NOTE: this method only supports *.tf files right now. Terraform code defined in *.json files is not currently
// supported.d
func findAllTerraformFilesInModules(terragruntOptions *options.TerragruntOptions) ([]string, error) {
	// Terraform downloads modules into the .terraform/modules folder. Unfortunately, it downloads not only the module
	// into that folder, but the entire repo it's in, which can contain lots of other unrelated code we probably don't
	// want to touch. To find the paths to the actual modules, we read the modules.json file in that folder, which is
	// a manifest file Terraform uses to track where the modules are within each repo. Note that this is an internal
	// API, so the way we parse/read this modules.json file may break in future Terraform versions.
	modulesJsonPath := util.JoinPath(terragruntOptions.DataDir(), "modules", "modules.json")

	if !util.FileExists(modulesJsonPath) {
		return nil, nil
	}

	modulesJsonContents, err := ioutil.ReadFile(modulesJsonPath)
	if err != nil {
		return nil, errors.WithStackTrace(err)
	}

	var terraformModulesJson TerraformModulesJson
	if err := json.Unmarshal(modulesJsonContents, &terraformModulesJson); err != nil {
		return nil, errors.WithStackTrace(err)
	}

	var terraformFiles []string

	for _, module := range terraformModulesJson.Modules {
		if module.Key != "" && module.Dir != "" {
			moduleAbsPath := module.Dir
			if !filepath.IsAbs(moduleAbsPath) {
				moduleAbsPath = util.JoinPath(terragruntOptions.WorkingDir, moduleAbsPath)
			}

			// Ideally, we'd use a builin Go library like filepath.Glob here, but per https://github.com/golang/go/issues/11862,
			// the current go implementation doesn't support treating ** as zero or more directories, just zero or one.
			// So we use a third-party library.
			matches, err := zglob.Glob(fmt.Sprintf("%s/**/*.tf", moduleAbsPath))
			if err != nil {
				return nil, errors.WithStackTrace(err)
			}

			terraformFiles = append(terraformFiles, matches...)
		}
	}

	return terraformFiles, nil
}

// patchAwsProviderInTerraformCode looks for provider "aws" { ... } blocks in the given Terraform code and overwrites
// the attributes in those provider blocks with the given attributes. It returns the new Terraform code and a boolean
// true if that code was updated.
//
// For example, if you passed in the following Terraform code:
//
// provider "aws" {
//    region = var.aws_region
// }
//
// And you set attributesToOverride to map[string]string{"region": "us-east-1"}, then this method will return:
//
// provider "aws" {
//    region = "us-east-1"
// }
//
// This is a temporary workaround for a Terraform bug (https://github.com/hashicorp/terraform/issues/13018) where
// any dynamic values in nested provider blocks are not handled correctly when you call 'terraform import', so by
// temporarily hard-coding them, we can allow 'import' to work.
func patchAwsProviderInTerraformCode(terraformCode string, terraformFilePath string, attributesToOverride map[string]string) (string, bool, error) {
	hclFile, err := hclwrite.ParseConfig([]byte(terraformCode), terraformFilePath, hcl.InitialPos)
	if err != nil {
		return "", false, errors.WithStackTrace(err)
	}

	codeWasUpdated := false

	for _, block := range hclFile.Body().Blocks() {
		tokens := block.BuildTokens(nil)
		if len(tokens) > 4 {
			// The tokens should be:
			//   - provider
			//   - "
			//   - aws
			//   - "
			maybeProvider := tokens[0]
			maybeAws := tokens[2]

			if maybeProvider.Type == hclsyntax.TokenIdent && string(maybeProvider.Bytes) == "provider" &&
				maybeAws.Type == hclsyntax.TokenQuotedLit && string(maybeAws.Bytes) == "aws" {

				codeWasUpdated = true
				for key, value := range attributesToOverride {
					block.Body().SetAttributeValue(key, cty.StringVal(value))
				}
			}
		}

	}

	return string(hclFile.Bytes()), codeWasUpdated, nil
}

// Custom error types

type MissingOverrides string

func (err MissingOverrides) Error() string {
	return fmt.Sprintf("You must specify at least one provider attribute to override via the --%s option.", string(err))
}
<commit_msg>Switch to debug logging<commit_after>package cli

import (
	"encoding/json"
	"fmt"
	"github.com/gruntwork-io/terragrunt/errors"
	"github.com/gruntwork-io/terragrunt/options"
	"github.com/gruntwork-io/terragrunt/util"
	"github.com/hashicorp/hcl2/hcl"
	"github.com/hashicorp/hcl2/hcl/hclsyntax"
	"github.com/hashicorp/hcl2/hclwrite"
	"github.com/mattn/go-zglob"
	"github.com/zclconf/go-cty/cty"
	"io/ioutil"
	"path/filepath"
)

func applyAwsProviderPatch(terragruntOptions *options.TerragruntOptions) error {
	if len(terragruntOptions.AwsProviderPatchOverrides) == 0 {
		return errors.WithStackTrace(MissingOverrides(OPT_TERRAGRUNT_OVERRIDE_ATTR))
	}

	terraformFilesInModules, err := findAllTerraformFilesInModules(terragruntOptions)
	if err != nil {
		return err
	}

	for _, terraformFile := range terraformFilesInModules {
		util.Debugf(terragruntOptions.Logger, "Looking at file %s", terraformFile)
		originalTerraformFileContents, err := util.ReadFileAsString(terraformFile)
		if err != nil {
			return err
		}

		updatedTerraformFileContents, codeWasUpdated, err := patchAwsProviderInTerraformCode(originalTerraformFileContents, terraformFile, terragruntOptions.AwsProviderPatchOverrides)
		if err != nil {
			return err
		}

		if codeWasUpdated {
			terragruntOptions.Logger.Printf("Patching AWS provider in %s", terraformFile)
			if err := util.WriteFileWithSamePermissions(terraformFile, terraformFile, []byte(updatedTerraformFileContents)); err != nil {
				return err
			}
		}
	}

	return nil
}

// The format we expect in the .terraform/modules/modules.json file
type TerraformModulesJson struct {
	Modules []TerraformModule
}

type TerraformModule struct {
	Key    string
	Source string
	Dir    string
}

// findAllTerraformFiles returns all Terraform source files within the modules being used by this Terragrunt
// configuration. To be more specific, it only returns the source files downloaded for module "xxx" { ... } blocks into
// the .terraform/modules folder; it does NOT return Terraform files for the top-level (AKA "root") module.
//
// NOTE: this method only supports *.tf files right now. Terraform code defined in *.json files is not currently
// supported.d
func findAllTerraformFilesInModules(terragruntOptions *options.TerragruntOptions) ([]string, error) {
	// Terraform downloads modules into the .terraform/modules folder. Unfortunately, it downloads not only the module
	// into that folder, but the entire repo it's in, which can contain lots of other unrelated code we probably don't
	// want to touch. To find the paths to the actual modules, we read the modules.json file in that folder, which is
	// a manifest file Terraform uses to track where the modules are within each repo. Note that this is an internal
	// API, so the way we parse/read this modules.json file may break in future Terraform versions.
	modulesJsonPath := util.JoinPath(terragruntOptions.DataDir(), "modules", "modules.json")

	if !util.FileExists(modulesJsonPath) {
		return nil, nil
	}

	modulesJsonContents, err := ioutil.ReadFile(modulesJsonPath)
	if err != nil {
		return nil, errors.WithStackTrace(err)
	}

	var terraformModulesJson TerraformModulesJson
	if err := json.Unmarshal(modulesJsonContents, &terraformModulesJson); err != nil {
		return nil, errors.WithStackTrace(err)
	}

	var terraformFiles []string

	for _, module := range terraformModulesJson.Modules {
		if module.Key != "" && module.Dir != "" {
			moduleAbsPath := module.Dir
			if !filepath.IsAbs(moduleAbsPath) {
				moduleAbsPath = util.JoinPath(terragruntOptions.WorkingDir, moduleAbsPath)
			}

			// Ideally, we'd use a builin Go library like filepath.Glob here, but per https://github.com/golang/go/issues/11862,
			// the current go implementation doesn't support treating ** as zero or more directories, just zero or one.
			// So we use a third-party library.
			matches, err := zglob.Glob(fmt.Sprintf("%s/**/*.tf", moduleAbsPath))
			if err != nil {
				return nil, errors.WithStackTrace(err)
			}

			terraformFiles = append(terraformFiles, matches...)
		}
	}

	return terraformFiles, nil
}

// patchAwsProviderInTerraformCode looks for provider "aws" { ... } blocks in the given Terraform code and overwrites
// the attributes in those provider blocks with the given attributes. It returns the new Terraform code and a boolean
// true if that code was updated.
//
// For example, if you passed in the following Terraform code:
//
// provider "aws" {
//    region = var.aws_region
// }
//
// And you set attributesToOverride to map[string]string{"region": "us-east-1"}, then this method will return:
//
// provider "aws" {
//    region = "us-east-1"
// }
//
// This is a temporary workaround for a Terraform bug (https://github.com/hashicorp/terraform/issues/13018) where
// any dynamic values in nested provider blocks are not handled correctly when you call 'terraform import', so by
// temporarily hard-coding them, we can allow 'import' to work.
func patchAwsProviderInTerraformCode(terraformCode string, terraformFilePath string, attributesToOverride map[string]string) (string, bool, error) {
	hclFile, err := hclwrite.ParseConfig([]byte(terraformCode), terraformFilePath, hcl.InitialPos)
	if err != nil {
		return "", false, errors.WithStackTrace(err)
	}

	codeWasUpdated := false

	for _, block := range hclFile.Body().Blocks() {
		tokens := block.BuildTokens(nil)
		if len(tokens) > 4 {
			// The tokens should be:
			//   - provider
			//   - "
			//   - aws
			//   - "
			maybeProvider := tokens[0]
			maybeAws := tokens[2]

			if maybeProvider.Type == hclsyntax.TokenIdent && string(maybeProvider.Bytes) == "provider" &&
				maybeAws.Type == hclsyntax.TokenQuotedLit && string(maybeAws.Bytes) == "aws" {

				codeWasUpdated = true
				for key, value := range attributesToOverride {
					block.Body().SetAttributeValue(key, cty.StringVal(value))
				}
			}
		}

	}

	return string(hclFile.Bytes()), codeWasUpdated, nil
}

// Custom error types

type MissingOverrides string

func (err MissingOverrides) Error() string {
	return fmt.Sprintf("You must specify at least one provider attribute to override via the --%s option.", string(err))
}
<|endoftext|>


<commit_before>package raftgorums

import (
	"fmt"
	"sync"

	"golang.org/x/net/context"

	"github.com/Sirupsen/logrus"
	"github.com/relab/raft"
	"github.com/relab/raft/commonpb"
	gorums "github.com/relab/raft/raftgorums/gorumspb"
)

type membership struct {
	id     uint64
	mgr    *gorums.Manager
	lookup map[uint64]int
	logger logrus.FieldLogger

	sync.RWMutex
	latest         *gorums.Configuration
	committed      *gorums.Configuration
	latestIndex    uint64
	committedIndex uint64
	pending        *commonpb.ReconfRequest
	stable         bool
	enabled        bool
}

func (m *membership) isActive() bool {
	m.RLock()
	defer m.RUnlock()

	return m.enabled
}

func (m *membership) startReconfiguration(req *commonpb.ReconfRequest) bool {
	m.Lock()
	defer m.Unlock()

	valid := true

	// Disallow servers not available in manager.
	if req.ServerID < 1 || req.ServerID > uint64(len(m.lookup)+1) {
		return false
	}

	switch req.ReconfType {
	case commonpb.ReconfAdd:
		conf, _ := m.addServer(req.ServerID)

		// Disallow configurations that do not result in a change.
		if conf == m.committed {
			valid = false
		}
	case commonpb.ReconfRemove:
		// Disallow configurations that result in a configuration
		// without nodes.
		if len(m.committed.NodeIDs()) == 1 {
			valid = false
			break
		}

		conf, enabled := m.removeServer(req.ServerID)

		// Disallow configurations that do not result in a change, but
		// only if we also don't step down.
		if conf == m.committed && enabled {
			valid = false
		}
	default:
		panic("malformed reconf request")
	}

	m.logger.WithFields(logrus.Fields{
		"pending": m.pending,
		"stable":  m.stable,
		"valid":   valid,
	}).Warnln("Attempt start reconfiguration")

	if m.pending == nil && m.stable && valid {
		m.pending = req
		return true
	}

	return false
}

func (m *membership) setPending(req *commonpb.ReconfRequest) {
	m.Lock()
	m.pending = req
	m.Unlock()
}

func (m *membership) setStable(stable bool) {
	m.Lock()
	// TODO Raft should setStable.
	m.stable = stable
	m.Unlock()
}

func (m *membership) set(index uint64) {
	m.Lock()
	defer m.Unlock()

	switch m.pending.ReconfType {
	case commonpb.ReconfAdd:
		m.latest, m.enabled = m.addServer(m.pending.ServerID)
	case commonpb.ReconfRemove:
		m.latest, m.enabled = m.removeServer(m.pending.ServerID)
	}
	m.latestIndex = index
	m.logger.WithField("latest", m.latest.NodeIDs()).Warnln("New configuration")
}

func (m *membership) commit() bool {
	m.Lock()
	defer m.Unlock()

	m.pending = nil
	m.committed = m.latest
	m.committedIndex = m.latestIndex

	return m.enabled
}

func (m *membership) rollback() {
	m.Lock()
	m.pending = nil
	m.latest = m.committed
	m.latestIndex = m.committedIndex
	m.Unlock()
}

func (m *membership) get() *gorums.Configuration {
	m.RLock()
	defer m.RUnlock()

	return m.latest
}

// TODO Return the same configuration if adding/removing self.

// addServer returns a new configuration including the given server.
func (m *membership) addServer(serverID uint64) (conf *gorums.Configuration, enabled bool) {
	// TODO Clean up.
	if m.enabled {
		enabled = true
	}

	nodeIDs := m.committed.NodeIDs()

	// TODO Not including self in the configuration seems to complicate
	// things. I Foresee a problem when removing the leader, and it is not
	// part of it's own latest configuration.
	if serverID != m.id {
		// Work around bug in Gorums. Duplicated node ids are not deduplicated.
		for _, nodeID := range nodeIDs {
			if nodeID == m.getNodeID(serverID) {
				var err error
				conf, err = m.mgr.NewConfiguration(nodeIDs, NewQuorumSpec(len(nodeIDs)+1))

				if err != nil {
					panic("addServer: " + err.Error())
				}

				return
			}
		}

		id := m.getNodeID(serverID)
		nodeIDs = append(nodeIDs, id)
	} else {
		enabled = true
	}

	// We can ignore the error as we are adding 1 server, and id is
	// guaranteed to be in the manager or getNodeID would have panicked.
	var err error
	conf, err = m.mgr.NewConfiguration(nodeIDs, NewQuorumSpec(len(nodeIDs)+1))

	if err != nil {
		panic("addServer: " + err.Error())
	}

	return
}

// removeServer returns a new configuration excluding the given server.
func (m *membership) removeServer(serverID uint64) (conf *gorums.Configuration, enabled bool) {
	// TODO Clean up.
	enabled = true

	oldIDs := m.committed.NodeIDs()

	if serverID == m.id {
		enabled = false

		var err error
		conf, err = m.mgr.NewConfiguration(oldIDs, NewQuorumSpec(len(oldIDs)+1))

		if err != nil {
			panic("removeServer: " + err.Error())
		}

		return
	}

	id := m.getNodeID(serverID)
	var nodeIDs []uint32

	for _, nodeID := range oldIDs {
		if nodeID == id {
			continue
		}

		nodeIDs = append(nodeIDs, nodeID)
	}

	// We can ignore the error as we do not allow cluster size < 2, and id
	// is guaranteed to be in the manager or getNodeID would have panicked.
	// Cluster size > 2 is a limitation of Gorums and how we have chosen not
	// to include ourselves in the manager.
	var err error
	conf, err = m.mgr.NewConfiguration(nodeIDs, NewQuorumSpec(len(nodeIDs)+1))

	if err != nil {
		panic("removeServer: " + err.Error())
	}

	return
}

func (m *membership) getNodeID(serverID uint64) uint32 {
	nodeID, ok := m.lookup[serverID]

	if !ok {
		panic(fmt.Sprintf("no lookup available for server %d", serverID))
	}

	return m.mgr.NodeIDs()[nodeID]
}

func (m *membership) getNode(serverID uint64) *gorums.Node {
	// Can ignore error because we looked up the node through the manager
	// first, therefore it exists.
	node, _ := m.mgr.Node(m.getNodeID(serverID))
	return node
}

func (r *Raft) replicate(serverID uint64, promise raft.PromiseEntry) {
	node := r.mem.getNode(serverID)
	var matchIndex uint64
	var errs int

	for {
		r.Lock()
		target := r.matchIndex
		// TODO We don't need lock on maxAppendEntries as it's only read
		// across all routines.
		maxEntries := r.maxAppendEntries

		entries := r.getNextEntries(matchIndex + 1)
		req := r.getAppendEntriesRequest(matchIndex+1, entries)
		r.Unlock()

		ctx, cancel := context.WithTimeout(context.Background(), r.electionTimeout)
		res, err := node.RaftClient.AppendEntries(ctx, req)
		cancel()

		// TODO handle better.
		if err != nil {
			errs++

			if errs > 3 {
				promise.Respond(&commonpb.ReconfResponse{
					Status: commonpb.ReconfTimeout,
				})
				r.mem.rollback()
				return
			}

			continue
		}

		r.Lock()
		state := r.state
		r.Unlock()

		if state != Leader {
			promise.Respond(&commonpb.ReconfResponse{
				Status: commonpb.ReconfNotLeader,
			})
			r.mem.rollback()
			return
		}

		if target-matchIndex < maxEntries {
			// TODO Context?
			r.queue <- promise
			return
		}

		if res.Success {
			matchIndex = res.MatchIndex
			continue
		}

		matchIndex = max(0, res.MatchIndex)
	}
}
<commit_msg>raftgorums/membership.go: Return early if pending not equal nil<commit_after>package raftgorums

import (
	"fmt"
	"sync"

	"golang.org/x/net/context"

	"github.com/Sirupsen/logrus"
	"github.com/relab/raft"
	"github.com/relab/raft/commonpb"
	gorums "github.com/relab/raft/raftgorums/gorumspb"
)

type membership struct {
	id     uint64
	mgr    *gorums.Manager
	lookup map[uint64]int
	logger logrus.FieldLogger

	sync.RWMutex
	latest         *gorums.Configuration
	committed      *gorums.Configuration
	latestIndex    uint64
	committedIndex uint64
	pending        *commonpb.ReconfRequest
	stable         bool
	enabled        bool
}

func (m *membership) isActive() bool {
	m.RLock()
	defer m.RUnlock()

	return m.enabled
}

func (m *membership) startReconfiguration(req *commonpb.ReconfRequest) bool {
	m.Lock()
	defer m.Unlock()

	if m.pending != nil {
		return false
	}

	valid := true

	// Disallow servers not available in manager.
	if req.ServerID < 1 || req.ServerID > uint64(len(m.lookup)+1) {
		return false
	}

	switch req.ReconfType {
	case commonpb.ReconfAdd:
		conf, _ := m.addServer(req.ServerID)

		// Disallow configurations that do not result in a change.
		if conf == m.committed {
			valid = false
		}
	case commonpb.ReconfRemove:
		// Disallow configurations that result in a configuration
		// without nodes.
		if len(m.committed.NodeIDs()) == 1 {
			valid = false
			break
		}

		conf, enabled := m.removeServer(req.ServerID)

		// Disallow configurations that do not result in a change, but
		// only if we also don't step down.
		if conf == m.committed && enabled {
			valid = false
		}
	default:
		panic("malformed reconf request")
	}

	m.logger.WithFields(logrus.Fields{
		"pending": m.pending,
		"stable":  m.stable,
		"valid":   valid,
	}).Warnln("Attempt start reconfiguration")

	if m.stable && valid {
		m.pending = req
		return true
	}

	return false
}

func (m *membership) setPending(req *commonpb.ReconfRequest) {
	m.Lock()
	m.pending = req
	m.Unlock()
}

func (m *membership) setStable(stable bool) {
	m.Lock()
	// TODO Raft should setStable.
	m.stable = stable
	m.Unlock()
}

func (m *membership) set(index uint64) {
	m.Lock()
	defer m.Unlock()

	switch m.pending.ReconfType {
	case commonpb.ReconfAdd:
		m.latest, m.enabled = m.addServer(m.pending.ServerID)
	case commonpb.ReconfRemove:
		m.latest, m.enabled = m.removeServer(m.pending.ServerID)
	}
	m.latestIndex = index
	m.logger.WithField("latest", m.latest.NodeIDs()).Warnln("New configuration")
}

func (m *membership) commit() bool {
	m.Lock()
	defer m.Unlock()

	m.pending = nil
	m.committed = m.latest
	m.committedIndex = m.latestIndex

	return m.enabled
}

func (m *membership) rollback() {
	m.Lock()
	m.pending = nil
	m.latest = m.committed
	m.latestIndex = m.committedIndex
	m.Unlock()
}

func (m *membership) get() *gorums.Configuration {
	m.RLock()
	defer m.RUnlock()

	return m.latest
}

// TODO Return the same configuration if adding/removing self.

// addServer returns a new configuration including the given server.
func (m *membership) addServer(serverID uint64) (conf *gorums.Configuration, enabled bool) {
	// TODO Clean up.
	if m.enabled {
		enabled = true
	}

	nodeIDs := m.committed.NodeIDs()

	// TODO Not including self in the configuration seems to complicate
	// things. I Foresee a problem when removing the leader, and it is not
	// part of it's own latest configuration.
	if serverID != m.id {
		// Work around bug in Gorums. Duplicated node ids are not deduplicated.
		for _, nodeID := range nodeIDs {
			if nodeID == m.getNodeID(serverID) {
				var err error
				conf, err = m.mgr.NewConfiguration(nodeIDs, NewQuorumSpec(len(nodeIDs)+1))

				if err != nil {
					panic("addServer: " + err.Error())
				}

				return
			}
		}

		id := m.getNodeID(serverID)
		nodeIDs = append(nodeIDs, id)
	} else {
		enabled = true
	}

	// We can ignore the error as we are adding 1 server, and id is
	// guaranteed to be in the manager or getNodeID would have panicked.
	var err error
	conf, err = m.mgr.NewConfiguration(nodeIDs, NewQuorumSpec(len(nodeIDs)+1))

	if err != nil {
		panic("addServer: " + err.Error())
	}

	return
}

// removeServer returns a new configuration excluding the given server.
func (m *membership) removeServer(serverID uint64) (conf *gorums.Configuration, enabled bool) {
	// TODO Clean up.
	enabled = true

	oldIDs := m.committed.NodeIDs()

	if serverID == m.id {
		enabled = false

		var err error
		conf, err = m.mgr.NewConfiguration(oldIDs, NewQuorumSpec(len(oldIDs)+1))

		if err != nil {
			panic("removeServer: " + err.Error())
		}

		return
	}

	id := m.getNodeID(serverID)
	var nodeIDs []uint32

	for _, nodeID := range oldIDs {
		if nodeID == id {
			continue
		}

		nodeIDs = append(nodeIDs, nodeID)
	}

	// We can ignore the error as we do not allow cluster size < 2, and id
	// is guaranteed to be in the manager or getNodeID would have panicked.
	// Cluster size > 2 is a limitation of Gorums and how we have chosen not
	// to include ourselves in the manager.
	var err error
	conf, err = m.mgr.NewConfiguration(nodeIDs, NewQuorumSpec(len(nodeIDs)+1))

	if err != nil {
		panic("removeServer: " + err.Error())
	}

	return
}

func (m *membership) getNodeID(serverID uint64) uint32 {
	nodeID, ok := m.lookup[serverID]

	if !ok {
		panic(fmt.Sprintf("no lookup available for server %d", serverID))
	}

	return m.mgr.NodeIDs()[nodeID]
}

func (m *membership) getNode(serverID uint64) *gorums.Node {
	// Can ignore error because we looked up the node through the manager
	// first, therefore it exists.
	node, _ := m.mgr.Node(m.getNodeID(serverID))
	return node
}

func (r *Raft) replicate(serverID uint64, promise raft.PromiseEntry) {
	node := r.mem.getNode(serverID)
	var matchIndex uint64
	var errs int

	for {
		r.Lock()
		target := r.matchIndex
		// TODO We don't need lock on maxAppendEntries as it's only read
		// across all routines.
		maxEntries := r.maxAppendEntries

		entries := r.getNextEntries(matchIndex + 1)
		req := r.getAppendEntriesRequest(matchIndex+1, entries)
		r.Unlock()

		ctx, cancel := context.WithTimeout(context.Background(), r.electionTimeout)
		res, err := node.RaftClient.AppendEntries(ctx, req)
		cancel()

		// TODO handle better.
		if err != nil {
			errs++

			if errs > 3 {
				promise.Respond(&commonpb.ReconfResponse{
					Status: commonpb.ReconfTimeout,
				})
				r.mem.rollback()
				return
			}

			continue
		}

		r.Lock()
		state := r.state
		r.Unlock()

		if state != Leader {
			promise.Respond(&commonpb.ReconfResponse{
				Status: commonpb.ReconfNotLeader,
			})
			r.mem.rollback()
			return
		}

		if target-matchIndex < maxEntries {
			// TODO Context?
			r.queue <- promise
			return
		}

		if res.Success {
			matchIndex = res.MatchIndex
			continue
		}

		matchIndex = max(0, res.MatchIndex)
	}
}
<|endoftext|>


<commit_before><commit_msg>interp: add env tests with duplicates and globals<commit_after><|endoftext|>


<commit_before>// Package http rate decorates an http.Handler and prevents too many requests being made
package http

import (
	"net/http"
	"strings"
)

const StatusTooManyRequests = 429

type RateLimitedHandler struct {
	http.Handler
	rateLimiter RateLimiter
}

type RateLimiter func(*http.Request) bool

func Decorate(delegate http.Handler, rateLimiter RateLimiter) RateLimitedHandler {
	if rateLimiter == nil {
		rateLimiter = DefaultRateLimiter
	}
	return RateLimitedHandler{delegate, rateLimiter}
}

func (lh RateLimitedHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
	if lh.rateLimiter(r) {
		lh.Handler.ServeHTTP(rw, r)
	} else {
		http.Error(rw, "Too many requests", StatusTooManyRequests)
	}
}
<commit_msg>Removed unused strings package from import.<commit_after>// Package http rate decorates an http.Handler and prevents too many requests being made
package http

import (
	"net/http"
)

const StatusTooManyRequests = 429

type RateLimitedHandler struct {
	http.Handler
	rateLimiter RateLimiter
}

type RateLimiter func(*http.Request) bool

func Decorate(delegate http.Handler, rateLimiter RateLimiter) RateLimitedHandler {
	if rateLimiter == nil {
		rateLimiter = DefaultRateLimiter
	}
	return RateLimitedHandler{delegate, rateLimiter}
}

func (lh RateLimitedHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
	if lh.rateLimiter(r) {
		lh.Handler.ServeHTTP(rw, r)
	} else {
		http.Error(rw, "Too many requests", StatusTooManyRequests)
	}
}
<|endoftext|>


<commit_before>package oplog

import (
	"encoding/base64"
	"expvar"
	"fmt"
	"io"
	"net/http"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
)

// SSEDaemon listens for events and send them to the oplog MongoDB capped collection
type SSEDaemon struct {
	s  *http.Server
	ol *OpLog
	// Password is the shared secret to connect to a password protected oplog.
	Password string
}

func NewSSEDaemon(addr string, ol *OpLog) *SSEDaemon {
	daemon := &SSEDaemon{nil, ol, ""}
	daemon.s = &http.Server{
		Addr:           addr,
		Handler:        daemon,
		MaxHeaderBytes: 1 << 20,
	}

	return daemon
}

// authenticate checks for HTTP basic authentication if an admin password is set.
func (daemon *SSEDaemon) authenticate(r *http.Request) bool {
	if daemon.Password == "" {
		return true
	}

	s := strings.SplitN(r.Header.Get("Authorization"), " ", 2)
	if len(s) != 2 || s[0] != "Basic" {
		return false
	}

	b, err := base64.StdEncoding.DecodeString(s[1])
	if err != nil {
		return false
	}
	pair := strings.SplitN(string(b), ":", 2)
	if len(pair) != 2 {
		return false
	}

	return daemon.Password == pair[1]
}

func (daemon *SSEDaemon) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		w.WriteHeader(405)
		return
	}
	switch r.URL.Path {
	case "/status":
		daemon.Status(w, r)
	case "/ops", "/":
		daemon.Ops(w, r)
	default:
		w.WriteHeader(404)
	}
}

func (daemon *SSEDaemon) Status(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintf(w, "{\"status\":\"OK\"")
	expvar.Do(func(kv expvar.KeyValue) {
		fmt.Fprintf(w, ",%q:%s", kv.Key, kv.Value)
	})
	fmt.Fprintf(w, "}")
}

func (daemon *SSEDaemon) Ops(w http.ResponseWriter, r *http.Request) {
	log.Info("SSE connection started")

	if r.Header.Get("Accept") != "text/event-stream" {
		// Not an event stream request, return a 406 Not Acceptable HTTP error
		w.WriteHeader(406)
		return
	}

	if !daemon.authenticate(r) {
		w.WriteHeader(401)
		return
	}

	h := w.Header()
	h.Set("Server", fmt.Sprintf("oplog/%s", VERSION))
	h.Set("Content-Type", "text/event-stream; charset=utf-8")
	h.Set("Cache-Control", "no-cache, no-store, must-revalidate")
	h.Set("Connection", "close")
	h.Set("Access-Control-Allow-Origin", "*")

	var lastId LastId
	var err error
	if r.Header.Get("Last-Event-ID") == "" {
		// No last id provided, use the very last id of the events collection
		lastId, err = daemon.ol.LastId()
		if err != nil {
			log.Warnf("SSE can't get last id: %s", err)
			w.WriteHeader(503)
			return
		}
	} else {
		if lastId, err = NewLastId(r.Header.Get("Last-Event-ID")); err != nil {
			log.Warnf("SSE invalid last id: %s", err)
			w.WriteHeader(400)
			return
		}
		found, err := daemon.ol.HasId(lastId)
		if err != nil {
			log.Warnf("SSE can't check last id: %s", err)
			w.WriteHeader(503)
			return
		}
		if !found {
			log.Debug("SSE last id not found, falling back to replication id: ", lastId.String())
			// If the requested event id is not found, fallback to a replication id
			olid := lastId.(*OperationLastId)
			lastId = olid.Fallback()
		}
		// Backward compat, remove when all oplogc will be updated
		h.Set("Last-Event-ID", r.Header.Get("Last-Event-ID"))
	}

	if lastId != nil {
		log.Debug("SSE using last id: ", lastId.String())
	}

	types := []string{}
	if r.URL.Query().Get("types") != "" {
		types = strings.Split(r.URL.Query().Get("types"), ",")
	}
	parents := []string{}
	if r.URL.Query().Get("parents") != "" {
		parents = strings.Split(r.URL.Query().Get("parents"), ",")
	}
	filter := OpLogFilter{
		Types:   types,
		Parents: parents,
	}

	flusher := w.(http.Flusher)
	notifier := w.(http.CloseNotifier)
	ops := make(chan io.WriterTo)
	stop := make(chan bool)
	flusher.Flush()

	go daemon.ol.Tail(lastId, filter, ops, stop)
	defer func() {
		// Stop the oplog tailer
		stop <- true
	}()

	daemon.ol.Stats.Clients.Add(1)
	daemon.ol.Stats.Connections.Add(1)
	defer daemon.ol.Stats.Clients.Add(-1)

	for {
		select {
		case <-notifier.CloseNotify():
			log.Info("SSE connection closed")
			return
		case op := <-ops:
			log.Debug("SSE sending event")
			daemon.ol.Stats.EventsSent.Add(1)
			if _, err := op.WriteTo(w); err != nil {
				log.Warn("SSE write error: ", err)
				return
			}
			flusher.Flush()
		case <-time.After(25 * time.Second):
			// Send "ping" data to prevent proxy/browsers from closing the connection
			// for inactivity
			log.Debug("SSE sending a keep alive ping")
			if _, err := w.Write([]byte{':', '\n'}); err != nil {
				log.Warn("SSE write error: ", err)
				return
			}
			flusher.Flush()
		}
	}
}

func (daemon *SSEDaemon) Run() error {
	return daemon.s.ListenAndServe()
}
<commit_msg>Add IP to SSE logs<commit_after>package oplog

import (
	"encoding/base64"
	"expvar"
	"fmt"
	"io"
	"net/http"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/sebest/xff"
)

// SSEDaemon listens for events and send them to the oplog MongoDB capped collection
type SSEDaemon struct {
	s  *http.Server
	ol *OpLog
	// Password is the shared secret to connect to a password protected oplog.
	Password string
}

func NewSSEDaemon(addr string, ol *OpLog) *SSEDaemon {
	daemon := &SSEDaemon{nil, ol, ""}
	daemon.s = &http.Server{
		Addr:           addr,
		Handler:        daemon,
		MaxHeaderBytes: 1 << 20,
	}

	return daemon
}

// authenticate checks for HTTP basic authentication if an admin password is set.
func (daemon *SSEDaemon) authenticate(r *http.Request) bool {
	if daemon.Password == "" {
		return true
	}

	s := strings.SplitN(r.Header.Get("Authorization"), " ", 2)
	if len(s) != 2 || s[0] != "Basic" {
		return false
	}

	b, err := base64.StdEncoding.DecodeString(s[1])
	if err != nil {
		return false
	}
	pair := strings.SplitN(string(b), ":", 2)
	if len(pair) != 2 {
		return false
	}

	return daemon.Password == pair[1]
}

func (daemon *SSEDaemon) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		w.WriteHeader(405)
		return
	}
	switch r.URL.Path {
	case "/status":
		daemon.Status(w, r)
	case "/ops", "/":
		daemon.Ops(w, r)
	default:
		w.WriteHeader(404)
	}
}

func (daemon *SSEDaemon) Status(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintf(w, "{\"status\":\"OK\"")
	expvar.Do(func(kv expvar.KeyValue) {
		fmt.Fprintf(w, ",%q:%s", kv.Key, kv.Value)
	})
	fmt.Fprintf(w, "}")
}

func (daemon *SSEDaemon) Ops(w http.ResponseWriter, r *http.Request) {
	ip := xff.GetRemoteAddr(r)
	log.Infof("SSE[%s] connection started", ip)

	if r.Header.Get("Accept") != "text/event-stream" {
		// Not an event stream request, return a 406 Not Acceptable HTTP error
		w.WriteHeader(406)
		return
	}

	if !daemon.authenticate(r) {
		w.WriteHeader(401)
		return
	}

	h := w.Header()
	h.Set("Server", fmt.Sprintf("oplog/%s", VERSION))
	h.Set("Content-Type", "text/event-stream; charset=utf-8")
	h.Set("Cache-Control", "no-cache, no-store, must-revalidate")
	h.Set("Connection", "close")
	h.Set("Access-Control-Allow-Origin", "*")

	var lastId LastId
	var err error
	if r.Header.Get("Last-Event-ID") == "" {
		// No last id provided, use the very last id of the events collection
		lastId, err = daemon.ol.LastId()
		if err != nil {
			log.Warnf("SSE[%s] can't get last id: %s", ip, err)
			w.WriteHeader(503)
			return
		}
	} else {
		if lastId, err = NewLastId(r.Header.Get("Last-Event-ID")); err != nil {
			log.Warnf("SSE[%s] invalid last id: %s", ip, err)
			w.WriteHeader(400)
			return
		}
		found, err := daemon.ol.HasId(lastId)
		if err != nil {
			log.Warnf("SSE[%s] can't check last id: %s", ip, err)
			w.WriteHeader(503)
			return
		}
		if !found {
			log.Debugf("SSE[%s] last id not found, falling back to replication id: %s", ip, lastId.String())
			// If the requested event id is not found, fallback to a replication id
			olid := lastId.(*OperationLastId)
			lastId = olid.Fallback()
		}
		// Backward compat, remove when all oplogc will be updated
		h.Set("Last-Event-ID", r.Header.Get("Last-Event-ID"))
	}

	if lastId != nil {
		log.Debugf("SSE[%s] using last id: %s", ip, lastId.String())
	}

	types := []string{}
	if r.URL.Query().Get("types") != "" {
		types = strings.Split(r.URL.Query().Get("types"), ",")
	}
	parents := []string{}
	if r.URL.Query().Get("parents") != "" {
		parents = strings.Split(r.URL.Query().Get("parents"), ",")
	}
	filter := OpLogFilter{
		Types:   types,
		Parents: parents,
	}

	flusher := w.(http.Flusher)
	notifier := w.(http.CloseNotifier)
	ops := make(chan io.WriterTo)
	stop := make(chan bool)
	flusher.Flush()

	go daemon.ol.Tail(lastId, filter, ops, stop)
	defer func() {
		// Stop the oplog tailer
		stop <- true
	}()

	daemon.ol.Stats.Clients.Add(1)
	daemon.ol.Stats.Connections.Add(1)
	defer daemon.ol.Stats.Clients.Add(-1)

	for {
		select {
		case <-notifier.CloseNotify():
			log.Infof("SSE[%s] connection closed", ip)
			return
		case op := <-ops:
			log.Debugf("SSE[%s] sending event", ip)
			daemon.ol.Stats.EventsSent.Add(1)
			if _, err := op.WriteTo(w); err != nil {
				log.Warnf("SSE[%s] write error: %s", ip, err)
				return
			}
			flusher.Flush()
		case <-time.After(25 * time.Second):
			// Send "ping" data to prevent proxy/browsers from closing the connection
			// for inactivity
			log.Debugf("SSE[%s] sending a keep alive ping", ip)
			if _, err := w.Write([]byte{':', '\n'}); err != nil {
				log.Warnf("SSE[%s] write error: ", ip, err)
				return
			}
			flusher.Flush()
		}
	}
}

func (daemon *SSEDaemon) Run() error {
	return daemon.s.ListenAndServe()
}
<|endoftext|>


<commit_before><commit_msg>add test re: issue 97<commit_after><|endoftext|>


<commit_before><commit_msg>Fix potential XSS in docserver.<commit_after><|endoftext|>


<commit_before>package abkleveldb

import (
  "fmt"

  "github.com/jmhodges/levigo"
)

func boohoo(errstring string, rise bool){
  fmt.Println(errstring)
  if rise == true{ panic(errstring) }
}

func CreateDB(dbname string) (*levigo.DB) {
  opts := levigo.NewOptions()
  opts.SetCache(levigo.NewLRUCache(1<<10))
  opts.SetCreateIfMissing(true)
  db, err := levigo.Open(dbname, opts)
  if err != nil { boohoo("DB " + dbname + " Creation failed.", true) }
  return db
}

/* Push KeyVal */
func PushKeyVal(key string, val string, db *levigo.DB) bool{
  writer := levigo.NewWriteOptions()
  defer writer.Close()

  keyname := []byte(key)
  value := []byte(val)
  fmt.Printf("Writing for %s = %s\n", key, val)
  err := db.Put(writer, keyname, value)
  if err != nil {
    boohoo("Key " + key + " insertion failed. It's value was " + val, false)
    return false
  }
  return true
}

/* Get Key */
func GetValues(key string, db *levigo.DB) string {
  reader := levigo.NewReadOptions()
  defer reader.Close()

  data, err := db.Get(reader, []byte(key))
  if err != nil { boohoo("Key " + key + " query failed.", false) }
  return string(data)
}
<commit_msg>noise printf removed from abkleveldb PushKeyVal operation<commit_after>package abkleveldb

import (
  "fmt"

  "github.com/jmhodges/levigo"
)

func boohoo(errstring string, rise bool){
  fmt.Println(errstring)
  if rise == true{ panic(errstring) }
}

func CreateDB(dbname string) (*levigo.DB) {
  opts := levigo.NewOptions()
  opts.SetCache(levigo.NewLRUCache(1<<10))
  opts.SetCreateIfMissing(true)
  db, err := levigo.Open(dbname, opts)
  if err != nil { boohoo("DB " + dbname + " Creation failed.", true) }
  return db
}

/* Push KeyVal */
func PushKeyVal(key string, val string, db *levigo.DB) bool{
  writer := levigo.NewWriteOptions()
  defer writer.Close()

  keyname := []byte(key)
  value := []byte(val)
  err := db.Put(writer, keyname, value)
  if err != nil {
    boohoo("Key " + key + " insertion failed. It's value was " + val, false)
    return false
  }
  return true
}

/* Get Key */
func GetValues(key string, db *levigo.DB) string {
  reader := levigo.NewReadOptions()
  defer reader.Close()

  data, err := db.Get(reader, []byte(key))
  if err != nil { boohoo("Key " + key + " query failed.", false) }
  return string(data)
}
<|endoftext|>


<commit_before>package filter

// Filter interface that all filter-like collections implement
type Filter interface {
	Add(value interface{})
	Has(value interface{}) bool
	Count() int
}
<commit_msg>filter: remove package (#95)<commit_after><|endoftext|>


<commit_before>/*
Copyright 2016 Google Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/
// Entry-point for Buildozer binary.

package main

import (
	"flag"
	"os"
	"strings"

	"github.com/bazelbuild/buildifier/edit"
)

var (
	stdout            = flag.Bool("stdout", false, "write changed BUILD file to stdout")
	buildifier        = flag.String("buildifier", "", "path to buildifier binary")
	parallelism       = flag.Int("P", 0, "number of cores to use for concurrent actions")
	numio             = flag.Int("numio", 200, "number of concurrent actions")
	commandsFile      = flag.String("f", "", "file name to read commands from, use '-' for stdin (format:|-separated command line arguments to buildozer, excluding flags)")
	keepGoing         = flag.Bool("k", false, "apply all commands, even if there are failures")
	filterRuleTypes   = stringList("types", "comma-separated list of rule types to change, the default empty list means all rules")
	preferEOLComments = flag.Bool("eol-comments", true, "when adding a new comment, put it on the same line if possible")
	rootDir           = flag.String("root_dir", "", "If present, use this folder rather than $PWD to find the root directory.")
	quiet             = flag.Bool("quiet", false, "suppress informational messages")
	editVariables     = flag.Bool("edit-variables", false, "For attributes that simply assign a variable (e.g. hdrs = LIB_HDRS), edit the build variable instead of appending to the attribute.")
	isPrintingProto   = flag.Bool("output_proto", false, "output serialized devtools.buildozer.Output protos instead of human-readable strings.")

	shortenLabelsFlag  = flag.Bool("shorten_labels", true, "convert added labels to short form, e.g. //foo:bar => :bar")
	deleteWithComments = flag.Bool("delete_with_comments", true, "If a list attribute should be deleted even if there is a comment attached to it")
)

func stringList(name, help string) func() []string {
	f := flag.String(name, "", help)
	return func() []string {
		res := strings.Split(*f, ",")
		for i := range res {
			res[i] = strings.TrimSpace(res[i])
		}
		return res
	}
}

func main() {
	flag.Parse()

	edit.ShortenLabelsFlag = *shortenLabelsFlag
	edit.DeleteWithComments = *deleteWithComments
	edit.Opts = edit.Options{
		Stdout:            *stdout,
		Buildifier:        *buildifier,
		Parallelism:       *parallelism,
		NumIO:             *numio,
		CommandsFile:      *commandsFile,
		KeepGoing:         *keepGoing,
		FilterRuleTypes:   filterRuleTypes(),
		PreferEOLComments: *preferEOLComments,
		RootDir:           *rootDir,
		Quiet:             *quiet,
		EditVariables:     *editVariables,
		IsPrintingProto:   *isPrintingProto,
	}
	os.Exit(edit.Buildozer(flag.Args()))
}
<commit_msg>bug fix for stringList, was returning [] instead of [] for empty input, causing buildozer to do nothing (#27)<commit_after>/*
Copyright 2016 Google Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/
// Entry-point for Buildozer binary.

package main

import (
	"flag"
	"os"
	"strings"

	"github.com/bazelbuild/buildifier/edit"
)

var (
	stdout            = flag.Bool("stdout", false, "write changed BUILD file to stdout")
	buildifier        = flag.String("buildifier", "", "path to buildifier binary")
	parallelism       = flag.Int("P", 0, "number of cores to use for concurrent actions")
	numio             = flag.Int("numio", 200, "number of concurrent actions")
	commandsFile      = flag.String("f", "", "file name to read commands from, use '-' for stdin (format:|-separated command line arguments to buildozer, excluding flags)")
	keepGoing         = flag.Bool("k", false, "apply all commands, even if there are failures")
	filterRuleTypes   = stringList("types", "comma-separated list of rule types to change, the default empty list means all rules")
	preferEOLComments = flag.Bool("eol-comments", true, "when adding a new comment, put it on the same line if possible")
	rootDir           = flag.String("root_dir", "", "If present, use this folder rather than $PWD to find the root directory.")
	quiet             = flag.Bool("quiet", false, "suppress informational messages")
	editVariables     = flag.Bool("edit-variables", false, "For attributes that simply assign a variable (e.g. hdrs = LIB_HDRS), edit the build variable instead of appending to the attribute.")
	isPrintingProto   = flag.Bool("output_proto", false, "output serialized devtools.buildozer.Output protos instead of human-readable strings.")

	shortenLabelsFlag  = flag.Bool("shorten_labels", true, "convert added labels to short form, e.g. //foo:bar => :bar")
	deleteWithComments = flag.Bool("delete_with_comments", true, "If a list attribute should be deleted even if there is a comment attached to it")
)

func stringList(name, help string) func() []string {
	f := flag.String(name, "", help)
	return func() []string {
	       if *f == "" {
	       	  return nil
}
		res := strings.Split(*f, ",")
		for i := range res {
			res[i] = strings.TrimSpace(res[i])
		}
		return res
	}
}

func main() {
	flag.Parse()

	edit.ShortenLabelsFlag = *shortenLabelsFlag
	edit.DeleteWithComments = *deleteWithComments
	edit.Opts = edit.Options{
		Stdout:            *stdout,
		Buildifier:        *buildifier,
		Parallelism:       *parallelism,
		NumIO:             *numio,
		CommandsFile:      *commandsFile,
		KeepGoing:         *keepGoing,
		FilterRuleTypes:   filterRuleTypes(),
		PreferEOLComments: *preferEOLComments,
		RootDir:           *rootDir,
		Quiet:             *quiet,
		EditVariables:     *editVariables,
		IsPrintingProto:   *isPrintingProto,
	}
	os.Exit(edit.Buildozer(flag.Args()))
}
<|endoftext|>