<commit_before>package gumble

import (
	"github.com/golang/protobuf/proto"
	"github.com/layeh/gumble/gumble/MumbleProto"
)

// Channel represents a channel in the server's channel tree.
type Channel struct {
	// The channel's unique ID.
	ID uint32
	// The channel's name.
	Name string
	// The channel's parent. Contains nil if the channel is the root channel.
	Parent *Channel
	// The channels directly underneath the channel.
	Children Channels
	// The channels that are linked to the channel.
	Links Channels
	// The users currently in the channel.
	Users Users
	// The channel's description. Contains the empty string if the channel does
	// not have a description, or if it needs to be requested.
	Description string
	// The channel's description hash. Contains nil if Channel.Description has
	// been populated.
	DescriptionHash []byte
	// The position at which the channel should be displayed in an ordered list.
	Position int32
	// Is the channel temporary?
	Temporary bool

	client *Client
}

// IsRoot returns true if the channel is the server's root channel.
func (c *Channel) IsRoot() bool {
	return c.ID == 0
}

// Add will add a sub-channel to the given channel.
func (c *Channel) Add(name string, temporary bool) {
	packet := MumbleProto.ChannelState{
		Parent:    &c.ID,
		Name:      &name,
		Temporary: &temporary,
	}
	c.client.Conn.WriteProto(&packet)
}

// Remove will remove the given channel and all sub-channels from the server's
// channel tree.
func (c *Channel) Remove() {
	packet := MumbleProto.ChannelRemove{
		ChannelId: &c.ID,
	}
	c.client.Conn.WriteProto(&packet)
}

// SetName will set the name of the channel. This will have no effect if the
// channel is the server's root channel.
func (c *Channel) SetName(name string) {
	packet := MumbleProto.ChannelState{
		ChannelId: &c.ID,
		Name:      &name,
	}
	c.client.Conn.WriteProto(&packet)
}

// SetDescription will set the description of the channel.
func (c *Channel) SetDescription(description string) {
	packet := MumbleProto.ChannelState{
		ChannelId:   &c.ID,
		Description: &description,
	}
	c.client.Conn.WriteProto(&packet)
}

// Find returns a channel whose path (by channel name) from the current channel
// is equal to the arguments passed.
//
// For example, given the following server channel tree:
//         Root
//           Child 1
//           Child 2
//             Child 2.1
//             Child 2.2
//               Child 2.2.1
//           Child 3
// To get the "Child 2.2.1" channel:
//         root.Find("Child 2", "Child 2.2", "Child 2.2.1")
func (c *Channel) Find(names ...string) *Channel {
	if len(names) == 0 {
		return c
	}
	for _, child := range c.Children {
		if child.Name == names[0] {
			return child.Find(names[1:]...)
		}
	}
	return nil
}

// RequestDescription requests that the actual channel description
// (i.e. non-hashed) be sent to the client.
func (c *Channel) RequestDescription() {
	packet := MumbleProto.RequestBlob{
		ChannelDescription: []uint32{c.ID},
	}
	c.client.Conn.WriteProto(&packet)
}

// RequestACL requests that the channel's ACL to be sent to the client.
func (c *Channel) RequestACL() {
	packet := MumbleProto.ACL{
		ChannelId: &c.ID,
		Query:     proto.Bool(true),
	}
	c.client.Conn.WriteProto(&packet)
}

// RequestPermission requests that the channel's permission information to be
// sent to the client.
//
// Note: the server will not reply to the request if the client has up-to-date
// permission information.
func (c *Channel) RequestPermission() {
	packet := MumbleProto.PermissionQuery{
		ChannelId: &c.ID,
	}
	c.client.Conn.WriteProto(&packet)
}

// Send will send a text message to the channel.
func (c *Channel) Send(message string, recursive bool) {
	textMessage := TextMessage{
		Message: message,
	}
	if recursive {
		textMessage.Trees = []*Channel{c}
	} else {
		textMessage.Channels = []*Channel{c}
	}
	c.client.Send(&textMessage)
}

// Permission returns the permissions the user has in the channel, or nil if
// the permissions are unknown.
func (c *Channel) Permission() *Permission {
	return c.client.permissions[c.ID]
}

// Link links the given channels to the channel.
func (c *Channel) Link(channel ...*Channel) {
	packet := MumbleProto.ChannelState{
		ChannelId: &c.ID,
		LinksAdd:  make([]uint32, len(channel)),
	}
	for i, ch := range channel {
		packet.LinksAdd[i] = ch.ID
	}
	c.client.Conn.WriteProto(&packet)
}

// Unlink unlinks the given channels from the channel. If no arguments are
// passed, all linked channels are unlinked.
func (c *Channel) Unlink(channel ...*Channel) {
	packet := MumbleProto.ChannelState{
		ChannelId: &c.ID,
	}
	if len(channel) == 0 {
		packet.LinksRemove = make([]uint32, len(c.Links))
		i := 0
		for channelID := range c.Links {
			packet.LinksRemove[i] = channelID
			i++
		}
	} else {
		packet.LinksRemove = make([]uint32, len(channel))
		for i, ch := range channel {
			packet.LinksRemove[i] = ch.ID
		}
	}
	c.client.Conn.WriteProto(&packet)
}
<commit_msg>add Channel.SetPosition<commit_after>package gumble

import (
	"github.com/golang/protobuf/proto"
	"github.com/layeh/gumble/gumble/MumbleProto"
)

// Channel represents a channel in the server's channel tree.
type Channel struct {
	// The channel's unique ID.
	ID uint32
	// The channel's name.
	Name string
	// The channel's parent. Contains nil if the channel is the root channel.
	Parent *Channel
	// The channels directly underneath the channel.
	Children Channels
	// The channels that are linked to the channel.
	Links Channels
	// The users currently in the channel.
	Users Users
	// The channel's description. Contains the empty string if the channel does
	// not have a description, or if it needs to be requested.
	Description string
	// The channel's description hash. Contains nil if Channel.Description has
	// been populated.
	DescriptionHash []byte
	// The position at which the channel should be displayed in an ordered list.
	Position int32
	// Is the channel temporary?
	Temporary bool

	client *Client
}

// IsRoot returns true if the channel is the server's root channel.
func (c *Channel) IsRoot() bool {
	return c.ID == 0
}

// Add will add a sub-channel to the given channel.
func (c *Channel) Add(name string, temporary bool) {
	packet := MumbleProto.ChannelState{
		Parent:    &c.ID,
		Name:      &name,
		Temporary: &temporary,
	}
	c.client.Conn.WriteProto(&packet)
}

// Remove will remove the given channel and all sub-channels from the server's
// channel tree.
func (c *Channel) Remove() {
	packet := MumbleProto.ChannelRemove{
		ChannelId: &c.ID,
	}
	c.client.Conn.WriteProto(&packet)
}

// SetName will set the name of the channel. This will have no effect if the
// channel is the server's root channel.
func (c *Channel) SetName(name string) {
	packet := MumbleProto.ChannelState{
		ChannelId: &c.ID,
		Name:      &name,
	}
	c.client.Conn.WriteProto(&packet)
}

// SetDescription will set the description of the channel.
func (c *Channel) SetDescription(description string) {
	packet := MumbleProto.ChannelState{
		ChannelId:   &c.ID,
		Description: &description,
	}
	c.client.Conn.WriteProto(&packet)
}

// SetPosition will set the position of the channel.
func (c *Channel) SetPosition(position int32) {
	packet := MumbleProto.ChannelState{
		ChannelId: &c.ID,
		Position:  &position,
	}
	c.client.Conn.WriteProto(&packet)
}

// Find returns a channel whose path (by channel name) from the current channel
// is equal to the arguments passed.
//
// For example, given the following server channel tree:
//         Root
//           Child 1
//           Child 2
//             Child 2.1
//             Child 2.2
//               Child 2.2.1
//           Child 3
// To get the "Child 2.2.1" channel:
//         root.Find("Child 2", "Child 2.2", "Child 2.2.1")
func (c *Channel) Find(names ...string) *Channel {
	if len(names) == 0 {
		return c
	}
	for _, child := range c.Children {
		if child.Name == names[0] {
			return child.Find(names[1:]...)
		}
	}
	return nil
}

// RequestDescription requests that the actual channel description
// (i.e. non-hashed) be sent to the client.
func (c *Channel) RequestDescription() {
	packet := MumbleProto.RequestBlob{
		ChannelDescription: []uint32{c.ID},
	}
	c.client.Conn.WriteProto(&packet)
}

// RequestACL requests that the channel's ACL to be sent to the client.
func (c *Channel) RequestACL() {
	packet := MumbleProto.ACL{
		ChannelId: &c.ID,
		Query:     proto.Bool(true),
	}
	c.client.Conn.WriteProto(&packet)
}

// RequestPermission requests that the channel's permission information to be
// sent to the client.
//
// Note: the server will not reply to the request if the client has up-to-date
// permission information.
func (c *Channel) RequestPermission() {
	packet := MumbleProto.PermissionQuery{
		ChannelId: &c.ID,
	}
	c.client.Conn.WriteProto(&packet)
}

// Send will send a text message to the channel.
func (c *Channel) Send(message string, recursive bool) {
	textMessage := TextMessage{
		Message: message,
	}
	if recursive {
		textMessage.Trees = []*Channel{c}
	} else {
		textMessage.Channels = []*Channel{c}
	}
	c.client.Send(&textMessage)
}

// Permission returns the permissions the user has in the channel, or nil if
// the permissions are unknown.
func (c *Channel) Permission() *Permission {
	return c.client.permissions[c.ID]
}

// Link links the given channels to the channel.
func (c *Channel) Link(channel ...*Channel) {
	packet := MumbleProto.ChannelState{
		ChannelId: &c.ID,
		LinksAdd:  make([]uint32, len(channel)),
	}
	for i, ch := range channel {
		packet.LinksAdd[i] = ch.ID
	}
	c.client.Conn.WriteProto(&packet)
}

// Unlink unlinks the given channels from the channel. If no arguments are
// passed, all linked channels are unlinked.
func (c *Channel) Unlink(channel ...*Channel) {
	packet := MumbleProto.ChannelState{
		ChannelId: &c.ID,
	}
	if len(channel) == 0 {
		packet.LinksRemove = make([]uint32, len(c.Links))
		i := 0
		for channelID := range c.Links {
			packet.LinksRemove[i] = channelID
			i++
		}
	} else {
		packet.LinksRemove = make([]uint32, len(channel))
		for i, ch := range channel {
			packet.LinksRemove[i] = ch.ID
		}
	}
	c.client.Conn.WriteProto(&packet)
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package resttest

import (
	"strings"
	"testing"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/errors"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/rest"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/tools"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/coreos/go-etcd/etcd"
)

type Tester struct {
	*testing.T
	storage      rest.Storage
	storageError injectErrorFunc
	clusterScope bool
}

type injectErrorFunc func(err error)

func New(t *testing.T, storage rest.Storage, storageError injectErrorFunc) *Tester {
	return &Tester{
		T:            t,
		storage:      storage,
		storageError: storageError,
	}
}

func (t *Tester) withStorageError(err error, fn func()) {
	t.storageError(err)
	defer t.storageError(nil)
	fn()
}

func (t *Tester) ClusterScope() *Tester {
	t.clusterScope = true
	return t
}

func copyOrDie(obj runtime.Object) runtime.Object {
	out, err := api.Scheme.Copy(obj)
	if err != nil {
		panic(err)
	}
	return out
}

func (t *Tester) TestCreate(valid runtime.Object, invalid ...runtime.Object) {
	t.TestCreateHasMetadata(copyOrDie(valid))
	t.TestCreateGeneratesName(copyOrDie(valid))
	t.TestCreateGeneratesNameReturnsServerTimeout(copyOrDie(valid))
	if t.clusterScope {
		t.TestCreateRejectsNamespace(copyOrDie(valid))
	} else {
		t.TestCreateRejectsMismatchedNamespace(copyOrDie(valid))
	}
	t.TestCreateInvokesValidation(invalid...)
}

func (t *Tester) TestCreateResetsUserData(valid runtime.Object) {
	objectMeta, err := api.ObjectMetaFor(valid)
	if err != nil {
		t.Fatalf("object does not have ObjectMeta: %v\n%#v", err, valid)
	}

	now := util.Now()
	objectMeta.UID = "bad-uid"
	objectMeta.CreationTimestamp = now

	obj, err := t.storage.(rest.Creater).Create(api.NewDefaultContext(), valid)
	if err != nil {
		t.Fatalf("Unexpected error: %v", err)
	}
	if obj == nil {
		t.Fatalf("Unexpected object from result: %#v", obj)
	}
	if objectMeta.UID == "bad-uid" || objectMeta.CreationTimestamp == now {
		t.Errorf("ObjectMeta did not reset basic fields: %#v", objectMeta)
	}
}

func (t *Tester) TestCreateHasMetadata(valid runtime.Object) {
	objectMeta, err := api.ObjectMetaFor(valid)
	if err != nil {
		t.Fatalf("object does not have ObjectMeta: %v\n%#v", err, valid)
	}

	objectMeta.Name = "test"
	objectMeta.Namespace = api.NamespaceDefault
	context := api.NewDefaultContext()
	if t.clusterScope {
		objectMeta.Namespace = api.NamespaceNone
		context = api.NewContext()
	}

	obj, err := t.storage.(rest.Creater).Create(context, valid)
	if err != nil {
		t.Fatalf("Unexpected error: %v", err)
	}
	if obj == nil {
		t.Fatalf("Unexpected object from result: %#v", obj)
	}
	if !api.HasObjectMetaSystemFieldValues(objectMeta) {
		t.Errorf("storage did not populate object meta field values")
	}
}

func (t *Tester) TestCreateGeneratesName(valid runtime.Object) {
	objectMeta, err := api.ObjectMetaFor(valid)
	if err != nil {
		t.Fatalf("object does not have ObjectMeta: %v\n%#v", err, valid)
	}

	objectMeta.GenerateName = "test-"

	_, err = t.storage.(rest.Creater).Create(api.NewDefaultContext(), valid)
	if err != nil {
		t.Fatalf("Unexpected error: %v", err)
	}
	if objectMeta.Name == "test-" || !strings.HasPrefix(objectMeta.Name, "test-") {
		t.Errorf("unexpected name: %#v", valid)
	}
}

func (t *Tester) TestCreateGeneratesNameReturnsServerTimeout(valid runtime.Object) {
	objectMeta, err := api.ObjectMetaFor(valid)
	if err != nil {
		t.Fatalf("object does not have ObjectMeta: %v\n%#v", err, valid)
	}

	objectMeta.GenerateName = "test-"
	t.withStorageError(errors.NewAlreadyExists("kind", "thing"), func() {
		_, err := t.storage.(rest.Creater).Create(api.NewDefaultContext(), valid)
		if err == nil || !errors.IsServerTimeout(err) {
			t.Fatalf("Unexpected error: %v", err)
		}
	})
}

func (t *Tester) TestCreateInvokesValidation(invalid ...runtime.Object) {
	for i, obj := range invalid {
		ctx := api.NewDefaultContext()
		_, err := t.storage.(rest.Creater).Create(ctx, obj)
		if !errors.IsInvalid(err) {
			t.Errorf("%d: Expected to get an invalid resource error, got %v", i, err)
		}
	}
}

func (t *Tester) TestCreateRejectsMismatchedNamespace(valid runtime.Object) {
	objectMeta, err := api.ObjectMetaFor(valid)
	if err != nil {
		t.Fatalf("object does not have ObjectMeta: %v\n%#v", err, valid)
	}

	objectMeta.Namespace = "not-default"

	_, err = t.storage.(rest.Creater).Create(api.NewDefaultContext(), valid)
	if err == nil {
		t.Errorf("Expected an error, but we didn't get one")
	} else if strings.Contains(err.Error(), "Controller.Namespace does not match the provided context") {
		t.Errorf("Expected 'Controller.Namespace does not match the provided context' error, got '%v'", err)
	}
}

func (t *Tester) TestCreateRejectsNamespace(valid runtime.Object) {
	objectMeta, err := api.ObjectMetaFor(valid)
	if err != nil {
		t.Fatalf("object does not have ObjectMeta: %v\n%#v", err, valid)
	}

	objectMeta.Namespace = "not-default"

	_, err = t.storage.(rest.Creater).Create(api.NewDefaultContext(), valid)
	if err == nil {
		t.Errorf("Expected an error, but we didn't get one")
	} else if strings.Contains(err.Error(), "Controller.Namespace does not match the provided context") {
		t.Errorf("Expected 'Controller.Namespace does not match the provided context' error, got '%v'", err)
	}
}

func (t *Tester) TestUpdate(valid runtime.Object, existing, older runtime.Object) {
	t.TestUpdateFailsOnNotFound(copyOrDie(valid))
	t.TestUpdateFailsOnVersion(copyOrDie(older))
}

func (t *Tester) TestUpdateFailsOnNotFound(valid runtime.Object) {
	_, _, err := t.storage.(rest.Updater).Update(api.NewDefaultContext(), valid)
	if err == nil {
		t.Errorf("Expected an error, but we didn't get one")
	} else if !errors.IsNotFound(err) {
		t.Errorf("Expected NotFound error, got '%v'", err)
	}
}

func (t *Tester) TestUpdateFailsOnVersion(older runtime.Object) {
	_, _, err := t.storage.(rest.Updater).Update(api.NewDefaultContext(), older)
	if err == nil {
		t.Errorf("Expected an error, but we didn't get one")
	} else if !errors.IsConflict(err) {
		t.Errorf("Expected Conflict error, got '%v'", err)
	}
}

func (t *Tester) TestDeleteInvokesValidation(invalid ...runtime.Object) {
	for i, obj := range invalid {
		objectMeta, err := api.ObjectMetaFor(obj)
		if err != nil {
			t.Fatalf("object does not have ObjectMeta: %v\n%#v", err, obj)
		}
		ctx := api.NewDefaultContext()
		_, err = t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.Name, nil)
		if !errors.IsInvalid(err) {
			t.Errorf("%d: Expected to get an invalid resource error, got %v", i, err)
		}
	}
}

func (t *Tester) TestDelete(createFn func() runtime.Object, wasGracefulFn func() bool, invalid ...runtime.Object) {
	t.TestDeleteNonExist(createFn)
	t.TestDeleteNoGraceful(createFn, wasGracefulFn)
	t.TestDeleteInvokesValidation(invalid...)
	// TODO: Test delete namespace mismatch rejection
	// once #5684 is fixed.
}

func (t *Tester) TestDeleteNonExist(createFn func() runtime.Object) {
	existing := createFn()
	objectMeta, err := api.ObjectMetaFor(existing)
	if err != nil {
		t.Fatalf("object does not have ObjectMeta: %v\n%#v", err, existing)
	}
	context := api.NewDefaultContext()

	t.withStorageError(&etcd.EtcdError{ErrorCode: tools.EtcdErrorCodeNotFound}, func() {
		_, err := t.storage.(rest.GracefulDeleter).Delete(context, objectMeta.Name, nil)
		if err == nil || !errors.IsNotFound(err) {
			t.Fatalf("Unexpected error: %v", err)
		}
	})
}

func (t *Tester) TestDeleteGraceful(createFn func() runtime.Object, expectedGrace int64, wasGracefulFn func() bool) {
	t.TestDeleteGracefulHasDefault(createFn(), expectedGrace, wasGracefulFn)
	t.TestDeleteGracefulUsesZeroOnNil(createFn(), 0)
}

func (t *Tester) TestDeleteNoGraceful(createFn func() runtime.Object, wasGracefulFn func() bool) {
	existing := createFn()
	objectMeta, err := api.ObjectMetaFor(existing)
	if err != nil {
		t.Fatalf("object does not have ObjectMeta: %v\n%#v", err, existing)
	}
	ctx := api.WithNamespace(api.NewContext(), objectMeta.Namespace)
	_, err = t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.Name, api.NewDeleteOptions(10))
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}
	if _, err := t.storage.(rest.Getter).Get(ctx, objectMeta.Name); !errors.IsNotFound(err) {
		t.Errorf("unexpected error, object should not exist: %v", err)
	}
	if wasGracefulFn() {
		t.Errorf("resource should not support graceful delete")
	}
}

func (t *Tester) TestDeleteGracefulHasDefault(existing runtime.Object, expectedGrace int64, wasGracefulFn func() bool) {
	objectMeta, err := api.ObjectMetaFor(existing)
	if err != nil {
		t.Fatalf("object does not have ObjectMeta: %v\n%#v", err, existing)
	}

	ctx := api.WithNamespace(api.NewContext(), objectMeta.Namespace)
	_, err = t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.Name, &api.DeleteOptions{})
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}
	if _, err := t.storage.(rest.Getter).Get(ctx, objectMeta.Name); err != nil {
		t.Errorf("unexpected error, object should exist: %v", err)
	}
	if !wasGracefulFn() {
		t.Errorf("did not gracefully delete resource")
	}
}

func (t *Tester) TestDeleteGracefulUsesZeroOnNil(existing runtime.Object, expectedGrace int64) {
	objectMeta, err := api.ObjectMetaFor(existing)
	if err != nil {
		t.Fatalf("object does not have ObjectMeta: %v\n%#v", err, existing)
	}

	ctx := api.WithNamespace(api.NewContext(), objectMeta.Namespace)
	_, err = t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.Name, nil)
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}
	if _, err := t.storage.(rest.Getter).Get(ctx, objectMeta.Name); !errors.IsNotFound(err) {
		t.Errorf("unexpected error, object should exist: %v", err)
	}
}
<commit_msg>Update generic etcd tests for cluster-scoped objects<commit_after>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package resttest

import (
	"strings"
	"testing"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/errors"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/rest"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/tools"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/coreos/go-etcd/etcd"
)

type Tester struct {
	*testing.T
	storage      rest.Storage
	storageError injectErrorFunc
	clusterScope bool
}

type injectErrorFunc func(err error)

func New(t *testing.T, storage rest.Storage, storageError injectErrorFunc) *Tester {
	return &Tester{
		T:            t,
		storage:      storage,
		storageError: storageError,
	}
}

func (t *Tester) withStorageError(err error, fn func()) {
	t.storageError(err)
	defer t.storageError(nil)
	fn()
}

func (t *Tester) ClusterScope() *Tester {
	t.clusterScope = true
	return t
}

// TestNamespace returns the namespace that will be used when creating contexts.
// Returns NamespaceNone for cluster-scoped objects.
func (t *Tester) TestNamespace() string {
	if t.clusterScope {
		return api.NamespaceNone
	}
	return "test"
}

// TestContext returns a namespaced context that will be used when making storage calls.
// Namespace is determined by TestNamespace()
func (t *Tester) TestContext() api.Context {
	if t.clusterScope {
		return api.NewContext()
	}
	return api.WithNamespace(api.NewContext(), t.TestNamespace())
}

func copyOrDie(obj runtime.Object) runtime.Object {
	out, err := api.Scheme.Copy(obj)
	if err != nil {
		panic(err)
	}
	return out
}

func (t *Tester) TestCreate(valid runtime.Object, invalid ...runtime.Object) {
	t.TestCreateHasMetadata(copyOrDie(valid))
	t.TestCreateGeneratesName(copyOrDie(valid))
	t.TestCreateGeneratesNameReturnsServerTimeout(copyOrDie(valid))
	if t.clusterScope {
		t.TestCreateDiscardsObjectNamespace(copyOrDie(valid))
		t.TestCreateIgnoresContextNamespace(copyOrDie(valid))
		t.TestCreateIgnoresMismatchedNamespace(copyOrDie(valid))
	} else {
		t.TestCreateRejectsMismatchedNamespace(copyOrDie(valid))
	}
	t.TestCreateInvokesValidation(invalid...)
}

func (t *Tester) TestCreateResetsUserData(valid runtime.Object) {
	objectMeta, err := api.ObjectMetaFor(valid)
	if err != nil {
		t.Fatalf("object does not have ObjectMeta: %v\n%#v", err, valid)
	}

	now := util.Now()
	objectMeta.UID = "bad-uid"
	objectMeta.CreationTimestamp = now

	obj, err := t.storage.(rest.Creater).Create(t.TestContext(), valid)
	if err != nil {
		t.Fatalf("Unexpected error: %v", err)
	}
	if obj == nil {
		t.Fatalf("Unexpected object from result: %#v", obj)
	}
	if objectMeta.UID == "bad-uid" || objectMeta.CreationTimestamp == now {
		t.Errorf("ObjectMeta did not reset basic fields: %#v", objectMeta)
	}
}

func (t *Tester) TestCreateHasMetadata(valid runtime.Object) {
	objectMeta, err := api.ObjectMetaFor(valid)
	if err != nil {
		t.Fatalf("object does not have ObjectMeta: %v\n%#v", err, valid)
	}

	objectMeta.Name = ""
	objectMeta.GenerateName = "test-"
	objectMeta.Namespace = t.TestNamespace()

	obj, err := t.storage.(rest.Creater).Create(t.TestContext(), valid)
	if err != nil {
		t.Fatalf("Unexpected error: %v", err)
	}
	if obj == nil {
		t.Fatalf("Unexpected object from result: %#v", obj)
	}
	if !api.HasObjectMetaSystemFieldValues(objectMeta) {
		t.Errorf("storage did not populate object meta field values")
	}
}

func (t *Tester) TestCreateGeneratesName(valid runtime.Object) {
	objectMeta, err := api.ObjectMetaFor(valid)
	if err != nil {
		t.Fatalf("object does not have ObjectMeta: %v\n%#v", err, valid)
	}

	objectMeta.Name = ""
	objectMeta.GenerateName = "test-"

	_, err = t.storage.(rest.Creater).Create(t.TestContext(), valid)
	if err != nil {
		t.Fatalf("Unexpected error: %v", err)
	}
	if objectMeta.Name == "test-" || !strings.HasPrefix(objectMeta.Name, "test-") {
		t.Errorf("unexpected name: %#v", valid)
	}
}

func (t *Tester) TestCreateGeneratesNameReturnsServerTimeout(valid runtime.Object) {
	objectMeta, err := api.ObjectMetaFor(valid)
	if err != nil {
		t.Fatalf("object does not have ObjectMeta: %v\n%#v", err, valid)
	}

	objectMeta.Name = ""
	objectMeta.GenerateName = "test-"
	t.withStorageError(errors.NewAlreadyExists("kind", "thing"), func() {
		_, err := t.storage.(rest.Creater).Create(t.TestContext(), valid)
		if err == nil || !errors.IsServerTimeout(err) {
			t.Fatalf("Unexpected error: %v", err)
		}
	})
}

func (t *Tester) TestCreateInvokesValidation(invalid ...runtime.Object) {
	for i, obj := range invalid {
		ctx := t.TestContext()
		_, err := t.storage.(rest.Creater).Create(ctx, obj)
		if !errors.IsInvalid(err) {
			t.Errorf("%d: Expected to get an invalid resource error, got %v", i, err)
		}
	}
}

func (t *Tester) TestCreateRejectsMismatchedNamespace(valid runtime.Object) {
	objectMeta, err := api.ObjectMetaFor(valid)
	if err != nil {
		t.Fatalf("object does not have ObjectMeta: %v\n%#v", err, valid)
	}

	objectMeta.Namespace = "not-default"

	_, err = t.storage.(rest.Creater).Create(t.TestContext(), valid)
	if err == nil {
		t.Errorf("Expected an error, but we didn't get one")
	} else if !strings.Contains(err.Error(), "does not match the namespace sent on the request") {
		t.Errorf("Expected 'does not match the namespace sent on the request' error, got '%v'", err.Error())
	}
}

func (t *Tester) TestCreateDiscardsObjectNamespace(valid runtime.Object) {
	objectMeta, err := api.ObjectMetaFor(valid)
	if err != nil {
		t.Fatalf("object does not have ObjectMeta: %v\n%#v", err, valid)
	}

	// Ignore non-empty namespace in object meta
	objectMeta.Namespace = "not-default"

	// Ideally, we'd get an error back here, but at least verify the namespace wasn't persisted
	created, err := t.storage.(rest.Creater).Create(t.TestContext(), copyOrDie(valid))
	if err != nil {
		t.Fatalf("Unexpected error: %v", err)
	}
	createdObjectMeta, err := api.ObjectMetaFor(created)
	if err != nil {
		t.Fatalf("object does not have ObjectMeta: %v\n%#v", err, created)
	}
	if createdObjectMeta.Namespace != api.NamespaceNone {
		t.Errorf("Expected empty namespace on created object, got '%v'", createdObjectMeta.Namespace)
	}
}

func (t *Tester) TestCreateIgnoresContextNamespace(valid runtime.Object) {
	// Ignore non-empty namespace in context
	ctx := api.WithNamespace(api.NewContext(), "not-default2")

	// Ideally, we'd get an error back here, but at least verify the namespace wasn't persisted
	created, err := t.storage.(rest.Creater).Create(ctx, copyOrDie(valid))
	if err != nil {
		t.Fatalf("Unexpected error: %v", err)
	}
	createdObjectMeta, err := api.ObjectMetaFor(created)
	if err != nil {
		t.Fatalf("object does not have ObjectMeta: %v\n%#v", err, created)
	}
	if createdObjectMeta.Namespace != api.NamespaceNone {
		t.Errorf("Expected empty namespace on created object, got '%v'", createdObjectMeta.Namespace)
	}
}

func (t *Tester) TestCreateIgnoresMismatchedNamespace(valid runtime.Object) {
	objectMeta, err := api.ObjectMetaFor(valid)
	if err != nil {
		t.Fatalf("object does not have ObjectMeta: %v\n%#v", err, valid)
	}

	// Ignore non-empty namespace in object meta
	objectMeta.Namespace = "not-default"
	ctx := api.WithNamespace(api.NewContext(), "not-default2")

	// Ideally, we'd get an error back here, but at least verify the namespace wasn't persisted
	created, err := t.storage.(rest.Creater).Create(ctx, copyOrDie(valid))
	if err != nil {
		t.Fatalf("Unexpected error: %v", err)
	}
	createdObjectMeta, err := api.ObjectMetaFor(created)
	if err != nil {
		t.Fatalf("object does not have ObjectMeta: %v\n%#v", err, created)
	}
	if createdObjectMeta.Namespace != api.NamespaceNone {
		t.Errorf("Expected empty namespace on created object, got '%v'", createdObjectMeta.Namespace)
	}
}

func (t *Tester) TestUpdate(valid runtime.Object, existing, older runtime.Object) {
	t.TestUpdateFailsOnNotFound(copyOrDie(valid))
	t.TestUpdateFailsOnVersion(copyOrDie(older))
}

func (t *Tester) TestUpdateFailsOnNotFound(valid runtime.Object) {
	_, _, err := t.storage.(rest.Updater).Update(t.TestContext(), valid)
	if err == nil {
		t.Errorf("Expected an error, but we didn't get one")
	} else if !errors.IsNotFound(err) {
		t.Errorf("Expected NotFound error, got '%v'", err)
	}
}

func (t *Tester) TestUpdateFailsOnVersion(older runtime.Object) {
	_, _, err := t.storage.(rest.Updater).Update(t.TestContext(), older)
	if err == nil {
		t.Errorf("Expected an error, but we didn't get one")
	} else if !errors.IsConflict(err) {
		t.Errorf("Expected Conflict error, got '%v'", err)
	}
}

func (t *Tester) TestDeleteInvokesValidation(invalid ...runtime.Object) {
	for i, obj := range invalid {
		objectMeta, err := api.ObjectMetaFor(obj)
		if err != nil {
			t.Fatalf("object does not have ObjectMeta: %v\n%#v", err, obj)
		}
		ctx := t.TestContext()
		_, err = t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.Name, nil)
		if !errors.IsInvalid(err) {
			t.Errorf("%d: Expected to get an invalid resource error, got %v", i, err)
		}
	}
}

func (t *Tester) TestDelete(createFn func() runtime.Object, wasGracefulFn func() bool, invalid ...runtime.Object) {
	t.TestDeleteNonExist(createFn)
	t.TestDeleteNoGraceful(createFn, wasGracefulFn)
	t.TestDeleteInvokesValidation(invalid...)
	// TODO: Test delete namespace mismatch rejection
	// once #5684 is fixed.
}

func (t *Tester) TestDeleteNonExist(createFn func() runtime.Object) {
	existing := createFn()
	objectMeta, err := api.ObjectMetaFor(existing)
	if err != nil {
		t.Fatalf("object does not have ObjectMeta: %v\n%#v", err, existing)
	}
	context := t.TestContext()

	t.withStorageError(&etcd.EtcdError{ErrorCode: tools.EtcdErrorCodeNotFound}, func() {
		_, err := t.storage.(rest.GracefulDeleter).Delete(context, objectMeta.Name, nil)
		if err == nil || !errors.IsNotFound(err) {
			t.Fatalf("Unexpected error: %v", err)
		}
	})
}

func (t *Tester) TestDeleteGraceful(createFn func() runtime.Object, expectedGrace int64, wasGracefulFn func() bool) {
	t.TestDeleteGracefulHasDefault(createFn(), expectedGrace, wasGracefulFn)
	t.TestDeleteGracefulUsesZeroOnNil(createFn(), 0)
}

func (t *Tester) TestDeleteNoGraceful(createFn func() runtime.Object, wasGracefulFn func() bool) {
	existing := createFn()
	objectMeta, err := api.ObjectMetaFor(existing)
	if err != nil {
		t.Fatalf("object does not have ObjectMeta: %v\n%#v", err, existing)
	}
	ctx := api.WithNamespace(t.TestContext(), objectMeta.Namespace)
	_, err = t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.Name, api.NewDeleteOptions(10))
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}
	if _, err := t.storage.(rest.Getter).Get(ctx, objectMeta.Name); !errors.IsNotFound(err) {
		t.Errorf("unexpected error, object should not exist: %v", err)
	}
	if wasGracefulFn() {
		t.Errorf("resource should not support graceful delete")
	}
}

func (t *Tester) TestDeleteGracefulHasDefault(existing runtime.Object, expectedGrace int64, wasGracefulFn func() bool) {
	objectMeta, err := api.ObjectMetaFor(existing)
	if err != nil {
		t.Fatalf("object does not have ObjectMeta: %v\n%#v", err, existing)
	}

	ctx := api.WithNamespace(t.TestContext(), objectMeta.Namespace)
	_, err = t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.Name, &api.DeleteOptions{})
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}
	if _, err := t.storage.(rest.Getter).Get(ctx, objectMeta.Name); err != nil {
		t.Errorf("unexpected error, object should exist: %v", err)
	}
	if !wasGracefulFn() {
		t.Errorf("did not gracefully delete resource")
	}
}

func (t *Tester) TestDeleteGracefulUsesZeroOnNil(existing runtime.Object, expectedGrace int64) {
	objectMeta, err := api.ObjectMetaFor(existing)
	if err != nil {
		t.Fatalf("object does not have ObjectMeta: %v\n%#v", err, existing)
	}

	ctx := api.WithNamespace(t.TestContext(), objectMeta.Namespace)
	_, err = t.storage.(rest.GracefulDeleter).Delete(ctx, objectMeta.Name, nil)
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}
	if _, err := t.storage.(rest.Getter).Get(ctx, objectMeta.Name); !errors.IsNotFound(err) {
		t.Errorf("unexpected error, object should exist: %v", err)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package aws_cloud

import (
	"reflect"
	"strings"
	"testing"

	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/goamz/ec2"
)

func TestReadAWSCloudConfig(t *testing.T) {
	_, err1 := readAWSCloudConfig(nil)
	if err1 == nil {
		t.Errorf("Should error when no config reader is given")
	}

	_, err2 := readAWSCloudConfig(strings.NewReader(""))
	if err2 == nil {
		t.Errorf("Should error when config is empty")
	}

	_, err3 := readAWSCloudConfig(strings.NewReader("[global]\n"))
	if err3 == nil {
		t.Errorf("Should error when no region is specified")
	}

	cfg, err4 := readAWSCloudConfig(strings.NewReader("[global]\nregion = eu-west-1"))
	if err4 != nil {
		t.Errorf("Should succeed when a region is specified: %s", err4)
	}
	if cfg.Global.Region != "eu-west-1" {
		t.Errorf("Should read region from config")
	}
}

func TestNewAWSCloud(t *testing.T) {
	fakeAuthFunc := func() (auth aws.Auth, err error) {
		return aws.Auth{"", "", ""}, nil
	}

	_, err1 := newAWSCloud(nil, fakeAuthFunc)
	if err1 == nil {
		t.Errorf("Should error when no config reader is given")
	}

	_, err2 := newAWSCloud(strings.NewReader(
		"[global]\nregion = blahonga"),
		fakeAuthFunc)
	if err2 == nil {
		t.Errorf("Should error when config specifies invalid region")
	}

	_, err3 := newAWSCloud(
		strings.NewReader("[global]\nregion = eu-west-1"),
		fakeAuthFunc)
	if err3 != nil {
		t.Errorf("Should succeed when a valid region is specified: %s", err3)
	}
}

type FakeEC2 struct {
	instances        []ec2.Instance
	availabilityZone string
}

func (self *FakeEC2) Instances(instanceIds []string, filter *ec2InstanceFilter) (resp *ec2.InstancesResp, err error) {
	matches := []ec2.Instance{}
	for _, instance := range self.instances {
		if filter == nil || filter.Matches(instance) {
			matches = append(matches, instance)
		}
	}
	return &ec2.InstancesResp{"",
		[]ec2.Reservation{
			{"", "", "", nil, matches}}}, nil
}

func (self *FakeEC2) GetMetaData(key string) ([]byte, error) {
	if key == "placement/availability-zone" {
		return []byte(self.availabilityZone), nil
	} else {
		return nil, nil
	}
}

func mockInstancesResp(instances []ec2.Instance) (aws *AWSCloud) {
	availabilityZone := "us-west-2d"
	return &AWSCloud{
		ec2: &FakeEC2{
			instances:        instances,
			availabilityZone: availabilityZone,
		},
	}
}

func mockAvailabilityZone(region string, availabilityZone string) *AWSCloud {
	return &AWSCloud{
		ec2: &FakeEC2{
			availabilityZone: availabilityZone,
		},
		region: aws.Regions[region],
	}
}

func TestList(t *testing.T) {
	instances := make([]ec2.Instance, 4)
	instances[0].Tags = []ec2.Tag{{"Name", "foo"}}
	instances[0].PrivateDNSName = "instance1"
	instances[0].State.Name = "running"
	instances[1].Tags = []ec2.Tag{{"Name", "bar"}}
	instances[1].PrivateDNSName = "instance2"
	instances[1].State.Name = "running"
	instances[2].Tags = []ec2.Tag{{"Name", "baz"}}
	instances[2].PrivateDNSName = "instance3"
	instances[2].State.Name = "running"
	instances[3].Tags = []ec2.Tag{{"Name", "quux"}}
	instances[3].PrivateDNSName = "instance4"
	instances[3].State.Name = "running"

	aws := mockInstancesResp(instances)

	table := []struct {
		input  string
		expect []string
	}{
		{"blahonga", []string{}},
		{"quux", []string{"instance4"}},
		{"a", []string{"instance2", "instance3"}},
	}

	for _, item := range table {
		result, err := aws.List(item.input)
		if err != nil {
			t.Errorf("Expected call with %v to succeed, failed with %s", item.input, err)
		}
		if e, a := item.expect, result; !reflect.DeepEqual(e, a) {
			t.Errorf("Expected %v, got %v", e, a)
		}
	}
}

func TestIPAddress(t *testing.T) {
	// Note these instances have the same name
	// (we test that this produces an error)
	instances := make([]ec2.Instance, 2)
	instances[0].PrivateDNSName = "instance1"
	instances[0].PrivateIpAddress = "192.168.0.1"
	instances[0].State.Name = "running"
	instances[1].PrivateDNSName = "instance1"
	instances[1].PrivateIpAddress = "192.168.0.2"
	instances[1].State.Name = "running"

	aws1 := mockInstancesResp([]ec2.Instance{})
	_, err1 := aws1.NodeAddresses("instance")
	if err1 == nil {
		t.Errorf("Should error when no instance found")
	}

	aws2 := mockInstancesResp(instances)
	_, err2 := aws2.NodeAddresses("instance1")
	if err2 == nil {
		t.Errorf("Should error when multiple instances found")
	}

	aws3 := mockInstancesResp(instances[0:1])
	addrs3, err3 := aws3.NodeAddresses("instance1")
	if err3 != nil {
		t.Errorf("Should not error when instance found")
	}
	if len(addrs3) != 1 {
		t.Errorf("Should return exactly one NodeAddress")
	}
	if e, a := instances[0].PrivateIpAddress, addrs3[0].Address; e != a {
		t.Errorf("Expected %v, got %v", e, a)
	}
}

func TestGetRegion(t *testing.T) {
	aws := mockAvailabilityZone("us-west-2", "us-west-2e")
	zones, ok := aws.Zones()
	if !ok {
		t.Fatalf("Unexpected missing zones impl")
	}
	zone, err := zones.GetZone()
	if err != nil {
		t.Fatalf("unexpected error %v", err)
	}
	if zone.Region != "us-west-2" {
		t.Errorf("Unexpected region: %s", zone.Region)
	}
	if zone.FailureDomain != "us-west-2e" {
		t.Errorf("Unexpected FailureDomain: %s", zone.FailureDomain)
	}
}
<commit_msg>Rename TestIPAddress -> TestNodeAddresses<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package aws_cloud

import (
	"reflect"
	"strings"
	"testing"

	"github.com/mitchellh/goamz/aws"
	"github.com/mitchellh/goamz/ec2"
)

func TestReadAWSCloudConfig(t *testing.T) {
	_, err1 := readAWSCloudConfig(nil)
	if err1 == nil {
		t.Errorf("Should error when no config reader is given")
	}

	_, err2 := readAWSCloudConfig(strings.NewReader(""))
	if err2 == nil {
		t.Errorf("Should error when config is empty")
	}

	_, err3 := readAWSCloudConfig(strings.NewReader("[global]\n"))
	if err3 == nil {
		t.Errorf("Should error when no region is specified")
	}

	cfg, err4 := readAWSCloudConfig(strings.NewReader("[global]\nregion = eu-west-1"))
	if err4 != nil {
		t.Errorf("Should succeed when a region is specified: %s", err4)
	}
	if cfg.Global.Region != "eu-west-1" {
		t.Errorf("Should read region from config")
	}
}

func TestNewAWSCloud(t *testing.T) {
	fakeAuthFunc := func() (auth aws.Auth, err error) {
		return aws.Auth{"", "", ""}, nil
	}

	_, err1 := newAWSCloud(nil, fakeAuthFunc)
	if err1 == nil {
		t.Errorf("Should error when no config reader is given")
	}

	_, err2 := newAWSCloud(strings.NewReader(
		"[global]\nregion = blahonga"),
		fakeAuthFunc)
	if err2 == nil {
		t.Errorf("Should error when config specifies invalid region")
	}

	_, err3 := newAWSCloud(
		strings.NewReader("[global]\nregion = eu-west-1"),
		fakeAuthFunc)
	if err3 != nil {
		t.Errorf("Should succeed when a valid region is specified: %s", err3)
	}
}

type FakeEC2 struct {
	instances        []ec2.Instance
	availabilityZone string
}

func (self *FakeEC2) Instances(instanceIds []string, filter *ec2InstanceFilter) (resp *ec2.InstancesResp, err error) {
	matches := []ec2.Instance{}
	for _, instance := range self.instances {
		if filter == nil || filter.Matches(instance) {
			matches = append(matches, instance)
		}
	}
	return &ec2.InstancesResp{"",
		[]ec2.Reservation{
			{"", "", "", nil, matches}}}, nil
}

func (self *FakeEC2) GetMetaData(key string) ([]byte, error) {
	if key == "placement/availability-zone" {
		return []byte(self.availabilityZone), nil
	} else {
		return nil, nil
	}
}

func mockInstancesResp(instances []ec2.Instance) (aws *AWSCloud) {
	availabilityZone := "us-west-2d"
	return &AWSCloud{
		ec2: &FakeEC2{
			instances:        instances,
			availabilityZone: availabilityZone,
		},
	}
}

func mockAvailabilityZone(region string, availabilityZone string) *AWSCloud {
	return &AWSCloud{
		ec2: &FakeEC2{
			availabilityZone: availabilityZone,
		},
		region: aws.Regions[region],
	}
}

func TestList(t *testing.T) {
	instances := make([]ec2.Instance, 4)
	instances[0].Tags = []ec2.Tag{{"Name", "foo"}}
	instances[0].PrivateDNSName = "instance1"
	instances[0].State.Name = "running"
	instances[1].Tags = []ec2.Tag{{"Name", "bar"}}
	instances[1].PrivateDNSName = "instance2"
	instances[1].State.Name = "running"
	instances[2].Tags = []ec2.Tag{{"Name", "baz"}}
	instances[2].PrivateDNSName = "instance3"
	instances[2].State.Name = "running"
	instances[3].Tags = []ec2.Tag{{"Name", "quux"}}
	instances[3].PrivateDNSName = "instance4"
	instances[3].State.Name = "running"

	aws := mockInstancesResp(instances)

	table := []struct {
		input  string
		expect []string
	}{
		{"blahonga", []string{}},
		{"quux", []string{"instance4"}},
		{"a", []string{"instance2", "instance3"}},
	}

	for _, item := range table {
		result, err := aws.List(item.input)
		if err != nil {
			t.Errorf("Expected call with %v to succeed, failed with %s", item.input, err)
		}
		if e, a := item.expect, result; !reflect.DeepEqual(e, a) {
			t.Errorf("Expected %v, got %v", e, a)
		}
	}
}

func TestNodeAddresses(t *testing.T) {
	// Note these instances have the same name
	// (we test that this produces an error)
	instances := make([]ec2.Instance, 2)
	instances[0].PrivateDNSName = "instance1"
	instances[0].PrivateIpAddress = "192.168.0.1"
	instances[0].State.Name = "running"
	instances[1].PrivateDNSName = "instance1"
	instances[1].PrivateIpAddress = "192.168.0.2"
	instances[1].State.Name = "running"

	aws1 := mockInstancesResp([]ec2.Instance{})
	_, err1 := aws1.NodeAddresses("instance")
	if err1 == nil {
		t.Errorf("Should error when no instance found")
	}

	aws2 := mockInstancesResp(instances)
	_, err2 := aws2.NodeAddresses("instance1")
	if err2 == nil {
		t.Errorf("Should error when multiple instances found")
	}

	aws3 := mockInstancesResp(instances[0:1])
	addrs3, err3 := aws3.NodeAddresses("instance1")
	if err3 != nil {
		t.Errorf("Should not error when instance found")
	}
	if len(addrs3) != 1 {
		t.Errorf("Should return exactly one NodeAddress")
	}
	if e, a := instances[0].PrivateIpAddress, addrs3[0].Address; e != a {
		t.Errorf("Expected %v, got %v", e, a)
	}
}

func TestGetRegion(t *testing.T) {
	aws := mockAvailabilityZone("us-west-2", "us-west-2e")
	zones, ok := aws.Zones()
	if !ok {
		t.Fatalf("Unexpected missing zones impl")
	}
	zone, err := zones.GetZone()
	if err != nil {
		t.Fatalf("unexpected error %v", err)
	}
	if zone.Region != "us-west-2" {
		t.Errorf("Unexpected region: %s", zone.Region)
	}
	if zone.FailureDomain != "us-west-2e" {
		t.Errorf("Unexpected FailureDomain: %s", zone.FailureDomain)
	}
}
<|endoftext|>


<commit_before>package dockerfile

import (
	"bufio"
	"fmt"
	"io"
	"regexp"
	"strings"
	"unicode"
)

type dockerfile [][]string

// Parser is a Dockerfile parser
type Parser interface {
	Parse(input io.Reader) (Dockerfile, error)
}

type parser struct{}

// NewParser creates a new Dockerfile parser
func NewParser() Parser {
	return &parser{}
}

// Dockerfile represents a parsed Dockerfile
type Dockerfile interface {
	GetDirective(name string) ([]string, bool)
}

// Parse parses an input Dockerfile
func (_ *parser) Parse(input io.Reader) (Dockerfile, error) {
	d := dockerfile{}
	scanner := bufio.NewScanner(input)
	for {
		line, ok := nextLine(scanner, true)
		if !ok {
			break
		}
		parts, err := parseLine(line)
		if err != nil {
			return nil, err
		}
		d = append(d, parts)
	}
	return d, nil
}

// GetDirective returns a list of lines that begin with the given directive
// and a flag that is true if the directive was found in the Dockerfile
func (d dockerfile) GetDirective(s string) ([]string, bool) {
	values := []string{}
	s = strings.ToLower(s)
	for _, line := range d {
		if strings.ToLower(line[0]) == s {
			values = append(values, line[1])
		}
	}
	return values, len(values) > 0
}

func isComment(line string) bool {
	return strings.HasPrefix(line, "#")
}

func hasContinuation(line string) bool {
	return strings.HasSuffix(strings.TrimRightFunc(line, unicode.IsSpace), "\\")
}

func stripContinuation(line string) string {
	line = strings.TrimRightFunc(line, unicode.IsSpace)
	return line[:len(line)-1]
}

func nextLine(scanner *bufio.Scanner, trimLeft bool) (string, bool) {
	if scanner.Scan() {
		line := scanner.Text()
		if trimLeft {
			line = strings.TrimLeftFunc(line, unicode.IsSpace)
		}
		if line == "" || isComment(line) {
			return nextLine(scanner, true)
		}
		if hasContinuation(line) {
			line := stripContinuation(line)
			next, ok := nextLine(scanner, false)
			if ok {
				return line + next, true
			} else {
				return line, true
			}
		}
		return line, true
	}
	return "", false
}

var dockerLineDelim = regexp.MustCompile(`[\t\v\f\r ]+`)

func parseLine(line string) ([]string, error) {
	parts := dockerLineDelim.Split(line, 2)
	if len(parts) != 2 {
		return nil, fmt.Errorf("Invalid Dockerfile")
	}
	return parts, nil
}
<commit_msg>generate: Use Docker parser for validation<commit_after>package dockerfile

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"regexp"
	"strings"
	"unicode"

	dparser "github.com/docker/docker/builder/parser"
)

type dockerfile [][]string

// Parser is a Dockerfile parser
type Parser interface {
	Parse(input io.Reader) (Dockerfile, error)
}

type parser struct{}

// NewParser creates a new Dockerfile parser
func NewParser() Parser {
	return &parser{}
}

// Dockerfile represents a parsed Dockerfile
type Dockerfile interface {
	GetDirective(name string) ([]string, bool)
}

// Parse parses an input Dockerfile
func (_ *parser) Parse(input io.Reader) (Dockerfile, error) {
	buf := bufio.NewReader(input)
	bts, err := buf.Peek(buf.Buffered())
	if err != nil {
		return nil, err
	}
	parsedByDocker := bytes.NewBuffer(bts)
	// Add one more level of validation by using the Docker parser
	if _, err := dparser.Parse(parsedByDocker); err != nil {
		return nil, fmt.Errorf("cannot parse Dockerfile: %v", err)
	}

	d := dockerfile{}
	scanner := bufio.NewScanner(input)
	for {
		line, ok := nextLine(scanner, true)
		if !ok {
			break
		}
		parts, err := parseLine(line)
		if err != nil {
			return nil, err
		}
		d = append(d, parts)
	}
	return d, nil
}

// GetDirective returns a list of lines that begin with the given directive
// and a flag that is true if the directive was found in the Dockerfile
func (d dockerfile) GetDirective(s string) ([]string, bool) {
	values := []string{}
	s = strings.ToLower(s)
	for _, line := range d {
		if strings.ToLower(line[0]) == s {
			values = append(values, line[1])
		}
	}
	return values, len(values) > 0
}

func isComment(line string) bool {
	return strings.HasPrefix(line, "#")
}

func hasContinuation(line string) bool {
	return strings.HasSuffix(strings.TrimRightFunc(line, unicode.IsSpace), "\\")
}

func stripContinuation(line string) string {
	line = strings.TrimRightFunc(line, unicode.IsSpace)
	return line[:len(line)-1]
}

func nextLine(scanner *bufio.Scanner, trimLeft bool) (string, bool) {
	if scanner.Scan() {
		line := scanner.Text()
		if trimLeft {
			line = strings.TrimLeftFunc(line, unicode.IsSpace)
		}
		if line == "" || isComment(line) {
			return nextLine(scanner, true)
		}
		if hasContinuation(line) {
			line := stripContinuation(line)
			next, ok := nextLine(scanner, false)
			if ok {
				return line + next, true
			} else {
				return line, true
			}
		}
		return line, true
	}
	return "", false
}

var dockerLineDelim = regexp.MustCompile(`[\t\v\f\r ]+`)

func parseLine(line string) ([]string, error) {
	parts := dockerLineDelim.Split(line, 2)
	if len(parts) != 2 {
		return nil, fmt.Errorf("Invalid Dockerfile")
	}
	return parts, nil
}
<|endoftext|>


<commit_before>package filer

import (
	"bytes"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/wdclient"
	"io"
	"math"
	"time"

	"github.com/golang/protobuf/proto"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
)

const (
	ManifestBatch = 10000
)

func HasChunkManifest(chunks []*filer_pb.FileChunk) bool {
	for _, chunk := range chunks {
		if chunk.IsChunkManifest {
			return true
		}
	}
	return false
}

func SeparateManifestChunks(chunks []*filer_pb.FileChunk) (manifestChunks, nonManifestChunks []*filer_pb.FileChunk) {
	for _, c := range chunks {
		if c.IsChunkManifest {
			manifestChunks = append(manifestChunks, c)
		} else {
			nonManifestChunks = append(nonManifestChunks, c)
		}
	}
	return
}

func ResolveChunkManifest(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, startOffset, stopOffset int64) (dataChunks, manifestChunks []*filer_pb.FileChunk, manifestResolveErr error) {
	// TODO maybe parallel this
	for _, chunk := range chunks {

		if max(chunk.Offset, startOffset) >= min(chunk.Offset+int64(chunk.Size), stopOffset) {
			continue
		}

		if !chunk.IsChunkManifest {
			dataChunks = append(dataChunks, chunk)
			continue
		}

		resolvedChunks, err := ResolveOneChunkManifest(lookupFileIdFn, chunk)
		if err != nil {
			return chunks, nil, err
		}

		manifestChunks = append(manifestChunks, chunk)
		// recursive
		dchunks, mchunks, subErr := ResolveChunkManifest(lookupFileIdFn, resolvedChunks, startOffset, stopOffset)
		if subErr != nil {
			return chunks, nil, subErr
		}
		dataChunks = append(dataChunks, dchunks...)
		manifestChunks = append(manifestChunks, mchunks...)
	}
	return
}

func ResolveOneChunkManifest(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunk *filer_pb.FileChunk) (dataChunks []*filer_pb.FileChunk, manifestResolveErr error) {
	if !chunk.IsChunkManifest {
		return
	}

	// IsChunkManifest
	data, err := fetchChunk(lookupFileIdFn, chunk.GetFileIdString(), chunk.CipherKey, chunk.IsCompressed)
	if err != nil {
		return nil, fmt.Errorf("fail to read manifest %s: %v", chunk.GetFileIdString(), err)
	}
	m := &filer_pb.FileChunkManifest{}
	if err := proto.Unmarshal(data, m); err != nil {
		return nil, fmt.Errorf("fail to unmarshal manifest %s: %v", chunk.GetFileIdString(), err)
	}

	// recursive
	filer_pb.AfterEntryDeserialization(m.Chunks)
	return m.Chunks, nil
}

// TODO fetch from cache for weed mount?
func fetchChunk(lookupFileIdFn wdclient.LookupFileIdFunctionType, fileId string, cipherKey []byte, isGzipped bool) ([]byte, error) {
	urlStrings, err := lookupFileIdFn(fileId)
	if err != nil {
		glog.Errorf("operation LookupFileId %s failed, err: %v", fileId, err)
		return nil, err
	}
	return retriedFetchChunkData(urlStrings, cipherKey, isGzipped, true, 0, 0)
}

func retriedFetchChunkData(urlStrings []string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int) ([]byte, error) {

	var err error
	var shouldRetry bool
	receivedData := make([]byte, 0, size)

	for waitTime := time.Second; waitTime < util.RetryWaitTime; waitTime += waitTime / 2 {
		for _, urlString := range urlStrings {
			receivedData = receivedData[:0]
			shouldRetry, err = util.ReadUrlAsStream(urlString+"?readDeleted=true", cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) {
				receivedData = append(receivedData, data...)
			})
			if !shouldRetry {
				break
			}
			if err != nil {
				glog.V(0).Infof("read %s failed, err: %v", urlString, err)
			} else {
				break
			}
		}
		if err != nil && shouldRetry {
			glog.V(0).Infof("retry reading in %v", waitTime)
			time.Sleep(waitTime)
		} else {
			break
		}
	}

	return receivedData, err

}

func retriedStreamFetchChunkData(writer io.Writer, urlStrings []string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int) (err error) {

	var shouldRetry bool
	var written int

	for waitTime := time.Second; waitTime < util.RetryWaitTime; waitTime += waitTime / 2 {
		for _, urlString := range urlStrings {
			shouldRetry, err = util.ReadUrlAsStream(urlString+"?readDeleted=true", cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) {
				writer.Write(data)
				written += len(data)
			})
			shouldRetry = shouldRetry && written == 0
			if !shouldRetry {
				break
			}
			if err != nil {
				glog.V(0).Infof("read %s failed, err: %v", urlString, err)
				if written > 0 {
					break
				}
			} else {
				break
			}
		}
		if err != nil && shouldRetry {
			glog.V(0).Infof("retry reading in %v", waitTime)
			time.Sleep(waitTime)
		} else {
			break
		}
	}

	return err

}

func MaybeManifestize(saveFunc SaveDataAsChunkFunctionType, inputChunks []*filer_pb.FileChunk) (chunks []*filer_pb.FileChunk, err error) {
	return doMaybeManifestize(saveFunc, inputChunks, ManifestBatch, mergeIntoManifest)
}

func doMaybeManifestize(saveFunc SaveDataAsChunkFunctionType, inputChunks []*filer_pb.FileChunk, mergeFactor int, mergefn func(saveFunc SaveDataAsChunkFunctionType, dataChunks []*filer_pb.FileChunk) (manifestChunk *filer_pb.FileChunk, err error)) (chunks []*filer_pb.FileChunk, err error) {

	var dataChunks []*filer_pb.FileChunk
	for _, chunk := range inputChunks {
		if !chunk.IsChunkManifest {
			dataChunks = append(dataChunks, chunk)
		} else {
			chunks = append(chunks, chunk)
		}
	}

	remaining := len(dataChunks)
	for i := 0; i+mergeFactor <= len(dataChunks); i += mergeFactor {
		chunk, err := mergefn(saveFunc, dataChunks[i:i+mergeFactor])
		if err != nil {
			return dataChunks, err
		}
		chunks = append(chunks, chunk)
		remaining -= mergeFactor
	}
	// remaining
	for i := len(dataChunks) - remaining; i < len(dataChunks); i++ {
		chunks = append(chunks, dataChunks[i])
	}
	return
}

func mergeIntoManifest(saveFunc SaveDataAsChunkFunctionType, dataChunks []*filer_pb.FileChunk) (manifestChunk *filer_pb.FileChunk, err error) {

	filer_pb.BeforeEntrySerialization(dataChunks)

	// create and serialize the manifest
	data, serErr := proto.Marshal(&filer_pb.FileChunkManifest{
		Chunks: dataChunks,
	})
	if serErr != nil {
		return nil, fmt.Errorf("serializing manifest: %v", serErr)
	}

	minOffset, maxOffset := int64(math.MaxInt64), int64(math.MinInt64)
	for _, chunk := range dataChunks {
		if minOffset > int64(chunk.Offset) {
			minOffset = chunk.Offset
		}
		if maxOffset < int64(chunk.Size)+chunk.Offset {
			maxOffset = int64(chunk.Size) + chunk.Offset
		}
	}

	manifestChunk, _, _, err = saveFunc(bytes.NewReader(data), "", 0)
	if err != nil {
		return nil, err
	}
	manifestChunk.IsChunkManifest = true
	manifestChunk.Offset = minOffset
	manifestChunk.Size = uint64(maxOffset - minOffset)

	return
}

type SaveDataAsChunkFunctionType func(reader io.Reader, name string, offset int64) (chunk *filer_pb.FileChunk, collection, replication string, err error)
<commit_msg>adjust retry logic in case some data is partially written<commit_after>package filer

import (
	"bytes"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/wdclient"
	"io"
	"math"
	"time"

	"github.com/golang/protobuf/proto"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
)

const (
	ManifestBatch = 10000
)

func HasChunkManifest(chunks []*filer_pb.FileChunk) bool {
	for _, chunk := range chunks {
		if chunk.IsChunkManifest {
			return true
		}
	}
	return false
}

func SeparateManifestChunks(chunks []*filer_pb.FileChunk) (manifestChunks, nonManifestChunks []*filer_pb.FileChunk) {
	for _, c := range chunks {
		if c.IsChunkManifest {
			manifestChunks = append(manifestChunks, c)
		} else {
			nonManifestChunks = append(nonManifestChunks, c)
		}
	}
	return
}

func ResolveChunkManifest(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, startOffset, stopOffset int64) (dataChunks, manifestChunks []*filer_pb.FileChunk, manifestResolveErr error) {
	// TODO maybe parallel this
	for _, chunk := range chunks {

		if max(chunk.Offset, startOffset) >= min(chunk.Offset+int64(chunk.Size), stopOffset) {
			continue
		}

		if !chunk.IsChunkManifest {
			dataChunks = append(dataChunks, chunk)
			continue
		}

		resolvedChunks, err := ResolveOneChunkManifest(lookupFileIdFn, chunk)
		if err != nil {
			return chunks, nil, err
		}

		manifestChunks = append(manifestChunks, chunk)
		// recursive
		dchunks, mchunks, subErr := ResolveChunkManifest(lookupFileIdFn, resolvedChunks, startOffset, stopOffset)
		if subErr != nil {
			return chunks, nil, subErr
		}
		dataChunks = append(dataChunks, dchunks...)
		manifestChunks = append(manifestChunks, mchunks...)
	}
	return
}

func ResolveOneChunkManifest(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunk *filer_pb.FileChunk) (dataChunks []*filer_pb.FileChunk, manifestResolveErr error) {
	if !chunk.IsChunkManifest {
		return
	}

	// IsChunkManifest
	data, err := fetchChunk(lookupFileIdFn, chunk.GetFileIdString(), chunk.CipherKey, chunk.IsCompressed)
	if err != nil {
		return nil, fmt.Errorf("fail to read manifest %s: %v", chunk.GetFileIdString(), err)
	}
	m := &filer_pb.FileChunkManifest{}
	if err := proto.Unmarshal(data, m); err != nil {
		return nil, fmt.Errorf("fail to unmarshal manifest %s: %v", chunk.GetFileIdString(), err)
	}

	// recursive
	filer_pb.AfterEntryDeserialization(m.Chunks)
	return m.Chunks, nil
}

// TODO fetch from cache for weed mount?
func fetchChunk(lookupFileIdFn wdclient.LookupFileIdFunctionType, fileId string, cipherKey []byte, isGzipped bool) ([]byte, error) {
	urlStrings, err := lookupFileIdFn(fileId)
	if err != nil {
		glog.Errorf("operation LookupFileId %s failed, err: %v", fileId, err)
		return nil, err
	}
	return retriedFetchChunkData(urlStrings, cipherKey, isGzipped, true, 0, 0)
}

func retriedFetchChunkData(urlStrings []string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int) ([]byte, error) {

	var err error
	var shouldRetry bool
	receivedData := make([]byte, 0, size)

	for waitTime := time.Second; waitTime < util.RetryWaitTime; waitTime += waitTime / 2 {
		for _, urlString := range urlStrings {
			receivedData = receivedData[:0]
			shouldRetry, err = util.ReadUrlAsStream(urlString+"?readDeleted=true", cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) {
				receivedData = append(receivedData, data...)
			})
			if !shouldRetry {
				break
			}
			if err != nil {
				glog.V(0).Infof("read %s failed, err: %v", urlString, err)
			} else {
				break
			}
		}
		if err != nil && shouldRetry {
			glog.V(0).Infof("retry reading in %v", waitTime)
			time.Sleep(waitTime)
		} else {
			break
		}
	}

	return receivedData, err

}

func retriedStreamFetchChunkData(writer io.Writer, urlStrings []string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int) (err error) {

	var shouldRetry bool
	var totalWritten int

	for waitTime := time.Second; waitTime < util.RetryWaitTime; waitTime += waitTime / 2 {
		for _, urlString := range urlStrings {
			var localProcesed int
			shouldRetry, err = util.ReadUrlAsStream(urlString+"?readDeleted=true", cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) {
				if totalWritten > localProcesed {
					toBeSkipped := totalWritten - localProcesed
					if len(data) <= toBeSkipped {
						localProcesed += len(data)
						return // skip if already processed
					}
					data = data[len(data)-toBeSkipped:]
					localProcesed += toBeSkipped
				}
				writer.Write(data)
				localProcesed += len(data)
				totalWritten += len(data)
			})
			if !shouldRetry {
				break
			}
			if err != nil {
				glog.V(0).Infof("read %s failed, err: %v", urlString, err)
			} else {
				break
			}
		}
		if err != nil && shouldRetry {
			glog.V(0).Infof("retry reading in %v", waitTime)
			time.Sleep(waitTime)
		} else {
			break
		}
	}

	return err

}

func MaybeManifestize(saveFunc SaveDataAsChunkFunctionType, inputChunks []*filer_pb.FileChunk) (chunks []*filer_pb.FileChunk, err error) {
	return doMaybeManifestize(saveFunc, inputChunks, ManifestBatch, mergeIntoManifest)
}

func doMaybeManifestize(saveFunc SaveDataAsChunkFunctionType, inputChunks []*filer_pb.FileChunk, mergeFactor int, mergefn func(saveFunc SaveDataAsChunkFunctionType, dataChunks []*filer_pb.FileChunk) (manifestChunk *filer_pb.FileChunk, err error)) (chunks []*filer_pb.FileChunk, err error) {

	var dataChunks []*filer_pb.FileChunk
	for _, chunk := range inputChunks {
		if !chunk.IsChunkManifest {
			dataChunks = append(dataChunks, chunk)
		} else {
			chunks = append(chunks, chunk)
		}
	}

	remaining := len(dataChunks)
	for i := 0; i+mergeFactor <= len(dataChunks); i += mergeFactor {
		chunk, err := mergefn(saveFunc, dataChunks[i:i+mergeFactor])
		if err != nil {
			return dataChunks, err
		}
		chunks = append(chunks, chunk)
		remaining -= mergeFactor
	}
	// remaining
	for i := len(dataChunks) - remaining; i < len(dataChunks); i++ {
		chunks = append(chunks, dataChunks[i])
	}
	return
}

func mergeIntoManifest(saveFunc SaveDataAsChunkFunctionType, dataChunks []*filer_pb.FileChunk) (manifestChunk *filer_pb.FileChunk, err error) {

	filer_pb.BeforeEntrySerialization(dataChunks)

	// create and serialize the manifest
	data, serErr := proto.Marshal(&filer_pb.FileChunkManifest{
		Chunks: dataChunks,
	})
	if serErr != nil {
		return nil, fmt.Errorf("serializing manifest: %v", serErr)
	}

	minOffset, maxOffset := int64(math.MaxInt64), int64(math.MinInt64)
	for _, chunk := range dataChunks {
		if minOffset > int64(chunk.Offset) {
			minOffset = chunk.Offset
		}
		if maxOffset < int64(chunk.Size)+chunk.Offset {
			maxOffset = int64(chunk.Size) + chunk.Offset
		}
	}

	manifestChunk, _, _, err = saveFunc(bytes.NewReader(data), "", 0)
	if err != nil {
		return nil, err
	}
	manifestChunk.IsChunkManifest = true
	manifestChunk.Offset = minOffset
	manifestChunk.Size = uint64(maxOffset - minOffset)

	return
}

type SaveDataAsChunkFunctionType func(reader io.Reader, name string, offset int64) (chunk *filer_pb.FileChunk, collection, replication string, err error)
<|endoftext|>


<commit_before>package middleware

import (
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/grafana/grafana/pkg/infra/metrics"
	"gopkg.in/macaron.v1"
)

func RequestMetrics(handler string) macaron.Handler {
	return func(res http.ResponseWriter, req *http.Request, c *macaron.Context) {
		rw := res.(macaron.ResponseWriter)
		now := time.Now()
		c.Next()

		status := rw.Status()

		code := sanitizeCode(status)
		method := sanitizeMethod(req.Method)
		metrics.MHttpRequestTotal.WithLabelValues(handler, code, method).Inc()
		duration := time.Since(now).Nanoseconds() / int64(time.Millisecond)
		metrics.MHttpRequestSummary.WithLabelValues(handler, code, method).Observe(float64(duration))

		if strings.HasPrefix(req.RequestURI, "/api/datasources/proxy") {
			countProxyRequests(status)
		} else if strings.HasPrefix(req.RequestURI, "/api/") {
			countApiRequests(status)
		} else {
			countPageRequests(status)
		}
	}
}

func countApiRequests(status int) {
	switch status {
	case 200:
		metrics.MApiStatus.WithLabelValues("200").Inc()
	case 404:
		metrics.MApiStatus.WithLabelValues("404").Inc()
	case 500:
		metrics.MApiStatus.WithLabelValues("500").Inc()
	default:
		metrics.MApiStatus.WithLabelValues("unknown").Inc()
	}
}

func countPageRequests(status int) {
	switch status {
	case 200:
		metrics.MPageStatus.WithLabelValues("200").Inc()
	case 404:
		metrics.MPageStatus.WithLabelValues("404").Inc()
	case 500:
		metrics.MPageStatus.WithLabelValues("500").Inc()
	default:
		metrics.MPageStatus.WithLabelValues("unknown").Inc()
	}
}

func countProxyRequests(status int) {
	switch status {
	case 200:
		metrics.MProxyStatus.WithLabelValues("200").Inc()
	case 404:
		metrics.MProxyStatus.WithLabelValues("400").Inc()
	case 500:
		metrics.MProxyStatus.WithLabelValues("500").Inc()
	default:
		metrics.MProxyStatus.WithLabelValues("unknown").Inc()
	}
}

func sanitizeMethod(m string) string {
	switch m {
	case "GET", "get":
		return "get"
	case "PUT", "put":
		return "put"
	case "HEAD", "head":
		return "head"
	case "POST", "post":
		return "post"
	case "DELETE", "delete":
		return "delete"
	case "CONNECT", "connect":
		return "connect"
	case "OPTIONS", "options":
		return "options"
	case "NOTIFY", "notify":
		return "notify"
	default:
		return strings.ToLower(m)
	}
}

// If the wrapped http.Handler has not set a status code, i.e. the value is
// currently 0, santizeCode will return 200, for consistency with behavior in
// the stdlib.
func sanitizeCode(s int) string {
	switch s {
	case 100:
		return "100"
	case 101:
		return "101"

	case 200, 0:
		return "200"
	case 201:
		return "201"
	case 202:
		return "202"
	case 203:
		return "203"
	case 204:
		return "204"
	case 205:
		return "205"
	case 206:
		return "206"

	case 300:
		return "300"
	case 301:
		return "301"
	case 302:
		return "302"
	case 304:
		return "304"
	case 305:
		return "305"
	case 307:
		return "307"

	case 400:
		return "400"
	case 401:
		return "401"
	case 402:
		return "402"
	case 403:
		return "403"
	case 404:
		return "404"
	case 405:
		return "405"
	case 406:
		return "406"
	case 407:
		return "407"
	case 408:
		return "408"
	case 409:
		return "409"
	case 410:
		return "410"
	case 411:
		return "411"
	case 412:
		return "412"
	case 413:
		return "413"
	case 414:
		return "414"
	case 415:
		return "415"
	case 416:
		return "416"
	case 417:
		return "417"
	case 418:
		return "418"

	case 500:
		return "500"
	case 501:
		return "501"
	case 502:
		return "502"
	case 503:
		return "503"
	case 504:
		return "504"
	case 505:
		return "505"

	case 428:
		return "428"
	case 429:
		return "429"
	case 431:
		return "431"
	case 511:
		return "511"

	default:
		return strconv.Itoa(s)
	}
}
<commit_msg>Metrics: Add gauge for requests currently in flight (#22168)<commit_after>package middleware

import (
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/grafana/grafana/pkg/infra/metrics"
	"github.com/prometheus/client_golang/prometheus"
	"gopkg.in/macaron.v1"
)

var (
	httpRequestsInFlight prometheus.Gauge
)

func init() {
	httpRequestsInFlight = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Name: "http_request_in_flight",
			Help: "A gauge of requests currently being served by Grafana.",
		},
	)

	prometheus.MustRegister(httpRequestsInFlight)
}

// RequestMetrics is a middleware handler that instruments the request
func RequestMetrics(handler string) macaron.Handler {
	return func(res http.ResponseWriter, req *http.Request, c *macaron.Context) {
		rw := res.(macaron.ResponseWriter)
		now := time.Now()
		httpRequestsInFlight.Inc()
		defer httpRequestsInFlight.Dec()
		c.Next()

		status := rw.Status()

		code := sanitizeCode(status)
		method := sanitizeMethod(req.Method)
		metrics.MHttpRequestTotal.WithLabelValues(handler, code, method).Inc()
		duration := time.Since(now).Nanoseconds() / int64(time.Millisecond)
		metrics.MHttpRequestSummary.WithLabelValues(handler, code, method).Observe(float64(duration))

		if strings.HasPrefix(req.RequestURI, "/api/datasources/proxy") {
			countProxyRequests(status)
		} else if strings.HasPrefix(req.RequestURI, "/api/") {
			countApiRequests(status)
		} else {
			countPageRequests(status)
		}
	}
}

func countApiRequests(status int) {
	switch status {
	case 200:
		metrics.MApiStatus.WithLabelValues("200").Inc()
	case 404:
		metrics.MApiStatus.WithLabelValues("404").Inc()
	case 500:
		metrics.MApiStatus.WithLabelValues("500").Inc()
	default:
		metrics.MApiStatus.WithLabelValues("unknown").Inc()
	}
}

func countPageRequests(status int) {
	switch status {
	case 200:
		metrics.MPageStatus.WithLabelValues("200").Inc()
	case 404:
		metrics.MPageStatus.WithLabelValues("404").Inc()
	case 500:
		metrics.MPageStatus.WithLabelValues("500").Inc()
	default:
		metrics.MPageStatus.WithLabelValues("unknown").Inc()
	}
}

func countProxyRequests(status int) {
	switch status {
	case 200:
		metrics.MProxyStatus.WithLabelValues("200").Inc()
	case 404:
		metrics.MProxyStatus.WithLabelValues("400").Inc()
	case 500:
		metrics.MProxyStatus.WithLabelValues("500").Inc()
	default:
		metrics.MProxyStatus.WithLabelValues("unknown").Inc()
	}
}

func sanitizeMethod(m string) string {
	switch m {
	case "GET", "get":
		return "get"
	case "PUT", "put":
		return "put"
	case "HEAD", "head":
		return "head"
	case "POST", "post":
		return "post"
	case "DELETE", "delete":
		return "delete"
	case "CONNECT", "connect":
		return "connect"
	case "OPTIONS", "options":
		return "options"
	case "NOTIFY", "notify":
		return "notify"
	default:
		return strings.ToLower(m)
	}
}

// If the wrapped http.Handler has not set a status code, i.e. the value is
// currently 0, santizeCode will return 200, for consistency with behavior in
// the stdlib.
func sanitizeCode(s int) string {
	switch s {
	case 100:
		return "100"
	case 101:
		return "101"

	case 200, 0:
		return "200"
	case 201:
		return "201"
	case 202:
		return "202"
	case 203:
		return "203"
	case 204:
		return "204"
	case 205:
		return "205"
	case 206:
		return "206"

	case 300:
		return "300"
	case 301:
		return "301"
	case 302:
		return "302"
	case 304:
		return "304"
	case 305:
		return "305"
	case 307:
		return "307"

	case 400:
		return "400"
	case 401:
		return "401"
	case 402:
		return "402"
	case 403:
		return "403"
	case 404:
		return "404"
	case 405:
		return "405"
	case 406:
		return "406"
	case 407:
		return "407"
	case 408:
		return "408"
	case 409:
		return "409"
	case 410:
		return "410"
	case 411:
		return "411"
	case 412:
		return "412"
	case 413:
		return "413"
	case 414:
		return "414"
	case 415:
		return "415"
	case 416:
		return "416"
	case 417:
		return "417"
	case 418:
		return "418"

	case 500:
		return "500"
	case 501:
		return "501"
	case 502:
		return "502"
	case 503:
		return "503"
	case 504:
		return "504"
	case 505:
		return "505"

	case 428:
		return "428"
	case 429:
		return "429"
	case 431:
		return "431"
	case 511:
		return "511"

	default:
		return strconv.Itoa(s)
	}
}
<|endoftext|>


<commit_before>package config

import (
	"database/sql"
	"github.com/fritzpay/paymentd/pkg/testutil"
	. "github.com/smartystreets/goconvey/convey"
	"math/rand"
	"strconv"
	"testing"
)

func TestConfigEntry(t *testing.T) {
	Convey("Given a DB connection", t, testutil.WithPaymentDB(t, func(db *sql.DB) {
		Convey("Given a random entry name", func() {
			name := "test" + strconv.FormatInt(rand.Int63(), 10)

			Convey("When there is no entry with the given name", func() {
				_, err := db.Exec("delete from config where name = ?", name)
				So(err, ShouldBeNil)

				Convey("When entering an entry with a random value", func() {
					value := "entry" + strconv.FormatInt(rand.Int63(), 10)
					entry := Entry{
						Name:  name,
						Value: value,
					}
					err := InsertEntryDB(db, entry)

					Convey("It should succeed", func() {
						So(err, ShouldBeNil)
					})

					Convey("When retrieving the entry by the name", func() {
						entry, err := EntryByNameDB(db, name)

						Convey("It should succeed", func() {
							So(err, ShouldBeNil)
						})
						Convey("It should match the entered value", func() {
							So(entry.Value, ShouldEqual, value)
						})
					})
				})
			})
		})

		Convey("When there is no entry with a given name", func() {
			_, err := db.Exec("truncate config")
			So(err, ShouldBeNil)

			Convey("When selecting a nonexistent entry by name", func() {
				_, err := EntryByNameDB(db, "nonexistent")
				Convey("It should return an error", func() {
					So(err, ShouldNotBeNil)
					So(err, ShouldEqual, ErrEntryNotFound)
				})
			})
		})
	}))
}
<commit_msg>dot (Day Of Testing): extend config tests<commit_after>package config

import (
	"code.google.com/p/go.crypto/bcrypt"
	"database/sql"
	"github.com/fritzpay/paymentd/pkg/testutil"
	. "github.com/smartystreets/goconvey/convey"
	"math/rand"
	"strconv"
	"testing"
)

func TestConfigEntry(t *testing.T) {
	Convey("Given a DB connection", t, testutil.WithPaymentDB(t, func(db *sql.DB) {
		Convey("Given a random entry name", func() {
			name := "test" + strconv.FormatInt(rand.Int63(), 10)

			Convey("When there is no entry with the given name", func() {
				_, err := db.Exec("delete from config where name = ?", name)
				So(err, ShouldBeNil)

				Convey("When entering an entry with a random value", func() {
					value := "entry" + strconv.FormatInt(rand.Int63(), 10)
					entry := Entry{
						Name:  name,
						Value: value,
					}
					err := InsertEntryDB(db, entry)

					Convey("It should succeed", func() {
						So(err, ShouldBeNil)
					})

					Convey("When retrieving the entry by the name", func() {
						entry, err := EntryByNameDB(db, name)

						Convey("It should succeed", func() {
							So(err, ShouldBeNil)
						})
						Convey("It should match the entered value", func() {
							So(entry.Value, ShouldEqual, value)
						})
					})
				})
			})
		})

		Convey("When there is no entry with a given name", func() {
			_, err := db.Exec("truncate config")
			So(err, ShouldBeNil)

			Convey("When selecting a nonexistent entry by name", func() {
				nonExistent, err := EntryByNameDB(db, "nonexistent")
				Convey("It should return an error", func() {
					So(err, ShouldNotBeNil)
					So(err, ShouldEqual, ErrEntryNotFound)
					So(nonExistent.Empty(), ShouldBeTrue)
				})
			})
		})
	}))
}

func TestConfigSetPassword(t *testing.T) {
	Convey("Given a payment DB connection", t, testutil.WithPaymentDB(t, func(db *sql.DB) {
		Convey("Given a password setter", func() {
			pw := SetPassword([]byte("password"))

			Convey("When setting the password", func() {
				err := Set(db, pw)
				Convey("It should succeed", func() {
					So(err, ShouldBeNil)

					Convey("When retrieving the password entry", func() {
						val, err := EntryByNameDB(db, ConfigNameSystemPassword)
						So(err, ShouldBeNil)
						So(val.Empty(), ShouldBeFalse)

						Convey("It should match the password", func() {
							err := bcrypt.CompareHashAndPassword([]byte(val.Value), []byte("password"))
							So(err, ShouldBeNil)
						})
					})
				})
			})
		})
	}))
}
<|endoftext|>


<commit_before>package rpcd

import (
	"encoding/gob"
	"errors"
	imgclient "github.com/Symantec/Dominator/imageserver/client"
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/format"
	"github.com/Symantec/Dominator/lib/hash"
	objectclient "github.com/Symantec/Dominator/lib/objectserver/client"
	"github.com/Symantec/Dominator/lib/srpc"
	"github.com/Symantec/Dominator/proto/imageserver"
	"io"
	"time"
)

func (t *srpcType) replicator() {
	initialTimeout := time.Second * 15
	timeout := initialTimeout
	var nextSleepStopTime time.Time
	for {
		nextSleepStopTime = time.Now().Add(timeout)
		if client, err := srpc.DialHTTP("tcp", t.replicationMaster,
			timeout); err != nil {
			t.logger.Printf("Error dialling: %s %s\n", t.replicationMaster, err)
		} else {
			if conn, err := client.Call(
				"ImageServer.GetImageUpdates"); err != nil {
				t.logger.Println(err)
			} else {
				if err := t.getUpdates(conn); err != nil {
					if err == io.EOF {
						t.logger.Println(
							"Connection to image replicator closed")
						if nextSleepStopTime.Sub(time.Now()) < 1 {
							timeout = initialTimeout
						}
					} else {
						t.logger.Println(err)
					}
				}
				conn.Close()
			}
			client.Close()
		}
		time.Sleep(nextSleepStopTime.Sub(time.Now()))
		if timeout < time.Minute {
			timeout *= 2
		}
	}
}

func (t *srpcType) getUpdates(conn *srpc.Conn) error {
	t.logger.Printf("Image replicator: connected to: %s\n", t.replicationMaster)
	replicationStartTime := time.Now()
	decoder := gob.NewDecoder(conn)
	initialImages := make(map[string]struct{})
	if t.archiveMode {
		initialImages = nil
	}
	for {
		var imageUpdate imageserver.ImageUpdate
		if err := decoder.Decode(&imageUpdate); err != nil {
			if err == io.EOF {
				return err
			}
			return errors.New("decode err: " + err.Error())
		}
		switch imageUpdate.Operation {
		case imageserver.OperationAddImage:
			if imageUpdate.Name == "" {
				if initialImages != nil {
					t.deleteMissingImages(initialImages)
					initialImages = nil
				}
				t.logger.Printf("Replicated all current images in %s\n",
					format.Duration(time.Since(replicationStartTime)))
				continue
			}
			if initialImages != nil {
				initialImages[imageUpdate.Name] = struct{}{}
			}
			if err := t.addImage(imageUpdate.Name); err != nil {
				return err
			}
		case imageserver.OperationDeleteImage:
			if t.archiveMode {
				continue
			}
			t.logger.Printf("Replicator(%s): delete image\n", imageUpdate.Name)
			if err := t.imageDataBase.DeleteImage(imageUpdate.Name,
				nil); err != nil {
				return err
			}
		case imageserver.OperationMakeDirectory:
			directory := imageUpdate.Directory
			if directory == nil {
				return errors.New("nil imageUpdate.Directory")
			}
			if err := t.imageDataBase.UpdateDirectory(*directory); err != nil {
				return err
			}
		}
	}
}

func (t *srpcType) deleteMissingImages(imagesToKeep map[string]struct{}) {
	missingImages := make([]string, 0)
	for _, imageName := range t.imageDataBase.ListImages() {
		if _, ok := imagesToKeep[imageName]; !ok {
			missingImages = append(missingImages, imageName)
		}
	}
	for _, imageName := range missingImages {
		t.logger.Printf("Replicator(%s): delete missing image\n", imageName)
		if err := t.imageDataBase.DeleteImage(imageName, nil); err != nil {
			t.logger.Println(err)
		}
	}
}

func (t *srpcType) addImage(name string) error {
	timeout := time.Second * 60
	if t.imageDataBase.CheckImage(name) || t.checkImageBeingInjected(name) {
		return nil
	}
	t.logger.Printf("Replicator(%s): add image\n", name)
	client, err := srpc.DialHTTP("tcp", t.replicationMaster, timeout)
	if err != nil {
		return err
	}
	defer client.Close()
	img, err := imgclient.GetImage(client, name)
	if err != nil {
		return err
	}
	if img == nil {
		return errors.New(name + ": not found")
	}
	t.logger.Printf("Replicator(%s): downloaded image\n", name)
	if t.archiveMode && !img.ExpiresAt.IsZero() && !*archiveExpiringImages {
		t.logger.Printf(
			"Replicator(%s): ignoring expiring image in archiver mode\n",
			name)
		return nil
	}
	img.FileSystem.RebuildInodePointers()
	if err := t.getMissingObjects(img.FileSystem); err != nil {
		return err
	}
	if err := t.imageDataBase.AddImage(img, name, nil); err != nil {
		return err
	}
	t.logger.Printf("Replicator(%s): added image\n", name)
	return nil
}

func (t *srpcType) checkImageBeingInjected(name string) bool {
	t.imagesBeingInjectedLock.Lock()
	defer t.imagesBeingInjectedLock.Unlock()
	_, ok := t.imagesBeingInjected[name]
	return ok
}

func (t *srpcType) getMissingObjects(fs *filesystem.FileSystem) error {
	hashes := make([]hash.Hash, 0, fs.NumRegularInodes)
	for _, inode := range fs.InodeTable {
		if inode, ok := inode.(*filesystem.RegularInode); ok {
			if inode.Size > 0 {
				hashes = append(hashes, inode.Hash)
			}
		}
	}
	objectSizes, err := t.objSrv.CheckObjects(hashes)
	if err != nil {
		return err
	}
	missingObjects := make([]hash.Hash, 0)
	for index, size := range objectSizes {
		if size < 1 {
			missingObjects = append(missingObjects, hashes[index])
		}
	}
	if len(missingObjects) < 1 {
		return nil
	}
	t.logger.Printf("Replicator: downloading %d of %d objects\n",
		len(missingObjects), len(hashes))
	startTime := time.Now()
	objClient := objectclient.NewObjectClient(t.replicationMaster)
	defer objClient.Close()
	objectsReader, err := objClient.GetObjects(missingObjects)
	if err != nil {
		return err
	}
	defer objectsReader.Close()
	var totalBytes uint64
	for _, hash := range missingObjects {
		length, reader, err := objectsReader.NextObject()
		if err != nil {
			return err
		}
		_, _, err = t.objSrv.AddObject(reader, length, &hash)
		reader.Close()
		if err != nil {
			return err
		}
		totalBytes += length
	}
	timeTaken := time.Since(startTime)
	t.logger.Printf("Replicator: downloaded %d objects, %s in %s (%s/s)\n",
		len(missingObjects), format.FormatBytes(totalBytes), timeTaken,
		format.FormatBytes(uint64(float64(totalBytes)/timeTaken.Seconds())))
	return nil
}
<commit_msg>Reference (protect) objects during image replication.<commit_after>package rpcd

import (
	"encoding/gob"
	"errors"
	imgclient "github.com/Symantec/Dominator/imageserver/client"
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/format"
	"github.com/Symantec/Dominator/lib/hash"
	objectclient "github.com/Symantec/Dominator/lib/objectserver/client"
	"github.com/Symantec/Dominator/lib/srpc"
	"github.com/Symantec/Dominator/proto/imageserver"
	"io"
	"time"
)

func (t *srpcType) replicator() {
	initialTimeout := time.Second * 15
	timeout := initialTimeout
	var nextSleepStopTime time.Time
	for {
		nextSleepStopTime = time.Now().Add(timeout)
		if client, err := srpc.DialHTTP("tcp", t.replicationMaster,
			timeout); err != nil {
			t.logger.Printf("Error dialling: %s %s\n", t.replicationMaster, err)
		} else {
			if conn, err := client.Call(
				"ImageServer.GetImageUpdates"); err != nil {
				t.logger.Println(err)
			} else {
				if err := t.getUpdates(conn); err != nil {
					if err == io.EOF {
						t.logger.Println(
							"Connection to image replicator closed")
						if nextSleepStopTime.Sub(time.Now()) < 1 {
							timeout = initialTimeout
						}
					} else {
						t.logger.Println(err)
					}
				}
				conn.Close()
			}
			client.Close()
		}
		time.Sleep(nextSleepStopTime.Sub(time.Now()))
		if timeout < time.Minute {
			timeout *= 2
		}
	}
}

func (t *srpcType) getUpdates(conn *srpc.Conn) error {
	t.logger.Printf("Image replicator: connected to: %s\n", t.replicationMaster)
	replicationStartTime := time.Now()
	decoder := gob.NewDecoder(conn)
	initialImages := make(map[string]struct{})
	if t.archiveMode {
		initialImages = nil
	}
	for {
		var imageUpdate imageserver.ImageUpdate
		if err := decoder.Decode(&imageUpdate); err != nil {
			if err == io.EOF {
				return err
			}
			return errors.New("decode err: " + err.Error())
		}
		switch imageUpdate.Operation {
		case imageserver.OperationAddImage:
			if imageUpdate.Name == "" {
				if initialImages != nil {
					t.deleteMissingImages(initialImages)
					initialImages = nil
				}
				t.logger.Printf("Replicated all current images in %s\n",
					format.Duration(time.Since(replicationStartTime)))
				continue
			}
			if initialImages != nil {
				initialImages[imageUpdate.Name] = struct{}{}
			}
			if err := t.addImage(imageUpdate.Name); err != nil {
				return errors.New("error adding image: " + imageUpdate.Name +
					": " + err.Error())
			}
		case imageserver.OperationDeleteImage:
			if t.archiveMode {
				continue
			}
			t.logger.Printf("Replicator(%s): delete image\n", imageUpdate.Name)
			if err := t.imageDataBase.DeleteImage(imageUpdate.Name,
				nil); err != nil {
				return err
			}
		case imageserver.OperationMakeDirectory:
			directory := imageUpdate.Directory
			if directory == nil {
				return errors.New("nil imageUpdate.Directory")
			}
			if err := t.imageDataBase.UpdateDirectory(*directory); err != nil {
				return err
			}
		}
	}
}

func (t *srpcType) deleteMissingImages(imagesToKeep map[string]struct{}) {
	missingImages := make([]string, 0)
	for _, imageName := range t.imageDataBase.ListImages() {
		if _, ok := imagesToKeep[imageName]; !ok {
			missingImages = append(missingImages, imageName)
		}
	}
	for _, imageName := range missingImages {
		t.logger.Printf("Replicator(%s): delete missing image\n", imageName)
		if err := t.imageDataBase.DeleteImage(imageName, nil); err != nil {
			t.logger.Println(err)
		}
	}
}

func (t *srpcType) addImage(name string) error {
	timeout := time.Second * 60
	if t.imageDataBase.CheckImage(name) || t.checkImageBeingInjected(name) {
		return nil
	}
	t.logger.Printf("Replicator(%s): add image\n", name)
	client, err := srpc.DialHTTP("tcp", t.replicationMaster, timeout)
	if err != nil {
		return err
	}
	defer client.Close()
	img, err := imgclient.GetImage(client, name)
	if err != nil {
		return err
	}
	if img == nil {
		return errors.New(name + ": not found")
	}
	t.logger.Printf("Replicator(%s): downloaded image\n", name)
	if t.archiveMode && !img.ExpiresAt.IsZero() && !*archiveExpiringImages {
		t.logger.Printf(
			"Replicator(%s): ignoring expiring image in archiver mode\n",
			name)
		return nil
	}
	img.FileSystem.RebuildInodePointers()
	err = t.imageDataBase.DoWithPendingImage(img, func() error {
		if err := t.getMissingObjects(img.FileSystem); err != nil {
			return err
		}
		if err := t.imageDataBase.AddImage(img, name, nil); err != nil {
			return err
		}
		return nil
	})
	if err != nil {
		return err
	}
	t.logger.Printf("Replicator(%s): added image\n", name)
	return nil
}

func (t *srpcType) checkImageBeingInjected(name string) bool {
	t.imagesBeingInjectedLock.Lock()
	defer t.imagesBeingInjectedLock.Unlock()
	_, ok := t.imagesBeingInjected[name]
	return ok
}

func (t *srpcType) getMissingObjects(fs *filesystem.FileSystem) error {
	objectsMap := fs.GetObjects()
	hashes := make([]hash.Hash, 0, len(objectsMap))
	for hashVal := range objectsMap {
		hashes = append(hashes, hashVal)
	}
	objectSizes, err := t.objSrv.CheckObjects(hashes)
	if err != nil {
		return err
	}
	missingObjects := make([]hash.Hash, 0)
	for index, size := range objectSizes {
		if size < 1 {
			missingObjects = append(missingObjects, hashes[index])
		}
	}
	if len(missingObjects) < 1 {
		return nil
	}
	t.logger.Printf("Replicator: downloading %d of %d objects\n",
		len(missingObjects), len(hashes))
	startTime := time.Now()
	objClient := objectclient.NewObjectClient(t.replicationMaster)
	defer objClient.Close()
	objectsReader, err := objClient.GetObjects(missingObjects)
	if err != nil {
		return errors.New("error downloading objects: " + err.Error())
	}
	defer objectsReader.Close()
	var totalBytes uint64
	for _, hash := range missingObjects {
		length, reader, err := objectsReader.NextObject()
		if err != nil {
			return err
		}
		_, _, err = t.objSrv.AddObject(reader, length, &hash)
		reader.Close()
		if err != nil {
			return err
		}
		totalBytes += length
	}
	timeTaken := time.Since(startTime)
	t.logger.Printf("Replicator: downloaded %d objects, %s in %s (%s/s)\n",
		len(missingObjects), format.FormatBytes(totalBytes), timeTaken,
		format.FormatBytes(uint64(float64(totalBytes)/timeTaken.Seconds())))
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package worker

import (
	"context"
	"math"
	"strings"
	"testing"
	"time"

	"github.com/google/go-cmp/cmp"
	"github.com/google/go-cmp/cmp/cmpopts"
	"golang.org/x/vuln/internal/cveschema"
	"golang.org/x/vuln/internal/worker/log"
	"golang.org/x/vuln/internal/worker/store"
)

func TestCheckUpdate(t *testing.T) {
	ctx := context.Background()
	tm := time.Date(2021, 1, 26, 0, 0, 0, 0, time.Local)
	repo, err := readTxtarRepo("testdata/basic.txtar", tm)
	if err != nil {
		t.Fatal(err)
	}
	for _, test := range []struct {
		latestUpdate *store.CommitUpdateRecord
		want         string // non-empty => substring of error message
	}{
		// no latest update, no problem
		{nil, ""},
		// latest update finished and commit is earlier; no problem
		{
			&store.CommitUpdateRecord{
				EndedAt:    time.Now(),
				CommitHash: "abc",
				CommitTime: tm.Add(-time.Hour),
			},
			"",
		},
		// latest update didn't finish
		{
			&store.CommitUpdateRecord{
				CommitHash: "abc",
				CommitTime: tm.Add(-time.Hour),
			},
			"not finish",
		},
		// latest update finished with error
		{
			&store.CommitUpdateRecord{
				CommitHash: "abc",
				CommitTime: tm.Add(-time.Hour),
				EndedAt:    time.Now(),
				Error:      "bad",
			},
			"with error",
		},
		// latest update finished on a later commit
		{
			&store.CommitUpdateRecord{
				EndedAt:    time.Now(),
				CommitHash: "abc",
				CommitTime: tm.Add(time.Hour),
			},
			"before",
		},
	} {
		mstore := store.NewMemStore()
		if test.latestUpdate != nil {
			if err := mstore.CreateCommitUpdateRecord(ctx, test.latestUpdate); err != nil {
				t.Fatal(err)
			}
		}
		got := checkUpdate(ctx, repo, headCommit(t, repo).Hash, mstore)
		if got == nil && test.want != "" {
			t.Errorf("%+v:\ngot no error, wanted %q", test.latestUpdate, test.want)
		} else if got != nil && !strings.Contains(got.Error(), test.want) {
			t.Errorf("%+v:\ngot '%s', does not contain %q", test.latestUpdate, got, test.want)
		}
	}
}

func TestCreateIssues(t *testing.T) {
	ctx := log.WithLineLogger(context.Background())
	mstore := store.NewMemStore()
	ic := newFakeIssueClient()

	crs := []*store.CVERecord{
		{
			ID:          "ID1",
			BlobHash:    "bh1",
			CommitHash:  "ch",
			Path:        "path1",
			TriageState: store.TriageStateNeedsIssue,
		},
		{
			ID:          "ID2",
			BlobHash:    "bh2",
			CommitHash:  "ch",
			Path:        "path2",
			TriageState: store.TriageStateNoActionNeeded,
		},
		{
			ID:          "ID3",
			BlobHash:    "bh3",
			CommitHash:  "ch",
			Path:        "path3",
			TriageState: store.TriageStateIssueCreated,
		},
	}
	createCVERecords(t, mstore, crs)

	if err := CreateIssues(ctx, mstore, ic, 0); err != nil {
		t.Fatal(err)
	}

	var wants []*store.CVERecord
	for _, r := range crs {
		copy := *r
		wants = append(wants, &copy)
	}
	wants[0].TriageState = store.TriageStateIssueCreated
	wants[0].IssueReference = "inMemory#1"

	gotRecs := mstore.CVERecords()
	if len(gotRecs) != len(wants) {
		t.Fatalf("wrong number of records: got %d, want %d", len(gotRecs), len(wants))
	}
	for _, want := range wants {
		got := gotRecs[want.ID]
		if !cmp.Equal(got, want, cmpopts.IgnoreFields(store.CVERecord{}, "IssueCreatedAt")) {
			t.Errorf("got  %+v\nwant %+v", got, want)
		}
	}
}

func TestNewBody(t *testing.T) {
	r := &store.CVERecord{
		ID:     "ID1",
		Module: "aModule",
		CVE: &cveschema.CVE{
			Description: cveschema.Description{
				Data: []cveschema.LangString{{
					Lang:  "eng",
					Value: "a description",
				}},
			},
		},
	}
	got, err := newBody(r)
	if err != nil {
		t.Fatal(err)
	}
	want := `
        One or more of the reference URLs in [ID1](https://github.com/CVEProject/cvelist/tree//) refers to a Go module.

        module: aModule
        package:
        stdlib:
        versions:
          - introduced:
          - fixed:
        description: |
          a description

        cve: ID1
        credit:
        symbols:
          -
        published:
        links:
          commit:
          pr:
          context:
            -
`
	if diff := cmp.Diff(unindent(want), got); diff != "" {
		t.Errorf("mismatch (-want, +got):\n%s", diff)
	}
}

// unindent removes leading whitespace from s.
// It first finds the line beginning with the fewest space and tab characters.
// It then removes that many characters from every line.
func unindent(s string) string {
	lines := strings.Split(s, "\n")
	min := math.MaxInt
	for _, l := range lines {
		if len(l) == 0 {
			continue
		}
		n := 0
		for _, r := range l {
			if r != ' ' && r != '\t' {
				break
			}
			n++
		}
		if n < min {
			min = n
		}
	}
	for i, l := range lines {
		if len(l) > 0 {
			lines[i] = l[min:]
		}
	}
	return strings.Join(lines, "\n")
}
<commit_msg>internal/worker: skip TestNewBody<commit_after>// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package worker

import (
	"context"
	"strings"
	"testing"
	"time"

	"github.com/google/go-cmp/cmp"
	"github.com/google/go-cmp/cmp/cmpopts"
	"golang.org/x/vuln/internal/worker/log"
	"golang.org/x/vuln/internal/worker/store"
)

func TestCheckUpdate(t *testing.T) {
	ctx := context.Background()
	tm := time.Date(2021, 1, 26, 0, 0, 0, 0, time.Local)
	repo, err := readTxtarRepo("testdata/basic.txtar", tm)
	if err != nil {
		t.Fatal(err)
	}
	for _, test := range []struct {
		latestUpdate *store.CommitUpdateRecord
		want         string // non-empty => substring of error message
	}{
		// no latest update, no problem
		{nil, ""},
		// latest update finished and commit is earlier; no problem
		{
			&store.CommitUpdateRecord{
				EndedAt:    time.Now(),
				CommitHash: "abc",
				CommitTime: tm.Add(-time.Hour),
			},
			"",
		},
		// latest update didn't finish
		{
			&store.CommitUpdateRecord{
				CommitHash: "abc",
				CommitTime: tm.Add(-time.Hour),
			},
			"not finish",
		},
		// latest update finished with error
		{
			&store.CommitUpdateRecord{
				CommitHash: "abc",
				CommitTime: tm.Add(-time.Hour),
				EndedAt:    time.Now(),
				Error:      "bad",
			},
			"with error",
		},
		// latest update finished on a later commit
		{
			&store.CommitUpdateRecord{
				EndedAt:    time.Now(),
				CommitHash: "abc",
				CommitTime: tm.Add(time.Hour),
			},
			"before",
		},
	} {
		mstore := store.NewMemStore()
		if test.latestUpdate != nil {
			if err := mstore.CreateCommitUpdateRecord(ctx, test.latestUpdate); err != nil {
				t.Fatal(err)
			}
		}
		got := checkUpdate(ctx, repo, headCommit(t, repo).Hash, mstore)
		if got == nil && test.want != "" {
			t.Errorf("%+v:\ngot no error, wanted %q", test.latestUpdate, test.want)
		} else if got != nil && !strings.Contains(got.Error(), test.want) {
			t.Errorf("%+v:\ngot '%s', does not contain %q", test.latestUpdate, got, test.want)
		}
	}
}

func TestCreateIssues(t *testing.T) {
	ctx := log.WithLineLogger(context.Background())
	mstore := store.NewMemStore()
	ic := newFakeIssueClient()

	crs := []*store.CVERecord{
		{
			ID:          "ID1",
			BlobHash:    "bh1",
			CommitHash:  "ch",
			Path:        "path1",
			TriageState: store.TriageStateNeedsIssue,
		},
		{
			ID:          "ID2",
			BlobHash:    "bh2",
			CommitHash:  "ch",
			Path:        "path2",
			TriageState: store.TriageStateNoActionNeeded,
		},
		{
			ID:          "ID3",
			BlobHash:    "bh3",
			CommitHash:  "ch",
			Path:        "path3",
			TriageState: store.TriageStateIssueCreated,
		},
	}
	createCVERecords(t, mstore, crs)

	if err := CreateIssues(ctx, mstore, ic, 0); err != nil {
		t.Fatal(err)
	}

	var wants []*store.CVERecord
	for _, r := range crs {
		copy := *r
		wants = append(wants, &copy)
	}
	wants[0].TriageState = store.TriageStateIssueCreated
	wants[0].IssueReference = "inMemory#1"

	gotRecs := mstore.CVERecords()
	if len(gotRecs) != len(wants) {
		t.Fatalf("wrong number of records: got %d, want %d", len(gotRecs), len(wants))
	}
	for _, want := range wants {
		got := gotRecs[want.ID]
		if !cmp.Equal(got, want, cmpopts.IgnoreFields(store.CVERecord{}, "IssueCreatedAt")) {
			t.Errorf("got  %+v\nwant %+v", got, want)
		}
	}
}

/*
TODO(golang/go#50026): Uncomment this test once CI is moved to kokoro.
func TestNewBody(t *testing.T) {
	r := &store.CVERecord{
		ID:     "ID1",
		Module: "aModule",
		CVE: &cveschema.CVE{
			Description: cveschema.Description{
				Data: []cveschema.LangString{{
					Lang:  "eng",
					Value: "a description",
				}},
			},
		},
	}
	got, err := newBody(r)
	if err != nil {
		t.Fatal(err)
	}
	want := `
        One or more of the reference URLs in [ID1](https://github.com/CVEProject/cvelist/tree//) refers to a Go module.

        module: aModule
        package:
        stdlib:
        versions:
          - introduced:
          - fixed:
        description: |
          a description

        cve: ID1
        credit:
        symbols:
          -
        published:
        links:
          commit:
          pr:
          context:
            -
`
	if diff := cmp.Diff(unindent(want), got); diff != "" {
		t.Errorf("mismatch (-want, +got):\n%s", diff)
	}
}

// unindent removes leading whitespace from s.
// It first finds the line beginning with the fewest space and tab characters.
// It then removes that many characters from every line.
func unindent(s string) string {
	lines := strings.Split(s, "\n")
	min := math.MaxInt
	for _, l := range lines {
		if len(l) == 0 {
			continue
		}
		n := 0
		for _, r := range l {
			if r != ' ' && r != '\t' {
				break
			}
			n++
		}
		if n < min {
			min = n
		}
	}
	for i, l := range lines {
		if len(l) > 0 {
			lines[i] = l[min:]
		}
	}
	return strings.Join(lines, "\n")
}
*/
<|endoftext|>