<commit_before>/*
 * Minio Cloud Storage, (C) 2015-2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package fs

import (
	"fmt"
	"net/url"
	"os"
	"path/filepath"
	"strings"

	"github.com/minio/minio/pkg/probe"
)

// ListObjects - lists all objects for a given prefix, returns up to
// maxKeys number of objects per call.
func (fs Filesystem) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsResult, *probe.Error) {
	result := ListObjectsResult{}

	// Input validation.
	if !IsValidBucketName(bucket) {
		return result, probe.NewError(BucketNameInvalid{Bucket: bucket})
	}

	bucket = fs.denormalizeBucket(bucket)

	if status, err := isDirExist(filepath.Join(fs.path, bucket)); !status {
		if err == nil {
			return result, probe.NewError(BucketNotFound{Bucket: bucket})
		} else if os.IsNotExist(err) {
			return result, probe.NewError(BucketNotFound{Bucket: bucket})
		} else {
			return result, probe.NewError(err)
		}
	}

	if delimiter != "" && delimiter != "/" {
		return result, probe.NewError(fmt.Errorf("delimiter '%s' is not supported", delimiter))
	}

	if marker != "" {
		if markerUnescaped, err := url.QueryUnescape(marker); err == nil {
			marker = markerUnescaped
		} else {
			return result, probe.NewError(err)
		}

		if !strings.HasPrefix(marker, prefix) {
			return result, probe.NewError(fmt.Errorf("marker '%s' and prefix '%s' do not match", marker, prefix))
		}
	}

	if maxKeys <= 0 || maxKeys > listObjectsLimit {
		maxKeys = listObjectsLimit
	}

	bucketDir := filepath.Join(fs.path, bucket)

	recursive := true
	skipDir := true
	if delimiter == "/" {
		skipDir = false
		recursive = false
	}

	prefixDir := filepath.Dir(filepath.FromSlash(prefix))
	rootDir := filepath.Join(bucketDir, prefixDir)

	objectInfoCh := fs.popListObjectCh(ListObjectParams{bucket, delimiter, marker, prefix})
	if objectInfoCh == nil {
		ch := treeWalk(rootDir, bucketDir, recursive)
		objectInfoCh = &ch
	}

	nextMarker := ""
	for i := 0; i < maxKeys; {
		objInfo, ok := objectInfoCh.Read()
		if !ok {
			// closed channel
			return result, nil
		}

		if objInfo.Err != nil {
			return ListObjectsResult{}, probe.NewError(objInfo.Err)
		}

		if strings.Contains(objInfo.Name, "$multiparts") || strings.Contains(objInfo.Name, "$tmpobject") {
			continue
		}

		if objInfo.IsDir && skipDir {
			continue
		}

		// Add the bucket.
		objInfo.Bucket = bucket

		if strings.HasPrefix(objInfo.Name, prefix) {
			if objInfo.Name > marker {
				if objInfo.IsDir {
					result.Prefixes = append(result.Prefixes, objInfo.Name)
				} else {
					result.Objects = append(result.Objects, objInfo)
				}
				nextMarker = objInfo.Name
				i++
			}
		}
	}

	if !objectInfoCh.IsClosed() {
		result.IsTruncated = true
		result.NextMarker = nextMarker
		fs.pushListObjectCh(ListObjectParams{bucket, delimiter, nextMarker, prefix}, *objectInfoCh)
	}

	return result, nil
}
<commit_msg>Handling maxKeys=0 case with a empty response<commit_after>/*
 * Minio Cloud Storage, (C) 2015-2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package fs

import (
	"fmt"
	"net/url"
	"os"
	"path/filepath"
	"strings"

	"github.com/minio/minio/pkg/probe"
)

// ListObjects - lists all objects for a given prefix, returns up to
// maxKeys number of objects per call.
func (fs Filesystem) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsResult, *probe.Error) {
	result := ListObjectsResult{}

	// Input validation.
	if !IsValidBucketName(bucket) {
		return result, probe.NewError(BucketNameInvalid{Bucket: bucket})
	}

	bucket = fs.denormalizeBucket(bucket)

	if status, err := isDirExist(filepath.Join(fs.path, bucket)); !status {
		if err == nil {
			return result, probe.NewError(BucketNotFound{Bucket: bucket})
		} else if os.IsNotExist(err) {
			return result, probe.NewError(BucketNotFound{Bucket: bucket})
		} else {
			return result, probe.NewError(err)
		}
	}

	if delimiter != "" && delimiter != "/" {
		return result, probe.NewError(fmt.Errorf("delimiter '%s' is not supported", delimiter))
	}

	if marker != "" {
		if markerUnescaped, err := url.QueryUnescape(marker); err == nil {
			marker = markerUnescaped
		} else {
			return result, probe.NewError(err)
		}

		if !strings.HasPrefix(marker, prefix) {
			return result, probe.NewError(fmt.Errorf("marker '%s' and prefix '%s' do not match", marker, prefix))
		}
	}

	// Return empty response for a valid request when maxKeys is 0.
	if maxKeys == 0 {
		return result, nil
	}

	if maxKeys < 0 || maxKeys > listObjectsLimit {
		maxKeys = listObjectsLimit
	}

	bucketDir := filepath.Join(fs.path, bucket)

	recursive := true
	skipDir := true
	if delimiter == "/" {
		skipDir = false
		recursive = false
	}

	prefixDir := filepath.Dir(filepath.FromSlash(prefix))
	rootDir := filepath.Join(bucketDir, prefixDir)

	objectInfoCh := fs.popListObjectCh(ListObjectParams{bucket, delimiter, marker, prefix})
	if objectInfoCh == nil {
		ch := treeWalk(rootDir, bucketDir, recursive)
		objectInfoCh = &ch
	}

	nextMarker := ""
	for i := 0; i < maxKeys; {
		objInfo, ok := objectInfoCh.Read()
		if !ok {
			// closed channel
			return result, nil
		}

		if objInfo.Err != nil {
			return ListObjectsResult{}, probe.NewError(objInfo.Err)
		}

		if strings.Contains(objInfo.Name, "$multiparts") || strings.Contains(objInfo.Name, "$tmpobject") {
			continue
		}

		if objInfo.IsDir && skipDir {
			continue
		}

		// Add the bucket.
		objInfo.Bucket = bucket

		if strings.HasPrefix(objInfo.Name, prefix) {
			if objInfo.Name > marker {
				if objInfo.IsDir {
					result.Prefixes = append(result.Prefixes, objInfo.Name)
				} else {
					result.Objects = append(result.Objects, objInfo)
				}
				nextMarker = objInfo.Name
				i++
			}
		}
	}

	if !objectInfoCh.IsClosed() {
		result.IsTruncated = true
		result.NextMarker = nextMarker
		fs.pushListObjectCh(ListObjectParams{bucket, delimiter, nextMarker, prefix}, *objectInfoCh)
	}

	return result, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package metadata

import (
	"fmt"
	"strings"

	kritisv1beta1 "github.com/grafeas/kritis/pkg/kritis/apis/kritis/v1beta1"
	"github.com/grafeas/kritis/pkg/kritis/secrets"
	grafeasv1beta1 "google.golang.org/genproto/googleapis/devtools/containeranalysis/v1beta1/grafeas"
)

// Read/write interface to access Occurrences and Notes using Grafeas API.
type Fetcher interface {
	// Vulnerabilities returns package vulnerabilities for a given image.
	Vulnerabilities(containerImage string) ([]Vulnerability, error)
	// CreateAttestationOccurrence creates an Attestation occurrence for a given image, secret, and project.
	CreateAttestationOccurrence(note *grafeasv1beta1.Note,
		containerImage string, pgpSigningKey *secrets.PGPSigningSecret,
		proj string) (*grafeasv1beta1.Occurrence, error)
	//AttestationNote fetches an Attestation note for an Attestation Authority.
	AttestationNote(aa *kritisv1beta1.AttestationAuthority) (*grafeasv1beta1.Note, error)
	// Create Attestation Note for an Attestation Authority.
	CreateAttestationNote(aa *kritisv1beta1.AttestationAuthority) (*grafeasv1beta1.Note, error)
	//Attestations get Attestation Occurrences for given image.
	Attestations(containerImage string, aa *kritisv1beta1.AttestationAuthority) ([]PGPAttestation, error)
	// Close closes client connections
	Close()
}

// Read-only interface to access Occurrences and Notes using Grafeas API.
type ReadOnlyClient interface {
	// Vulnerabilities returns package vulnerabilities for a given image.
	Vulnerabilities(containerImage string) ([]Vulnerability, error)
	//AttestationNote fetches an Attestation note for an Attestation Authority.
	AttestationNote(aa *kritisv1beta1.AttestationAuthority) (*grafeasv1beta1.Note, error)
	//Attestations get Attestation Occurrences for given image.
	Attestations(containerImage string, aa *kritisv1beta1.AttestationAuthority) ([]PGPAttestation, error)
	// Close closes client connections
	Close()
}

type Vulnerability struct {
	Severity        string
	HasFixAvailable bool
	CVE             string
}

// PGPAttestation represents the Signature and the Signer Key Id from the
// containeranalysis Occurrence_Attestation instance.
type PGPAttestation struct {
	Signature string
	KeyID     string
	// OccID is the occurrence ID for containeranalysis Occurrence_Attestation instance
	OccID string
}

// GetProjectFromNoteReference extracts the project ID form the NoteReference
func GetProjectFromNoteReference(ref string) (string, error) {
	str := strings.Split(ref, "/")
	if len(str) != 2 {
		return "", fmt.Errorf("invalid Note Reference, should be in format projects/<project_id>")
	}
	if str[0] != "projects" {
		return "", fmt.Errorf("invalid Note Reference, should be in format projects/<project_id>")
	}
	return str[1], nil
}
<commit_msg>Remove the unused GetAttestationNote method from the readonly client.<commit_after>/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package metadata

import (
	"fmt"
	"strings"

	kritisv1beta1 "github.com/grafeas/kritis/pkg/kritis/apis/kritis/v1beta1"
	"github.com/grafeas/kritis/pkg/kritis/secrets"
	grafeasv1beta1 "google.golang.org/genproto/googleapis/devtools/containeranalysis/v1beta1/grafeas"
)

// Read/write interface to access Occurrences and Notes using Grafeas API.
type Fetcher interface {
	// Vulnerabilities returns package vulnerabilities for a given image.
	Vulnerabilities(containerImage string) ([]Vulnerability, error)
	// CreateAttestationOccurrence creates an Attestation occurrence for a given image, secret, and project.
	CreateAttestationOccurrence(note *grafeasv1beta1.Note,
		containerImage string, pgpSigningKey *secrets.PGPSigningSecret,
		proj string) (*grafeasv1beta1.Occurrence, error)
	//AttestationNote fetches an Attestation note for an Attestation Authority.
	AttestationNote(aa *kritisv1beta1.AttestationAuthority) (*grafeasv1beta1.Note, error)
	// Create Attestation Note for an Attestation Authority.
	CreateAttestationNote(aa *kritisv1beta1.AttestationAuthority) (*grafeasv1beta1.Note, error)
	//Attestations get Attestation Occurrences for given image.
	Attestations(containerImage string, aa *kritisv1beta1.AttestationAuthority) ([]PGPAttestation, error)
	// Close closes client connections
	Close()
}

// Read-only interface to access Occurrences and Notes using Grafeas API.
type ReadOnlyClient interface {
	// Vulnerabilities returns package vulnerabilities for a given image.
	Vulnerabilities(containerImage string) ([]Vulnerability, error)
	//Attestations get Attestation Occurrences for given image.
	Attestations(containerImage string, aa *kritisv1beta1.AttestationAuthority) ([]PGPAttestation, error)
	// Close closes client connections
	Close()
}

type Vulnerability struct {
	Severity        string
	HasFixAvailable bool
	CVE             string
}

// PGPAttestation represents the Signature and the Signer Key Id from the
// containeranalysis Occurrence_Attestation instance.
type PGPAttestation struct {
	Signature string
	KeyID     string
	// OccID is the occurrence ID for containeranalysis Occurrence_Attestation instance
	OccID string
}

// GetProjectFromNoteReference extracts the project ID form the NoteReference
func GetProjectFromNoteReference(ref string) (string, error) {
	str := strings.Split(ref, "/")
	if len(str) != 2 {
		return "", fmt.Errorf("invalid Note Reference, should be in format projects/<project_id>")
	}
	if str[0] != "projects" {
		return "", fmt.Errorf("invalid Note Reference, should be in format projects/<project_id>")
	}
	return str[1], nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cpuset

import (
	"bytes"
	"fmt"
	"os"
	"reflect"
	"sort"
	"strconv"
	"strings"

	"k8s.io/klog/v2"
)

// Builder is a mutable builder for CPUSet. Functions that mutate instances
// of this type are not thread-safe.
type Builder struct {
	result CPUSet
	done   bool
}

// NewBuilder returns a mutable CPUSet builder.
func NewBuilder() *Builder {
	return &Builder{
		result: CPUSet{
			elems: map[int]struct{}{},
		},
	}
}

// Add adds the supplied elements to the result. Calling Add after calling
// Result has no effect.
func (b *Builder) Add(elems ...int) {
	if b.done {
		return
	}
	for _, elem := range elems {
		b.result.elems[elem] = struct{}{}
	}
}

// Result returns the result CPUSet containing all elements that were
// previously added to this builder. Subsequent calls to Add have no effect.
func (b *Builder) Result() CPUSet {
	b.done = true
	return b.result
}

// CPUSet is a thread-safe, immutable set-like data structure for CPU IDs.
type CPUSet struct {
	elems map[int]struct{}
}

// NewCPUSet returns a new CPUSet containing the supplied elements.
func NewCPUSet(cpus ...int) CPUSet {
	b := NewBuilder()
	for _, c := range cpus {
		b.Add(c)
	}
	return b.Result()
}

// NewCPUSet returns a new CPUSet containing the supplied elements, as slice of int64.
func NewCPUSetInt64(cpus ...int64) CPUSet {
	b := NewBuilder()
	for _, c := range cpus {
		b.Add(int(c))
	}
	return b.Result()
}

// Size returns the number of elements in this set.
func (s CPUSet) Size() int {
	return len(s.elems)
}

// IsEmpty returns true if there are zero elements in this set.
func (s CPUSet) IsEmpty() bool {
	return s.Size() == 0
}

// Contains returns true if the supplied element is present in this set.
func (s CPUSet) Contains(cpu int) bool {
	_, found := s.elems[cpu]
	return found
}

// Equals returns true if the supplied set contains exactly the same elements
// as this set (s IsSubsetOf s2 and s2 IsSubsetOf s).
func (s CPUSet) Equals(s2 CPUSet) bool {
	return reflect.DeepEqual(s.elems, s2.elems)
}

// Filter returns a new CPU set that contains all of the elements from this
// set that match the supplied predicate, without mutating the source set.
func (s CPUSet) Filter(predicate func(int) bool) CPUSet {
	b := NewBuilder()
	for cpu := range s.elems {
		if predicate(cpu) {
			b.Add(cpu)
		}
	}
	return b.Result()
}

// FilterNot returns a new CPU set that contains all of the elements from this
// set that do not match the supplied predicate, without mutating the source
// set.
func (s CPUSet) FilterNot(predicate func(int) bool) CPUSet {
	b := NewBuilder()
	for cpu := range s.elems {
		if !predicate(cpu) {
			b.Add(cpu)
		}
	}
	return b.Result()
}

// IsSubsetOf returns true if the supplied set contains all the elements
func (s CPUSet) IsSubsetOf(s2 CPUSet) bool {
	result := true
	for cpu := range s.elems {
		if !s2.Contains(cpu) {
			result = false
			break
		}
	}
	return result
}

// Union returns a new CPU set that contains all of the elements from this
// set and all of the elements from the supplied set, without mutating
// either source set.
func (s CPUSet) Union(s2 CPUSet) CPUSet {
	b := NewBuilder()
	for cpu := range s.elems {
		b.Add(cpu)
	}
	for cpu := range s2.elems {
		b.Add(cpu)
	}
	return b.Result()
}

// UnionAll returns a new CPU set that contains all of the elements from this
// set and all of the elements from the supplied sets, without mutating
// either source set.
func (s CPUSet) UnionAll(s2 []CPUSet) CPUSet {
	b := NewBuilder()
	for cpu := range s.elems {
		b.Add(cpu)
	}
	for _, cs := range s2 {
		for cpu := range cs.elems {
			b.Add(cpu)
		}
	}
	return b.Result()
}

// Intersection returns a new CPU set that contains all of the elements
// that are present in both this set and the supplied set, without mutating
// either source set.
func (s CPUSet) Intersection(s2 CPUSet) CPUSet {
	return s.Filter(func(cpu int) bool { return s2.Contains(cpu) })
}

// Difference returns a new CPU set that contains all of the elements that
// are present in this set and not the supplied set, without mutating either
// source set.
func (s CPUSet) Difference(s2 CPUSet) CPUSet {
	return s.FilterNot(func(cpu int) bool { return s2.Contains(cpu) })
}

// ToSlice returns a slice of integers that contains all elements from
// this set.
func (s CPUSet) ToSlice() []int {
	result := []int{}
	for cpu := range s.elems {
		result = append(result, cpu)
	}
	sort.Ints(result)
	return result
}

// ToSliceNoSort returns a slice of integers that contains all elements from
// this set.
func (s CPUSet) ToSliceNoSort() []int {
	result := []int{}
	for cpu := range s.elems {
		result = append(result, cpu)
	}
	return result
}

// ToSliceInt64 returns an ordered slice of int64 that contains all elements from
// this set
func (s CPUSet) ToSliceInt64() []int64 {
	var result []int64
	for cpu := range s.elems {
		result = append(result, int64(cpu))
	}
	sort.Slice(result, func(i, j int) bool { return result[i] < result[j] })
	return result
}

// ToSliceNoSortInt64 returns a slice of int64 that contains all elements from
// this set.
func (s CPUSet) ToSliceNoSortInt64() []int64 {
	var result []int64
	for cpu := range s.elems {
		result = append(result, int64(cpu))
	}
	return result
}

// String returns a new string representation of the elements in this CPU set
// in canonical linux CPU list format.
//
// See: http://man7.org/linux/man-pages/man7/cpuset.7.html#FORMATS
func (s CPUSet) String() string {
	if s.IsEmpty() {
		return ""
	}

	elems := s.ToSlice()

	type rng struct {
		start int
		end   int
	}

	ranges := []rng{{elems[0], elems[0]}}

	for i := 1; i < len(elems); i++ {
		lastRange := &ranges[len(ranges)-1]
		// if this element is adjacent to the high end of the last range
		if elems[i] == lastRange.end+1 {
			// then extend the last range to include this element
			lastRange.end = elems[i]
			continue
		}
		// otherwise, start a new range beginning with this element
		ranges = append(ranges, rng{elems[i], elems[i]})
	}

	// construct string from ranges
	var result bytes.Buffer
	for _, r := range ranges {
		if r.start == r.end {
			result.WriteString(strconv.Itoa(r.start))
		} else {
			result.WriteString(fmt.Sprintf("%d-%d", r.start, r.end))
		}
		result.WriteString(",")
	}
	return strings.TrimRight(result.String(), ",")
}

// MustParse CPUSet constructs a new CPU set from a Linux CPU list formatted
// string. Unlike Parse, it does not return an error but rather panics if the
// input cannot be used to construct a CPU set.
func MustParse(s string) CPUSet {
	res, err := Parse(s)
	if err != nil {
		klog.ErrorS(err, "Failed to parse input as CPUSet", "input", s)
		os.Exit(1)
	}
	return res
}

// Parse CPUSet constructs a new CPU set from a Linux CPU list formatted string.
//
// See: http://man7.org/linux/man-pages/man7/cpuset.7.html#FORMATS
func Parse(s string) (CPUSet, error) {
	b := NewBuilder()

	// Handle empty string.
	if s == "" {
		return b.Result(), nil
	}

	// Split CPU list string:
	// "0-5,34,46-48" => ["0-5", "34", "46-48"]
	ranges := strings.Split(s, ",")

	for _, r := range ranges {
		boundaries := strings.SplitN(r, "-", 2)
		if len(boundaries) == 1 {
			// Handle ranges that consist of only one element like "34".
			elem, err := strconv.Atoi(boundaries[0])
			if err != nil {
				return NewCPUSet(), err
			}
			b.Add(elem)
		} else if len(boundaries) == 2 {
			// Handle multi-element ranges like "0-5".
			start, err := strconv.Atoi(boundaries[0])
			if err != nil {
				return NewCPUSet(), err
			}
			end, err := strconv.Atoi(boundaries[1])
			if err != nil {
				return NewCPUSet(), err
			}
			if start > end {
				return NewCPUSet(), fmt.Errorf("invalid range %q (%d >= %d)", r, start, end)
			}
			// start == end is acceptable (1-1 -> 1)

			// Add all elements to the result.
			// e.g. "0-5", "46-48" => [0, 1, 2, 3, 4, 5, 46, 47, 48].
			for e := start; e <= end; e++ {
				b.Add(e)
			}
		}
	}
	return b.Result(), nil
}

// Clone returns a copy of this CPU set.
func (s CPUSet) Clone() CPUSet {
	b := NewBuilder()
	for elem := range s.elems {
		b.Add(elem)
	}
	return b.Result()
}
<commit_msg>Fix comment typo<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cpuset

import (
	"bytes"
	"fmt"
	"os"
	"reflect"
	"sort"
	"strconv"
	"strings"

	"k8s.io/klog/v2"
)

// Builder is a mutable builder for CPUSet. Functions that mutate instances
// of this type are not thread-safe.
type Builder struct {
	result CPUSet
	done   bool
}

// NewBuilder returns a mutable CPUSet builder.
func NewBuilder() *Builder {
	return &Builder{
		result: CPUSet{
			elems: map[int]struct{}{},
		},
	}
}

// Add adds the supplied elements to the result. Calling Add after calling
// Result has no effect.
func (b *Builder) Add(elems ...int) {
	if b.done {
		return
	}
	for _, elem := range elems {
		b.result.elems[elem] = struct{}{}
	}
}

// Result returns the result CPUSet containing all elements that were
// previously added to this builder. Subsequent calls to Add have no effect.
func (b *Builder) Result() CPUSet {
	b.done = true
	return b.result
}

// CPUSet is a thread-safe, immutable set-like data structure for CPU IDs.
type CPUSet struct {
	elems map[int]struct{}
}

// NewCPUSet returns a new CPUSet containing the supplied elements.
func NewCPUSet(cpus ...int) CPUSet {
	b := NewBuilder()
	for _, c := range cpus {
		b.Add(c)
	}
	return b.Result()
}

// NewCPUSetInt64 returns a new CPUSet containing the supplied elements, as slice of int64.
func NewCPUSetInt64(cpus ...int64) CPUSet {
	b := NewBuilder()
	for _, c := range cpus {
		b.Add(int(c))
	}
	return b.Result()
}

// Size returns the number of elements in this set.
func (s CPUSet) Size() int {
	return len(s.elems)
}

// IsEmpty returns true if there are zero elements in this set.
func (s CPUSet) IsEmpty() bool {
	return s.Size() == 0
}

// Contains returns true if the supplied element is present in this set.
func (s CPUSet) Contains(cpu int) bool {
	_, found := s.elems[cpu]
	return found
}

// Equals returns true if the supplied set contains exactly the same elements
// as this set (s IsSubsetOf s2 and s2 IsSubsetOf s).
func (s CPUSet) Equals(s2 CPUSet) bool {
	return reflect.DeepEqual(s.elems, s2.elems)
}

// Filter returns a new CPU set that contains all of the elements from this
// set that match the supplied predicate, without mutating the source set.
func (s CPUSet) Filter(predicate func(int) bool) CPUSet {
	b := NewBuilder()
	for cpu := range s.elems {
		if predicate(cpu) {
			b.Add(cpu)
		}
	}
	return b.Result()
}

// FilterNot returns a new CPU set that contains all of the elements from this
// set that do not match the supplied predicate, without mutating the source
// set.
func (s CPUSet) FilterNot(predicate func(int) bool) CPUSet {
	b := NewBuilder()
	for cpu := range s.elems {
		if !predicate(cpu) {
			b.Add(cpu)
		}
	}
	return b.Result()
}

// IsSubsetOf returns true if the supplied set contains all the elements
func (s CPUSet) IsSubsetOf(s2 CPUSet) bool {
	result := true
	for cpu := range s.elems {
		if !s2.Contains(cpu) {
			result = false
			break
		}
	}
	return result
}

// Union returns a new CPU set that contains all of the elements from this
// set and all of the elements from the supplied set, without mutating
// either source set.
func (s CPUSet) Union(s2 CPUSet) CPUSet {
	b := NewBuilder()
	for cpu := range s.elems {
		b.Add(cpu)
	}
	for cpu := range s2.elems {
		b.Add(cpu)
	}
	return b.Result()
}

// UnionAll returns a new CPU set that contains all of the elements from this
// set and all of the elements from the supplied sets, without mutating
// either source set.
func (s CPUSet) UnionAll(s2 []CPUSet) CPUSet {
	b := NewBuilder()
	for cpu := range s.elems {
		b.Add(cpu)
	}
	for _, cs := range s2 {
		for cpu := range cs.elems {
			b.Add(cpu)
		}
	}
	return b.Result()
}

// Intersection returns a new CPU set that contains all of the elements
// that are present in both this set and the supplied set, without mutating
// either source set.
func (s CPUSet) Intersection(s2 CPUSet) CPUSet {
	return s.Filter(func(cpu int) bool { return s2.Contains(cpu) })
}

// Difference returns a new CPU set that contains all of the elements that
// are present in this set and not the supplied set, without mutating either
// source set.
func (s CPUSet) Difference(s2 CPUSet) CPUSet {
	return s.FilterNot(func(cpu int) bool { return s2.Contains(cpu) })
}

// ToSlice returns a slice of integers that contains all elements from
// this set.
func (s CPUSet) ToSlice() []int {
	result := []int{}
	for cpu := range s.elems {
		result = append(result, cpu)
	}
	sort.Ints(result)
	return result
}

// ToSliceNoSort returns a slice of integers that contains all elements from
// this set.
func (s CPUSet) ToSliceNoSort() []int {
	result := []int{}
	for cpu := range s.elems {
		result = append(result, cpu)
	}
	return result
}

// ToSliceInt64 returns an ordered slice of int64 that contains all elements from
// this set
func (s CPUSet) ToSliceInt64() []int64 {
	var result []int64
	for cpu := range s.elems {
		result = append(result, int64(cpu))
	}
	sort.Slice(result, func(i, j int) bool { return result[i] < result[j] })
	return result
}

// ToSliceNoSortInt64 returns a slice of int64 that contains all elements from
// this set.
func (s CPUSet) ToSliceNoSortInt64() []int64 {
	var result []int64
	for cpu := range s.elems {
		result = append(result, int64(cpu))
	}
	return result
}

// String returns a new string representation of the elements in this CPU set
// in canonical linux CPU list format.
//
// See: http://man7.org/linux/man-pages/man7/cpuset.7.html#FORMATS
func (s CPUSet) String() string {
	if s.IsEmpty() {
		return ""
	}

	elems := s.ToSlice()

	type rng struct {
		start int
		end   int
	}

	ranges := []rng{{elems[0], elems[0]}}

	for i := 1; i < len(elems); i++ {
		lastRange := &ranges[len(ranges)-1]
		// if this element is adjacent to the high end of the last range
		if elems[i] == lastRange.end+1 {
			// then extend the last range to include this element
			lastRange.end = elems[i]
			continue
		}
		// otherwise, start a new range beginning with this element
		ranges = append(ranges, rng{elems[i], elems[i]})
	}

	// construct string from ranges
	var result bytes.Buffer
	for _, r := range ranges {
		if r.start == r.end {
			result.WriteString(strconv.Itoa(r.start))
		} else {
			result.WriteString(fmt.Sprintf("%d-%d", r.start, r.end))
		}
		result.WriteString(",")
	}
	return strings.TrimRight(result.String(), ",")
}

// MustParse CPUSet constructs a new CPU set from a Linux CPU list formatted
// string. Unlike Parse, it does not return an error but rather panics if the
// input cannot be used to construct a CPU set.
func MustParse(s string) CPUSet {
	res, err := Parse(s)
	if err != nil {
		klog.ErrorS(err, "Failed to parse input as CPUSet", "input", s)
		os.Exit(1)
	}
	return res
}

// Parse CPUSet constructs a new CPU set from a Linux CPU list formatted string.
//
// See: http://man7.org/linux/man-pages/man7/cpuset.7.html#FORMATS
func Parse(s string) (CPUSet, error) {
	b := NewBuilder()

	// Handle empty string.
	if s == "" {
		return b.Result(), nil
	}

	// Split CPU list string:
	// "0-5,34,46-48" => ["0-5", "34", "46-48"]
	ranges := strings.Split(s, ",")

	for _, r := range ranges {
		boundaries := strings.SplitN(r, "-", 2)
		if len(boundaries) == 1 {
			// Handle ranges that consist of only one element like "34".
			elem, err := strconv.Atoi(boundaries[0])
			if err != nil {
				return NewCPUSet(), err
			}
			b.Add(elem)
		} else if len(boundaries) == 2 {
			// Handle multi-element ranges like "0-5".
			start, err := strconv.Atoi(boundaries[0])
			if err != nil {
				return NewCPUSet(), err
			}
			end, err := strconv.Atoi(boundaries[1])
			if err != nil {
				return NewCPUSet(), err
			}
			if start > end {
				return NewCPUSet(), fmt.Errorf("invalid range %q (%d >= %d)", r, start, end)
			}
			// start == end is acceptable (1-1 -> 1)

			// Add all elements to the result.
			// e.g. "0-5", "46-48" => [0, 1, 2, 3, 4, 5, 46, 47, 48].
			for e := start; e <= end; e++ {
				b.Add(e)
			}
		}
	}
	return b.Result(), nil
}

// Clone returns a copy of this CPU set.
func (s CPUSet) Clone() CPUSet {
	b := NewBuilder()
	for elem := range s.elems {
		b.Add(elem)
	}
	return b.Result()
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cm

import (
	"fmt"

	libcontainercgroups "github.com/opencontainers/runc/libcontainer/cgroups"
)

// GetCgroupSubsystems returns information about the mounted cgroup subsystems
func GetCgroupSubsystems() (*CgroupSubsystems, error) {
	// get all cgroup mounts.
	allCgroups, err := libcontainercgroups.GetCgroupMounts()
	if err != nil {
		return &CgroupSubsystems{}, err
	}
	if len(allCgroups) == 0 {
		return &CgroupSubsystems{}, fmt.Errorf("failed to find cgroup mounts")
	}

	mountPoints := make(map[string]string, len(allCgroups))
	for _, mount := range allCgroups {
		for _, subsystem := range mount.Subsystems {
			mountPoints[subsystem] = mount.Mountpoint
		}
	}
	return &CgroupSubsystems{
		Mounts:      allCgroups,
		MountPoints: mountPoints,
	}, nil
}
<commit_msg>Fix build with runc updates<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cm

import (
	"fmt"

	libcontainercgroups "github.com/opencontainers/runc/libcontainer/cgroups"
)

// GetCgroupSubsystems returns information about the mounted cgroup subsystems
func GetCgroupSubsystems() (*CgroupSubsystems, error) {
	// get all cgroup mounts.
	allCgroups, err := libcontainercgroups.GetCgroupMounts(true)
	if err != nil {
		return &CgroupSubsystems{}, err
	}
	if len(allCgroups) == 0 {
		return &CgroupSubsystems{}, fmt.Errorf("failed to find cgroup mounts")
	}

	mountPoints := make(map[string]string, len(allCgroups))
	for _, mount := range allCgroups {
		for _, subsystem := range mount.Subsystems {
			mountPoints[subsystem] = mount.Mountpoint
		}
	}
	return &CgroupSubsystems{
		Mounts:      allCgroups,
		MountPoints: mountPoints,
	}, nil
}
<|endoftext|>


<commit_before>// +build !windows,!solaris

package listeners

import (
	"crypto/tls"
	"fmt"
	"net"
	"strconv"

	"github.com/Sirupsen/logrus"
	"github.com/coreos/go-systemd/activation"
	"github.com/docker/go-connections/sockets"
)

// Init creates new listeners for the server.
// TODO: Clean up the fact that socketGroup and tlsConfig aren't always used.
func Init(proto, addr, socketGroup string, tlsConfig *tls.Config) ([]net.Listener, error) {
	ls := []net.Listener{}

	switch proto {
	case "fd":
		fds, err := listenFD(addr, tlsConfig)
		if err != nil {
			return nil, err
		}
		ls = append(ls, fds...)
	case "tcp":
		l, err := sockets.NewTCPSocket(addr, tlsConfig)
		if err != nil {
			return nil, err
		}
		ls = append(ls, l)
	case "unix":
		l, err := sockets.NewUnixSocket(addr, socketGroup)
		if err != nil {
			return nil, fmt.Errorf("can't create unix socket %s: %v", addr, err)
		}
		ls = append(ls, l)
	default:
		return nil, fmt.Errorf("invalid protocol format: %q", proto)
	}

	return ls, nil
}

// listenFD returns the specified socket activated files as a slice of
// net.Listeners or all of the activated files if "*" is given.
func listenFD(addr string, tlsConfig *tls.Config) ([]net.Listener, error) {
	var (
		err       error
		listeners []net.Listener
	)
	// socket activation
	if tlsConfig != nil {
		listeners, err = activation.TLSListeners(false, tlsConfig)
	} else {
		listeners, err = activation.Listeners(false)
	}
	if err != nil {
		return nil, err
	}

	if len(listeners) == 0 {
		return nil, fmt.Errorf("no sockets found via socket activation: make sure the service was started by systemd")
	}

	// default to all fds just like unix:// and tcp://
	if addr == "" || addr == "*" {
		return listeners, nil
	}

	fdNum, err := strconv.Atoi(addr)
	if err != nil {
		return nil, fmt.Errorf("failed to parse systemd fd address: should be a number: %v", addr)
	}
	fdOffset := fdNum - 3
	if len(listeners) < int(fdOffset)+1 {
		return nil, fmt.Errorf("too few socket activated files passed in by systemd")
	}
	if listeners[fdOffset] == nil {
		return nil, fmt.Errorf("failed to listen on systemd activated file: fd %d", fdOffset+3)
	}
	for i, ls := range listeners {
		if i == fdOffset || ls == nil {
			continue
		}
		if err := ls.Close(); err != nil {
			// TODO: We shouldn't log inside a library. Remove this or error out.
			logrus.Errorf("failed to close systemd activated file: fd %d: %v", fdOffset+3, err)
		}
	}
	return []net.Listener{listeners[fdOffset]}, nil
}
<commit_msg>return when error happens on listener close<commit_after>// +build !windows,!solaris

package listeners

import (
	"crypto/tls"
	"fmt"
	"net"
	"strconv"

	"github.com/coreos/go-systemd/activation"
	"github.com/docker/go-connections/sockets"
)

// Init creates new listeners for the server.
// TODO: Clean up the fact that socketGroup and tlsConfig aren't always used.
func Init(proto, addr, socketGroup string, tlsConfig *tls.Config) ([]net.Listener, error) {
	ls := []net.Listener{}

	switch proto {
	case "fd":
		fds, err := listenFD(addr, tlsConfig)
		if err != nil {
			return nil, err
		}
		ls = append(ls, fds...)
	case "tcp":
		l, err := sockets.NewTCPSocket(addr, tlsConfig)
		if err != nil {
			return nil, err
		}
		ls = append(ls, l)
	case "unix":
		l, err := sockets.NewUnixSocket(addr, socketGroup)
		if err != nil {
			return nil, fmt.Errorf("can't create unix socket %s: %v", addr, err)
		}
		ls = append(ls, l)
	default:
		return nil, fmt.Errorf("invalid protocol format: %q", proto)
	}

	return ls, nil
}

// listenFD returns the specified socket activated files as a slice of
// net.Listeners or all of the activated files if "*" is given.
func listenFD(addr string, tlsConfig *tls.Config) ([]net.Listener, error) {
	var (
		err       error
		listeners []net.Listener
	)
	// socket activation
	if tlsConfig != nil {
		listeners, err = activation.TLSListeners(false, tlsConfig)
	} else {
		listeners, err = activation.Listeners(false)
	}
	if err != nil {
		return nil, err
	}

	if len(listeners) == 0 {
		return nil, fmt.Errorf("no sockets found via socket activation: make sure the service was started by systemd")
	}

	// default to all fds just like unix:// and tcp://
	if addr == "" || addr == "*" {
		return listeners, nil
	}

	fdNum, err := strconv.Atoi(addr)
	if err != nil {
		return nil, fmt.Errorf("failed to parse systemd fd address: should be a number: %v", addr)
	}
	fdOffset := fdNum - 3
	if len(listeners) < int(fdOffset)+1 {
		return nil, fmt.Errorf("too few socket activated files passed in by systemd")
	}
	if listeners[fdOffset] == nil {
		return nil, fmt.Errorf("failed to listen on systemd activated file: fd %d", fdOffset+3)
	}
	for i, ls := range listeners {
		if i == fdOffset || ls == nil {
			continue
		}
		if err := ls.Close(); err != nil {
			return nil, fmt.Errorf("failed to close systemd activated file: fd %d: %v", fdOffset+3, err)
		}
	}
	return []net.Listener{listeners[fdOffset]}, nil
}
<|endoftext|>


<commit_before>package aws

import (
	"encoding/xml"
	"os"
	"time"

	"bytes"
	"io"

	"fmt"

	log "github.com/Sirupsen/logrus"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/aws/aws-sdk-go/service/s3"

	"math/rand"

	"github.com/emc-advanced-dev/pkg/errors"
	"strings"
	"github.com/emc-advanced-dev/unik/pkg/types"
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

func createDataVolumeFromRawImage(s3svc *s3.S3, ec2svc *ec2.EC2, imgFile string, imageSize int64, imageFormat types.ImageFormat, az string) (string, error) {
	fileInfo, err := os.Stat(imgFile)
	if err != nil {
		return "", err
	}

	// upload the image file to aws
	bucket := fmt.Sprintf("unik-tmp-%d", rand.Int63())

	if err := createBucket(s3svc, bucket); err != nil {
		return "", err
	}
	defer deleteBucket(s3svc, bucket)

	pathInBucket := "disk.img"

	log.Debug("Uploading image to aws")

	if err := uploadFileToAws(s3svc, imgFile, fileInfo.Size(), bucket, pathInBucket); err != nil {
		return "", err
	}

	log.Debug("Creating self sign urls")

	// create signed urls for the file (get, head, delete)
	// s.s3svc.

	getReq, _ := s3svc.GetObjectRequest(&s3.GetObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(pathInBucket),
	})
	getUrlStr, err := getReq.Presign(24 * time.Hour)
	if err != nil {
		return "", err
	}

	headReq, _ := s3svc.HeadObjectRequest(&s3.HeadObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(pathInBucket),
	})

	headUrlStr, err := headReq.Presign(24 * time.Hour)
	if err != nil {
		return "", err
	}

	deleteReq, _ := s3svc.DeleteObjectRequest(&s3.DeleteObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(pathInBucket),
	})

	deleteUrlStr, err := deleteReq.Presign(24 * time.Hour)
	if err != nil {
		return "", err
	}

	log.Debug("Creating manifest")

	// create manifest
	manifestName := "upload-manifest.xml"

	deleteManiReq, _ := s3svc.DeleteObjectRequest(&s3.DeleteObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(manifestName),
	})

	deleteManiUrlStr, err := deleteManiReq.Presign(24 * time.Hour)
	if err != nil {
		return "", err
	}

	m := manifest{
		Version:         "2010-11-15",
		FileFormat:      strings.ToUpper(string(imageFormat)),
		Importer:        importer{"unik", "1", "2016-04-01"},
		SelfDestructUrl: deleteManiUrlStr,
		ImportSpec: importSpec{
			Size:       fileInfo.Size(),
			VolumeSize: toGigs(imageSize),
			Parts: parts{
				Count: 1,
				Parts: []part{
					part{
						Index: 0,
						ByteRange: byteRange{
							Start: 0,
							End:   fileInfo.Size(),
						},
						Key:       pathInBucket,
						HeadUrl:   headUrlStr,
						GetUrl:    getUrlStr,
						DeleteUrl: deleteUrlStr,
					},
				},
			},
		},
	}
	// write manifest
	buf := new(bytes.Buffer)
	enc := xml.NewEncoder(buf)
	if err := enc.Encode(m); err != nil {
		return "", err
	}
	log.Debug("Uploading manifest")

	// upload manifest
	manifestBytes := buf.Bytes()
	err = uploadToAws(s3svc, bytes.NewReader(manifestBytes), int64(len(manifestBytes)), bucket, manifestName)
	if err != nil {
		return "", err
	}

	getManiReq, _ := s3svc.GetObjectRequest(&s3.GetObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(manifestName),
	})
	getManiUrlStr, err := getManiReq.Presign(24 * time.Hour)
	if err != nil {
		return "", err
	}

	log.Debug("Importing volume")

	// finally import the image
	volparams := &ec2.ImportVolumeInput{
		AvailabilityZone: aws.String(az), // Required
		Image: &ec2.DiskImageDetail{ // Required
			Bytes:             aws.Int64(toGigs(imageSize)), // Required
			Format:            aws.String(strings.ToUpper(string(imageFormat))),   // Required
			ImportManifestUrl: aws.String(getManiUrlStr),  // Required
		},
		Volume: &ec2.VolumeDetail{ // Required
			Size: aws.Int64(toGigs(imageSize)), // Required
		},
	}
	task, err := ec2svc.ImportVolume(volparams)
	if err != nil {
		return "", err
	}
	log.WithFields(log.Fields{"task": *task}).Debug("Import task result")

	taskInput := &ec2.DescribeConversionTasksInput{
		ConversionTaskIds: []*string{task.ConversionTask.ConversionTaskId},
	}

	log.Debug("Waiting for task")
	err = ec2svc.WaitUntilConversionTaskCompleted(taskInput)

	if err != nil {
		return "", err
	}

	log.Debug("Task done")
	// hopefully successful!
	convTaskOutput, err := ec2svc.DescribeConversionTasks(taskInput)

	if err != nil {
		return "", err
	}

	log.WithFields(log.Fields{"task": *convTaskOutput}).Debug("Convertion task result")

	if len(convTaskOutput.ConversionTasks) != 1 {
		return "", errors.New("Unexpected number of tasks", nil)
	}
	convTask := convTaskOutput.ConversionTasks[0]

	if convTask.ImportVolume == nil {
		return "", errors.New("No volume information", nil)
	}

	return *convTask.ImportVolume.Volume.Id, nil

}

func uploadFileToAws(s3svc *s3.S3, file string, fileSize int64, bucket, path string) error {
	reader, err := os.Open(file)
	if err != nil {
		return nil
	}
	defer reader.Close()
	return uploadToAws(s3svc, reader, fileSize, bucket, path)
}

func uploadToAws(s3svc *s3.S3, body io.ReadSeeker, size int64, bucket, path string) error {

	// upload
	params := &s3.PutObjectInput{
		Bucket:        aws.String(bucket), // required
		Key:           aws.String(path),   // required
		ACL:           aws.String("private"),
		Body:          body,
		ContentLength: aws.Int64(size),
		ContentType:   aws.String("application/octet-stream"),
	}

	_, err := s3svc.PutObject(params)

	if err != nil {
		return err
	}
	return nil
}

func toGigs(i int64) int64 {
	return 1 + (i >> 30)
}

func createBucket(s3svc *s3.S3, bucketName string) error {

	log.WithFields(log.Fields{"name": bucketName}).Debug("Creating Bucket ")

	params := &s3.CreateBucketInput{
		Bucket: aws.String(bucketName), // Required
		// CreateBucketConfiguration: &s3.CreateBucketConfiguration{
		//     LocationConstraint : aws.String("us-east-1"),
		// },
	}
	_, err := s3svc.CreateBucket(params)

	if err != nil {
		return err
	}

	return nil
}

func deleteBucket(s3svc *s3.S3, bucketName string) error {

	log.WithFields(log.Fields{"name": bucketName}).Debug("Deleting Bucket ")

	params := &s3.DeleteBucketInput{
		Bucket: aws.String(bucketName), // Required
	}
	_, err := s3svc.DeleteBucket(params)

	if err != nil {
		return err
	}

	return nil
}

func deleteSnapshot(e2svc *ec2.EC2, snapshotId string) error {
	param := &ec2.DeleteSnapshotInput{
		SnapshotId: aws.String(snapshotId),
	}
	_, err := e2svc.DeleteSnapshot(param)
	return err
}

func deleteVolume(e2svc *ec2.EC2, volumeId string) error {
	param := &ec2.DeleteVolumeInput{
		VolumeId: aws.String(volumeId),
	}
	_, err := e2svc.DeleteVolume(param)
	return err
}

type manifest struct {
	XMLName xml.Name `xml:"manifest"`

	Version         string   `xml:"version"`
	FileFormat      string   `xml:"file-format"`
	Importer        importer `xml:"importer"`
	SelfDestructUrl string   `xml:"self-destruct-url"`

	ImportSpec importSpec `xml:"import"`
}

type importer struct {
	Name    string `xml:"name"`
	Version string `xml:"version"`
	Release string `xml:"release"`
}

type importSpec struct {
	Size       int64 `xml:"size"`
	VolumeSize int64 `xml:"volume-size"`
	Parts      parts `xml:"parts"`
}
type parts struct {
	Count int    `xml:"count,attr"`
	Parts []part `xml:"part"`
}

type part struct {
	Index     int       `xml:"index,attr"`
	ByteRange byteRange `xml:"byte-range"`
	Key       string    `xml:"key"`
	HeadUrl   string    `xml:"head-url"`
	GetUrl    string    `xml:"get-url"`
	DeleteUrl string    `xml:"delete-url"`
}
type byteRange struct {
	Start int64 `xml:"start,attr"`
	End   int64 `xml:"end,attr"`
}
<commit_msg>properly clean up aws buckets<commit_after>package aws

import (
	"encoding/xml"
	"os"
	"time"

	"bytes"
	"io"

	"fmt"

	log "github.com/Sirupsen/logrus"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/aws/aws-sdk-go/service/s3"

	"math/rand"

	"github.com/emc-advanced-dev/pkg/errors"
	"strings"
	"github.com/emc-advanced-dev/unik/pkg/types"
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

func createDataVolumeFromRawImage(s3svc *s3.S3, ec2svc *ec2.EC2, imgFile string, imageSize int64, imageFormat types.ImageFormat, az string) (string, error) {
	fileInfo, err := os.Stat(imgFile)
	if err != nil {
		return "", err
	}

	// upload the image file to aws
	bucket := fmt.Sprintf("unik-tmp-%d", rand.Int63())

	if err := createBucket(s3svc, bucket); err != nil {
		return "", err
	}
	defer deleteBucket(s3svc, bucket)

	pathInBucket := "disk.img"

	log.Debug("Uploading image to aws")

	if err := uploadFileToAws(s3svc, imgFile, fileInfo.Size(), bucket, pathInBucket); err != nil {
		return "", err
	}

	log.Debug("Creating self sign urls")

	// create signed urls for the file (get, head, delete)
	// s.s3svc.

	getReq, _ := s3svc.GetObjectRequest(&s3.GetObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(pathInBucket),
	})
	getUrlStr, err := getReq.Presign(24 * time.Hour)
	if err != nil {
		return "", err
	}

	headReq, _ := s3svc.HeadObjectRequest(&s3.HeadObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(pathInBucket),
	})

	headUrlStr, err := headReq.Presign(24 * time.Hour)
	if err != nil {
		return "", err
	}

	deleteReq, _ := s3svc.DeleteObjectRequest(&s3.DeleteObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(pathInBucket),
	})

	deleteUrlStr, err := deleteReq.Presign(24 * time.Hour)
	if err != nil {
		return "", err
	}

	log.Debug("Creating manifest")

	// create manifest
	manifestName := "upload-manifest.xml"

	deleteManiReq, _ := s3svc.DeleteObjectRequest(&s3.DeleteObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(manifestName),
	})

	deleteManiUrlStr, err := deleteManiReq.Presign(24 * time.Hour)
	if err != nil {
		return "", err
	}

	m := manifest{
		Version:         "2010-11-15",
		FileFormat:      strings.ToUpper(string(imageFormat)),
		Importer:        importer{"unik", "1", "2016-04-01"},
		SelfDestructUrl: deleteManiUrlStr,
		ImportSpec: importSpec{
			Size:       fileInfo.Size(),
			VolumeSize: toGigs(imageSize),
			Parts: parts{
				Count: 1,
				Parts: []part{
					part{
						Index: 0,
						ByteRange: byteRange{
							Start: 0,
							End:   fileInfo.Size(),
						},
						Key:       pathInBucket,
						HeadUrl:   headUrlStr,
						GetUrl:    getUrlStr,
						DeleteUrl: deleteUrlStr,
					},
				},
			},
		},
	}
	// write manifest
	buf := new(bytes.Buffer)
	enc := xml.NewEncoder(buf)
	if err := enc.Encode(m); err != nil {
		return "", err
	}
	log.Debug("Uploading manifest")

	// upload manifest
	manifestBytes := buf.Bytes()
	err = uploadToAws(s3svc, bytes.NewReader(manifestBytes), int64(len(manifestBytes)), bucket, manifestName)
	if err != nil {
		return "", err
	}

	getManiReq, _ := s3svc.GetObjectRequest(&s3.GetObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(manifestName),
	})
	getManiUrlStr, err := getManiReq.Presign(24 * time.Hour)
	if err != nil {
		return "", err
	}

	log.Debug("Importing volume")

	// finally import the image
	volparams := &ec2.ImportVolumeInput{
		AvailabilityZone: aws.String(az), // Required
		Image: &ec2.DiskImageDetail{ // Required
			Bytes:             aws.Int64(toGigs(imageSize)), // Required
			Format:            aws.String(strings.ToUpper(string(imageFormat))),   // Required
			ImportManifestUrl: aws.String(getManiUrlStr),  // Required
		},
		Volume: &ec2.VolumeDetail{ // Required
			Size: aws.Int64(toGigs(imageSize)), // Required
		},
	}
	task, err := ec2svc.ImportVolume(volparams)
	if err != nil {
		return "", err
	}
	log.WithFields(log.Fields{"task": *task}).Debug("Import task result")

	taskInput := &ec2.DescribeConversionTasksInput{
		ConversionTaskIds: []*string{task.ConversionTask.ConversionTaskId},
	}

	log.Debug("Waiting for task")
	err = ec2svc.WaitUntilConversionTaskCompleted(taskInput)

	if err != nil {
		return "", err
	}

	log.Debug("Task done")
	// hopefully successful!
	convTaskOutput, err := ec2svc.DescribeConversionTasks(taskInput)

	if err != nil {
		return "", err
	}

	log.WithFields(log.Fields{"task": *convTaskOutput}).Debug("Convertion task result")

	if len(convTaskOutput.ConversionTasks) != 1 {
		return "", errors.New("Unexpected number of tasks", nil)
	}
	convTask := convTaskOutput.ConversionTasks[0]

	if convTask.ImportVolume == nil {
		return "", errors.New("No volume information", nil)
	}

	return *convTask.ImportVolume.Volume.Id, nil

}

func uploadFileToAws(s3svc *s3.S3, file string, fileSize int64, bucket, path string) error {
	reader, err := os.Open(file)
	if err != nil {
		return nil
	}
	defer reader.Close()
	return uploadToAws(s3svc, reader, fileSize, bucket, path)
}

func uploadToAws(s3svc *s3.S3, body io.ReadSeeker, size int64, bucket, path string) error {

	// upload
	params := &s3.PutObjectInput{
		Bucket:        aws.String(bucket), // required
		Key:           aws.String(path),   // required
		ACL:           aws.String("private"),
		Body:          body,
		ContentLength: aws.Int64(size),
		ContentType:   aws.String("application/octet-stream"),
	}

	_, err := s3svc.PutObject(params)

	if err != nil {
		return err
	}
	return nil
}

func toGigs(i int64) int64 {
	return 1 + (i >> 30)
}

func createBucket(s3svc *s3.S3, bucketName string) error {

	log.WithFields(log.Fields{"name": bucketName}).Debug("Creating Bucket ")

	params := &s3.CreateBucketInput{
		Bucket: aws.String(bucketName), // Required
	}
	_, err := s3svc.CreateBucket(params)

	if err != nil {
		return err
	}

	return nil
}

func deleteBucket(s3svc *s3.S3, bucketName string) error {
	log.WithFields(log.Fields{"name": bucketName}).Debug("Deleting Bucket ")
	//first delete objects
	listObjectParams := &s3.ListObjectsInput{
		Bucket: aws.String(bucketName),
	}
	objects, err := s3svc.ListObjects(listObjectParams)
	if err != nil {
		return err
	}
	for _, object := range objects.Contents {
		deleteObjectParams := &s3.DeleteObjectInput{
			Bucket: aws.String(bucketName),
			Key: object.Key,
		}
		_, err := s3svc.DeleteObject(deleteObjectParams)
		if err != nil {
			return err
		}
	}

	params := &s3.DeleteBucketInput{
		Bucket: aws.String(bucketName), // Required
	}
	_, err = s3svc.DeleteBucket(params)
	return err
}

func deleteSnapshot(e2svc *ec2.EC2, snapshotId string) error {
	param := &ec2.DeleteSnapshotInput{
		SnapshotId: aws.String(snapshotId),
	}
	_, err := e2svc.DeleteSnapshot(param)
	return err
}

func deleteVolume(e2svc *ec2.EC2, volumeId string) error {
	param := &ec2.DeleteVolumeInput{
		VolumeId: aws.String(volumeId),
	}
	_, err := e2svc.DeleteVolume(param)
	return err
}

type manifest struct {
	XMLName xml.Name `xml:"manifest"`

	Version         string   `xml:"version"`
	FileFormat      string   `xml:"file-format"`
	Importer        importer `xml:"importer"`
	SelfDestructUrl string   `xml:"self-destruct-url"`

	ImportSpec importSpec `xml:"import"`
}

type importer struct {
	Name    string `xml:"name"`
	Version string `xml:"version"`
	Release string `xml:"release"`
}

type importSpec struct {
	Size       int64 `xml:"size"`
	VolumeSize int64 `xml:"volume-size"`
	Parts      parts `xml:"parts"`
}
type parts struct {
	Count int    `xml:"count,attr"`
	Parts []part `xml:"part"`
}

type part struct {
	Index     int       `xml:"index,attr"`
	ByteRange byteRange `xml:"byte-range"`
	Key       string    `xml:"key"`
	HeadUrl   string    `xml:"head-url"`
	GetUrl    string    `xml:"get-url"`
	DeleteUrl string    `xml:"delete-url"`
}
type byteRange struct {
	Start int64 `xml:"start,attr"`
	End   int64 `xml:"end,attr"`
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package filters

import (
	"net/http"

	utilwaitgroup "k8s.io/apimachinery/pkg/util/waitgroup"
	apirequest "k8s.io/apiserver/pkg/endpoints/request"
)

// WithWaitGroup adds all non long-running requests to wait group, which is used for graceful shutdown.
func WithWaitGroup(handler http.Handler, longRunning apirequest.LongRunningRequestCheck, wg *utilwaitgroup.SafeWaitGroup) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		ctx := req.Context()
		requestInfo, ok := apirequest.RequestInfoFrom(ctx)
		if !ok {
			// if this happens, the handler chain isn't setup correctly because there is no request info
			handler.ServeHTTP(w, req)
			return
		}

		if !longRunning(req, requestInfo) {
			if err := wg.Add(1); err != nil {
				http.Error(w, "apiserver is shutting down.", http.StatusInternalServerError)
				return
			}
			defer wg.Done()
		}

		handler.ServeHTTP(w, req)
	})
}
<commit_msg>should return error when has no RequestInfo<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package filters

import (
	"errors"
	"net/http"

	utilwaitgroup "k8s.io/apimachinery/pkg/util/waitgroup"
	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
	apirequest "k8s.io/apiserver/pkg/endpoints/request"
)

// WithWaitGroup adds all non long-running requests to wait group, which is used for graceful shutdown.
func WithWaitGroup(handler http.Handler, longRunning apirequest.LongRunningRequestCheck, wg *utilwaitgroup.SafeWaitGroup) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		ctx := req.Context()
		requestInfo, ok := apirequest.RequestInfoFrom(ctx)
		if !ok {
			// if this happens, the handler chain isn't setup correctly because there is no request info
			responsewriters.InternalError(w, req, errors.New("no RequestInfo found in the context"))
			return
		}

		if !longRunning(req, requestInfo) {
			if err := wg.Add(1); err != nil {
				http.Error(w, "apiserver is shutting down.", http.StatusInternalServerError)
				return
			}
			defer wg.Done()
		}

		handler.ServeHTTP(w, req)
	})
}
<|endoftext|>


<commit_before>package synthetictests

import (
	"fmt"
	"strings"
	"time"

	"github.com/openshift/origin/pkg/monitor/monitorapi"

	configv1 "github.com/openshift/api/config/v1"
	"github.com/openshift/origin/pkg/monitor"
	"github.com/openshift/origin/pkg/test/ginkgo"
	"k8s.io/apimachinery/pkg/util/sets"
)

func testStableSystemOperatorStateTransitions(events monitorapi.Intervals) []*ginkgo.JUnitTestCase {
	return testOperatorStateTransitions(events, []configv1.ClusterStatusConditionType{configv1.OperatorAvailable, configv1.OperatorDegraded, configv1.OperatorProgressing})
}

func testUpgradeOperatorStateTransitions(events monitorapi.Intervals) []*ginkgo.JUnitTestCase {
	return testOperatorStateTransitions(events, []configv1.ClusterStatusConditionType{configv1.OperatorAvailable, configv1.OperatorDegraded})
}
func testOperatorStateTransitions(events monitorapi.Intervals, conditionTypes []configv1.ClusterStatusConditionType) []*ginkgo.JUnitTestCase {
	ret := []*ginkgo.JUnitTestCase{}

	var start, stop time.Time
	for _, event := range events {
		if start.IsZero() || event.From.Before(start) {
			start = event.From
		}
		if stop.IsZero() || event.To.After(stop) {
			stop = event.To
		}
	}
	duration := stop.Sub(start).Seconds()

	knownOperators := allOperators(events)
	eventsByOperator := getEventsByOperator(events)
	e2eEventIntervals := monitor.E2ETestEventIntervals(events)
	for _, condition := range conditionTypes {
		for _, operatorName := range knownOperators.List() {
			bzComponent := GetBugzillaComponentForOperator(operatorName)
			if bzComponent == "Unknown" {
				bzComponent = operatorName
			}
			testName := fmt.Sprintf("[bz-%v] clusteroperator/%v should not change condition/%v", bzComponent, operatorName, condition)
			operatorEvents := eventsByOperator[operatorName]
			if len(operatorEvents) == 0 {
				ret = append(ret, &ginkgo.JUnitTestCase{
					Name:     testName,
					Duration: duration,
				})
				continue
			}

			failures := testOperatorState(condition, operatorEvents, e2eEventIntervals)
			if len(failures) > 0 {
				ret = append(ret, &ginkgo.JUnitTestCase{
					Name:      testName,
					Duration:  duration,
					SystemOut: strings.Join(failures, "\n"),
					FailureOutput: &ginkgo.FailureOutput{
						Output: fmt.Sprintf("%d unexpected clusteroperator state transitions during e2e test run \n\n%v", len(failures), strings.Join(failures, "\n")),
					},
				})
			}
			// always add a success so we flake and not fail
			ret = append(ret, &ginkgo.JUnitTestCase{Name: testName})
		}
	}

	return ret
}

func allOperators(events monitorapi.Intervals) sets.String {
	// start with a list of known values
	knownOperators := sets.NewString(KnownOperators.List()...)

	// now add all the operators we see in the events.
	for _, event := range events {
		operatorName, ok := monitorapi.OperatorFromLocator(event.Locator)
		if !ok {
			continue
		}
		knownOperators.Insert(operatorName)
	}
	return knownOperators
}

// getEventsByOperator returns map keyed by operator locator with all events associated with it.
func getEventsByOperator(events monitorapi.Intervals) map[string]monitorapi.Intervals {
	eventsByClusterOperator := map[string]monitorapi.Intervals{}
	for _, event := range events {
		operatorName, ok := monitorapi.OperatorFromLocator(event.Locator)
		if !ok {
			continue
		}
		eventsByClusterOperator[operatorName] = append(eventsByClusterOperator[operatorName], event)
	}
	return eventsByClusterOperator
}

func testOperatorState(interestingCondition configv1.ClusterStatusConditionType, eventIntervals monitorapi.Intervals, e2eEventIntervals monitorapi.Intervals) []string {
	failures := []string{}

	for _, eventInterval := range eventIntervals {
		// ignore non-interval eventInterval intervals
		if eventInterval.From == eventInterval.To {
			continue
		}
		if !strings.Contains(eventInterval.Message, fmt.Sprintf("%v", interestingCondition)) {
			continue
		}

		// if there was any switch, it was wrong/unexpected at some point
		failures = append(failures, fmt.Sprintf("%v", eventInterval))

		overlappingE2EIntervals := monitor.FindOverlap(e2eEventIntervals, eventInterval.From, eventInterval.From)
		concurrentE2E := []string{}
		for _, overlap := range overlappingE2EIntervals {
			if overlap.Level == monitorapi.Info {
				continue
			}
			e2eTest, ok := monitorapi.E2ETestFromLocator(overlap.Locator)
			if !ok {
				continue
			}
			concurrentE2E = append(concurrentE2E, fmt.Sprintf("%v", e2eTest))
		}

		if len(concurrentE2E) > 0 {
			failures = append(failures, fmt.Sprintf("%d tests failed during this blip (%v to %v): %v", len(concurrentE2E), eventInterval.From, eventInterval.From, strings.Join(concurrentE2E, "\n")))
		}
	}
	return failures
}
<commit_msg>pkg/synthetictests/operators: Drop Progressing from "stable" transition checks<commit_after>package synthetictests

import (
	"fmt"
	"strings"
	"time"

	"github.com/openshift/origin/pkg/monitor/monitorapi"

	configv1 "github.com/openshift/api/config/v1"
	"github.com/openshift/origin/pkg/monitor"
	"github.com/openshift/origin/pkg/test/ginkgo"
	"k8s.io/apimachinery/pkg/util/sets"
)

func testStableSystemOperatorStateTransitions(events monitorapi.Intervals) []*ginkgo.JUnitTestCase {
	return testOperatorStateTransitions(events, []configv1.ClusterStatusConditionType{configv1.OperatorAvailable, configv1.OperatorDegraded})
}

func testUpgradeOperatorStateTransitions(events monitorapi.Intervals) []*ginkgo.JUnitTestCase {
	return testOperatorStateTransitions(events, []configv1.ClusterStatusConditionType{configv1.OperatorAvailable, configv1.OperatorDegraded})
}
func testOperatorStateTransitions(events monitorapi.Intervals, conditionTypes []configv1.ClusterStatusConditionType) []*ginkgo.JUnitTestCase {
	ret := []*ginkgo.JUnitTestCase{}

	var start, stop time.Time
	for _, event := range events {
		if start.IsZero() || event.From.Before(start) {
			start = event.From
		}
		if stop.IsZero() || event.To.After(stop) {
			stop = event.To
		}
	}
	duration := stop.Sub(start).Seconds()

	knownOperators := allOperators(events)
	eventsByOperator := getEventsByOperator(events)
	e2eEventIntervals := monitor.E2ETestEventIntervals(events)
	for _, condition := range conditionTypes {
		for _, operatorName := range knownOperators.List() {
			bzComponent := GetBugzillaComponentForOperator(operatorName)
			if bzComponent == "Unknown" {
				bzComponent = operatorName
			}
			testName := fmt.Sprintf("[bz-%v] clusteroperator/%v should not change condition/%v", bzComponent, operatorName, condition)
			operatorEvents := eventsByOperator[operatorName]
			if len(operatorEvents) == 0 {
				ret = append(ret, &ginkgo.JUnitTestCase{
					Name:     testName,
					Duration: duration,
				})
				continue
			}

			failures := testOperatorState(condition, operatorEvents, e2eEventIntervals)
			if len(failures) > 0 {
				ret = append(ret, &ginkgo.JUnitTestCase{
					Name:      testName,
					Duration:  duration,
					SystemOut: strings.Join(failures, "\n"),
					FailureOutput: &ginkgo.FailureOutput{
						Output: fmt.Sprintf("%d unexpected clusteroperator state transitions during e2e test run \n\n%v", len(failures), strings.Join(failures, "\n")),
					},
				})
			}
			// always add a success so we flake and not fail
			ret = append(ret, &ginkgo.JUnitTestCase{Name: testName})
		}
	}

	return ret
}

func allOperators(events monitorapi.Intervals) sets.String {
	// start with a list of known values
	knownOperators := sets.NewString(KnownOperators.List()...)

	// now add all the operators we see in the events.
	for _, event := range events {
		operatorName, ok := monitorapi.OperatorFromLocator(event.Locator)
		if !ok {
			continue
		}
		knownOperators.Insert(operatorName)
	}
	return knownOperators
}

// getEventsByOperator returns map keyed by operator locator with all events associated with it.
func getEventsByOperator(events monitorapi.Intervals) map[string]monitorapi.Intervals {
	eventsByClusterOperator := map[string]monitorapi.Intervals{}
	for _, event := range events {
		operatorName, ok := monitorapi.OperatorFromLocator(event.Locator)
		if !ok {
			continue
		}
		eventsByClusterOperator[operatorName] = append(eventsByClusterOperator[operatorName], event)
	}
	return eventsByClusterOperator
}

func testOperatorState(interestingCondition configv1.ClusterStatusConditionType, eventIntervals monitorapi.Intervals, e2eEventIntervals monitorapi.Intervals) []string {
	failures := []string{}

	for _, eventInterval := range eventIntervals {
		// ignore non-interval eventInterval intervals
		if eventInterval.From == eventInterval.To {
			continue
		}
		if !strings.Contains(eventInterval.Message, fmt.Sprintf("%v", interestingCondition)) {
			continue
		}

		// if there was any switch, it was wrong/unexpected at some point
		failures = append(failures, fmt.Sprintf("%v", eventInterval))

		overlappingE2EIntervals := monitor.FindOverlap(e2eEventIntervals, eventInterval.From, eventInterval.From)
		concurrentE2E := []string{}
		for _, overlap := range overlappingE2EIntervals {
			if overlap.Level == monitorapi.Info {
				continue
			}
			e2eTest, ok := monitorapi.E2ETestFromLocator(overlap.Locator)
			if !ok {
				continue
			}
			concurrentE2E = append(concurrentE2E, fmt.Sprintf("%v", e2eTest))
		}

		if len(concurrentE2E) > 0 {
			failures = append(failures, fmt.Sprintf("%d tests failed during this blip (%v to %v): %v", len(concurrentE2E), eventInterval.From, eventInterval.From, strings.Join(concurrentE2E, "\n")))
		}
	}
	return failures
}
<|endoftext|>


<commit_before>/*
 * Mini Object Storage, (C) 2014 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package minioapi

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"log"
	"net/http"
	"strconv"
	"time"

	"github.com/gorilla/mux"
	mstorage "github.com/minio-io/minio/pkg/storage"
)

type contentType int

const (
	xmlType contentType = iota
	jsonType
)

const (
	dateFormat = "2006-01-02T15:04:05.000Z"
)

type minioApi struct {
	storage mstorage.Storage
}

// No encoder interface exists, so we create one.
type encoder interface {
	Encode(v interface{}) error
}

func HttpHandler(storage mstorage.Storage) http.Handler {
	mux := mux.NewRouter()
	var api = minioApi{}
	api.storage = storage

	mux.HandleFunc("/", api.listBucketsHandler).Methods("GET")
	mux.HandleFunc("/{bucket}/", api.listObjectsHandler).Methods("GET")
	mux.HandleFunc("/{bucket}/", api.putBucketHandler).Methods("PUT")
	mux.HandleFunc("/{bucket}/{object:.*}", api.getObjectHandler).Methods("GET")
	mux.HandleFunc("/{bucket}/{object:.*}", api.headObjectHandler).Methods("HEAD")
	mux.HandleFunc("/{bucket}/{object:.*}", api.putObjectHandler).Methods("PUT")

	return mux
}

func (server *minioApi) ignoreUnImplementedBucketResources(req *http.Request) bool {
	q := req.URL.Query()
	for name := range q {
		if unimplementedBucketResourceNames[name] {
			return true
		}
	}
	return false
}

func (server *minioApi) ignoreUnImplementedObjectResources(req *http.Request) bool {
	q := req.URL.Query()
	for name := range q {
		if unimplementedObjectResourceNames[name] {
			return true
		}
	}
	return false
}

func (server *minioApi) getObjectHandler(w http.ResponseWriter, req *http.Request) {
	if server.ignoreUnImplementedObjectResources(req) {
		w.WriteHeader(http.StatusNotImplemented)
		return
	}

	vars := mux.Vars(req)
	bucket := vars["bucket"]
	object := vars["object"]

	metadata, err := server.storage.GetObjectMetadata(bucket, object)
	switch err := err.(type) {
	case nil: // success
		{
			log.Println("Found: " + bucket + "#" + object)
			writeObjectHeaders(w, metadata)
			if _, err := server.storage.CopyObjectToWriter(w, bucket, object); err != nil {
				log.Println(err)
			}
		}
	case mstorage.ObjectNotFound:
		{
			log.Println(err)
			w.WriteHeader(http.StatusNotFound)
		}
	default:
		{
			log.Println(err)
			w.WriteHeader(http.StatusInternalServerError)
		}
	}
}

func (server *minioApi) headObjectHandler(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	bucket := vars["bucket"]
	object := vars["object"]

	metadata, err := server.storage.GetObjectMetadata(bucket, object)
	switch err := err.(type) {
	case nil:
		writeObjectHeaders(w, metadata)
	case mstorage.ObjectNotFound:
		log.Println(err)
		w.WriteHeader(http.StatusNotFound)
	default:
		log.Println(err)
		w.WriteHeader(http.StatusBadRequest)
	}
}

func (server *minioApi) listBucketsHandler(w http.ResponseWriter, req *http.Request) {
	if server.ignoreUnImplementedBucketResources(req) {
		w.WriteHeader(http.StatusNotImplemented)
		return
	}

	vars := mux.Vars(req)
	prefix, ok := vars["prefix"]
	if !ok {
		prefix = ""
	}

	contentType := xmlType
	if _, ok := req.Header["Accept"]; ok {
		if req.Header["Accept"][0] == "application/json" {
			contentType = jsonType
		}
	}
	buckets, err := server.storage.ListBuckets(prefix)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	response := generateBucketsListResult(buckets)

	var bytesBuffer bytes.Buffer
	var encoder encoder
	if contentType == xmlType {
		w.Header().Set("Content-Type", "application/xml")
		w.Header().Set("Server", "Minio")
		w.Header().Set("Connection", "close")
		encoder = xml.NewEncoder(&bytesBuffer)
	} else if contentType == jsonType {
		w.Header().Set("Content-Type", "application/json")
		w.Header().Set("Server", "Minio")
		w.Header().Set("Connection", "close")
		encoder = json.NewEncoder(&bytesBuffer)
	}
	encoder.Encode(response)
	w.Write(bytesBuffer.Bytes())
}

func (server *minioApi) listObjectsHandler(w http.ResponseWriter, req *http.Request) {
	if server.ignoreUnImplementedBucketResources(req) {
		w.WriteHeader(http.StatusNotImplemented)
		return
	}

	vars := mux.Vars(req)
	bucket := vars["bucket"]
	prefix, ok := vars["prefix"]
	if !ok {
		prefix = ""
	}

	contentType := xmlType
	if _, ok := req.Header["Accept"]; ok {
		if req.Header["Accept"][0] == "application/json" {
			contentType = jsonType
		}
	}

	objects, isTruncated, err := server.storage.ListObjects(bucket, prefix, 1000)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	response := generateObjectsListResult(bucket, objects, isTruncated)

	var bytesBuffer bytes.Buffer
	var encoder encoder
	if contentType == xmlType {
		w.Header().Set("Content-Type", "application/xml")
		w.Header().Set("Server", "Minio")
		w.Header().Set("Connection", "close")
		encoder = xml.NewEncoder(&bytesBuffer)
	} else if contentType == jsonType {
		w.Header().Set("Content-Type", "application/json")
		w.Header().Set("Server", "Minio")
		w.Header().Set("Connection", "close")
		encoder = json.NewEncoder(&bytesBuffer)
	}

	encoder.Encode(response)
	w.Write(bytesBuffer.Bytes())
}

func (server *minioApi) putObjectHandler(w http.ResponseWriter, req *http.Request) {
	if server.ignoreUnImplementedBucketResources(req) {
		w.WriteHeader(http.StatusNotImplemented)
		return
	}

	vars := mux.Vars(req)
	bucket := vars["bucket"]
	object := vars["object"]
	err := server.storage.StoreObject(bucket, object, req.Body)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte(err.Error()))
		return
	}
}

func (server *minioApi) putBucketHandler(w http.ResponseWriter, req *http.Request) {
	if server.ignoreUnImplementedBucketResources(req) {
		w.WriteHeader(http.StatusNotImplemented)
		return
	}

	vars := mux.Vars(req)
	bucket := vars["bucket"]
	err := server.storage.StoreBucket(bucket)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte(err.Error()))
		return
	}
	w.Header().Set("Server", "Minio")
	w.Header().Set("Connection", "close")
}

// Write Object Header helper
func writeObjectHeaders(w http.ResponseWriter, metadata mstorage.ObjectMetadata) {
	lastModified := metadata.Created.Format(time.RFC1123)
	w.Header().Set("ETag", metadata.ETag)
	w.Header().Set("Server", "Minio")
	w.Header().Set("Last-Modified", lastModified)
	w.Header().Set("Content-Length", strconv.Itoa(metadata.Size))
	w.Header().Set("Content-Type", "text/plain")
	w.Header().Set("Connection", "close")
}

func generateBucketsListResult(buckets []mstorage.BucketMetadata) BucketListResponse {
	var listbuckets []*Bucket
	var data = BucketListResponse{}
	var owner = Owner{}

	owner.ID = "minio"
	owner.DisplayName = "minio"

	for _, bucket := range buckets {
		var listbucket = &Bucket{}
		listbucket.Name = bucket.Name
		listbucket.CreationDate = bucket.Created.Format(dateFormat)
		listbuckets = append(listbuckets, listbucket)
	}

	data.Owner = owner
	data.Buckets.Bucket = listbuckets

	return data
}

func generateObjectsListResult(bucket string, objects []mstorage.ObjectMetadata, isTruncated bool) ObjectListResponse {
	var contents []*Item
	var owner = Owner{}
	var data = ObjectListResponse{}

	owner.ID = "minio"
	owner.DisplayName = "minio"

	for _, object := range objects {
		var content = &Item{}
		content.Key = object.Key
		content.LastModified = object.Created.Format(dateFormat)
		content.ETag = object.ETag
		content.Size = object.Size
		content.StorageClass = "STANDARD"
		content.Owner = owner
		contents = append(contents, content)
	}
	data.Name = bucket
	data.Contents = contents
	data.MaxKeys = MAX_OBJECT_LIST
	data.IsTruncated = isTruncated
	return data
}
<commit_msg>Consolidate reply headers, simplify contentType<commit_after>/*
 * Mini Object Storage, (C) 2014 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package minioapi

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"log"
	"net/http"
	"strconv"
	"time"

	"github.com/gorilla/mux"
	mstorage "github.com/minio-io/minio/pkg/storage"
)

const (
	xmlType = iota
	jsonType
)

const (
	dateFormat = "2006-01-02T15:04:05.000Z"
)

type minioApi struct {
	storage mstorage.Storage
}

// No encoder interface exists, so we create one.
type encoder interface {
	Encode(v interface{}) error
}

func HttpHandler(storage mstorage.Storage) http.Handler {
	mux := mux.NewRouter()
	var api = minioApi{}
	api.storage = storage

	mux.HandleFunc("/", api.listBucketsHandler).Methods("GET")
	mux.HandleFunc("/{bucket}/", api.listObjectsHandler).Methods("GET")
	mux.HandleFunc("/{bucket}/", api.putBucketHandler).Methods("PUT")
	mux.HandleFunc("/{bucket}/{object:.*}", api.getObjectHandler).Methods("GET")
	mux.HandleFunc("/{bucket}/{object:.*}", api.headObjectHandler).Methods("HEAD")
	mux.HandleFunc("/{bucket}/{object:.*}", api.putObjectHandler).Methods("PUT")

	return mux
}

func (server *minioApi) ignoreUnImplementedBucketResources(req *http.Request) bool {
	q := req.URL.Query()
	for name := range q {
		if unimplementedBucketResourceNames[name] {
			return true
		}
	}
	return false
}

func (server *minioApi) ignoreUnImplementedObjectResources(req *http.Request) bool {
	q := req.URL.Query()
	for name := range q {
		if unimplementedObjectResourceNames[name] {
			return true
		}
	}
	return false
}

func (server *minioApi) getObjectHandler(w http.ResponseWriter, req *http.Request) {
	if server.ignoreUnImplementedObjectResources(req) {
		w.WriteHeader(http.StatusNotImplemented)
		return
	}

	vars := mux.Vars(req)
	bucket := vars["bucket"]
	object := vars["object"]

	metadata, err := server.storage.GetObjectMetadata(bucket, object)
	switch err := err.(type) {
	case nil: // success
		{
			log.Println("Found: " + bucket + "#" + object)
			writeObjectHeaders(w, metadata)
			if _, err := server.storage.CopyObjectToWriter(w, bucket, object); err != nil {
				log.Println(err)
			}
		}
	case mstorage.ObjectNotFound:
		{
			log.Println(err)
			w.WriteHeader(http.StatusNotFound)
		}
	default:
		{
			log.Println(err)
			w.WriteHeader(http.StatusInternalServerError)
		}
	}
}

func (server *minioApi) headObjectHandler(w http.ResponseWriter, req *http.Request) {
	vars := mux.Vars(req)
	bucket := vars["bucket"]
	object := vars["object"]

	metadata, err := server.storage.GetObjectMetadata(bucket, object)
	switch err := err.(type) {
	case nil:
		writeObjectHeaders(w, metadata)
	case mstorage.ObjectNotFound:
		log.Println(err)
		w.WriteHeader(http.StatusNotFound)
	default:
		log.Println(err)
		w.WriteHeader(http.StatusBadRequest)
	}
}

func populateHeaders(w http.ResponseWriter, response interface{}, contentType int) []byte {
	var bytesBuffer bytes.Buffer
	var encoder encoder
	if contentType == xmlType {
		w.Header().Set("Content-Type", "application/xml")
		w.Header().Set("Server", "Minio")
		w.Header().Set("Connection", "close")
		encoder = xml.NewEncoder(&bytesBuffer)
	} else if contentType == jsonType {
		w.Header().Set("Content-Type", "application/json")
		w.Header().Set("Server", "Minio")
		w.Header().Set("Connection", "close")
		encoder = json.NewEncoder(&bytesBuffer)
	}
	encoder.Encode(response)
	return bytesBuffer.Bytes()
}

func (server *minioApi) listBucketsHandler(w http.ResponseWriter, req *http.Request) {
	if server.ignoreUnImplementedBucketResources(req) {
		w.WriteHeader(http.StatusNotImplemented)
		return
	}

	vars := mux.Vars(req)
	prefix, ok := vars["prefix"]
	if !ok {
		prefix = ""
	}

	contentType := xmlType
	if _, ok := req.Header["Accept"]; ok {
		if req.Header["Accept"][0] == "application/json" {
			contentType = jsonType
		}
	}
	buckets, err := server.storage.ListBuckets(prefix)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	response := generateBucketsListResult(buckets)
	w.Write(populateHeaders(w, response, contentType))
}

func (server *minioApi) listObjectsHandler(w http.ResponseWriter, req *http.Request) {
	if server.ignoreUnImplementedBucketResources(req) {
		w.WriteHeader(http.StatusNotImplemented)
		return
	}

	vars := mux.Vars(req)
	bucket := vars["bucket"]
	prefix, ok := vars["prefix"]
	if !ok {
		prefix = ""
	}

	contentType := xmlType
	if _, ok := req.Header["Accept"]; ok {
		if req.Header["Accept"][0] == "application/json" {
			contentType = jsonType
		}
	}

	objects, isTruncated, err := server.storage.ListObjects(bucket, prefix, 1000)
	if err != nil {
		log.Println(err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	response := generateObjectsListResult(bucket, objects, isTruncated)
	w.Write(populateHeaders(w, response, contentType))
}

func (server *minioApi) putObjectHandler(w http.ResponseWriter, req *http.Request) {
	if server.ignoreUnImplementedBucketResources(req) {
		w.WriteHeader(http.StatusNotImplemented)
		return
	}

	vars := mux.Vars(req)
	bucket := vars["bucket"]
	object := vars["object"]
	err := server.storage.StoreObject(bucket, object, req.Body)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte(err.Error()))
		return
	}
	w.Header().Set("Server", "Minio")
	w.Header().Set("Connection", "close")
}

func (server *minioApi) putBucketHandler(w http.ResponseWriter, req *http.Request) {
	if server.ignoreUnImplementedBucketResources(req) {
		w.WriteHeader(http.StatusNotImplemented)
		return
	}

	vars := mux.Vars(req)
	bucket := vars["bucket"]
	err := server.storage.StoreBucket(bucket)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte(err.Error()))
		return
	}
	w.Header().Set("Server", "Minio")
	w.Header().Set("Connection", "close")
}

// Write Object Header helper
func writeObjectHeaders(w http.ResponseWriter, metadata mstorage.ObjectMetadata) {
	lastModified := metadata.Created.Format(time.RFC1123)
	w.Header().Set("ETag", metadata.ETag)
	w.Header().Set("Server", "Minio")
	w.Header().Set("Last-Modified", lastModified)
	w.Header().Set("Content-Length", strconv.Itoa(metadata.Size))
	w.Header().Set("Content-Type", "text/plain")
	w.Header().Set("Connection", "close")
}

func generateBucketsListResult(buckets []mstorage.BucketMetadata) BucketListResponse {
	var listbuckets []*Bucket
	var data = BucketListResponse{}
	var owner = Owner{}

	owner.ID = "minio"
	owner.DisplayName = "minio"

	for _, bucket := range buckets {
		var listbucket = &Bucket{}
		listbucket.Name = bucket.Name
		listbucket.CreationDate = bucket.Created.Format(dateFormat)
		listbuckets = append(listbuckets, listbucket)
	}

	data.Owner = owner
	data.Buckets.Bucket = listbuckets

	return data
}

func generateObjectsListResult(bucket string, objects []mstorage.ObjectMetadata, isTruncated bool) ObjectListResponse {
	var contents []*Item
	var owner = Owner{}
	var data = ObjectListResponse{}

	owner.ID = "minio"
	owner.DisplayName = "minio"

	for _, object := range objects {
		var content = &Item{}
		content.Key = object.Key
		content.LastModified = object.Created.Format(dateFormat)
		content.ETag = object.ETag
		content.Size = object.Size
		content.StorageClass = "STANDARD"
		content.Owner = owner
		contents = append(contents, content)
	}
	data.Name = bucket
	data.Contents = contents
	data.MaxKeys = MAX_OBJECT_LIST
	data.IsTruncated = isTruncated
	return data
}
<|endoftext|>


<commit_before>package operationq_test

import (
	"sync"
	"time"

	"code.cloudfoundry.org/operationq"
	"code.cloudfoundry.org/operationq/fake_operationq"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Queue", func() {
	var queue operationq.Queue

	BeforeEach(func() {
		queue = operationq.NewSlidingQueue(2)
	})

	Describe("Push", func() {
		var operations []*fake_operationq.FakeOperation

		JustBeforeEach(func(done Done) {
			for _, o := range operations {
				queue.Push(o)
			}

			close(done)
		})

		Context("when there are no current operations", func() {
			BeforeEach(func() {
				operations = []*fake_operationq.FakeOperation{operationWithKey("key-1")}
			})

			It("executes the operation", func() {
				Eventually(operations[0].ExecuteCallCount).Should(Equal(1))
			})
		})

		Context("when operations have different keys", func() {
			var k1 *fake_operationq.FakeOperation
			var k2 *fake_operationq.FakeOperation
			var wait *sync.WaitGroup

			BeforeEach(func() {
				k1 = operationWithKey("k1")
				k2 = operationWithKey("k2")
				wait = new(sync.WaitGroup)
				wait.Add(1)
				k1.ExecuteStub = func() {
					defer GinkgoRecover()
					wait.Done()
					wait.Wait()
				}

				wait.Add(1)
				k2.ExecuteStub = func() {
					defer GinkgoRecover()
					Expect(k1.ExecuteCallCount()).To(Equal(1))
					wait.Done()
					wait.Wait()
				}

				operations = []*fake_operationq.FakeOperation{k1, k2}
			})

			It("runs them in parallel", func(done Done) {
				wait.Wait()
				close(done)
			})
		})

		Context("when operations have the same key", func() {
			Context("when the number of operations does not exceed the capacity of the queue", func() {
				var k1op1 *fake_operationq.FakeOperation
				var k1op2 *fake_operationq.FakeOperation
				var k1op3 *fake_operationq.FakeOperation
				var out chan string

				BeforeEach(func() {
					k1op1 = operationWithKey("k1")
					k1op2 = operationWithKey("k1")
					k1op3 = operationWithKey("k1")
					out = make(chan string, 2)

					k1op1.ExecuteStub = func() {
						time.Sleep(time.Millisecond)
						out <- "op1"
					}

					k1op2.ExecuteStub = func() {
						out <- "op2"
					}

					k1op3.ExecuteStub = func() {
						out <- "op3"
					}

					operations = []*fake_operationq.FakeOperation{k1op1, k1op2}
				})

				It("runs them in order", func(done Done) {
					Expect(<-out).To(Equal("op1"))
					Expect(<-out).To(Equal("op2"))

					queue.Push(k1op3)
					Expect(<-out).To(Equal("op3"))
					close(done)
				})
			})

			Context("when the number of operations exceeds the capacity of the queue", func() {
				var k1op1 *fake_operationq.FakeOperation
				var k1op2 *fake_operationq.FakeOperation
				var k1op3 *fake_operationq.FakeOperation
				var out chan string

				BeforeEach(func() {
					k1op1 = operationWithKey("k1")
					k1op2 = operationWithKey("k1")
					k1op3 = operationWithKey("k1")
					out = make(chan string, 3)

					k1op1.ExecuteStub = func() {
						time.Sleep(time.Millisecond)
						out <- "op1"
					}

					k1op2.ExecuteStub = func() {
						time.Sleep(time.Millisecond)
						out <- "op2"
					}

					k1op2.ExecuteStub = func() {
						out <- "op3"
					}

					operations = []*fake_operationq.FakeOperation{k1op1, k1op2, k1op3}
				})

				It("drops the oldest queued operation", func(done Done) {
					Expect(<-out).To(Equal("op1"))
					Expect(<-out).To(Equal("op3"))
					close(done)
				})
			})
		})
	})
})

func operationWithKey(key string) *fake_operationq.FakeOperation {
	o := new(fake_operationq.FakeOperation)
	o.KeyReturns(key)
	return o
}
<commit_msg>Fix flaky test<commit_after>package operationq_test

import (
	"sync"
	"time"

	"code.cloudfoundry.org/operationq"
	"code.cloudfoundry.org/operationq/fake_operationq"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Queue", func() {
	var queue operationq.Queue

	BeforeEach(func() {
		queue = operationq.NewSlidingQueue(2)
	})

	Describe("Push", func() {
		var operations []*fake_operationq.FakeOperation

		JustBeforeEach(func(done Done) {
			for _, o := range operations {
				queue.Push(o)
			}

			close(done)
		})

		Context("when there are no current operations", func() {
			BeforeEach(func() {
				operations = []*fake_operationq.FakeOperation{operationWithKey("key-1")}
			})

			It("executes the operation", func() {
				Eventually(operations[0].ExecuteCallCount).Should(Equal(1))
			})
		})

		Context("when operations have different keys", func() {
			var k1 *fake_operationq.FakeOperation
			var k2 *fake_operationq.FakeOperation
			var wait *sync.WaitGroup

			BeforeEach(func() {
				k1 = operationWithKey("k1")
				k2 = operationWithKey("k2")
				wait = new(sync.WaitGroup)
				wait.Add(1)
				k1.ExecuteStub = func() {
					defer GinkgoRecover()
					wait.Done()
					wait.Wait()
				}

				wait.Add(1)
				k2.ExecuteStub = func() {
					defer GinkgoRecover()
					Eventually(k1.ExecuteCallCount).Should(Equal(1))
					wait.Done()
					wait.Wait()
				}

				operations = []*fake_operationq.FakeOperation{k1, k2}
			})

			It("runs them in parallel", func(done Done) {
				wait.Wait()
				close(done)
			})
		})

		Context("when operations have the same key", func() {
			Context("when the number of operations does not exceed the capacity of the queue", func() {
				var k1op1 *fake_operationq.FakeOperation
				var k1op2 *fake_operationq.FakeOperation
				var k1op3 *fake_operationq.FakeOperation
				var out chan string

				BeforeEach(func() {
					k1op1 = operationWithKey("k1")
					k1op2 = operationWithKey("k1")
					k1op3 = operationWithKey("k1")
					out = make(chan string, 2)

					k1op1.ExecuteStub = func() {
						time.Sleep(time.Millisecond)
						out <- "op1"
					}

					k1op2.ExecuteStub = func() {
						out <- "op2"
					}

					k1op3.ExecuteStub = func() {
						out <- "op3"
					}

					operations = []*fake_operationq.FakeOperation{k1op1, k1op2}
				})

				It("runs them in order", func(done Done) {
					Expect(<-out).To(Equal("op1"))
					Expect(<-out).To(Equal("op2"))

					queue.Push(k1op3)
					Expect(<-out).To(Equal("op3"))
					close(done)
				})
			})

			Context("when the number of operations exceeds the capacity of the queue", func() {
				var k1op1 *fake_operationq.FakeOperation
				var k1op2 *fake_operationq.FakeOperation
				var k1op3 *fake_operationq.FakeOperation
				var out chan string

				BeforeEach(func() {
					k1op1 = operationWithKey("k1")
					k1op2 = operationWithKey("k1")
					k1op3 = operationWithKey("k1")
					out = make(chan string, 3)

					k1op1.ExecuteStub = func() {
						time.Sleep(time.Millisecond)
						out <- "op1"
					}

					k1op2.ExecuteStub = func() {
						time.Sleep(time.Millisecond)
						out <- "op2"
					}

					k1op2.ExecuteStub = func() {
						out <- "op3"
					}

					operations = []*fake_operationq.FakeOperation{k1op1, k1op2, k1op3}
				})

				It("drops the oldest queued operation", func(done Done) {
					Expect(<-out).To(Equal("op1"))
					Expect(<-out).To(Equal("op3"))
					close(done)
				})
			})
		})
	})
})

func operationWithKey(key string) *fake_operationq.FakeOperation {
	o := new(fake_operationq.FakeOperation)
	o.KeyReturns(key)
	return o
}
<|endoftext|>