<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcs

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"math/rand"
	"net"
	"net/url"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/api/googleapi"
)

// A bucket that wraps another, calling its methods in a retry loop with
// randomized exponential backoff.
type retryBucket struct {
	maxSleep time.Duration
	wrapped  Bucket
}

func newRetryBucket(
	maxSleep time.Duration,
	wrapped Bucket) (b Bucket) {
	b = &retryBucket{
		maxSleep: maxSleep,
		wrapped:  wrapped,
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func shouldRetry(err error) (b bool) {
	// HTTP 50x errors.
	if typed, ok := err.(*googleapi.Error); ok {
		if typed.Code >= 500 && typed.Code < 600 {
			b = true
			return
		}
	}

	// HTTP 429 errors (GCS uses these for rate limiting).
	if typed, ok := err.(*googleapi.Error); ok {
		if typed.Code == 429 {
			b = true
			return
		}
	}

	// Network errors, which tend to show up transiently when doing lots of
	// operations in parallel. For example:
	//
	//     dial tcp 74.125.203.95:443: too many open files
	//
	if _, ok := err.(*net.OpError); ok {
		b = true
		return
	}

	// The HTTP package returns ErrUnexpectedEOF in several places. This seems to
	// come up when the server terminates the connection in the middle of an
	// object read.
	if err == io.ErrUnexpectedEOF {
		b = true
		return
	}

	// The HTTP library also appears to leak EOF errors from... somewhere in its
	// guts as URL errors sometimes.
	if urlErr, ok := err.(*url.Error); ok {
		if urlErr.Err == io.EOF {
			b = true
			return
		}
	}

	// Sometimes the HTTP package helpfully encapsulates the real error in a URL
	// error.
	if urlErr, ok := err.(*url.Error); ok {
		b = shouldRetry(urlErr.Err)
		return
	}

	return
}

// Choose an appropriate delay for exponential backoff, given that we have
// already slept the given number of times for this logical request.
func chooseDelay(prevSleepCount uint) (d time.Duration) {
	const baseDelay = time.Millisecond

	// Choose a a delay in [0, 2^prevSleepCount * baseDelay).
	d = (1 << prevSleepCount) * baseDelay
	d = time.Duration(rand.Int63n(int64(d)))

	return
}

// Exponential backoff for a function that might fail.
//
// This is essentially what is described in the "Best practices" section of the
// "Upload Objects" docs:
//
//	https://cloud.google.com/storage/docs/json_api/v1/how-tos/upload
//
// with the following exceptions:
//
//   - We perform backoff for all operations.
//
//   - The random component scales with the delay, so that the first sleep
//     cannot be as long as one second. The algorithm used matches the
//     description at http://en.wikipedia.org/wiki/Exponential_backoff.
//
//   - We retry more types of errors; see shouldRetry above.
//
// State for total sleep time and number of previous sleeps is housed outside
// of this function to allow it to be "resumed" by multiple invocations of
// retryObjectReader.Read.
func expBackoff(
	ctx context.Context,
	desc string,
	maxSleep time.Duration,
	f func() error,
	prevSleepCount *uint,
	prevSleepDuration *time.Duration) (err error) {
	for {
		// Make an attempt. Stop if successful.
		err = f()
		if err == nil {
			return
		}

		// Do we want to retry?
		if !shouldRetry(err) {
			// Special case: don't spam up the logs for EOF, which io.Reader returns
			// in the normal course of things.
			if err != io.EOF {
				err = fmt.Errorf("not retrying %s: %w", desc, err)
			}

			return
		}

		// Choose a a delay.
		d := chooseDelay(*prevSleepCount)
		*prevSleepCount++

		// Are we out of credit?
		if *prevSleepDuration+d > maxSleep {
			// Return the most recent error.
			return
		}

		// Sleep, returning early if cancelled.
		select {
		case <-ctx.Done():
			// On cancellation, return the last error we saw.
			return

		case <-time.After(d):
			*prevSleepDuration += d
			continue
		}
	}
}

// Like expBackoff, but assumes that we've never slept before (and won't need
// to sleep again).
func oneShotExpBackoff(
	ctx context.Context,
	desc string,
	maxSleep time.Duration,
	f func() error) (err error) {
	var prevSleepCount uint
	var prevSleepDuration time.Duration

	err = expBackoff(
		ctx,
		desc,
		maxSleep,
		f,
		&prevSleepCount,
		&prevSleepDuration)

	return
}

////////////////////////////////////////////////////////////////////////
// Read support
////////////////////////////////////////////////////////////////////////

type retryObjectReader struct {
	bucket *retryBucket

	// The context we should watch when sleeping for retries.
	ctx context.Context

	// What we are trying to read.
	name       string
	generation int64
	byteRange  ByteRange

	// nil when we start or have seen a permanent error.
	wrapped io.ReadCloser

	// If we've seen an error that we shouldn't retry for, this will be non-nil
	// and should be returned permanently.
	permanentErr error

	// The number of times we've slept so far, and the total amount of time we've
	// spent sleeping.
	sleepCount    uint
	sleepDuration time.Duration
}

// Set up the wrapped reader.
func (rc *retryObjectReader) setUpWrapped() (err error) {
	// Call through to create the reader.
	req := &ReadObjectRequest{
		Name:       rc.name,
		Generation: rc.generation,
		Range:      &rc.byteRange,
	}

	wrapped, err := rc.bucket.wrapped.NewReader(rc.ctx, req)
	if err != nil {
		return
	}

	rc.wrapped = wrapped
	return
}

// Set up the wrapped reader if necessary, and make one attempt to read through
// it.
//
// Clears the wrapped reader on error.
func (rc *retryObjectReader) readOnce(p []byte) (n int, err error) {
	// Set up the wrapped reader if it's not already around.
	if rc.wrapped == nil {
		err = rc.setUpWrapped()
		if err != nil {
			return
		}
	}

	// Attempt to read from it.
	n, err = rc.wrapped.Read(p)
	if err != nil {
		rc.wrapped.Close()
		rc.wrapped = nil
		return
	}

	return
}

// Invariant: we never return an error from this function unless we've given up
// on retrying. In particular, we won't return a short read because the wrapped
// reader returned a short read and an error.
func (rc *retryObjectReader) Read(p []byte) (n int, err error) {
	// Whatever we do, accumulate the bytes that we're returning to the user.
	defer func() {
		if n < 0 {
			panic(fmt.Sprintf("Negative byte count: %d", n))
		}

		rc.byteRange.Start += uint64(n)
	}()

	// If we've already decided on a permanent error, return that.
	if rc.permanentErr != nil {
		err = rc.permanentErr
		return
	}

	// If we let an error escape below, it must be a permanent one.
	defer func() {
		if err != nil {
			rc.permanentErr = err
		}
	}()

	// We will repeatedly make single attempts until we get a successful request.
	// Don't forget to accumulate the result each time.
	tryOnce := func() (err error) {
		var bytesRead int
		bytesRead, err = rc.readOnce(p)
		n += bytesRead
		p = p[bytesRead:]

		return
	}

	err = expBackoff(
		rc.ctx,
		fmt.Sprintf("Read(%q, %d)", rc.name, rc.generation),
		rc.bucket.maxSleep,
		tryOnce,
		&rc.sleepCount,
		&rc.sleepDuration)

	return
}

func (rc *retryObjectReader) Close() (err error) {
	// If we don't have a wrapped reader, there is nothing useful that we can or
	// need to do here.
	if rc.wrapped == nil {
		return
	}

	// Call through.
	err = rc.wrapped.Close()

	return
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

func (rb *retryBucket) Name() (name string) {
	name = rb.wrapped.Name()
	return
}

func (rb *retryBucket) NewReader(
	ctx context.Context,
	req *ReadObjectRequest) (rc io.ReadCloser, err error) {
	// If the user specified the latest generation, we need to figure out what
	// that is so that we can create a reader that knows how to keep a stable
	// generation despite retrying repeatedly.
	var generation int64 = req.Generation
	var sleepCount uint
	var sleepDuration time.Duration

	if generation == 0 {
		findGeneration := func() (err error) {
			o, err := rb.wrapped.StatObject(
				ctx,
				&StatObjectRequest{
					Name: req.Name,
				})

			if err != nil {
				return
			}

			generation = o.Generation
			return
		}

		err = expBackoff(
			ctx,
			fmt.Sprintf("FindLatestGeneration(%q)", req.Name),
			rb.maxSleep,
			findGeneration,
			&sleepCount,
			&sleepDuration)

		if err != nil {
			return
		}
	}

	// Choose an appropriate byte range.
	byteRange := ByteRange{0, math.MaxUint64}
	if req.Range != nil {
		byteRange = *req.Range
	}

	// Now that we know what generation we're looking for, return an appropriate
	// reader that knows how to retry when the connection fails. Make sure to
	// inherit the time spent sleeping above.
	rc = &retryObjectReader{
		bucket: rb,
		ctx:    ctx,

		name:       req.Name,
		generation: generation,
		byteRange:  byteRange,

		sleepCount:    sleepCount,
		sleepDuration: sleepDuration,
	}

	return
}

func (rb *retryBucket) CreateObject(
	ctx context.Context,
	req *CreateObjectRequest) (o *Object, err error) {
	var seeker io.ReadSeeker
	if readSeeker, ok := req.Contents.(io.ReadSeeker); ok {
		seeker = readSeeker
	} else {
		// We can't simply replay the request multiple times, because the first
		// attempt might exhaust some of the req.Contents reader, leaving
		// missing contents for the second attempt.
		//
		// So, copy out all contents and create a copy of the request that we
		// will modify to serve from memory for each call.
		data, err := ioutil.ReadAll(req.Contents)
		if err != nil {
			err = fmt.Errorf("ioutil.ReadAll: %v", err)
			return nil, err
		}
		seeker = bytes.NewReader(data)
	}

	reqCopy := *req

	// Call through with that request.
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("CreateObject(%q)", req.Name),
		rb.maxSleep,
		func() (err error) {
			if _, err = seeker.Seek(0, io.SeekStart); err != nil {
				return
			}
			reqCopy.Contents = seeker
			o, err = rb.wrapped.CreateObject(ctx, &reqCopy)
			return
		})

	return
}

func (rb *retryBucket) CopyObject(
	ctx context.Context,
	req *CopyObjectRequest) (o *Object, err error) {
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("CopyObject(%q, %q)", req.SrcName, req.DstName),
		rb.maxSleep,
		func() (err error) {
			o, err = rb.wrapped.CopyObject(ctx, req)
			return
		})

	return
}

func (rb *retryBucket) ComposeObjects(
	ctx context.Context,
	req *ComposeObjectsRequest) (o *Object, err error) {
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("ComposeObjects(%q)", req.DstName),
		rb.maxSleep,
		func() (err error) {
			o, err = rb.wrapped.ComposeObjects(ctx, req)
			return
		})

	return
}

func (rb *retryBucket) StatObject(
	ctx context.Context,
	req *StatObjectRequest) (o *Object, err error) {
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("StatObject(%q)", req.Name),
		rb.maxSleep,
		func() (err error) {
			o, err = rb.wrapped.StatObject(ctx, req)
			return
		})

	return
}

func (rb *retryBucket) ListObjects(
	ctx context.Context,
	req *ListObjectsRequest) (listing *Listing, err error) {
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("ListObjects(%q)", req.Prefix),
		rb.maxSleep,
		func() (err error) {
			listing, err = rb.wrapped.ListObjects(ctx, req)
			return
		})
	return
}

func (rb *retryBucket) UpdateObject(
	ctx context.Context,
	req *UpdateObjectRequest) (o *Object, err error) {
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("UpdateObject(%q)", req.Name),
		rb.maxSleep,
		func() (err error) {
			o, err = rb.wrapped.UpdateObject(ctx, req)
			return
		})

	return
}

func (rb *retryBucket) DeleteObject(
	ctx context.Context,
	req *DeleteObjectRequest) (err error) {
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("DeleteObject(%q)", req.Name),
		rb.maxSleep,
		func() (err error) {
			err = rb.wrapped.DeleteObject(ctx, req)
			return
		})

	return
}
<commit_msg>Unit Test for BucketManager<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcs

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"math/rand"
	"net"
	"net/url"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/api/googleapi"
)

// A bucket that wraps another, calling its methods in a retry loop with
// randomized exponential backoff.
type retryBucket struct {
	maxSleep time.Duration
	wrapped  Bucket
}

func newRetryBucket(
	maxSleep time.Duration,
	wrapped Bucket) (b Bucket) {
	b = &retryBucket{
		maxSleep: maxSleep,
		wrapped:  wrapped,
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func shouldRetry(err error) (b bool) {
	// HTTP 50x errors.
	if typed, ok := err.(*googleapi.Error); ok {
		if typed.Code >= 500 && typed.Code < 600 {
			b = true
			return
		}
	}

	// HTTP 429 errors (GCS uses these for rate limiting).
	if typed, ok := err.(*googleapi.Error); ok {
		if typed.Code == 429 {
			b = true
			return
		}
	}

	// Network errors, which tend to show up transiently when doing lots of
	// operations in parallel. For example:
	//
	//     dial tcp 74.125.203.95:443: too many open files
	//
	if _, ok := err.(*net.OpError); ok {
		b = true
		return
	}

	// The HTTP package returns ErrUnexpectedEOF in several places. This seems to
	// come up when the server terminates the connection in the middle of an
	// object read.
	if err == io.ErrUnexpectedEOF {
		b = true
		return
	}

	// The HTTP library also appears to leak EOF errors from... somewhere in its
	// guts as URL errors sometimes.
	if urlErr, ok := err.(*url.Error); ok {
		if urlErr.Err == io.EOF {
			b = true
			return
		}
	}

	// Sometimes the HTTP package helpfully encapsulates the real error in a URL
	// error.
	if urlErr, ok := err.(*url.Error); ok {
		b = shouldRetry(urlErr.Err)
		return
	}

	return
}

// Choose an appropriate delay for exponential backoff, given that we have
// already slept the given number of times for this logical request.
func chooseDelay(prevSleepCount uint) (d time.Duration) {
	const baseDelay = time.Millisecond

	// Choose a a delay in [0, 2^prevSleepCount * baseDelay).
	d = (1 << prevSleepCount) * baseDelay
	d = time.Duration(rand.Int63n(int64(d)))

	return
}

// Exponential backoff for a function that might fail.
//
// This is essentially what is described in the "Best practices" section of the
// "Upload Objects" docs:
//
//     https://cloud.google.com/storage/docs/json_api/v1/how-tos/upload
//
// with the following exceptions:
//
//  *  We perform backoff for all operations.
//
//  *  The random component scales with the delay, so that the first sleep
//     cannot be as long as one second. The algorithm used matches the
//     description at http://en.wikipedia.org/wiki/Exponential_backoff.
//
//  *  We retry more types of errors; see shouldRetry above.
//
// State for total sleep time and number of previous sleeps is housed outside
// of this function to allow it to be "resumed" by multiple invocations of
// retryObjectReader.Read.
func expBackoff(
	ctx context.Context,
	desc string,
	maxSleep time.Duration,
	f func() error,
	prevSleepCount *uint,
	prevSleepDuration *time.Duration) (err error) {
	for {
		// Make an attempt. Stop if successful.
		err = f()
		if err == nil {
			return
		}

		// Do we want to retry?
		if !shouldRetry(err) {
			// Special case: don't spam up the logs for EOF, which io.Reader returns
			// in the normal course of things.
			if err != io.EOF {
				err = fmt.Errorf("not retrying %s: %w", desc, err)
			}

			return
		}

		// Choose a a delay.
		d := chooseDelay(*prevSleepCount)
		*prevSleepCount++

		// Are we out of credit?
		if *prevSleepDuration+d > maxSleep {
			// Return the most recent error.
			return
		}

		// Sleep, returning early if cancelled.
		select {
		case <-ctx.Done():
			// On cancellation, return the last error we saw.
			return

		case <-time.After(d):
			*prevSleepDuration += d
			continue
		}
	}
}

// Like expBackoff, but assumes that we've never slept before (and won't need
// to sleep again).
func oneShotExpBackoff(
	ctx context.Context,
	desc string,
	maxSleep time.Duration,
	f func() error) (err error) {
	var prevSleepCount uint
	var prevSleepDuration time.Duration

	err = expBackoff(
		ctx,
		desc,
		maxSleep,
		f,
		&prevSleepCount,
		&prevSleepDuration)

	return
}

////////////////////////////////////////////////////////////////////////
// Read support
////////////////////////////////////////////////////////////////////////

type retryObjectReader struct {
	bucket *retryBucket

	// The context we should watch when sleeping for retries.
	ctx context.Context

	// What we are trying to read.
	name       string
	generation int64
	byteRange  ByteRange

	// nil when we start or have seen a permanent error.
	wrapped io.ReadCloser

	// If we've seen an error that we shouldn't retry for, this will be non-nil
	// and should be returned permanently.
	permanentErr error

	// The number of times we've slept so far, and the total amount of time we've
	// spent sleeping.
	sleepCount    uint
	sleepDuration time.Duration
}

// Set up the wrapped reader.
func (rc *retryObjectReader) setUpWrapped() (err error) {
	// Call through to create the reader.
	req := &ReadObjectRequest{
		Name:       rc.name,
		Generation: rc.generation,
		Range:      &rc.byteRange,
	}

	wrapped, err := rc.bucket.wrapped.NewReader(rc.ctx, req)
	if err != nil {
		return
	}

	rc.wrapped = wrapped
	return
}

// Set up the wrapped reader if necessary, and make one attempt to read through
// it.
//
// Clears the wrapped reader on error.
func (rc *retryObjectReader) readOnce(p []byte) (n int, err error) {
	// Set up the wrapped reader if it's not already around.
	if rc.wrapped == nil {
		err = rc.setUpWrapped()
		if err != nil {
			return
		}
	}

	// Attempt to read from it.
	n, err = rc.wrapped.Read(p)
	if err != nil {
		rc.wrapped.Close()
		rc.wrapped = nil
		return
	}

	return
}

// Invariant: we never return an error from this function unless we've given up
// on retrying. In particular, we won't return a short read because the wrapped
// reader returned a short read and an error.
func (rc *retryObjectReader) Read(p []byte) (n int, err error) {
	// Whatever we do, accumulate the bytes that we're returning to the user.
	defer func() {
		if n < 0 {
			panic(fmt.Sprintf("Negative byte count: %d", n))
		}

		rc.byteRange.Start += uint64(n)
	}()

	// If we've already decided on a permanent error, return that.
	if rc.permanentErr != nil {
		err = rc.permanentErr
		return
	}

	// If we let an error escape below, it must be a permanent one.
	defer func() {
		if err != nil {
			rc.permanentErr = err
		}
	}()

	// We will repeatedly make single attempts until we get a successful request.
	// Don't forget to accumulate the result each time.
	tryOnce := func() (err error) {
		var bytesRead int
		bytesRead, err = rc.readOnce(p)
		n += bytesRead
		p = p[bytesRead:]

		return
	}

	err = expBackoff(
		rc.ctx,
		fmt.Sprintf("Read(%q, %d)", rc.name, rc.generation),
		rc.bucket.maxSleep,
		tryOnce,
		&rc.sleepCount,
		&rc.sleepDuration)

	return
}

func (rc *retryObjectReader) Close() (err error) {
	// If we don't have a wrapped reader, there is nothing useful that we can or
	// need to do here.
	if rc.wrapped == nil {
		return
	}

	// Call through.
	err = rc.wrapped.Close()

	return
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

func (rb *retryBucket) Name() (name string) {
	name = rb.wrapped.Name()
	return
}

func (rb *retryBucket) NewReader(
	ctx context.Context,
	req *ReadObjectRequest) (rc io.ReadCloser, err error) {
	// If the user specified the latest generation, we need to figure out what
	// that is so that we can create a reader that knows how to keep a stable
	// generation despite retrying repeatedly.
	var generation int64 = req.Generation
	var sleepCount uint
	var sleepDuration time.Duration

	if generation == 0 {
		findGeneration := func() (err error) {
			o, err := rb.wrapped.StatObject(
				ctx,
				&StatObjectRequest{
					Name: req.Name,
				})

			if err != nil {
				return
			}

			generation = o.Generation
			return
		}

		err = expBackoff(
			ctx,
			fmt.Sprintf("FindLatestGeneration(%q)", req.Name),
			rb.maxSleep,
			findGeneration,
			&sleepCount,
			&sleepDuration)

		if err != nil {
			return
		}
	}

	// Choose an appropriate byte range.
	byteRange := ByteRange{0, math.MaxUint64}
	if req.Range != nil {
		byteRange = *req.Range
	}

	// Now that we know what generation we're looking for, return an appropriate
	// reader that knows how to retry when the connection fails. Make sure to
	// inherit the time spent sleeping above.
	rc = &retryObjectReader{
		bucket: rb,
		ctx:    ctx,

		name:       req.Name,
		generation: generation,
		byteRange:  byteRange,

		sleepCount:    sleepCount,
		sleepDuration: sleepDuration,
	}

	return
}

func (rb *retryBucket) CreateObject(
	ctx context.Context,
	req *CreateObjectRequest) (o *Object, err error) {
	var seeker io.ReadSeeker
	if readSeeker, ok := req.Contents.(io.ReadSeeker); ok {
		seeker = readSeeker
	} else {
		// We can't simply replay the request multiple times, because the first
		// attempt might exhaust some of the req.Contents reader, leaving
		// missing contents for the second attempt.
		//
		// So, copy out all contents and create a copy of the request that we
		// will modify to serve from memory for each call.
		data, err := ioutil.ReadAll(req.Contents)
		if err != nil {
			err = fmt.Errorf("ioutil.ReadAll: %v", err)
			return nil, err
		}
		seeker = bytes.NewReader(data)
	}

	reqCopy := *req

	// Call through with that request.
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("CreateObject(%q)", req.Name),
		rb.maxSleep,
		func() (err error) {
			if _, err = seeker.Seek(0, io.SeekStart); err != nil {
				return
			}
			reqCopy.Contents = seeker
			o, err = rb.wrapped.CreateObject(ctx, &reqCopy)
			return
		})

	return
}

func (rb *retryBucket) CopyObject(
	ctx context.Context,
	req *CopyObjectRequest) (o *Object, err error) {
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("CopyObject(%q, %q)", req.SrcName, req.DstName),
		rb.maxSleep,
		func() (err error) {
			o, err = rb.wrapped.CopyObject(ctx, req)
			return
		})

	return
}

func (rb *retryBucket) ComposeObjects(
	ctx context.Context,
	req *ComposeObjectsRequest) (o *Object, err error) {
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("ComposeObjects(%q)", req.DstName),
		rb.maxSleep,
		func() (err error) {
			o, err = rb.wrapped.ComposeObjects(ctx, req)
			return
		})

	return
}

func (rb *retryBucket) StatObject(
	ctx context.Context,
	req *StatObjectRequest) (o *Object, err error) {
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("StatObject(%q)", req.Name),
		rb.maxSleep,
		func() (err error) {
			o, err = rb.wrapped.StatObject(ctx, req)
			return
		})

	return
}

func (rb *retryBucket) ListObjects(
	ctx context.Context,
	req *ListObjectsRequest) (listing *Listing, err error) {
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("ListObjects(%q)", req.Prefix),
		rb.maxSleep,
		func() (err error) {
			listing, err = rb.wrapped.ListObjects(ctx, req)
			return
		})
	return
}

func (rb *retryBucket) UpdateObject(
	ctx context.Context,
	req *UpdateObjectRequest) (o *Object, err error) {
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("UpdateObject(%q)", req.Name),
		rb.maxSleep,
		func() (err error) {
			o, err = rb.wrapped.UpdateObject(ctx, req)
			return
		})

	return
}

func (rb *retryBucket) DeleteObject(
	ctx context.Context,
	req *DeleteObjectRequest) (err error) {
	err = oneShotExpBackoff(
		ctx,
		fmt.Sprintf("DeleteObject(%q)", req.Name),
		rb.maxSleep,
		func() (err error) {
			err = rb.wrapped.DeleteObject(ctx, req)
			return
		})

	return
}
<|endoftext|>


<commit_before>package certdata

import "encoding/asn1"

// Source GlobalSign CP and Cabforum BR
// https://www.globalsign.com/en/repository/GlobalSign_CP_v5.3.pdf
var polOidType []oidType

type oidType struct {
	ObjectIdentifier asn1.ObjectIdentifier
	Type             string
}

/*
func getType(oid asn1.ObjectIdentifier) string {
	for _, oidt := range polOidType {
		if oid.Equal(oidt.ObjectIdentifier) {
			return oidt.Type
		}
	}
	return ""
}*/

func getType(oid []asn1.ObjectIdentifier) string {
	for _, poid := range oid {
		for _, oidt := range polOidType {
			if poid.Equal(oidt.ObjectIdentifier) {
				return oidt.Type
			}
		}
	}
	return ""
}

// TODO: Can we handle this differently, we might want to use a constant here?
func init() {
	// Extended Validation
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 1}, "EV"}) // Extended Validation Certificates Policy – SSL
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 2}, "CS"}) // Extended Validation Certificates Policy – Code Signing

	// Domain Validation
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 10}, "DV"})     // Domain Validation Certificates Policy
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 10, 10}, "DV"}) // Domain Validation Certificates Policy – AlphaSSL

	// Organization Validation
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 20}, "OV"}) // Organization Validation Certificates Policy
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 21}, "-"})  // Untrusted OneClickSSL Test Certificate (not in cp)

	// Intranet Validation
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 25}, "IN"}) // IntranetSSL Validation Certificates Policy

	// Time Stamping
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 30}, "TS"}) // Time Stamping Certificates Policy
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 31}, "TS"}) // Time Stamping Certificates Policy – AATL

	// Client Certificates
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 40}, "PS"})     // Client Certificates Policy (Generic)
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 40, 10}, "PS"}) // Client Certificates Policy (ePKI – Enterprise PKI)
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 40, 20}, "PS"}) // Client Certificates Policy (JCAN – Japan CA Network)
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 40, 30}, "PS"}) // Client Certificates Policy (AATL)
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 40, 40}, "PS"}) // Client Certificates Policy (ePKI for private CAs)

	// Code Signing
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 50}, "CS"}) // Code Signing Certificates Policy

	// CA Chaining and Cross Signing
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 60}, "CA"})    // CA Chaining Policy – Trusted Root and Hosted Root
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 60, 1}, "CA"}) // CA Chaining Policy – Trusted Root (Baseline Requirements Compatible)

	// Others
	/*polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1,3,6,1,4,1,4146,1,80}, "XX"}) // Retail Industry Electronic Data Interchange Client Certificate Policy
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1,3,6,1,4,1,4146,1,81}, "XX"}) // Retail Industry Electronic Data Interchange Server Certificate Policy
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1,3,6,1,4,1,4146,1,90}, "XX"}) // Trusted Root TPM Policy
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1,3,6,1,4,1,4146,1,95}, "XX"}) // Online Certificate Status Protocol Policy
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1,3,6,1,4,1,4146,1,70}, "XX"}) // High Volume CA Policy
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1,3,6,1,4,1,4146,1,26}, "XX"}) // Test Certificate Policy (Should not be trusted)

	// In addition to these identifiers, all Certificates that comply with the NAESB Business
	// Practice Standards will include one of the following additional identifiers:-
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{2,16,840,1,114505,1,12,1,2}, "XX"}) // NAESB Rudimentary Assurance
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{2,16,840,1,114505,1,12,2,2}, "XX"}) // NAESB Basic Assurance
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{2,16,840,1,114505,1,12,3,2}, "XX"}) // NAESB Medium Assurance
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{2,16,840,1,114505,1,12,4,2}, "XX"}) // NAESB High Assurance
	*/
	// In addition to these identifiers, all Certificates that comply with the Baseline
	// Requirements will include the following additional identifiers:-
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{2, 23, 140, 1, 1}, "EV"}) // Extended Validation Certificate Policy
	//polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{2,23,140,1,2}, ""}) // BR Compliance Certificate Policy
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{2, 23, 140, 1, 3}, "EVCS"}) // Extended Validation Code Signing Certificates Policy
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{2, 23, 140, 1, 4}, "CS"})   // BR Compliance Code Signing Certificates Policy

	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{2, 23, 140, 1, 2, 1}, "DV"}) // Domain Validation Certificates Policy
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{2, 23, 140, 1, 2, 2}, "OV"}) // Organization Validation Certificates Policy
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{2, 23, 140, 1, 2, 3}, "IV"}) // Individual Validation Certificates Policy

	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 2, 840, 113583, 1, 2, 1}, "PS"}) // Adobe Certificate Policy Attribute Object Identifier (PDF)
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 2, 840, 113583, 1, 2, 2}, "PS"}) // Test Adobe Certificate Policy Attribute Object Identifier

	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 40, 30, 2}, "PS"}) // AATL Adobe Certificate Policy Attribute Object Identifier
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 2, 392, 200063, 30, 5300}, "PS"})          // JCAN
}
<commit_msg>Remove unused fuction<commit_after>package certdata

import "encoding/asn1"

// Source GlobalSign CP and Cabforum BR
// https://www.globalsign.com/en/repository/GlobalSign_CP_v5.3.pdf
var polOidType []oidType

type oidType struct {
	ObjectIdentifier asn1.ObjectIdentifier
	Type             string
}

func getType(oid []asn1.ObjectIdentifier) string {
	for _, poid := range oid {
		for _, oidt := range polOidType {
			if poid.Equal(oidt.ObjectIdentifier) {
				return oidt.Type
			}
		}
	}
	return ""
}

// TODO: Can we handle this differently, we might want to use a constant here?
func init() {
	// Extended Validation
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 1}, "EV"}) // Extended Validation Certificates Policy – SSL
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 2}, "CS"}) // Extended Validation Certificates Policy – Code Signing

	// Domain Validation
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 10}, "DV"})     // Domain Validation Certificates Policy
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 10, 10}, "DV"}) // Domain Validation Certificates Policy – AlphaSSL

	// Organization Validation
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 20}, "OV"}) // Organization Validation Certificates Policy
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 21}, "-"})  // Untrusted OneClickSSL Test Certificate (not in cp)

	// Intranet Validation
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 25}, "IN"}) // IntranetSSL Validation Certificates Policy

	// Time Stamping
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 30}, "TS"}) // Time Stamping Certificates Policy
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 31}, "TS"}) // Time Stamping Certificates Policy – AATL

	// Client Certificates
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 40}, "PS"})     // Client Certificates Policy (Generic)
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 40, 10}, "PS"}) // Client Certificates Policy (ePKI – Enterprise PKI)
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 40, 20}, "PS"}) // Client Certificates Policy (JCAN – Japan CA Network)
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 40, 30}, "PS"}) // Client Certificates Policy (AATL)
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 40, 40}, "PS"}) // Client Certificates Policy (ePKI for private CAs)

	// Code Signing
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 50}, "CS"}) // Code Signing Certificates Policy

	// CA Chaining and Cross Signing
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 60}, "CA"})    // CA Chaining Policy – Trusted Root and Hosted Root
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 60, 1}, "CA"}) // CA Chaining Policy – Trusted Root (Baseline Requirements Compatible)

	// Others
	/*polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1,3,6,1,4,1,4146,1,80}, "XX"}) // Retail Industry Electronic Data Interchange Client Certificate Policy
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1,3,6,1,4,1,4146,1,81}, "XX"}) // Retail Industry Electronic Data Interchange Server Certificate Policy
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1,3,6,1,4,1,4146,1,90}, "XX"}) // Trusted Root TPM Policy
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1,3,6,1,4,1,4146,1,95}, "XX"}) // Online Certificate Status Protocol Policy
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1,3,6,1,4,1,4146,1,70}, "XX"}) // High Volume CA Policy
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1,3,6,1,4,1,4146,1,26}, "XX"}) // Test Certificate Policy (Should not be trusted)

	// In addition to these identifiers, all Certificates that comply with the NAESB Business
	// Practice Standards will include one of the following additional identifiers:-
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{2,16,840,1,114505,1,12,1,2}, "XX"}) // NAESB Rudimentary Assurance
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{2,16,840,1,114505,1,12,2,2}, "XX"}) // NAESB Basic Assurance
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{2,16,840,1,114505,1,12,3,2}, "XX"}) // NAESB Medium Assurance
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{2,16,840,1,114505,1,12,4,2}, "XX"}) // NAESB High Assurance
	*/
	// In addition to these identifiers, all Certificates that comply with the Baseline
	// Requirements will include the following additional identifiers:-
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{2, 23, 140, 1, 1}, "EV"}) // Extended Validation Certificate Policy
	//polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{2,23,140,1,2}, ""}) // BR Compliance Certificate Policy
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{2, 23, 140, 1, 3}, "EVCS"}) // Extended Validation Code Signing Certificates Policy
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{2, 23, 140, 1, 4}, "CS"})   // BR Compliance Code Signing Certificates Policy

	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{2, 23, 140, 1, 2, 1}, "DV"}) // Domain Validation Certificates Policy
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{2, 23, 140, 1, 2, 2}, "OV"}) // Organization Validation Certificates Policy
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{2, 23, 140, 1, 2, 3}, "IV"}) // Individual Validation Certificates Policy

	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 2, 840, 113583, 1, 2, 1}, "PS"}) // Adobe Certificate Policy Attribute Object Identifier (PDF)
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 2, 840, 113583, 1, 2, 2}, "PS"}) // Test Adobe Certificate Policy Attribute Object Identifier

	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 3, 6, 1, 4, 1, 4146, 1, 40, 30, 2}, "PS"}) // AATL Adobe Certificate Policy Attribute Object Identifier
	polOidType = append(polOidType, oidType{asn1.ObjectIdentifier{1, 2, 392, 200063, 30, 5300}, "PS"})          // JCAN
}
<|endoftext|>


<commit_before>package api

import (
	"fmt"
	"strings"
	"time"

	"github.com/cloudfoundry/cli/cf/configuration/core_config"
	"github.com/cloudfoundry/cli/cf/models"
	"github.com/cloudfoundry/cli/cf/net"
)

type ApplicationSummaries struct {
	Apps []ApplicationFromSummary
}

func (resource ApplicationSummaries) ToModels() (apps []models.ApplicationFields) {
	for _, application := range resource.Apps {
		apps = append(apps, application.ToFields())
	}
	return
}

type ApplicationFromSummary struct {
	Guid             string
	Name             string
	Routes           []RouteSummary
	Services         []ServicePlanSummary
	RunningInstances int `json:"running_instances"`
	Memory           int64
	Instances        int
	DiskQuota        int64 `json:"disk_quota"`
	Urls             []string
	State            string
	SpaceGuid        string     `json:"space_guid"`
	PackageUpdatedAt *time.Time `json:"package_updated_at"`
}

func (resource ApplicationFromSummary) ToFields() (app models.ApplicationFields) {
	app = models.ApplicationFields{}
	app.Guid = resource.Guid
	app.Name = resource.Name
	app.State = strings.ToLower(resource.State)
	app.InstanceCount = resource.Instances
	app.DiskQuota = resource.DiskQuota
	app.RunningInstances = resource.RunningInstances
	app.Memory = resource.Memory
	app.SpaceGuid = resource.SpaceGuid
	app.PackageUpdatedAt = resource.PackageUpdatedAt

	return
}

func (resource ApplicationFromSummary) ToModel() (app models.Application) {
	app.ApplicationFields = resource.ToFields()
	routes := []models.RouteSummary{}
	for _, route := range resource.Routes {
		routes = append(routes, route.ToModel())
	}
	app.Routes = routes

	services := []models.ServicePlanSummary{}
	for _, service := range resource.Services {
		services = append(services, service.ToModel())
	}
	app.Routes = routes
	app.Services = services

	return
}

type RouteSummary struct {
	Guid   string
	Host   string
	Domain DomainSummary
}

func (resource RouteSummary) ToModel() (route models.RouteSummary) {
	domain := models.DomainFields{}
	domain.Guid = resource.Domain.Guid
	domain.Name = resource.Domain.Name
	domain.Shared = resource.Domain.OwningOrganizationGuid != ""

	route.Guid = resource.Guid
	route.Host = resource.Host
	route.Domain = domain
	return
}

func (resource ServicePlanSummary) ToModel() (route models.ServicePlanSummary) {
	route.Guid = resource.Guid
	route.Name = resource.Name
	return
}

type DomainSummary struct {
	Guid                   string
	Name                   string
	OwningOrganizationGuid string
}

type AppSummaryRepository interface {
	GetSummariesInCurrentSpace() (apps []models.Application, apiErr error)
	GetSummary(appGuid string) (summary models.Application, apiErr error)
}

type CloudControllerAppSummaryRepository struct {
	config  core_config.Reader
	gateway net.Gateway
}

func NewCloudControllerAppSummaryRepository(config core_config.Reader, gateway net.Gateway) (repo CloudControllerAppSummaryRepository) {
	repo.config = config
	repo.gateway = gateway
	return
}

func (repo CloudControllerAppSummaryRepository) GetSummariesInCurrentSpace() (apps []models.Application, apiErr error) {
	resources := new(ApplicationSummaries)

	path := fmt.Sprintf("%s/v2/spaces/%s/summary", repo.config.ApiEndpoint(), repo.config.SpaceFields().Guid)
	apiErr = repo.gateway.GetResource(path, resources)
	if apiErr != nil {
		return
	}

	for _, resource := range resources.Apps {
		apps = append(apps, resource.ToModel())
	}
	return
}

func (repo CloudControllerAppSummaryRepository) GetSummary(appGuid string) (summary models.Application, apiErr error) {
	path := fmt.Sprintf("%s/v2/apps/%s/summary", repo.config.ApiEndpoint(), appGuid)
	summaryResponse := new(ApplicationFromSummary)
	apiErr = repo.gateway.GetResource(path, summaryResponse)
	if apiErr != nil {
		return
	}

	summary = summaryResponse.ToModel()
	return
}
<commit_msg>populates EnvironmentVars when hitting app/summary endpoint<commit_after>package api

import (
	"fmt"
	"strings"
	"time"

	"github.com/cloudfoundry/cli/cf/configuration/core_config"
	"github.com/cloudfoundry/cli/cf/models"
	"github.com/cloudfoundry/cli/cf/net"
)

type ApplicationSummaries struct {
	Apps []ApplicationFromSummary
}

func (resource ApplicationSummaries) ToModels() (apps []models.ApplicationFields) {
	for _, application := range resource.Apps {
		apps = append(apps, application.ToFields())
	}
	return
}

type ApplicationFromSummary struct {
	Guid             string
	Name             string
	Routes           []RouteSummary
	Services         []ServicePlanSummary
	RunningInstances int `json:"running_instances"`
	Memory           int64
	Instances        int
	DiskQuota        int64 `json:"disk_quota"`
	Urls             []string
	EnvironmentVars  map[string]interface{} `json:"environment_json,omitempty"`
	State            string
	SpaceGuid        string     `json:"space_guid"`
	PackageUpdatedAt *time.Time `json:"package_updated_at"`
}

func (resource ApplicationFromSummary) ToFields() (app models.ApplicationFields) {
	app = models.ApplicationFields{}
	app.Guid = resource.Guid
	app.Name = resource.Name
	app.State = strings.ToLower(resource.State)
	app.InstanceCount = resource.Instances
	app.DiskQuota = resource.DiskQuota
	app.RunningInstances = resource.RunningInstances
	app.Memory = resource.Memory
	app.SpaceGuid = resource.SpaceGuid
	app.PackageUpdatedAt = resource.PackageUpdatedAt

	return
}

func (resource ApplicationFromSummary) ToModel() (app models.Application) {
	app.ApplicationFields = resource.ToFields()
	routes := []models.RouteSummary{}
	for _, route := range resource.Routes {
		routes = append(routes, route.ToModel())
	}
	app.Routes = routes

	services := []models.ServicePlanSummary{}
	for _, service := range resource.Services {
		services = append(services, service.ToModel())
	}

	app.EnvironmentVars = resource.EnvironmentVars
	app.Routes = routes
	app.Services = services

	return
}

type RouteSummary struct {
	Guid   string
	Host   string
	Domain DomainSummary
}

func (resource RouteSummary) ToModel() (route models.RouteSummary) {
	domain := models.DomainFields{}
	domain.Guid = resource.Domain.Guid
	domain.Name = resource.Domain.Name
	domain.Shared = resource.Domain.OwningOrganizationGuid != ""

	route.Guid = resource.Guid
	route.Host = resource.Host
	route.Domain = domain
	return
}

func (resource ServicePlanSummary) ToModel() (route models.ServicePlanSummary) {
	route.Guid = resource.Guid
	route.Name = resource.Name
	return
}

type DomainSummary struct {
	Guid                   string
	Name                   string
	OwningOrganizationGuid string
}

type AppSummaryRepository interface {
	GetSummariesInCurrentSpace() (apps []models.Application, apiErr error)
	GetSummary(appGuid string) (summary models.Application, apiErr error)
}

type CloudControllerAppSummaryRepository struct {
	config  core_config.Reader
	gateway net.Gateway
}

func NewCloudControllerAppSummaryRepository(config core_config.Reader, gateway net.Gateway) (repo CloudControllerAppSummaryRepository) {
	repo.config = config
	repo.gateway = gateway
	return
}

func (repo CloudControllerAppSummaryRepository) GetSummariesInCurrentSpace() (apps []models.Application, apiErr error) {
	resources := new(ApplicationSummaries)

	path := fmt.Sprintf("%s/v2/spaces/%s/summary", repo.config.ApiEndpoint(), repo.config.SpaceFields().Guid)
	apiErr = repo.gateway.GetResource(path, resources)
	if apiErr != nil {
		return
	}

	for _, resource := range resources.Apps {
		apps = append(apps, resource.ToModel())
	}
	return
}

func (repo CloudControllerAppSummaryRepository) GetSummary(appGuid string) (summary models.Application, apiErr error) {
	path := fmt.Sprintf("%s/v2/apps/%s/summary", repo.config.ApiEndpoint(), appGuid)
	summaryResponse := new(ApplicationFromSummary)
	apiErr = repo.gateway.GetResource(path, summaryResponse)
	if apiErr != nil {
		return
	}

	summary = summaryResponse.ToModel()
	return
}
<|endoftext|>


<commit_before>package discovery

import (
	"context"
	"time"

	"github.com/libp2p/go-libp2p-core/peer"

	logging "github.com/ipfs/go-log"
)

var log = logging.Logger("discovery")

// FindPeers is a utility function that synchronously collects peers from a Discoverer.
func FindPeers(ctx context.Context, d Discoverer, ns string, opts ...Option) ([]peer.AddrInfo, error) {
	var res []peer.AddrInfo

	ch, err := d.FindPeers(ctx, ns, opts...)
	if err != nil {
		return nil, err
	}

	for pi := range ch {
		res = append(res, pi)
	}

	return res, nil
}

// Advertise is a utility function that persistently advertises a service through an Advertiser.
func Advertise(ctx context.Context, a Advertiser, ns string, opts ...Option) {
	go func() {
		for {
			ttl, err := a.Advertise(ctx, ns, opts...)
			if err != nil {
				log.Debugf("Error advertising %s: %s", ns, err.Error())
				if ctx.Err() != nil {
					return
				}

				select {
				case <-time.After(2 * time.Minute):
					continue
				case <-ctx.Done():
					return
				}
			}

			wait := 7 * ttl / 8
			select {
			case <-time.After(wait):
			case <-ctx.Done():
				return
			}
		}
	}()
}
<commit_msg>remove deprecated types<commit_after>package discovery

import (
	"context"
	"time"

	"github.com/libp2p/go-libp2p-core/discovery"
	"github.com/libp2p/go-libp2p-core/peer"

	logging "github.com/ipfs/go-log"
)

var log = logging.Logger("discovery")

// FindPeers is a utility function that synchronously collects peers from a Discoverer.
func FindPeers(ctx context.Context, d discovery.Discoverer, ns string, opts ...discovery.Option) ([]peer.AddrInfo, error) {
	var res []peer.AddrInfo

	ch, err := d.FindPeers(ctx, ns, opts...)
	if err != nil {
		return nil, err
	}

	for pi := range ch {
		res = append(res, pi)
	}

	return res, nil
}

// Advertise is a utility function that persistently advertises a service through an Advertiser.
func Advertise(ctx context.Context, a discovery.Advertiser, ns string, opts ...discovery.Option) {
	go func() {
		for {
			ttl, err := a.Advertise(ctx, ns, opts...)
			if err != nil {
				log.Debugf("Error advertising %s: %s", ns, err.Error())
				if ctx.Err() != nil {
					return
				}

				select {
				case <-time.After(2 * time.Minute):
					continue
				case <-ctx.Done():
					return
				}
			}

			wait := 7 * ttl / 8
			select {
			case <-time.After(wait):
			case <-ctx.Done():
				return
			}
		}
	}()
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"strings"

	"sync"

	"errors"

	"github.com/piotrkowalczuk/charon"
)

// Permission returns Permission value that is concatenated
// using entity properties like subsystem, module and action.
func (pe *permissionEntity) Permission() charon.Permission {
	return charon.Permission(pe.Subsystem + ":" + pe.Module + ":" + pe.Action)
}

// PermissionRepository ...
type PermissionRepository interface {
	Find(criteria *permissionCriteria) ([]*permissionEntity, error)
	FindOneByID(id int64) (entity *permissionEntity, err error)
	FindByUserID(userID int64) (entities []*permissionEntity, err error)
	Register(permissions charon.Permissions) (created, untouched, removed int64, err error)
	Insert(entity *permissionEntity) (*permissionEntity, error)
}

func newPermissionRepository(dbPool *sql.DB) *permissionRepository {
	return &permissionRepository{
		db:      dbPool,
		table:   tablePermission,
		columns: tablePermissionColumns,
	}
}

// FindByUserID retrieves all permissions for user represented by given id.
func (pr *permissionRepository) FindByUserID(userID int64) ([]*permissionEntity, error) {
	query := `
		SELECT DISTINCT ON (p.id)
			` + columns(tablePermissionColumns, "p") + `
		FROM ` + pr.table + ` AS p
		LEFT JOIN ` + tableUserPermissions + ` AS up ON up.permission_id = p.id AND up.user_id = $1
		LEFT JOIN ` + tableUserGroups + ` AS ug ON ug.user_id = $1
		LEFT JOIN ` + tableGroupPermissions + ` AS gp ON gp.permission_id = p.id AND gp.group_id = ug.group_id
		WHERE up.user_id = $1 OR ug.user_id = $1
	`

	rows, err := pr.db.Query(query, userID)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	permissions := []*permissionEntity{}
	for rows.Next() {
		var p permissionEntity
		err = rows.Scan(
			&p.Action,
			&p.CreatedAt,
			&p.ID,
			&p.Module,
			&p.Subsystem,
			&p.UpdatedAt,
		)
		if err != nil {
			return nil, err
		}

		permissions = append(permissions, &p)
	}
	if rows.Err() != nil {
		return nil, rows.Err()
	}

	return permissions, nil
}

func (pr *permissionRepository) findOneStmt() (*sql.Stmt, error) {
	return pr.db.Prepare(
		"SELECT " + strings.Join(tablePermissionColumns, ",") + " " +
			"FROM " + pr.table + " AS p " +
			"WHERE p.subsystem = $1 AND p.module = $2 AND p.action = $3",
	)
}

func (pr *permissionRepository) Register(permissions charon.Permissions) (created, unt, removed int64, err error) {
	var (
		tx             *sql.Tx
		insert, delete *sql.Stmt
		rows           *sql.Rows
		res            sql.Result
		subsystem      string
		entities       []*permissionEntity
		affected       int64
	)
	if len(permissions) == 0 {
		return 0, 0, 0, errors.New("charond: empty slice, permissions cannot be registered")
	}

	subsystem = permissions[0].Subsystem()
	if subsystem == "" {
		return 0, 0, 0, errors.New("charond: subsystem name is empty string, permissions cannot be registered")
	}

	for _, p := range permissions {
		if p.Subsystem() != subsystem {
			return 0, 0, 0, errors.New("charond: provided permissions do not belong to one subsystem, permissions cannot be registered")
		}
	}

	tx, err = pr.db.Begin()
	if err != nil {
		return
	}
	defer func() {
		if err != nil {
			tx.Rollback()
		} else {
			err = tx.Commit()
			unt = untouched(int64(len(permissions)), created, removed)
		}
	}()

	rows, err = tx.Query("SELECT "+strings.Join(tablePermissionColumns, ",")+" FROM "+pr.table+" AS p WHERE p.subsystem = $1", subsystem)
	if err != nil {
		return
	}
	defer rows.Close()

	entities = []*permissionEntity{}
	for rows.Next() {
		var entity permissionEntity
		err = rows.Scan(
			&entity.Action,
			&entity.CreatedAt,
			&entity.ID,
			&entity.Module,
			&entity.Subsystem,
			&entity.UpdatedAt,
		)
		if err != nil {
			return
		}
		entities = append(entities, &entity)
	}
	if rows.Err() != nil {
		return 0, 0, 0, rows.Err()
	}

	insert, err = tx.Prepare("INSERT INTO " + pr.table + " (subsystem, module, action) VALUES ($1, $2, $3)")
	if err != nil {
		return
	}

MissingPermissionsLoop:
	for _, p := range permissions {
		for _, e := range entities {
			if p == e.Permission() {
				continue MissingPermissionsLoop
			}
		}

		if res, err = insert.Exec(p.Split()); err != nil {
			return
		}
		if affected, err = res.RowsAffected(); err != nil {
			return
		}
		created += affected
	}

	delete, err = tx.Prepare("DELETE FROM " + pr.table + " AS p WHERE p.id = $1")
	if err != nil {
		return
	}

RedundantPermissionsLoop:
	for _, e := range entities {
		for _, p := range permissions {
			if e.Permission() == p {
				continue RedundantPermissionsLoop
			}
		}

		if res, err = delete.Exec(e.ID); err != nil {
			return
		}
		if affected, err = res.RowsAffected(); err != nil {
			return
		}

		removed += affected
	}

	return
}

// PermissionRegistry is an interface that describes in memory storage that holds information
// about permissions that was registered by 3rd party services.
// Should be only used as a proxy for registration process to avoid multiple sql hits.
type PermissionRegistry interface {
	// Exists returns true if given Permission was already registered.
	Exists(permission charon.Permission) (exists bool)
	// Register checks if given collection is valid and
	// calls PermissionRepository to store provided permissions
	// in persistent way.
	Register(permissions charon.Permissions) (created, untouched, removed int64, err error)
}

type permissionRegistry struct {
	sync.RWMutex
	repository  PermissionRepository
	permissions map[charon.Permission]struct{}
}

func newPermissionRegistry(r PermissionRepository) PermissionRegistry {
	return &permissionRegistry{
		repository:  r,
		permissions: make(map[charon.Permission]struct{}),
	}
}

// Exists implements PermissionRegistry interface.
func (pr *permissionRegistry) Exists(permission charon.Permission) (ok bool) {
	pr.RLock()
	pr.RUnlock()

	_, ok = pr.permissions[permission]
	return
}

// Register implements PermissionRegistry interface.
func (pr *permissionRegistry) Register(permissions charon.Permissions) (created, untouched, removed int64, err error) {
	pr.Lock()
	defer pr.Unlock()

	nb := 0
	for _, p := range permissions {
		if _, ok := pr.permissions[p]; !ok {
			pr.permissions[p] = struct{}{}
			nb++
		}
	}

	if nb > 0 {
		return pr.repository.Register(permissions)
	}

	return 0, 0, 0, nil
}
<commit_msg>PermissionsRepository comments cleanup<commit_after>package main

import (
	"database/sql"
	"strings"

	"sync"

	"errors"

	"github.com/piotrkowalczuk/charon"
)

// Permission returns Permission value that is concatenated
// using entity properties like subsystem, module and action.
func (pe *permissionEntity) Permission() charon.Permission {
	return charon.Permission(pe.Subsystem + ":" + pe.Module + ":" + pe.Action)
}

// PermissionRepository ...
type PermissionRepository interface {
	Find(criteria *permissionCriteria) ([]*permissionEntity, error)
	FindOneByID(id int64) (entity *permissionEntity, err error)
	// FindByUserID retrieves all permissions for user represented by given id.
	FindByUserID(userID int64) (entities []*permissionEntity, err error)
	Register(permissions charon.Permissions) (created, untouched, removed int64, err error)
	Insert(entity *permissionEntity) (*permissionEntity, error)
}

func newPermissionRepository(dbPool *sql.DB) *permissionRepository {
	return &permissionRepository{
		db:      dbPool,
		table:   tablePermission,
		columns: tablePermissionColumns,
	}
}

// FindByUserID implements PermissionRepository interface.
func (pr *permissionRepository) FindByUserID(userID int64) ([]*permissionEntity, error) {
	query := `
		SELECT DISTINCT ON (p.id)
			` + columns(tablePermissionColumns, "p") + `
		FROM ` + pr.table + ` AS p
		LEFT JOIN ` + tableUserPermissions + ` AS up ON up.permission_id = p.id AND up.user_id = $1
		LEFT JOIN ` + tableUserGroups + ` AS ug ON ug.user_id = $1
		LEFT JOIN ` + tableGroupPermissions + ` AS gp ON gp.permission_id = p.id AND gp.group_id = ug.group_id
		WHERE up.user_id = $1 OR ug.user_id = $1
	`

	rows, err := pr.db.Query(query, userID)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	permissions := []*permissionEntity{}
	for rows.Next() {
		var p permissionEntity
		err = rows.Scan(
			&p.Action,
			&p.CreatedAt,
			&p.ID,
			&p.Module,
			&p.Subsystem,
			&p.UpdatedAt,
		)
		if err != nil {
			return nil, err
		}

		permissions = append(permissions, &p)
	}
	if rows.Err() != nil {
		return nil, rows.Err()
	}

	return permissions, nil
}

func (pr *permissionRepository) findOneStmt() (*sql.Stmt, error) {
	return pr.db.Prepare(
		"SELECT " + strings.Join(tablePermissionColumns, ",") + " " +
			"FROM " + pr.table + " AS p " +
			"WHERE p.subsystem = $1 AND p.module = $2 AND p.action = $3",
	)
}

// Register implements PermissionRepository interface.
func (pr *permissionRepository) Register(permissions charon.Permissions) (created, unt, removed int64, err error) {
	var (
		tx             *sql.Tx
		insert, delete *sql.Stmt
		rows           *sql.Rows
		res            sql.Result
		subsystem      string
		entities       []*permissionEntity
		affected       int64
	)
	if len(permissions) == 0 {
		return 0, 0, 0, errors.New("charond: empty slice, permissions cannot be registered")
	}

	subsystem = permissions[0].Subsystem()
	if subsystem == "" {
		return 0, 0, 0, errors.New("charond: subsystem name is empty string, permissions cannot be registered")
	}

	for _, p := range permissions {
		if p.Subsystem() != subsystem {
			return 0, 0, 0, errors.New("charond: provided permissions do not belong to one subsystem, permissions cannot be registered")
		}
	}

	tx, err = pr.db.Begin()
	if err != nil {
		return
	}
	defer func() {
		if err != nil {
			tx.Rollback()
		} else {
			err = tx.Commit()
			unt = untouched(int64(len(permissions)), created, removed)
		}
	}()

	rows, err = tx.Query("SELECT "+strings.Join(tablePermissionColumns, ",")+" FROM "+pr.table+" AS p WHERE p.subsystem = $1", subsystem)
	if err != nil {
		return
	}
	defer rows.Close()

	entities = []*permissionEntity{}
	for rows.Next() {
		var entity permissionEntity
		err = rows.Scan(
			&entity.Action,
			&entity.CreatedAt,
			&entity.ID,
			&entity.Module,
			&entity.Subsystem,
			&entity.UpdatedAt,
		)
		if err != nil {
			return
		}
		entities = append(entities, &entity)
	}
	if rows.Err() != nil {
		return 0, 0, 0, rows.Err()
	}

	insert, err = tx.Prepare("INSERT INTO " + pr.table + " (subsystem, module, action) VALUES ($1, $2, $3)")
	if err != nil {
		return
	}

MissingPermissionsLoop:
	for _, p := range permissions {
		for _, e := range entities {
			if p == e.Permission() {
				continue MissingPermissionsLoop
			}
		}

		if res, err = insert.Exec(p.Split()); err != nil {
			return
		}
		if affected, err = res.RowsAffected(); err != nil {
			return
		}
		created += affected
	}

	delete, err = tx.Prepare("DELETE FROM " + pr.table + " AS p WHERE p.id = $1")
	if err != nil {
		return
	}

RedundantPermissionsLoop:
	for _, e := range entities {
		for _, p := range permissions {
			if e.Permission() == p {
				continue RedundantPermissionsLoop
			}
		}

		if res, err = delete.Exec(e.ID); err != nil {
			return
		}
		if affected, err = res.RowsAffected(); err != nil {
			return
		}

		removed += affected
	}

	return
}

// PermissionRegistry is an interface that describes in memory storage that holds information
// about permissions that was registered by 3rd party services.
// Should be only used as a proxy for registration process to avoid multiple sql hits.
type PermissionRegistry interface {
	// Exists returns true if given Permission was already registered.
	Exists(permission charon.Permission) (exists bool)
	// Register checks if given collection is valid and
	// calls PermissionRepository to store provided permissions
	// in persistent way.
	Register(permissions charon.Permissions) (created, untouched, removed int64, err error)
}

type permissionRegistry struct {
	sync.RWMutex
	repository  PermissionRepository
	permissions map[charon.Permission]struct{}
}

func newPermissionRegistry(r PermissionRepository) PermissionRegistry {
	return &permissionRegistry{
		repository:  r,
		permissions: make(map[charon.Permission]struct{}),
	}
}

// Exists implements PermissionRegistry interface.
func (pr *permissionRegistry) Exists(permission charon.Permission) (ok bool) {
	pr.RLock()
	pr.RUnlock()

	_, ok = pr.permissions[permission]
	return
}

// Register implements PermissionRegistry interface.
func (pr *permissionRegistry) Register(permissions charon.Permissions) (created, untouched, removed int64, err error) {
	pr.Lock()
	defer pr.Unlock()

	nb := 0
	for _, p := range permissions {
		if _, ok := pr.permissions[p]; !ok {
			pr.permissions[p] = struct{}{}
			nb++
		}
	}

	if nb > 0 {
		return pr.repository.Register(permissions)
	}

	return 0, 0, 0, nil
}

// FindByTag implements PermissionRepository interface.
func (pr *permissionRepository) FindByTag(userID int64) ([]*permissionEntity, error) {
	query := `
		SELECT DISTINCT ON (p.id)
			` + columns(tablePermissionColumns, "p") + `
		FROM ` + pr.table + ` AS p
		LEFT JOIN ` + tableUserPermissions + ` AS up ON up.permission_id = p.id AND up.user_id = $1
		LEFT JOIN ` + tableUserGroups + ` AS ug ON ug.user_id = $1
		LEFT JOIN ` + tableGroupPermissions + ` AS gp ON gp.permission_id = p.id AND gp.group_id = ug.group_id
		WHERE up.user_id = $1 OR ug.user_id = $1
	`

	rows, err := pr.db.Query(query, userID)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	permissions := []*permissionEntity{}
	for rows.Next() {
		var p permissionEntity
		err = rows.Scan(
			&p.Action,
			&p.CreatedAt,
			&p.ID,
			&p.Module,
			&p.Subsystem,
			&p.UpdatedAt,
		)
		if err != nil {
			return nil, err
		}

		permissions = append(permissions, &p)
	}
	if rows.Err() != nil {
		return nil, rows.Err()
	}

	return permissions, nil
}<|endoftext|>


<commit_before>// Copyright 2017 The nvim-go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package autocmd

import (
	"time"

	"nvim-go/nvimutil"
)

// bufEnterEval represents the current buffer number, windows ID and buffer files directory.
type bufEnterEval struct {
	BufNr int    `eval:"bufnr('%')"`
	WinID int    `eval:"win_getid()"`
	Dir   string `eval:"expand('%:p:h')"`
}

// BufEnter gets the current buffer number, windows ID and set context from the directory structure on BufEnter autocmd.
func (a *Autocmd) BufEnter(eval *bufEnterEval) error {
	defer nvimutil.Profile(time.Now(), "BufEnter")

	a.mu.Lock()
	a.buildctxt.BufNr = eval.BufNr
	a.buildctxt.WinID = eval.WinID
	a.buildctxt.Dir = eval.Dir
	a.mu.Unlock()

	a.buildctxt.SetContext(eval.Dir)
	return nil
}
<commit_msg>autocmd/bufenter: cache prevDir<commit_after>// Copyright 2017 The nvim-go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package autocmd

import (
	"time"

	"nvim-go/nvimutil"
)

// bufEnterEval represents the current buffer number, windows ID and buffer files directory.
type bufEnterEval struct {
	BufNr int    `eval:"bufnr('%')"`
	WinID int    `eval:"win_getid()"`
	Dir   string `eval:"expand('%:p:h')"`
}

// BufEnter gets the current buffer number, windows ID and set context from the directory structure on BufEnter autocmd.
func (a *Autocmd) BufEnter(eval *bufEnterEval) error {
	defer nvimutil.Profile(time.Now(), "BufEnter")

	a.mu.Lock()
	a.buildctxt.BufNr = eval.BufNr
	a.buildctxt.WinID = eval.WinID
	a.buildctxt.Dir = eval.Dir
	a.mu.Unlock()

	if eval.Dir != "" && a.buildctxt.PrevDir != eval.Dir {
		a.buildctxt.SetContext(eval.Dir)
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016-2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"
	"regexp"
	"strconv"
	"strings"
	"text/tabwriter"

	"github.com/cilium/cilium/pkg/bpf"
	"github.com/cilium/cilium/pkg/endpoint"
	"github.com/cilium/cilium/pkg/identity"
	"github.com/cilium/cilium/pkg/maps/policymap"
	"github.com/cilium/cilium/pkg/u8proto"

	"github.com/spf13/cobra"
)

// Fatalf prints the Printf formatted message to stderr and exits the program
// Note: os.Exit(1) is not recoverable
func Fatalf(msg string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "Error: %s\n", fmt.Sprintf(msg, args...))
	os.Exit(1)
}

// Usagef prints the Printf formatted message to stderr, prints usage help and
// exits the program
// Note: os.Exit(1) is not recoverable
func Usagef(cmd *cobra.Command, msg string, args ...interface{}) {
	txt := fmt.Sprintf(msg, args...)
	fmt.Fprintf(os.Stderr, "Error: %s\n\n", txt)
	cmd.Help()
	os.Exit(1)
}

func requireEndpointID(cmd *cobra.Command, args []string) {
	if len(args) < 1 {
		Usagef(cmd, "Missing endpoint id argument")
	}

	if id := identity.ReservedIdentities[args[0]]; id == identity.IdentityUnknown {
		_, _, err := endpoint.ValidateID(args[0])

		if err != nil {
			Fatalf("Cannot parse endpoint id \"%s\": %s", args[0], err)
		}
	}
}

func requireEndpointIDorGlobal(cmd *cobra.Command, args []string) {
	if len(args) < 1 {
		Usagef(cmd, "Missing endpoint id or 'global' argument")
	}

	if args[0] != "global" {
		requireEndpointID(cmd, args)
	}
}

func requirePath(cmd *cobra.Command, args []string) {
	if len(args) < 1 {
		Usagef(cmd, "Missing path argument")
	}

	if args[0] == "" {
		Usagef(cmd, "Empty path argument")
	}
}

func requireServiceID(cmd *cobra.Command, args []string) {
	if len(args) < 1 {
		Usagef(cmd, "Missing service id argument")
	}

	if args[0] == "" {
		Usagef(cmd, "Empty service id argument")
	}
}

// TablePrinter prints the map[string][]string, which is an usual representation
// of dumped BPF map, using tabwriter.
func TablePrinter(firstTitle, secondTitle string, data map[string][]string) {
	w := tabwriter.NewWriter(os.Stdout, 5, 0, 3, ' ', 0)

	fmt.Fprintf(w, "%s\t%s\t\n", firstTitle, secondTitle)

	for key, value := range data {
		for k, v := range value {
			if k == 0 {
				fmt.Fprintf(w, "%s\t%s\t\n", key, v)
			} else {
				fmt.Fprintf(w, "%s\t%s\t\n", "", v)
			}
		}
	}

	w.Flush()
}

// Search 'result' for strings with escaped JSON inside, and expand the JSON.
func expandNestedJSON(result bytes.Buffer) (bytes.Buffer, error) {
	reStringWithJSON := regexp.MustCompile(`"[^"\\{]*{.*[^\\]"`)
	reJSON := regexp.MustCompile(`{.*}`)
	for {
		var (
			loc    []int
			indent string
		)

		// Search for nested JSON; if we don't find any, then break.
		resBytes := result.Bytes()
		if loc = reStringWithJSON.FindIndex(resBytes); loc == nil {
			break
		}

		// Determine the current indentation
		for i := 0; i < loc[0]-1; i++ {
			idx := loc[0] - i - 1
			if resBytes[idx] != ' ' {
				break
			}
			indent = fmt.Sprintf("%s ", indent)
		}

		stringStart := loc[0]
		stringEnd := loc[1]

		// Unquote the string with the nested json.
		quotedBytes := resBytes[stringStart:stringEnd]
		unquoted, err := strconv.Unquote(string(quotedBytes))
		if err != nil {
			return bytes.Buffer{}, fmt.Errorf("Failed to Unquote string: %s\n%s", err.Error(), string(quotedBytes))
		}

		// Find the JSON within the quoted string.
		nestedStart := 0
		nestedEnd := 0
		if locs := reJSON.FindAllStringIndex(unquoted, -1); locs != nil {
			// The last match is the longest one.
			last := len(locs) - 1
			nestedStart = locs[last][0]
			nestedEnd = locs[last][1]
		} else if reJSON.Match(quotedBytes) {
			// The entire string is JSON
			nestedEnd = len(unquoted)
		}

		// Decode the nested JSON
		decoded := ""
		if nestedEnd != 0 {
			m := make(map[string]interface{})
			nested := bytes.NewBufferString(unquoted[nestedStart:nestedEnd])
			if err := json.NewDecoder(nested).Decode(&m); err != nil {
				return bytes.Buffer{}, fmt.Errorf("Failed to decode nested JSON: %s", err.Error())
			}
			decodedBytes, err := json.MarshalIndent(m, indent, "  ")
			if err != nil {
				return bytes.Buffer{}, fmt.Errorf("Cannot marshal nested JSON: %s", err.Error())
			}
			decoded = string(decodedBytes)
		}

		// Serialize
		nextResult := bytes.Buffer{}
		nextResult.Write(resBytes[0:stringStart])
		nextResult.WriteString(string(unquoted[:nestedStart]))
		nextResult.WriteString(string(decoded))
		nextResult.WriteString(string(unquoted[nestedEnd:]))
		nextResult.Write(resBytes[stringEnd:])
		result = nextResult
	}

	return result, nil
}

// parseTrafficString converts the provided string to its corresponding
// TrafficDirection. If the string does not correspond to a valid TrafficDirection
// type, returns Invalid and a corresponding error.
func parseTrafficString(td string) (policymap.TrafficDirection, error) {
	lowered := strings.ToLower(td)
	if lowered == "ingress" {
		return policymap.Ingress, nil
	} else if lowered == "egress" {
		return policymap.Egress, nil
	} else {
		return policymap.Invalid, fmt.Errorf("invalid direction %q provided", td)
	}

}

func updatePolicyKey(cmd *cobra.Command, args []string, add bool) {
	if len(args) < 3 {
		Usagef(cmd, "<endpoint id>, <traffic-direction>, and <identity> required")
	}

	trafficDirection := args[1]
	parsedTd, err := parseTrafficString(trafficDirection)
	if err != nil {
		Fatalf("Failed to convert %s to a valid traffic direction: %s", args[1], err)
	}

	endpointID := args[0]
	if numericIdentity := identity.GetReservedID(endpointID); numericIdentity != identity.IdentityUnknown {
		endpointID = "reserved_" + strconv.FormatUint(uint64(numericIdentity), 10)
	}

	policyMapPath := bpf.MapPath(policymap.MapName + endpointID)
	policyMap, _, err := policymap.OpenMap(policyMapPath)
	if err != nil {
		Fatalf("Cannot open policymap '%s' : %s", policyMapPath, err)
	}

	peerLbl, err := strconv.ParseUint(args[2], 10, 32)
	if err != nil {
		Fatalf("Failed to convert %s", args[2])
	}

	port := uint16(0)
	protos := []uint8{}
	if len(args) > 3 {
		pp, err := parseL4PortsSlice([]string{args[3]})
		if err != nil {
			Fatalf("Failed to parse L4: %s", err)
		}
		port = pp[0].Port
		if port != 0 {
			proto, _ := u8proto.ParseProtocol(pp[0].Protocol)
			if proto == 0 {
				for _, proto := range u8proto.ProtoIDs {
					protos = append(protos, uint8(proto))
				}
			} else {
				protos = append(protos, uint8(proto))
			}
		}
	}
	if len(protos) == 0 {
		protos = append(protos, 0)
	}

	label := uint32(peerLbl)
	for _, proto := range protos {
		u8p := u8proto.U8proto(proto)
		entry := fmt.Sprintf("%d %d/%s", label, port, u8p.String())
		if add == true {
			if err := policyMap.AllowL4(label, port, proto, parsedTd); err != nil {
				Fatalf("Cannot add policy key '%s': %s\n", entry, err)
			}
		} else {
			if err := policyMap.DeleteL4(label, port, proto, parsedTd); err != nil {
				Fatalf("Cannot delete policy key '%s': %s\n", entry, err)
			}
		}
	}
}
<commit_msg>cilium/cmd: refactor parseTrafficString<commit_after>// Copyright 2016-2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"
	"regexp"
	"strconv"
	"strings"
	"text/tabwriter"

	"github.com/cilium/cilium/pkg/bpf"
	"github.com/cilium/cilium/pkg/endpoint"
	"github.com/cilium/cilium/pkg/identity"
	"github.com/cilium/cilium/pkg/maps/policymap"
	"github.com/cilium/cilium/pkg/u8proto"

	"github.com/spf13/cobra"
)

// Fatalf prints the Printf formatted message to stderr and exits the program
// Note: os.Exit(1) is not recoverable
func Fatalf(msg string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "Error: %s\n", fmt.Sprintf(msg, args...))
	os.Exit(1)
}

// Usagef prints the Printf formatted message to stderr, prints usage help and
// exits the program
// Note: os.Exit(1) is not recoverable
func Usagef(cmd *cobra.Command, msg string, args ...interface{}) {
	txt := fmt.Sprintf(msg, args...)
	fmt.Fprintf(os.Stderr, "Error: %s\n\n", txt)
	cmd.Help()
	os.Exit(1)
}

func requireEndpointID(cmd *cobra.Command, args []string) {
	if len(args) < 1 {
		Usagef(cmd, "Missing endpoint id argument")
	}

	if id := identity.ReservedIdentities[args[0]]; id == identity.IdentityUnknown {
		_, _, err := endpoint.ValidateID(args[0])

		if err != nil {
			Fatalf("Cannot parse endpoint id \"%s\": %s", args[0], err)
		}
	}
}

func requireEndpointIDorGlobal(cmd *cobra.Command, args []string) {
	if len(args) < 1 {
		Usagef(cmd, "Missing endpoint id or 'global' argument")
	}

	if args[0] != "global" {
		requireEndpointID(cmd, args)
	}
}

func requirePath(cmd *cobra.Command, args []string) {
	if len(args) < 1 {
		Usagef(cmd, "Missing path argument")
	}

	if args[0] == "" {
		Usagef(cmd, "Empty path argument")
	}
}

func requireServiceID(cmd *cobra.Command, args []string) {
	if len(args) < 1 {
		Usagef(cmd, "Missing service id argument")
	}

	if args[0] == "" {
		Usagef(cmd, "Empty service id argument")
	}
}

// TablePrinter prints the map[string][]string, which is an usual representation
// of dumped BPF map, using tabwriter.
func TablePrinter(firstTitle, secondTitle string, data map[string][]string) {
	w := tabwriter.NewWriter(os.Stdout, 5, 0, 3, ' ', 0)

	fmt.Fprintf(w, "%s\t%s\t\n", firstTitle, secondTitle)

	for key, value := range data {
		for k, v := range value {
			if k == 0 {
				fmt.Fprintf(w, "%s\t%s\t\n", key, v)
			} else {
				fmt.Fprintf(w, "%s\t%s\t\n", "", v)
			}
		}
	}

	w.Flush()
}

// Search 'result' for strings with escaped JSON inside, and expand the JSON.
func expandNestedJSON(result bytes.Buffer) (bytes.Buffer, error) {
	reStringWithJSON := regexp.MustCompile(`"[^"\\{]*{.*[^\\]"`)
	reJSON := regexp.MustCompile(`{.*}`)
	for {
		var (
			loc    []int
			indent string
		)

		// Search for nested JSON; if we don't find any, then break.
		resBytes := result.Bytes()
		if loc = reStringWithJSON.FindIndex(resBytes); loc == nil {
			break
		}

		// Determine the current indentation
		for i := 0; i < loc[0]-1; i++ {
			idx := loc[0] - i - 1
			if resBytes[idx] != ' ' {
				break
			}
			indent = fmt.Sprintf("%s ", indent)
		}

		stringStart := loc[0]
		stringEnd := loc[1]

		// Unquote the string with the nested json.
		quotedBytes := resBytes[stringStart:stringEnd]
		unquoted, err := strconv.Unquote(string(quotedBytes))
		if err != nil {
			return bytes.Buffer{}, fmt.Errorf("Failed to Unquote string: %s\n%s", err.Error(), string(quotedBytes))
		}

		// Find the JSON within the quoted string.
		nestedStart := 0
		nestedEnd := 0
		if locs := reJSON.FindAllStringIndex(unquoted, -1); locs != nil {
			// The last match is the longest one.
			last := len(locs) - 1
			nestedStart = locs[last][0]
			nestedEnd = locs[last][1]
		} else if reJSON.Match(quotedBytes) {
			// The entire string is JSON
			nestedEnd = len(unquoted)
		}

		// Decode the nested JSON
		decoded := ""
		if nestedEnd != 0 {
			m := make(map[string]interface{})
			nested := bytes.NewBufferString(unquoted[nestedStart:nestedEnd])
			if err := json.NewDecoder(nested).Decode(&m); err != nil {
				return bytes.Buffer{}, fmt.Errorf("Failed to decode nested JSON: %s", err.Error())
			}
			decodedBytes, err := json.MarshalIndent(m, indent, "  ")
			if err != nil {
				return bytes.Buffer{}, fmt.Errorf("Cannot marshal nested JSON: %s", err.Error())
			}
			decoded = string(decodedBytes)
		}

		// Serialize
		nextResult := bytes.Buffer{}
		nextResult.Write(resBytes[0:stringStart])
		nextResult.WriteString(string(unquoted[:nestedStart]))
		nextResult.WriteString(string(decoded))
		nextResult.WriteString(string(unquoted[nestedEnd:]))
		nextResult.Write(resBytes[stringEnd:])
		result = nextResult
	}

	return result, nil
}

// parseTrafficString converts the provided string to its corresponding
// TrafficDirection. If the string does not correspond to a valid TrafficDirection
// type, returns Invalid and a corresponding error.
func parseTrafficString(td string) (policymap.TrafficDirection, error) {
	lowered := strings.ToLower(td)

	switch lowered {
	case "ingress":
		return policymap.Ingress, nil
	case "egress":
		return policymap.Egress, nil
	default:
		return policymap.Invalid, fmt.Errorf("invalid direction %q provided", td)
	}

}

func updatePolicyKey(cmd *cobra.Command, args []string, add bool) {
	if len(args) < 3 {
		Usagef(cmd, "<endpoint id>, <traffic-direction>, and <identity> required")
	}

	trafficDirection := args[1]
	parsedTd, err := parseTrafficString(trafficDirection)
	if err != nil {
		Fatalf("Failed to convert %s to a valid traffic direction: %s", args[1], err)
	}

	endpointID := args[0]
	if numericIdentity := identity.GetReservedID(endpointID); numericIdentity != identity.IdentityUnknown {
		endpointID = "reserved_" + strconv.FormatUint(uint64(numericIdentity), 10)
	}

	policyMapPath := bpf.MapPath(policymap.MapName + endpointID)
	policyMap, _, err := policymap.OpenMap(policyMapPath)
	if err != nil {
		Fatalf("Cannot open policymap '%s' : %s", policyMapPath, err)
	}

	peerLbl, err := strconv.ParseUint(args[2], 10, 32)
	if err != nil {
		Fatalf("Failed to convert %s", args[2])
	}

	port := uint16(0)
	protos := []uint8{}
	if len(args) > 3 {
		pp, err := parseL4PortsSlice([]string{args[3]})
		if err != nil {
			Fatalf("Failed to parse L4: %s", err)
		}
		port = pp[0].Port
		if port != 0 {
			proto, _ := u8proto.ParseProtocol(pp[0].Protocol)
			if proto == 0 {
				for _, proto := range u8proto.ProtoIDs {
					protos = append(protos, uint8(proto))
				}
			} else {
				protos = append(protos, uint8(proto))
			}
		}
	}
	if len(protos) == 0 {
		protos = append(protos, 0)
	}

	label := uint32(peerLbl)
	for _, proto := range protos {
		u8p := u8proto.U8proto(proto)
		entry := fmt.Sprintf("%d %d/%s", label, port, u8p.String())
		if add == true {
			if err := policyMap.AllowL4(label, port, proto, parsedTd); err != nil {
				Fatalf("Cannot add policy key '%s': %s\n", entry, err)
			}
		} else {
			if err := policyMap.DeleteL4(label, port, proto, parsedTd); err != nil {
				Fatalf("Cannot delete policy key '%s': %s\n", entry, err)
			}
		}
	}
}
<|endoftext|>


<commit_before>// +build integ
// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package helmupgrade

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"istio.io/istio/pkg/test/env"
	"istio.io/istio/pkg/test/framework"
	"istio.io/istio/pkg/test/framework/components/cluster"
	kubecluster "istio.io/istio/pkg/test/framework/components/cluster/kube"
	"istio.io/istio/pkg/test/framework/image"
	"istio.io/istio/pkg/test/helm"
	kubetest "istio.io/istio/pkg/test/kube"
	"istio.io/istio/pkg/test/scopes"
	"istio.io/istio/pkg/test/util/retry"
	helmtest "istio.io/istio/tests/integration/helm"
	"istio.io/istio/tests/util/sanitycheck"
)

const (
	gcrHub = "gcr.io/istio-release"

	defaultValues = `
global:
  hub: %s
  tag: %s
`
	revisionValues = `
global:
  hub: %s
  tag: %s

revision: canary
`
	tarGzSuffix = ".tar.gz"

	revisionLabel       = "canary"
	revisionChartSuffix = "-canary"
)

// previousChartPath is path of Helm charts for previous Istio deployments.
var previousChartPath = filepath.Join(env.IstioSrc, "tests/integration/helm/testdata/")

// upgradeCharts upgrades Istio using Helm charts with the provided
// override values file to the latest charts in $ISTIO_SRC/manifests
func upgradeCharts(ctx framework.TestContext, h *helm.Helm, overrideValuesFile string) {
	// Upgrade base chart
	err := h.UpgradeChart(helmtest.BaseReleaseName, filepath.Join(helmtest.ChartPath, helmtest.BaseChart),
		helmtest.IstioNamespace, overrideValuesFile, helmtest.Timeout)
	if err != nil {
		ctx.Fatalf("failed to upgrade istio %s chart", helmtest.BaseChart)
	}

	// Upgrade discovery chart
	err = h.UpgradeChart(helmtest.IstiodReleaseName, filepath.Join(helmtest.ChartPath, helmtest.ControlChartsDir, helmtest.DiscoveryChart),
		helmtest.IstioNamespace, overrideValuesFile, helmtest.Timeout)
	if err != nil {
		ctx.Fatalf("failed to upgrade istio %s chart", helmtest.DiscoveryChart)
	}

	// Upgrade ingress gateway chart
	err = h.UpgradeChart(helmtest.IngressReleaseName, filepath.Join(helmtest.ChartPath, helmtest.GatewayChartsDir, helmtest.IngressGatewayChart),
		helmtest.IstioNamespace, overrideValuesFile, helmtest.Timeout)
	if err != nil {
		ctx.Fatalf("failed to upgrade istio %s chart", helmtest.IngressGatewayChart)
	}

	// Upgrade egress gateway chart
	err = h.UpgradeChart(helmtest.EgressReleaseName, filepath.Join(helmtest.ChartPath, helmtest.GatewayChartsDir, helmtest.EgressGatewayChart),
		helmtest.IstioNamespace, overrideValuesFile, helmtest.Timeout)
	if err != nil {
		ctx.Fatalf("failed to upgrade istio %s chart", helmtest.EgressGatewayChart)
	}
}

// deleteIstio deletes installed Istio Helm charts and resources
func deleteIstio(cs cluster.Cluster, h *helm.Helm) error {
	scopes.Framework.Infof("cleaning up resources")
	if err := h.DeleteChart(helmtest.EgressReleaseName, helmtest.IstioNamespace); err != nil {
		return fmt.Errorf("failed to delete %s release", helmtest.EgressReleaseName)
	}
	if err := h.DeleteChart(helmtest.IngressReleaseName, helmtest.IstioNamespace); err != nil {
		return fmt.Errorf("failed to delete %s release", helmtest.IngressReleaseName)
	}
	if err := h.DeleteChart(helmtest.IstiodReleaseName, helmtest.IstioNamespace); err != nil {
		return fmt.Errorf("failed to delete %s release", helmtest.IngressReleaseName)
	}
	if err := h.DeleteChart(helmtest.BaseReleaseName, helmtest.IstioNamespace); err != nil {
		return fmt.Errorf("failed to delete %s release", helmtest.BaseReleaseName)
	}
	if err := cs.CoreV1().Namespaces().Delete(context.TODO(), helmtest.IstioNamespace, metav1.DeleteOptions{}); err != nil {
		return fmt.Errorf("failed to delete istio namespace: %v", err)
	}
	if err := kubetest.WaitForNamespaceDeletion(cs, helmtest.IstioNamespace, retry.Timeout(helmtest.RetryTimeOut)); err != nil {
		return fmt.Errorf("wating for istio namespace to be deleted: %v", err)
	}

	return nil
}

// deleteIstioCanary deletes installed Istio Helm charts and resources
func deleteIstioRevision(h *helm.Helm, revision string) error {
	scopes.Framework.Infof("cleaning up canary resources")
	if err := h.DeleteChart(helmtest.IstiodReleaseName+revision, helmtest.IstioNamespace); err != nil {
		return fmt.Errorf("failed to delete %s release", helmtest.IngressReleaseName)
	}

	return nil
}

// getValuesOverrides returns the the values file created to pass into Helm override default values
// for the hub and tag
func getValuesOverrides(ctx framework.TestContext, valuesStr, hub, tag string) string {
	workDir := ctx.CreateTmpDirectoryOrFail("helm")
	overrideValues := fmt.Sprintf(valuesStr, hub, tag)
	overrideValuesFile := filepath.Join(workDir, "values.yaml")
	if err := ioutil.WriteFile(overrideValuesFile, []byte(overrideValues), os.ModePerm); err != nil {
		ctx.Fatalf("failed to write iop cr file: %v", err)
	}

	return overrideValuesFile
}

// performInPlaceUpgradeFunc returns the provided function necessary to run inside of a integration test
// for upgrade capability
func performInPlaceUpgradeFunc(previousVersion string) func(framework.TestContext) {
	return func(t framework.TestContext) {
		cs := t.Clusters().Default().(*kubecluster.Cluster)
		h := helm.New(cs.Filename(), filepath.Join(previousChartPath, previousVersion))

		t.ConditionalCleanup(func() {
			// only need to do call this once as helm doesn't need to remove
			// all versions
			err := deleteIstio(cs, h)
			if err != nil {
				t.Fatalf("could not delete istio: %v", err)
			}
		})

		overrideValuesFile := getValuesOverrides(t, defaultValues, gcrHub, previousVersion)
		helmtest.InstallIstio(t, cs, h, tarGzSuffix, overrideValuesFile)
		helmtest.VerifyInstallation(t, cs)

		oldClient, oldServer := sanitycheck.SetupTrafficTest(t, t, "")
		sanitycheck.RunTrafficTestClientServer(t, oldClient, oldServer)

		// now upgrade istio to the latest version found in this branch
		// use the command line or environmental vars from the user to set
		// the hub/tag
		s, err := image.SettingsFromCommandLine()
		if err != nil {
			t.Fatal(err)
		}

		overrideValuesFile = getValuesOverrides(t, defaultValues, s.Hub, s.Tag)
		upgradeCharts(t, h, overrideValuesFile)
		helmtest.VerifyInstallation(t, cs)

		newClient, newServer := sanitycheck.SetupTrafficTest(t, t, "")
		sanitycheck.RunTrafficTestClientServer(t, newClient, newServer)

		// now check that we are compatible with N-1 proxy with N proxy
		sanitycheck.RunTrafficTestClientServer(t, oldClient, newServer)
	}
}

// performRevisionUpgradeFunc returns the provided function necessary to run inside of a integration test
// for upgrade capability with revisions
func performRevisionUpgradeFunc(previousVersion string) func(framework.TestContext) {
	return func(t framework.TestContext) {
		cs := t.Clusters().Default().(*kubecluster.Cluster)
		h := helm.New(cs.Filename(), filepath.Join(previousChartPath, previousVersion))

		t.ConditionalCleanup(func() {
			err := deleteIstioRevision(h, revisionChartSuffix)
			if err != nil {
				t.Fatalf("could not delete istio: %v", err)
			}
			err = deleteIstio(cs, h)
			if err != nil {
				t.Fatalf("could not delete istio: %v", err)
			}
		})

		overrideValuesFile := getValuesOverrides(t, defaultValues, gcrHub, previousVersion)
		helmtest.InstallIstio(t, cs, h, tarGzSuffix, overrideValuesFile)
		helmtest.VerifyInstallation(t, cs)

		oldClient, oldServer := sanitycheck.SetupTrafficTest(t, t, "")
		sanitycheck.RunTrafficTestClientServer(t, oldClient, oldServer)

		// now upgrade istio to the latest version found in this branch
		// use the command line or environmental vars from the user to set
		// the hub/tag
		s, err := image.SettingsFromCommandLine()
		if err != nil {
			t.Fatal(err)
		}

		overrideValuesFile = getValuesOverrides(t, revisionValues, s.Hub, s.Tag)
		helmtest.InstallIstioWithRevision(t, cs, h, tarGzSuffix, revisionChartSuffix, overrideValuesFile)
		helmtest.VerifyInstallation(t, cs)

		newClient, newServer := sanitycheck.SetupTrafficTest(t, t, revisionLabel)
		sanitycheck.RunTrafficTestClientServer(t, newClient, newServer)

		// now check that we are compatible with N-1 proxy with N proxy
		sanitycheck.RunTrafficTestClientServer(t, oldClient, newServer)
	}
}
<commit_msg>modify parameters in helm_upgrade (#33465)<commit_after>// +build integ
// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package helmupgrade

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"istio.io/istio/pkg/test/env"
	"istio.io/istio/pkg/test/framework"
	"istio.io/istio/pkg/test/framework/components/cluster"
	kubecluster "istio.io/istio/pkg/test/framework/components/cluster/kube"
	"istio.io/istio/pkg/test/framework/image"
	"istio.io/istio/pkg/test/helm"
	kubetest "istio.io/istio/pkg/test/kube"
	"istio.io/istio/pkg/test/scopes"
	"istio.io/istio/pkg/test/util/retry"
	helmtest "istio.io/istio/tests/integration/helm"
	"istio.io/istio/tests/util/sanitycheck"
)

const (
	gcrHub = "gcr.io/istio-release"

	defaultValues = `
global:
  hub: %s
  tag: %s
`
	revisionValues = `
global:
  hub: %s
  tag: %s

revision: canary
`
	tarGzSuffix = ".tar.gz"

	revisionLabel       = "canary"
	revisionChartSuffix = "-canary"
)

// previousChartPath is path of Helm charts for previous Istio deployments.
var previousChartPath = filepath.Join(env.IstioSrc, "tests/integration/helm/testdata/")

// upgradeCharts upgrades Istio using Helm charts with the provided
// override values file to the latest charts in $ISTIO_SRC/manifests
func upgradeCharts(ctx framework.TestContext, h *helm.Helm, overrideValuesFile string) {
	// Upgrade base chart
	err := h.UpgradeChart(helmtest.BaseReleaseName, filepath.Join(helmtest.ChartPath, helmtest.BaseChart),
		helmtest.IstioNamespace, overrideValuesFile, helmtest.Timeout)
	if err != nil {
		ctx.Fatalf("failed to upgrade istio %s chart", helmtest.BaseChart)
	}

	// Upgrade discovery chart
	err = h.UpgradeChart(helmtest.IstiodReleaseName, filepath.Join(helmtest.ChartPath, helmtest.ControlChartsDir, helmtest.DiscoveryChart),
		helmtest.IstioNamespace, overrideValuesFile, helmtest.Timeout)
	if err != nil {
		ctx.Fatalf("failed to upgrade istio %s chart", helmtest.DiscoveryChart)
	}

	// Upgrade ingress gateway chart
	err = h.UpgradeChart(helmtest.IngressReleaseName, filepath.Join(helmtest.ChartPath, helmtest.GatewayChartsDir, helmtest.IngressGatewayChart),
		helmtest.IstioNamespace, overrideValuesFile, helmtest.Timeout)
	if err != nil {
		ctx.Fatalf("failed to upgrade istio %s chart", helmtest.IngressGatewayChart)
	}

	// Upgrade egress gateway chart
	err = h.UpgradeChart(helmtest.EgressReleaseName, filepath.Join(helmtest.ChartPath, helmtest.GatewayChartsDir, helmtest.EgressGatewayChart),
		helmtest.IstioNamespace, overrideValuesFile, helmtest.Timeout)
	if err != nil {
		ctx.Fatalf("failed to upgrade istio %s chart", helmtest.EgressGatewayChart)
	}
}

// deleteIstio deletes installed Istio Helm charts and resources
func deleteIstio(cs cluster.Cluster, h *helm.Helm) error {
	scopes.Framework.Infof("cleaning up resources")
	if err := h.DeleteChart(helmtest.EgressReleaseName, helmtest.IstioNamespace); err != nil {
		return fmt.Errorf("failed to delete %s release", helmtest.EgressReleaseName)
	}
	if err := h.DeleteChart(helmtest.IngressReleaseName, helmtest.IstioNamespace); err != nil {
		return fmt.Errorf("failed to delete %s release", helmtest.IngressReleaseName)
	}
	if err := h.DeleteChart(helmtest.IstiodReleaseName, helmtest.IstioNamespace); err != nil {
		return fmt.Errorf("failed to delete %s release", helmtest.IstiodReleaseName)
	}
	if err := h.DeleteChart(helmtest.BaseReleaseName, helmtest.IstioNamespace); err != nil {
		return fmt.Errorf("failed to delete %s release", helmtest.BaseReleaseName)
	}
	if err := cs.CoreV1().Namespaces().Delete(context.TODO(), helmtest.IstioNamespace, metav1.DeleteOptions{}); err != nil {
		return fmt.Errorf("failed to delete istio namespace: %v", err)
	}
	if err := kubetest.WaitForNamespaceDeletion(cs, helmtest.IstioNamespace, retry.Timeout(helmtest.RetryTimeOut)); err != nil {
		return fmt.Errorf("wating for istio namespace to be deleted: %v", err)
	}

	return nil
}

// deleteIstioCanary deletes installed Istio Helm charts and resources
func deleteIstioRevision(h *helm.Helm, revision string) error {
	scopes.Framework.Infof("cleaning up canary resources")
	if err := h.DeleteChart(helmtest.IstiodReleaseName+revision, helmtest.IstioNamespace); err != nil {
		return fmt.Errorf("failed to delete %s release", helmtest.IstiodReleaseName+revision)
	}

	return nil
}

// getValuesOverrides returns the the values file created to pass into Helm override default values
// for the hub and tag
func getValuesOverrides(ctx framework.TestContext, valuesStr, hub, tag string) string {
	workDir := ctx.CreateTmpDirectoryOrFail("helm")
	overrideValues := fmt.Sprintf(valuesStr, hub, tag)
	overrideValuesFile := filepath.Join(workDir, "values.yaml")
	if err := ioutil.WriteFile(overrideValuesFile, []byte(overrideValues), os.ModePerm); err != nil {
		ctx.Fatalf("failed to write iop cr file: %v", err)
	}

	return overrideValuesFile
}

// performInPlaceUpgradeFunc returns the provided function necessary to run inside of a integration test
// for upgrade capability
func performInPlaceUpgradeFunc(previousVersion string) func(framework.TestContext) {
	return func(t framework.TestContext) {
		cs := t.Clusters().Default().(*kubecluster.Cluster)
		h := helm.New(cs.Filename(), filepath.Join(previousChartPath, previousVersion))

		t.ConditionalCleanup(func() {
			// only need to do call this once as helm doesn't need to remove
			// all versions
			err := deleteIstio(cs, h)
			if err != nil {
				t.Fatalf("could not delete istio: %v", err)
			}
		})

		overrideValuesFile := getValuesOverrides(t, defaultValues, gcrHub, previousVersion)
		helmtest.InstallIstio(t, cs, h, tarGzSuffix, overrideValuesFile)
		helmtest.VerifyInstallation(t, cs)

		oldClient, oldServer := sanitycheck.SetupTrafficTest(t, t, "")
		sanitycheck.RunTrafficTestClientServer(t, oldClient, oldServer)

		// now upgrade istio to the latest version found in this branch
		// use the command line or environmental vars from the user to set
		// the hub/tag
		s, err := image.SettingsFromCommandLine()
		if err != nil {
			t.Fatal(err)
		}

		overrideValuesFile = getValuesOverrides(t, defaultValues, s.Hub, s.Tag)
		upgradeCharts(t, h, overrideValuesFile)
		helmtest.VerifyInstallation(t, cs)

		newClient, newServer := sanitycheck.SetupTrafficTest(t, t, "")
		sanitycheck.RunTrafficTestClientServer(t, newClient, newServer)

		// now check that we are compatible with N-1 proxy with N proxy
		sanitycheck.RunTrafficTestClientServer(t, oldClient, newServer)
	}
}

// performRevisionUpgradeFunc returns the provided function necessary to run inside of a integration test
// for upgrade capability with revisions
func performRevisionUpgradeFunc(previousVersion string) func(framework.TestContext) {
	return func(t framework.TestContext) {
		cs := t.Clusters().Default().(*kubecluster.Cluster)
		h := helm.New(cs.Filename(), filepath.Join(previousChartPath, previousVersion))

		t.ConditionalCleanup(func() {
			err := deleteIstioRevision(h, revisionChartSuffix)
			if err != nil {
				t.Fatalf("could not delete istio: %v", err)
			}
			err = deleteIstio(cs, h)
			if err != nil {
				t.Fatalf("could not delete istio: %v", err)
			}
		})

		overrideValuesFile := getValuesOverrides(t, defaultValues, gcrHub, previousVersion)
		helmtest.InstallIstio(t, cs, h, tarGzSuffix, overrideValuesFile)
		helmtest.VerifyInstallation(t, cs)

		oldClient, oldServer := sanitycheck.SetupTrafficTest(t, t, "")
		sanitycheck.RunTrafficTestClientServer(t, oldClient, oldServer)

		// now upgrade istio to the latest version found in this branch
		// use the command line or environmental vars from the user to set
		// the hub/tag
		s, err := image.SettingsFromCommandLine()
		if err != nil {
			t.Fatal(err)
		}

		overrideValuesFile = getValuesOverrides(t, revisionValues, s.Hub, s.Tag)
		helmtest.InstallIstioWithRevision(t, cs, h, tarGzSuffix, revisionChartSuffix, overrideValuesFile)
		helmtest.VerifyInstallation(t, cs)

		newClient, newServer := sanitycheck.SetupTrafficTest(t, t, revisionLabel)
		sanitycheck.RunTrafficTestClientServer(t, newClient, newServer)

		// now check that we are compatible with N-1 proxy with N proxy
		sanitycheck.RunTrafficTestClientServer(t, oldClient, newServer)
	}
}
<|endoftext|>


<commit_before>package api

import (
       "io/ioutil"
       "net/http"
       "net/url"
       "fmt"
)

type API struct {
     token string
     endpoint string
}

func New(token string) *API {

     return &API{
     	    token: token,
	    endpoint: "https://api.collection.cooperhewitt.org/rest/",
     }
}

function ExecuteMethod (api *API, method string, params *url.Values) ([]byte error) {

	params.Set("method",  method)
	params.Set("access_token", api.token)

	req, err := http.NewRequest("POST", api.endpoint, params)

	client := &http.Client{}
	rsp, err := client.Do(req)

	if err != nil {
        	panic(err)
	}

	defer rsp.Body.Close()

	fmt.Println("response Status:", rsp.Status)
	fmt.Println("response Headers:", rsp.Header)

	body, _ := ioutil.ReadAll(resp.Body)
	return body
}

<commit_msg>build but still doesn't quite work<commit_after>package api

import (
       "io/ioutil"
       "net/http"
       "net/url"
       "fmt"
)

type API struct {
     token string
     endpoint string
}

func Client(token string) *API {

     return &API{
     	    token: token,
	    endpoint: "https://api.collection.cooperhewitt.org/rest/",
     }
}

func (api *API) ExecuteMethod (method string, params *url.Values) ([]byte, error) {

	params.Set("method",  method)
	params.Set("access_token", api.token)

	req, err := http.NewRequest("POST", api.endpoint, nil)
	req.URL.RawQuery = (*params).Encode()

	client := &http.Client{}
	rsp, err := client.Do(req)

	if err != nil {
        	panic(err)
	}

	defer rsp.Body.Close()

	fmt.Println("response Status:", rsp.Status)
	fmt.Println("response Headers:", rsp.Header)

	body, _ := ioutil.ReadAll(rsp.Body)
	return body, nil
}

<|endoftext|>