<commit_before>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Based on github.com/grpc-ecosystem/go-grpc-middleware/retry, but modified to support the more
// fine grained error checking required by write-at-most-once retry semantics of etcd.

package clientv3

import (
	"context"
	"io"
	"sync"
	"time"

	"go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes"
	"go.uber.org/zap"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/metadata"
	"google.golang.org/grpc/status"
)

// unaryClientInterceptor returns a new retrying unary client interceptor.
//
// The default configuration of the interceptor is to not retry *at all*. This behaviour can be
// changed through options (e.g. WithMax) on creation of the interceptor or on call (through grpc.CallOptions).
func (c *Client) unaryClientInterceptor(logger *zap.Logger, optFuncs ...retryOption) grpc.UnaryClientInterceptor {
	intOpts := reuseOrNewWithCallOptions(defaultOptions, optFuncs)
	return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
		grpcOpts, retryOpts := filterCallOptions(opts)
		callOpts := reuseOrNewWithCallOptions(intOpts, retryOpts)
		// short circuit for simplicity, and avoiding allocations.
		if callOpts.max == 0 {
			return invoker(ctx, method, req, reply, cc, grpcOpts...)
		}
		var lastErr error
		for attempt := uint(0); attempt < callOpts.max; attempt++ {
			if err := waitRetryBackoff(ctx, attempt, callOpts); err != nil {
				return err
			}
			logger.Debug(
				"retrying of unary invoker",
				zap.String("target", cc.Target()),
				zap.Uint("attempt", attempt),
			)
			lastErr = invoker(ctx, method, req, reply, cc, grpcOpts...)
			if lastErr == nil {
				return nil
			}
			logger.Warn(
				"retrying of unary invoker failed",
				zap.String("target", cc.Target()),
				zap.Uint("attempt", attempt),
				zap.Error(lastErr),
			)
			if isContextError(lastErr) {
				if ctx.Err() != nil {
					// its the context deadline or cancellation.
					return lastErr
				}
				// its the callCtx deadline or cancellation, in which case try again.
				continue
			}
			if callOpts.retryAuth && rpctypes.Error(lastErr) == rpctypes.ErrInvalidAuthToken {
				gterr := c.getToken(ctx)
				if gterr != nil {
					logger.Warn(
						"retrying of unary invoker failed to fetch new auth token",
						zap.String("target", cc.Target()),
						zap.Error(gterr),
					)
					return gterr // lastErr must be invalid auth token
				}
				continue
			}
			if !isSafeRetry(c.lg, lastErr, callOpts) {
				return lastErr
			}
		}
		return lastErr
	}
}

// streamClientInterceptor returns a new retrying stream client interceptor for server side streaming calls.
//
// The default configuration of the interceptor is to not retry *at all*. This behaviour can be
// changed through options (e.g. WithMax) on creation of the interceptor or on call (through grpc.CallOptions).
//
// Retry logic is available *only for ServerStreams*, i.e. 1:n streams, as the internal logic needs
// to buffer the messages sent by the client. If retry is enabled on any other streams (ClientStreams,
// BidiStreams), the retry interceptor will fail the call.
func (c *Client) streamClientInterceptor(logger *zap.Logger, optFuncs ...retryOption) grpc.StreamClientInterceptor {
	intOpts := reuseOrNewWithCallOptions(defaultOptions, optFuncs)
	return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
		grpcOpts, retryOpts := filterCallOptions(opts)
		callOpts := reuseOrNewWithCallOptions(intOpts, retryOpts)
		// short circuit for simplicity, and avoiding allocations.
		if callOpts.max == 0 {
			return streamer(ctx, desc, cc, method, grpcOpts...)
		}
		if desc.ClientStreams {
			return nil, status.Errorf(codes.Unimplemented, "clientv3/retry_interceptor: cannot retry on ClientStreams, set Disable()")
		}
		newStreamer, err := streamer(ctx, desc, cc, method, grpcOpts...)
		logger.Warn("retry stream intercept", zap.Error(err))
		if err != nil {
			// TODO(mwitkow): Maybe dial and transport errors should be retriable?
			return nil, err
		}
		retryingStreamer := &serverStreamingRetryingStream{
			client:       c,
			ClientStream: newStreamer,
			callOpts:     callOpts,
			ctx:          ctx,
			streamerCall: func(ctx context.Context) (grpc.ClientStream, error) {
				return streamer(ctx, desc, cc, method, grpcOpts...)
			},
		}
		return retryingStreamer, nil
	}
}

// type serverStreamingRetryingStream is the implementation of grpc.ClientStream that acts as a
// proxy to the underlying call. If any of the RecvMsg() calls fail, it will try to reestablish
// a new ClientStream according to the retry policy.
type serverStreamingRetryingStream struct {
	grpc.ClientStream
	client        *Client
	bufferedSends []interface{} // single message that the client can sen
	receivedGood  bool          // indicates whether any prior receives were successful
	wasClosedSend bool          // indicates that CloseSend was closed
	ctx           context.Context
	callOpts      *options
	streamerCall  func(ctx context.Context) (grpc.ClientStream, error)
	mu            sync.RWMutex
}

func (s *serverStreamingRetryingStream) setStream(clientStream grpc.ClientStream) {
	s.mu.Lock()
	s.ClientStream = clientStream
	s.mu.Unlock()
}

func (s *serverStreamingRetryingStream) getStream() grpc.ClientStream {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.ClientStream
}

func (s *serverStreamingRetryingStream) SendMsg(m interface{}) error {
	s.mu.Lock()
	s.bufferedSends = append(s.bufferedSends, m)
	s.mu.Unlock()
	return s.getStream().SendMsg(m)
}

func (s *serverStreamingRetryingStream) CloseSend() error {
	s.mu.Lock()
	s.wasClosedSend = true
	s.mu.Unlock()
	return s.getStream().CloseSend()
}

func (s *serverStreamingRetryingStream) Header() (metadata.MD, error) {
	return s.getStream().Header()
}

func (s *serverStreamingRetryingStream) Trailer() metadata.MD {
	return s.getStream().Trailer()
}

func (s *serverStreamingRetryingStream) RecvMsg(m interface{}) error {
	attemptRetry, lastErr := s.receiveMsgAndIndicateRetry(m)
	if !attemptRetry {
		return lastErr // success or hard failure
	}
	// We start off from attempt 1, because zeroth was already made on normal SendMsg().
	for attempt := uint(1); attempt < s.callOpts.max; attempt++ {
		if err := waitRetryBackoff(s.ctx, attempt, s.callOpts); err != nil {
			return err
		}
		newStream, err := s.reestablishStreamAndResendBuffer(s.ctx)
		if err != nil {
			// TODO(mwitkow): Maybe dial and transport errors should be retriable?
			return err
		}
		s.setStream(newStream)
		attemptRetry, lastErr = s.receiveMsgAndIndicateRetry(m)
		//fmt.Printf("Received message and indicate: %v  %v\n", attemptRetry, lastErr)
		if !attemptRetry {
			return lastErr
		}
	}
	return lastErr
}

func (s *serverStreamingRetryingStream) receiveMsgAndIndicateRetry(m interface{}) (bool, error) {
	s.mu.RLock()
	wasGood := s.receivedGood
	s.mu.RUnlock()
	err := s.getStream().RecvMsg(m)
	if err == nil || err == io.EOF {
		s.mu.Lock()
		s.receivedGood = true
		s.mu.Unlock()
		return false, err
	} else if wasGood {
		// previous RecvMsg in the stream succeeded, no retry logic should interfere
		return false, err
	}
	if isContextError(err) {
		if s.ctx.Err() != nil {
			return false, err
		}
		// its the callCtx deadline or cancellation, in which case try again.
		return true, err
	}
	if s.callOpts.retryAuth && rpctypes.Error(err) == rpctypes.ErrInvalidAuthToken {
		gterr := s.client.getToken(s.ctx)
		if gterr != nil {
			s.client.lg.Warn("retry failed to fetch new auth token", zap.Error(gterr))
			return false, err // return the original error for simplicity
		}
		return true, err

	}
	return isSafeRetry(s.client.lg, err, s.callOpts), err
}

func (s *serverStreamingRetryingStream) reestablishStreamAndResendBuffer(callCtx context.Context) (grpc.ClientStream, error) {
	s.mu.RLock()
	bufferedSends := s.bufferedSends
	s.mu.RUnlock()
	newStream, err := s.streamerCall(callCtx)
	if err != nil {
		return nil, err
	}
	for _, msg := range bufferedSends {
		if err := newStream.SendMsg(msg); err != nil {
			return nil, err
		}
	}
	if err := newStream.CloseSend(); err != nil {
		return nil, err
	}
	return newStream, nil
}

func waitRetryBackoff(ctx context.Context, attempt uint, callOpts *options) error {
	waitTime := time.Duration(0)
	if attempt > 0 {
		waitTime = callOpts.backoffFunc(attempt)
	}
	if waitTime > 0 {
		timer := time.NewTimer(waitTime)
		select {
		case <-ctx.Done():
			timer.Stop()
			return contextErrToGrpcErr(ctx.Err())
		case <-timer.C:
		}
	}
	return nil
}

// isSafeRetry returns "true", if request is safe for retry with the given error.
func isSafeRetry(lg *zap.Logger, err error, callOpts *options) bool {
	if isContextError(err) {
		return false
	}
	switch callOpts.retryPolicy {
	case repeatable:
		return isSafeRetryImmutableRPC(err)
	case nonRepeatable:
		return isSafeRetryMutableRPC(err)
	default:
		lg.Warn("unrecognized retry policy", zap.String("retryPolicy", callOpts.retryPolicy.String()))
		return false
	}
}

func isContextError(err error) bool {
	return grpc.Code(err) == codes.DeadlineExceeded || grpc.Code(err) == codes.Canceled
}

func contextErrToGrpcErr(err error) error {
	switch err {
	case context.DeadlineExceeded:
		return status.Errorf(codes.DeadlineExceeded, err.Error())
	case context.Canceled:
		return status.Errorf(codes.Canceled, err.Error())
	default:
		return status.Errorf(codes.Unknown, err.Error())
	}
}

var (
	defaultOptions = &options{
		retryPolicy: nonRepeatable,
		max:         0, // disable
		backoffFunc: backoffLinearWithJitter(50*time.Millisecond /*jitter*/, 0.10),
		retryAuth:   true,
	}
)

// backoffFunc denotes a family of functions that control the backoff duration between call retries.
//
// They are called with an identifier of the attempt, and should return a time the system client should
// hold off for. If the time returned is longer than the `context.Context.Deadline` of the request
// the deadline of the request takes precedence and the wait will be interrupted before proceeding
// with the next iteration.
type backoffFunc func(attempt uint) time.Duration

// withRetryPolicy sets the retry policy of this call.
func withRetryPolicy(rp retryPolicy) retryOption {
	return retryOption{applyFunc: func(o *options) {
		o.retryPolicy = rp
	}}
}

// withMax sets the maximum number of retries on this call, or this interceptor.
func withMax(maxRetries uint) retryOption {
	return retryOption{applyFunc: func(o *options) {
		o.max = maxRetries
	}}
}

// WithBackoff sets the `BackoffFunc `used to control time between retries.
func withBackoff(bf backoffFunc) retryOption {
	return retryOption{applyFunc: func(o *options) {
		o.backoffFunc = bf
	}}
}

type options struct {
	retryPolicy retryPolicy
	max         uint
	backoffFunc backoffFunc
	retryAuth   bool
}

// retryOption is a grpc.CallOption that is local to clientv3's retry interceptor.
type retryOption struct {
	grpc.EmptyCallOption // make sure we implement private after() and before() fields so we don't panic.
	applyFunc            func(opt *options)
}

func reuseOrNewWithCallOptions(opt *options, retryOptions []retryOption) *options {
	if len(retryOptions) == 0 {
		return opt
	}
	optCopy := &options{}
	*optCopy = *opt
	for _, f := range retryOptions {
		f.applyFunc(optCopy)
	}
	return optCopy
}

func filterCallOptions(callOptions []grpc.CallOption) (grpcOptions []grpc.CallOption, retryOptions []retryOption) {
	for _, opt := range callOptions {
		if co, ok := opt.(retryOption); ok {
			retryOptions = append(retryOptions, co)
		} else {
			grpcOptions = append(grpcOptions, opt)
		}
	}
	return grpcOptions, retryOptions
}

// BackoffLinearWithJitter waits a set period of time, allowing for jitter (fractional adjustment).
//
// For example waitBetween=1s and jitter=0.10 can generate waits between 900ms and 1100ms.
func backoffLinearWithJitter(waitBetween time.Duration, jitterFraction float64) backoffFunc {
	return func(attempt uint) time.Duration {
		return jitterUp(waitBetween, jitterFraction)
	}
}
<commit_msg>clientv3: fix retry/streamer error message<commit_after>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Based on github.com/grpc-ecosystem/go-grpc-middleware/retry, but modified to support the more
// fine grained error checking required by write-at-most-once retry semantics of etcd.

package clientv3

import (
	"context"
	"io"
	"sync"
	"time"

	"go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes"
	"go.uber.org/zap"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/metadata"
	"google.golang.org/grpc/status"
)

// unaryClientInterceptor returns a new retrying unary client interceptor.
//
// The default configuration of the interceptor is to not retry *at all*. This behaviour can be
// changed through options (e.g. WithMax) on creation of the interceptor or on call (through grpc.CallOptions).
func (c *Client) unaryClientInterceptor(logger *zap.Logger, optFuncs ...retryOption) grpc.UnaryClientInterceptor {
	intOpts := reuseOrNewWithCallOptions(defaultOptions, optFuncs)
	return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
		grpcOpts, retryOpts := filterCallOptions(opts)
		callOpts := reuseOrNewWithCallOptions(intOpts, retryOpts)
		// short circuit for simplicity, and avoiding allocations.
		if callOpts.max == 0 {
			return invoker(ctx, method, req, reply, cc, grpcOpts...)
		}
		var lastErr error
		for attempt := uint(0); attempt < callOpts.max; attempt++ {
			if err := waitRetryBackoff(ctx, attempt, callOpts); err != nil {
				return err
			}
			logger.Debug(
				"retrying of unary invoker",
				zap.String("target", cc.Target()),
				zap.Uint("attempt", attempt),
			)
			lastErr = invoker(ctx, method, req, reply, cc, grpcOpts...)
			if lastErr == nil {
				return nil
			}
			logger.Warn(
				"retrying of unary invoker failed",
				zap.String("target", cc.Target()),
				zap.Uint("attempt", attempt),
				zap.Error(lastErr),
			)
			if isContextError(lastErr) {
				if ctx.Err() != nil {
					// its the context deadline or cancellation.
					return lastErr
				}
				// its the callCtx deadline or cancellation, in which case try again.
				continue
			}
			if callOpts.retryAuth && rpctypes.Error(lastErr) == rpctypes.ErrInvalidAuthToken {
				gterr := c.getToken(ctx)
				if gterr != nil {
					logger.Warn(
						"retrying of unary invoker failed to fetch new auth token",
						zap.String("target", cc.Target()),
						zap.Error(gterr),
					)
					return gterr // lastErr must be invalid auth token
				}
				continue
			}
			if !isSafeRetry(c.lg, lastErr, callOpts) {
				return lastErr
			}
		}
		return lastErr
	}
}

// streamClientInterceptor returns a new retrying stream client interceptor for server side streaming calls.
//
// The default configuration of the interceptor is to not retry *at all*. This behaviour can be
// changed through options (e.g. WithMax) on creation of the interceptor or on call (through grpc.CallOptions).
//
// Retry logic is available *only for ServerStreams*, i.e. 1:n streams, as the internal logic needs
// to buffer the messages sent by the client. If retry is enabled on any other streams (ClientStreams,
// BidiStreams), the retry interceptor will fail the call.
func (c *Client) streamClientInterceptor(logger *zap.Logger, optFuncs ...retryOption) grpc.StreamClientInterceptor {
	intOpts := reuseOrNewWithCallOptions(defaultOptions, optFuncs)
	return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
		grpcOpts, retryOpts := filterCallOptions(opts)
		callOpts := reuseOrNewWithCallOptions(intOpts, retryOpts)
		// short circuit for simplicity, and avoiding allocations.
		if callOpts.max == 0 {
			return streamer(ctx, desc, cc, method, grpcOpts...)
		}
		if desc.ClientStreams {
			return nil, status.Errorf(codes.Unimplemented, "clientv3/retry_interceptor: cannot retry on ClientStreams, set Disable()")
		}
		newStreamer, err := streamer(ctx, desc, cc, method, grpcOpts...)
		if err != nil {
			logger.Error("streamer failed to create ClientStream", zap.Error(err))
			return nil, err // TODO(mwitkow): Maybe dial and transport errors should be retriable?
		}
		retryingStreamer := &serverStreamingRetryingStream{
			client:       c,
			ClientStream: newStreamer,
			callOpts:     callOpts,
			ctx:          ctx,
			streamerCall: func(ctx context.Context) (grpc.ClientStream, error) {
				return streamer(ctx, desc, cc, method, grpcOpts...)
			},
		}
		return retryingStreamer, nil
	}
}

// type serverStreamingRetryingStream is the implementation of grpc.ClientStream that acts as a
// proxy to the underlying call. If any of the RecvMsg() calls fail, it will try to reestablish
// a new ClientStream according to the retry policy.
type serverStreamingRetryingStream struct {
	grpc.ClientStream
	client        *Client
	bufferedSends []interface{} // single message that the client can sen
	receivedGood  bool          // indicates whether any prior receives were successful
	wasClosedSend bool          // indicates that CloseSend was closed
	ctx           context.Context
	callOpts      *options
	streamerCall  func(ctx context.Context) (grpc.ClientStream, error)
	mu            sync.RWMutex
}

func (s *serverStreamingRetryingStream) setStream(clientStream grpc.ClientStream) {
	s.mu.Lock()
	s.ClientStream = clientStream
	s.mu.Unlock()
}

func (s *serverStreamingRetryingStream) getStream() grpc.ClientStream {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.ClientStream
}

func (s *serverStreamingRetryingStream) SendMsg(m interface{}) error {
	s.mu.Lock()
	s.bufferedSends = append(s.bufferedSends, m)
	s.mu.Unlock()
	return s.getStream().SendMsg(m)
}

func (s *serverStreamingRetryingStream) CloseSend() error {
	s.mu.Lock()
	s.wasClosedSend = true
	s.mu.Unlock()
	return s.getStream().CloseSend()
}

func (s *serverStreamingRetryingStream) Header() (metadata.MD, error) {
	return s.getStream().Header()
}

func (s *serverStreamingRetryingStream) Trailer() metadata.MD {
	return s.getStream().Trailer()
}

func (s *serverStreamingRetryingStream) RecvMsg(m interface{}) error {
	attemptRetry, lastErr := s.receiveMsgAndIndicateRetry(m)
	if !attemptRetry {
		return lastErr // success or hard failure
	}

	// We start off from attempt 1, because zeroth was already made on normal SendMsg().
	for attempt := uint(1); attempt < s.callOpts.max; attempt++ {
		if err := waitRetryBackoff(s.ctx, attempt, s.callOpts); err != nil {
			return err
		}
		newStream, err := s.reestablishStreamAndResendBuffer(s.ctx)
		if err != nil {
			s.client.lg.Error("failed reestablishStreamAndResendBuffer", zap.Error(err))
			return err // TODO(mwitkow): Maybe dial and transport errors should be retriable?
		}
		s.setStream(newStream)

		s.client.lg.Warn("retrying RecvMsg", zap.Error(lastErr))
		attemptRetry, lastErr = s.receiveMsgAndIndicateRetry(m)
		if !attemptRetry {
			return lastErr
		}
	}
	return lastErr
}

func (s *serverStreamingRetryingStream) receiveMsgAndIndicateRetry(m interface{}) (bool, error) {
	s.mu.RLock()
	wasGood := s.receivedGood
	s.mu.RUnlock()
	err := s.getStream().RecvMsg(m)
	if err == nil || err == io.EOF {
		s.mu.Lock()
		s.receivedGood = true
		s.mu.Unlock()
		return false, err
	} else if wasGood {
		// previous RecvMsg in the stream succeeded, no retry logic should interfere
		return false, err
	}
	if isContextError(err) {
		if s.ctx.Err() != nil {
			return false, err
		}
		// its the callCtx deadline or cancellation, in which case try again.
		return true, err
	}
	if s.callOpts.retryAuth && rpctypes.Error(err) == rpctypes.ErrInvalidAuthToken {
		gterr := s.client.getToken(s.ctx)
		if gterr != nil {
			s.client.lg.Warn("retry failed to fetch new auth token", zap.Error(gterr))
			return false, err // return the original error for simplicity
		}
		return true, err

	}
	return isSafeRetry(s.client.lg, err, s.callOpts), err
}

func (s *serverStreamingRetryingStream) reestablishStreamAndResendBuffer(callCtx context.Context) (grpc.ClientStream, error) {
	s.mu.RLock()
	bufferedSends := s.bufferedSends
	s.mu.RUnlock()
	newStream, err := s.streamerCall(callCtx)
	if err != nil {
		return nil, err
	}
	for _, msg := range bufferedSends {
		if err := newStream.SendMsg(msg); err != nil {
			return nil, err
		}
	}
	if err := newStream.CloseSend(); err != nil {
		return nil, err
	}
	return newStream, nil
}

func waitRetryBackoff(ctx context.Context, attempt uint, callOpts *options) error {
	waitTime := time.Duration(0)
	if attempt > 0 {
		waitTime = callOpts.backoffFunc(attempt)
	}
	if waitTime > 0 {
		timer := time.NewTimer(waitTime)
		select {
		case <-ctx.Done():
			timer.Stop()
			return contextErrToGrpcErr(ctx.Err())
		case <-timer.C:
		}
	}
	return nil
}

// isSafeRetry returns "true", if request is safe for retry with the given error.
func isSafeRetry(lg *zap.Logger, err error, callOpts *options) bool {
	if isContextError(err) {
		return false
	}
	switch callOpts.retryPolicy {
	case repeatable:
		return isSafeRetryImmutableRPC(err)
	case nonRepeatable:
		return isSafeRetryMutableRPC(err)
	default:
		lg.Warn("unrecognized retry policy", zap.String("retryPolicy", callOpts.retryPolicy.String()))
		return false
	}
}

func isContextError(err error) bool {
	return grpc.Code(err) == codes.DeadlineExceeded || grpc.Code(err) == codes.Canceled
}

func contextErrToGrpcErr(err error) error {
	switch err {
	case context.DeadlineExceeded:
		return status.Errorf(codes.DeadlineExceeded, err.Error())
	case context.Canceled:
		return status.Errorf(codes.Canceled, err.Error())
	default:
		return status.Errorf(codes.Unknown, err.Error())
	}
}

var (
	defaultOptions = &options{
		retryPolicy: nonRepeatable,
		max:         0, // disable
		backoffFunc: backoffLinearWithJitter(50*time.Millisecond /*jitter*/, 0.10),
		retryAuth:   true,
	}
)

// backoffFunc denotes a family of functions that control the backoff duration between call retries.
//
// They are called with an identifier of the attempt, and should return a time the system client should
// hold off for. If the time returned is longer than the `context.Context.Deadline` of the request
// the deadline of the request takes precedence and the wait will be interrupted before proceeding
// with the next iteration.
type backoffFunc func(attempt uint) time.Duration

// withRetryPolicy sets the retry policy of this call.
func withRetryPolicy(rp retryPolicy) retryOption {
	return retryOption{applyFunc: func(o *options) {
		o.retryPolicy = rp
	}}
}

// withMax sets the maximum number of retries on this call, or this interceptor.
func withMax(maxRetries uint) retryOption {
	return retryOption{applyFunc: func(o *options) {
		o.max = maxRetries
	}}
}

// WithBackoff sets the `BackoffFunc `used to control time between retries.
func withBackoff(bf backoffFunc) retryOption {
	return retryOption{applyFunc: func(o *options) {
		o.backoffFunc = bf
	}}
}

type options struct {
	retryPolicy retryPolicy
	max         uint
	backoffFunc backoffFunc
	retryAuth   bool
}

// retryOption is a grpc.CallOption that is local to clientv3's retry interceptor.
type retryOption struct {
	grpc.EmptyCallOption // make sure we implement private after() and before() fields so we don't panic.
	applyFunc            func(opt *options)
}

func reuseOrNewWithCallOptions(opt *options, retryOptions []retryOption) *options {
	if len(retryOptions) == 0 {
		return opt
	}
	optCopy := &options{}
	*optCopy = *opt
	for _, f := range retryOptions {
		f.applyFunc(optCopy)
	}
	return optCopy
}

func filterCallOptions(callOptions []grpc.CallOption) (grpcOptions []grpc.CallOption, retryOptions []retryOption) {
	for _, opt := range callOptions {
		if co, ok := opt.(retryOption); ok {
			retryOptions = append(retryOptions, co)
		} else {
			grpcOptions = append(grpcOptions, opt)
		}
	}
	return grpcOptions, retryOptions
}

// BackoffLinearWithJitter waits a set period of time, allowing for jitter (fractional adjustment).
//
// For example waitBetween=1s and jitter=0.10 can generate waits between 900ms and 1100ms.
func backoffLinearWithJitter(waitBetween time.Duration, jitterFraction float64) backoffFunc {
	return func(attempt uint) time.Duration {
		return jitterUp(waitBetween, jitterFraction)
	}
}
<|endoftext|>


<commit_before>package scan

import (
	"fmt"
	"go/ast"
	"reflect"
	"strconv"
	"strings"

	"golang.org/x/tools/go/loader"

	"github.com/go-swagger/go-swagger/spec"
)

type responseTypable struct {
	in       string
	header   *spec.Header
	response *spec.Response
}

func (ht responseTypable) Typed(tpe, format string) {
	ht.header.Typed(tpe, format)
}

func (ht responseTypable) Items() swaggerTypable {
	if ht.in == "body" {
		// get the schema for items on the schema property
		if ht.response.Schema == nil {
			ht.response.Schema = new(spec.Schema)
		}
		if ht.response.Schema.Items == nil {
			ht.response.Schema.Items = new(spec.SchemaOrArray)
		}
		if ht.response.Schema.Items.Schema == nil {
			ht.response.Schema.Items.Schema = new(spec.Schema)
		}
		ht.response.Schema.Typed("array", "")
		return schemaTypable{ht.response.Schema.Items.Schema}
	}

	if ht.header.Items == nil {
		ht.header.Items = new(spec.Items)
	}
	ht.header.Type = "array"
	return itemsTypable{ht.header.Items}
}

func (ht responseTypable) SetRef(ref spec.Ref) {
	// having trouble seeing the usefulness of this one here
}

func (ht responseTypable) Schema() *spec.Schema {
	if ht.response.Schema == nil {
		ht.response.Schema = new(spec.Schema)
	}
	return ht.response.Schema
}

func (ht responseTypable) SetSchema(schema *spec.Schema) {
	ht.response.Schema = schema
}
func (ht responseTypable) CollectionOf(items *spec.Items, format string) {
	ht.header.CollectionOf(items, format)
}

type headerValidations struct {
	current *spec.Header
}

func (sv headerValidations) SetMaximum(val float64, exclusive bool) {
	sv.current.Maximum = &val
	sv.current.ExclusiveMaximum = exclusive
}
func (sv headerValidations) SetMinimum(val float64, exclusive bool) {
	sv.current.Minimum = &val
	sv.current.ExclusiveMinimum = exclusive
}
func (sv headerValidations) SetMultipleOf(val float64)      { sv.current.MultipleOf = &val }
func (sv headerValidations) SetMinItems(val int64)          { sv.current.MinItems = &val }
func (sv headerValidations) SetMaxItems(val int64)          { sv.current.MaxItems = &val }
func (sv headerValidations) SetMinLength(val int64)         { sv.current.MinLength = &val }
func (sv headerValidations) SetMaxLength(val int64)         { sv.current.MaxLength = &val }
func (sv headerValidations) SetPattern(val string)          { sv.current.Pattern = val }
func (sv headerValidations) SetUnique(val bool)             { sv.current.UniqueItems = val }
func (sv headerValidations) SetCollectionFormat(val string) { sv.current.CollectionFormat = val }

func newResponseDecl(file *ast.File, decl *ast.GenDecl, ts *ast.TypeSpec) responseDecl {
	var rd responseDecl
	rd.File = file
	rd.Decl = decl
	rd.TypeSpec = ts
	rd.inferNames()
	return rd
}

type responseDecl struct {
	File      *ast.File
	Decl      *ast.GenDecl
	TypeSpec  *ast.TypeSpec
	GoName    string
	Name      string
	annotated bool
}

func (sd *responseDecl) hasAnnotation() bool {
	sd.inferNames()
	return sd.annotated
}

func (sd *responseDecl) inferNames() (goName string, name string) {
	if sd.GoName != "" {
		goName, name = sd.GoName, sd.Name
		return
	}
	goName = sd.TypeSpec.Name.Name
	name = goName
	if sd.Decl.Doc != nil {
	DECLS:
		for _, cmt := range sd.Decl.Doc.List {
			for _, ln := range strings.Split(cmt.Text, "\n") {
				matches := rxResponseOverride.FindStringSubmatch(ln)
				if len(matches) > 0 {
					sd.annotated = true
				}
				if len(matches) > 1 && len(matches[1]) > 0 {
					name = matches[1]
					break DECLS
				}
			}
		}
	}
	sd.GoName = goName
	sd.Name = name
	return
}

func newResponseParser(prog *loader.Program) *responseParser {
	return &responseParser{prog, nil, newSchemaParser(prog)}
}

type responseParser struct {
	program   *loader.Program
	postDecls []schemaDecl
	scp       *schemaParser
}

func (rp *responseParser) Parse(gofile *ast.File, target interface{}) error {
	tgt := target.(map[string]spec.Response)
	for _, decl := range gofile.Decls {
		gd, ok := decl.(*ast.GenDecl)
		if !ok {
			continue
		}
		for _, spc := range gd.Specs {
			if ts, ok := spc.(*ast.TypeSpec); ok {
				sd := newResponseDecl(gofile, gd, ts)
				if sd.hasAnnotation() {
					if err := rp.parseDecl(tgt, sd); err != nil {
						return err
					}
				}
			}
		}
	}
	return nil
}

func (rp *responseParser) parseDecl(responses map[string]spec.Response, decl responseDecl) error {
	// check if there is a swagger:parameters tag that is followed by one or more words,
	// these words are the ids of the operations this parameter struct applies to
	// once type name is found convert it to a schema, by looking up the schema in the
	// parameters dictionary that got passed into this parse method
	response := responses[decl.Name]
	resPtr := &response

	// analyze struct body for fields etc
	// each exported struct field:
	// * gets a type mapped to a go primitive
	// * perhaps gets a format
	// * has to document the validations that apply for the type and the field
	// * when the struct field points to a model it becomes a ref: #/definitions/ModelName
	// * comments that aren't tags is used as the description
	if tpe, ok := decl.TypeSpec.Type.(*ast.StructType); ok {
		if err := rp.parseStructType(decl.File, resPtr, tpe, make(map[string]struct{})); err != nil {
			return err
		}
	}

	responses[decl.Name] = response
	return nil
}

func (rp *responseParser) parseEmbeddedStruct(gofile *ast.File, response *spec.Response, expr ast.Expr, seenPreviously map[string]struct{}) error {
	switch tpe := expr.(type) {
	case *ast.Ident:
		// do lookup of type
		// take primitives into account, they should result in an error for swagger
		pkg, err := rp.scp.packageForFile(gofile)
		if err != nil {
			return fmt.Errorf("embedded struct: %v", err)
		}
		file, _, ts, err := findSourceFile(pkg, tpe.Name)
		if err != nil {
			return fmt.Errorf("embedded struct: %v", err)
		}
		if st, ok := ts.Type.(*ast.StructType); ok {
			return rp.parseStructType(file, response, st, seenPreviously)
		}
	case *ast.SelectorExpr:
		// look up package, file and then type
		pkg, err := rp.scp.packageForSelector(gofile, tpe.X)
		if err != nil {
			return fmt.Errorf("embedded struct: %v", err)
		}
		file, _, ts, err := findSourceFile(pkg, tpe.Sel.Name)
		if err != nil {
			return fmt.Errorf("embedded struct: %v", err)
		}
		if st, ok := ts.Type.(*ast.StructType); ok {
			return rp.parseStructType(file, response, st, seenPreviously)
		}
	}
	return fmt.Errorf("unable to resolve embedded struct for: %v\n", expr)
}

func (rp *responseParser) parseStructType(gofile *ast.File, response *spec.Response, tpe *ast.StructType, seenPreviously map[string]struct{}) error {
	if tpe.Fields != nil {

		seenProperties := seenPreviously

		for _, fld := range tpe.Fields.List {
			if len(fld.Names) == 0 {
				// when the embedded struct is annotated with swagger:allOf it will be used as allOf property
				// otherwise the fields will just be included as normal properties
				if err := rp.parseEmbeddedStruct(gofile, response, fld.Type, seenProperties); err != nil {
					return err
				}
			}
		}

		for _, fld := range tpe.Fields.List {
			var nm string
			if len(fld.Names) > 0 && fld.Names[0] != nil && fld.Names[0].IsExported() {
				nm = fld.Names[0].Name
				if fld.Tag != nil && len(strings.TrimSpace(fld.Tag.Value)) > 0 {
					tv, err := strconv.Unquote(fld.Tag.Value)
					if err != nil {
						return err
					}

					if strings.TrimSpace(tv) != "" {
						st := reflect.StructTag(tv)
						if st.Get("json") != "" {
							nm = strings.Split(st.Get("json"), ",")[0]
						}
					}
				}

				var in string
				// scan for param location first, this changes some behavior down the line
				if fld.Doc != nil {
					for _, cmt := range fld.Doc.List {
						for _, line := range strings.Split(cmt.Text, "\n") {
							matches := rxIn.FindStringSubmatch(line)
							if len(matches) > 0 && len(strings.TrimSpace(matches[1])) > 0 {
								in = strings.TrimSpace(matches[1])
							}
						}
					}
				}

				ps := response.Headers[nm]
				if err := parseProperty(rp.scp, gofile, fld.Type, responseTypable{in, &ps, response}); err != nil {
					return err
				}

				sp := new(sectionedParser)
				sp.setDescription = func(lines []string) { ps.Description = joinDropLast(lines) }
				sp.taggers = []tagParser{
					newSingleLineTagParser("maximum", &setMaximum{headerValidations{&ps}, rxf(rxMaximumFmt, "")}),
					newSingleLineTagParser("minimum", &setMinimum{headerValidations{&ps}, rxf(rxMinimumFmt, "")}),
					newSingleLineTagParser("multipleOf", &setMultipleOf{headerValidations{&ps}, rxf(rxMultipleOfFmt, "")}),
					newSingleLineTagParser("minLength", &setMinLength{headerValidations{&ps}, rxf(rxMinLengthFmt, "")}),
					newSingleLineTagParser("maxLength", &setMaxLength{headerValidations{&ps}, rxf(rxMaxLengthFmt, "")}),
					newSingleLineTagParser("pattern", &setPattern{headerValidations{&ps}, rxf(rxPatternFmt, "")}),
					newSingleLineTagParser("collectionFormat", &setCollectionFormat{headerValidations{&ps}, rxf(rxCollectionFormatFmt, "")}),
					newSingleLineTagParser("minItems", &setMinItems{headerValidations{&ps}, rxf(rxMinItemsFmt, "")}),
					newSingleLineTagParser("maxItems", &setMaxItems{headerValidations{&ps}, rxf(rxMaxItemsFmt, "")}),
					newSingleLineTagParser("unique", &setUnique{headerValidations{&ps}, rxf(rxUniqueFmt, "")}),
				}
				itemsTaggers := func() []tagParser {
					return []tagParser{
						newSingleLineTagParser("itemsMaximum", &setMaximum{itemsValidations{ps.Items}, rxf(rxMaximumFmt, rxItemsPrefix)}),
						newSingleLineTagParser("itemsMinimum", &setMinimum{itemsValidations{ps.Items}, rxf(rxMinimumFmt, rxItemsPrefix)}),
						newSingleLineTagParser("itemsMultipleOf", &setMultipleOf{itemsValidations{ps.Items}, rxf(rxMultipleOfFmt, rxItemsPrefix)}),
						newSingleLineTagParser("itemsMinLength", &setMinLength{itemsValidations{ps.Items}, rxf(rxMinLengthFmt, rxItemsPrefix)}),
						newSingleLineTagParser("itemsMaxLength", &setMaxLength{itemsValidations{ps.Items}, rxf(rxMaxLengthFmt, rxItemsPrefix)}),
						newSingleLineTagParser("itemsPattern", &setPattern{itemsValidations{ps.Items}, rxf(rxPatternFmt, rxItemsPrefix)}),
						newSingleLineTagParser("itemsCollectionFormat", &setCollectionFormat{itemsValidations{ps.Items}, rxf(rxCollectionFormatFmt, rxItemsPrefix)}),
						newSingleLineTagParser("itemsMinItems", &setMinItems{itemsValidations{ps.Items}, rxf(rxMinItemsFmt, rxItemsPrefix)}),
						newSingleLineTagParser("itemsMaxItems", &setMaxItems{itemsValidations{ps.Items}, rxf(rxMaxItemsFmt, rxItemsPrefix)}),
						newSingleLineTagParser("itemsUnique", &setUnique{itemsValidations{ps.Items}, rxf(rxUniqueFmt, rxItemsPrefix)}),
					}
				}

				// check if this is a primitive, if so parse the validations from the
				// doc comments of the slice declaration.
				if ftpe, ok := fld.Type.(*ast.ArrayType); ok {
					if iftpe, ok := ftpe.Elt.(*ast.Ident); ok && iftpe.Obj == nil {
						if ps.Items != nil {
							// items matchers should go before the default matchers so they match first
							sp.taggers = append(itemsTaggers(), sp.taggers...)
						}
					}
				}

				if err := sp.Parse(fld.Doc); err != nil {
					return err
				}

				if in != "body" {
					seenProperties[nm] = struct{}{}
					if response.Headers == nil {
						response.Headers = make(map[string]spec.Header)
					}
					response.Headers[nm] = ps
				}
			}
		}

		for k := range response.Headers {
			if _, ok := seenProperties[k]; !ok {
				delete(response.Headers, k)
			}
		}
	}

	return nil
}
<commit_msg>Fix setting a description for a response model<commit_after>package scan

import (
	"fmt"
	"go/ast"
	"reflect"
	"strconv"
	"strings"

	"golang.org/x/tools/go/loader"

	"github.com/go-swagger/go-swagger/spec"
)

type responseTypable struct {
	in       string
	header   *spec.Header
	response *spec.Response
}

func (ht responseTypable) Typed(tpe, format string) {
	ht.header.Typed(tpe, format)
}

func (ht responseTypable) Items() swaggerTypable {
	if ht.in == "body" {
		// get the schema for items on the schema property
		if ht.response.Schema == nil {
			ht.response.Schema = new(spec.Schema)
		}
		if ht.response.Schema.Items == nil {
			ht.response.Schema.Items = new(spec.SchemaOrArray)
		}
		if ht.response.Schema.Items.Schema == nil {
			ht.response.Schema.Items.Schema = new(spec.Schema)
		}
		ht.response.Schema.Typed("array", "")
		return schemaTypable{ht.response.Schema.Items.Schema}
	}

	if ht.header.Items == nil {
		ht.header.Items = new(spec.Items)
	}
	ht.header.Type = "array"
	return itemsTypable{ht.header.Items}
}

func (ht responseTypable) SetRef(ref spec.Ref) {
	// having trouble seeing the usefulness of this one here
}

func (ht responseTypable) Schema() *spec.Schema {
	if ht.response.Schema == nil {
		ht.response.Schema = new(spec.Schema)
	}
	return ht.response.Schema
}

func (ht responseTypable) SetSchema(schema *spec.Schema) {
	ht.response.Schema = schema
}
func (ht responseTypable) CollectionOf(items *spec.Items, format string) {
	ht.header.CollectionOf(items, format)
}

type headerValidations struct {
	current *spec.Header
}

func (sv headerValidations) SetMaximum(val float64, exclusive bool) {
	sv.current.Maximum = &val
	sv.current.ExclusiveMaximum = exclusive
}
func (sv headerValidations) SetMinimum(val float64, exclusive bool) {
	sv.current.Minimum = &val
	sv.current.ExclusiveMinimum = exclusive
}
func (sv headerValidations) SetMultipleOf(val float64)      { sv.current.MultipleOf = &val }
func (sv headerValidations) SetMinItems(val int64)          { sv.current.MinItems = &val }
func (sv headerValidations) SetMaxItems(val int64)          { sv.current.MaxItems = &val }
func (sv headerValidations) SetMinLength(val int64)         { sv.current.MinLength = &val }
func (sv headerValidations) SetMaxLength(val int64)         { sv.current.MaxLength = &val }
func (sv headerValidations) SetPattern(val string)          { sv.current.Pattern = val }
func (sv headerValidations) SetUnique(val bool)             { sv.current.UniqueItems = val }
func (sv headerValidations) SetCollectionFormat(val string) { sv.current.CollectionFormat = val }

func newResponseDecl(file *ast.File, decl *ast.GenDecl, ts *ast.TypeSpec) responseDecl {
	var rd responseDecl
	rd.File = file
	rd.Decl = decl
	rd.TypeSpec = ts
	rd.inferNames()
	return rd
}

type responseDecl struct {
	File      *ast.File
	Decl      *ast.GenDecl
	TypeSpec  *ast.TypeSpec
	GoName    string
	Name      string
	annotated bool
}

func (sd *responseDecl) hasAnnotation() bool {
	sd.inferNames()
	return sd.annotated
}

func (sd *responseDecl) inferNames() (goName string, name string) {
	if sd.GoName != "" {
		goName, name = sd.GoName, sd.Name
		return
	}
	goName = sd.TypeSpec.Name.Name
	name = goName
	if sd.Decl.Doc != nil {
	DECLS:
		for _, cmt := range sd.Decl.Doc.List {
			for _, ln := range strings.Split(cmt.Text, "\n") {
				matches := rxResponseOverride.FindStringSubmatch(ln)
				if len(matches) > 0 {
					sd.annotated = true
				}
				if len(matches) > 1 && len(matches[1]) > 0 {
					name = matches[1]
					break DECLS
				}
			}
		}
	}
	sd.GoName = goName
	sd.Name = name
	return
}

func newResponseParser(prog *loader.Program) *responseParser {
	return &responseParser{prog, nil, newSchemaParser(prog)}
}

type responseParser struct {
	program   *loader.Program
	postDecls []schemaDecl
	scp       *schemaParser
}

func (rp *responseParser) Parse(gofile *ast.File, target interface{}) error {
	tgt := target.(map[string]spec.Response)
	for _, decl := range gofile.Decls {
		gd, ok := decl.(*ast.GenDecl)
		if !ok {
			continue
		}
		for _, spc := range gd.Specs {
			if ts, ok := spc.(*ast.TypeSpec); ok {
				sd := newResponseDecl(gofile, gd, ts)
				if sd.hasAnnotation() {
					if err := rp.parseDecl(tgt, sd); err != nil {
						return err
					}
				}
			}
		}
	}
	return nil
}

func (rp *responseParser) parseDecl(responses map[string]spec.Response, decl responseDecl) error {
	// check if there is a swagger:parameters tag that is followed by one or more words,
	// these words are the ids of the operations this parameter struct applies to
	// once type name is found convert it to a schema, by looking up the schema in the
	// parameters dictionary that got passed into this parse method
	response := responses[decl.Name]
	resPtr := &response

	// analyze doc comment for the model
	sp := new(sectionedParser)
	sp.setDescription = func(lines []string) { resPtr.Description = joinDropLast(lines) }
	if err := sp.Parse(decl.Decl.Doc); err != nil {
		return err
	}

	// analyze struct body for fields etc
	// each exported struct field:
	// * gets a type mapped to a go primitive
	// * perhaps gets a format
	// * has to document the validations that apply for the type and the field
	// * when the struct field points to a model it becomes a ref: #/definitions/ModelName
	// * comments that aren't tags is used as the description
	if tpe, ok := decl.TypeSpec.Type.(*ast.StructType); ok {
		if err := rp.parseStructType(decl.File, resPtr, tpe, make(map[string]struct{})); err != nil {
			return err
		}
	}

	responses[decl.Name] = response
	return nil
}

func (rp *responseParser) parseEmbeddedStruct(gofile *ast.File, response *spec.Response, expr ast.Expr, seenPreviously map[string]struct{}) error {
	switch tpe := expr.(type) {
	case *ast.Ident:
		// do lookup of type
		// take primitives into account, they should result in an error for swagger
		pkg, err := rp.scp.packageForFile(gofile)
		if err != nil {
			return fmt.Errorf("embedded struct: %v", err)
		}
		file, _, ts, err := findSourceFile(pkg, tpe.Name)
		if err != nil {
			return fmt.Errorf("embedded struct: %v", err)
		}
		if st, ok := ts.Type.(*ast.StructType); ok {
			return rp.parseStructType(file, response, st, seenPreviously)
		}
	case *ast.SelectorExpr:
		// look up package, file and then type
		pkg, err := rp.scp.packageForSelector(gofile, tpe.X)
		if err != nil {
			return fmt.Errorf("embedded struct: %v", err)
		}
		file, _, ts, err := findSourceFile(pkg, tpe.Sel.Name)
		if err != nil {
			return fmt.Errorf("embedded struct: %v", err)
		}
		if st, ok := ts.Type.(*ast.StructType); ok {
			return rp.parseStructType(file, response, st, seenPreviously)
		}
	}
	return fmt.Errorf("unable to resolve embedded struct for: %v\n", expr)
}

func (rp *responseParser) parseStructType(gofile *ast.File, response *spec.Response, tpe *ast.StructType, seenPreviously map[string]struct{}) error {
	if tpe.Fields != nil {

		seenProperties := seenPreviously

		for _, fld := range tpe.Fields.List {
			if len(fld.Names) == 0 {
				// when the embedded struct is annotated with swagger:allOf it will be used as allOf property
				// otherwise the fields will just be included as normal properties
				if err := rp.parseEmbeddedStruct(gofile, response, fld.Type, seenProperties); err != nil {
					return err
				}
			}
		}

		for _, fld := range tpe.Fields.List {
			var nm string
			if len(fld.Names) > 0 && fld.Names[0] != nil && fld.Names[0].IsExported() {
				nm = fld.Names[0].Name
				if fld.Tag != nil && len(strings.TrimSpace(fld.Tag.Value)) > 0 {
					tv, err := strconv.Unquote(fld.Tag.Value)
					if err != nil {
						return err
					}

					if strings.TrimSpace(tv) != "" {
						st := reflect.StructTag(tv)
						if st.Get("json") != "" {
							nm = strings.Split(st.Get("json"), ",")[0]
						}
					}
				}

				var in string
				// scan for param location first, this changes some behavior down the line
				if fld.Doc != nil {
					for _, cmt := range fld.Doc.List {
						for _, line := range strings.Split(cmt.Text, "\n") {
							matches := rxIn.FindStringSubmatch(line)
							if len(matches) > 0 && len(strings.TrimSpace(matches[1])) > 0 {
								in = strings.TrimSpace(matches[1])
							}
						}
					}
				}

				ps := response.Headers[nm]
				if err := parseProperty(rp.scp, gofile, fld.Type, responseTypable{in, &ps, response}); err != nil {
					return err
				}

				sp := new(sectionedParser)
				sp.setDescription = func(lines []string) { ps.Description = joinDropLast(lines) }
				sp.taggers = []tagParser{
					newSingleLineTagParser("maximum", &setMaximum{headerValidations{&ps}, rxf(rxMaximumFmt, "")}),
					newSingleLineTagParser("minimum", &setMinimum{headerValidations{&ps}, rxf(rxMinimumFmt, "")}),
					newSingleLineTagParser("multipleOf", &setMultipleOf{headerValidations{&ps}, rxf(rxMultipleOfFmt, "")}),
					newSingleLineTagParser("minLength", &setMinLength{headerValidations{&ps}, rxf(rxMinLengthFmt, "")}),
					newSingleLineTagParser("maxLength", &setMaxLength{headerValidations{&ps}, rxf(rxMaxLengthFmt, "")}),
					newSingleLineTagParser("pattern", &setPattern{headerValidations{&ps}, rxf(rxPatternFmt, "")}),
					newSingleLineTagParser("collectionFormat", &setCollectionFormat{headerValidations{&ps}, rxf(rxCollectionFormatFmt, "")}),
					newSingleLineTagParser("minItems", &setMinItems{headerValidations{&ps}, rxf(rxMinItemsFmt, "")}),
					newSingleLineTagParser("maxItems", &setMaxItems{headerValidations{&ps}, rxf(rxMaxItemsFmt, "")}),
					newSingleLineTagParser("unique", &setUnique{headerValidations{&ps}, rxf(rxUniqueFmt, "")}),
				}
				itemsTaggers := func() []tagParser {
					return []tagParser{
						newSingleLineTagParser("itemsMaximum", &setMaximum{itemsValidations{ps.Items}, rxf(rxMaximumFmt, rxItemsPrefix)}),
						newSingleLineTagParser("itemsMinimum", &setMinimum{itemsValidations{ps.Items}, rxf(rxMinimumFmt, rxItemsPrefix)}),
						newSingleLineTagParser("itemsMultipleOf", &setMultipleOf{itemsValidations{ps.Items}, rxf(rxMultipleOfFmt, rxItemsPrefix)}),
						newSingleLineTagParser("itemsMinLength", &setMinLength{itemsValidations{ps.Items}, rxf(rxMinLengthFmt, rxItemsPrefix)}),
						newSingleLineTagParser("itemsMaxLength", &setMaxLength{itemsValidations{ps.Items}, rxf(rxMaxLengthFmt, rxItemsPrefix)}),
						newSingleLineTagParser("itemsPattern", &setPattern{itemsValidations{ps.Items}, rxf(rxPatternFmt, rxItemsPrefix)}),
						newSingleLineTagParser("itemsCollectionFormat", &setCollectionFormat{itemsValidations{ps.Items}, rxf(rxCollectionFormatFmt, rxItemsPrefix)}),
						newSingleLineTagParser("itemsMinItems", &setMinItems{itemsValidations{ps.Items}, rxf(rxMinItemsFmt, rxItemsPrefix)}),
						newSingleLineTagParser("itemsMaxItems", &setMaxItems{itemsValidations{ps.Items}, rxf(rxMaxItemsFmt, rxItemsPrefix)}),
						newSingleLineTagParser("itemsUnique", &setUnique{itemsValidations{ps.Items}, rxf(rxUniqueFmt, rxItemsPrefix)}),
					}
				}

				// check if this is a primitive, if so parse the validations from the
				// doc comments of the slice declaration.
				if ftpe, ok := fld.Type.(*ast.ArrayType); ok {
					if iftpe, ok := ftpe.Elt.(*ast.Ident); ok && iftpe.Obj == nil {
						if ps.Items != nil {
							// items matchers should go before the default matchers so they match first
							sp.taggers = append(itemsTaggers(), sp.taggers...)
						}
					}
				}

				if err := sp.Parse(fld.Doc); err != nil {
					return err
				}

				if in != "body" {
					seenProperties[nm] = struct{}{}
					if response.Headers == nil {
						response.Headers = make(map[string]spec.Header)
					}
					response.Headers[nm] = ps
				}
			}
		}

		for k := range response.Headers {
			if _, ok := seenProperties[k]; !ok {
				delete(response.Headers, k)
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"golang.org/x/net/context"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"

	// Google App Engine
	"google.golang.org/appengine/aetest"

	// Request routing
	"github.com/gorilla/mux"
)

type TestHandler struct {
	t *testing.T

	// The HTTP router to be tested.
	router func() *mux.Router

	ctx context.Context
}

// Build an HTTP request, pass it to the HTTP handler, and return the response.
func (handler TestHandler) request(method, path string, headers map[string]string) TestResponse {
	inst, err := aetest.NewInstance(nil)
	if err != nil {
		handler.t.Fatalf("Failed to create instance: %v", err)
	}
	defer inst.Close()

	request, err := inst.NewRequest(method, path, nil)
	if err != nil {
		handler.t.Fatal(err)
	}

	// Set request headers, if any.
	for header, value := range headers {
		request.Header.Set(header, value)
	}

	// Set an arbitrary remote address for logging purposes, etc.
	request.RemoteAddr = "1.2.3.4:80"

	response := httptest.NewRecorder()
	handler.router().ServeHTTP(response, request)
	return TestResponse{handler.t, response}
}

// Make a GET request to the HTTP handler, and return the response.
func (handler TestHandler) Get(path string, headers map[string]string) TestResponse {
	return handler.request(http.MethodGet, path, headers)
}

// Make a POST request to the HTTP handler, and return the response.
func (handler TestHandler) Post(path string, headers map[string]string) TestResponse {
	return handler.request(http.MethodPost, path, headers)
}

type TestResponse struct {
	t *testing.T

	// The HTTP response being asserted against.
	r *httptest.ResponseRecorder
}

// Ensure that the response contains the expected status code.
func (response TestResponse) AssertStatusEquals(expected int) {
	if response.r.Code != expected {
		response.t.Errorf(
			"Handler returned unexpected status code: got `%v` want `%v`",
			response.r.Code, expected)
	}
}

// Ensure that the response body is exactly as expected.
func (response TestResponse) AssertBodyEquals(expected string) {
	if actual := response.r.Body.String(); actual != expected {
		response.t.Errorf(
			"Handler returned unexpected body: got `%v` want `%v`",
			actual, expected)
	}
}

// Ensure that the response body contains a substring.
func (response TestResponse) AssertBodyContains(substr string) {
	if actual := response.r.Body.String(); !strings.Contains(actual, substr) {
		response.t.Errorf(
			"Handler returned unexpected body: did not find `%v` in `%v`",
			substr, actual)
	}
}

// Ensure that the response contains a specific header.
func (response TestResponse) AssertHeaderExists(header string) {
	if _, ok := response.r.Header()[header]; !ok {
		response.t.Errorf(
			"Handler did not set header `%v`",
			header)
	}
}

// Ensure that the response contains a specific header-value pair.
func (response TestResponse) AssertHeaderContains(header, expected string) {
	response.AssertHeaderExists(header)
	actuals, _ := response.r.Header()[header]
	for _, actual := range actuals {
		if actual == expected {
			return
		}
	}

	response.t.Errorf(
		"Handler returned unexpected %v: got `%v` want `%v`",
		header, actuals, expected)
}

func TestGetIndex(t *testing.T) {
	ctx, done, err := aetest.NewContext()
	if err != nil {
		t.Fatal(err)
	}
	defer done()

	response := TestHandler{t, Router, ctx}.Get("/", nil)
	response.AssertStatusEquals(http.StatusOK)
	response.AssertBodyEquals("1.2.3.4\n")
	response.AssertHeaderContains("Content-Type", "text/plain; charset=UTF-8")
}

func TestGetInvalidUrl(t *testing.T) {
	ctx, done, err := aetest.NewContext()
	if err != nil {
		t.Fatal(err)
	}
	defer done()

	response := TestHandler{t, Router, ctx}.Get("/non-existant", nil)
	response.AssertStatusEquals(http.StatusNotFound)
	response.AssertBodyEquals("404 Not Found\n")
	response.AssertHeaderContains("Content-Type", "text/plain; charset=UTF-8")
}
<commit_msg>Use verbose variable name<commit_after>package main

import (
	"golang.org/x/net/context"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"

	// Google App Engine
	"google.golang.org/appengine/aetest"

	// Request routing
	"github.com/gorilla/mux"
)

type TestHandler struct {
	t *testing.T

	// The HTTP router to be tested.
	router func() *mux.Router

	ctx context.Context
}

// Build an HTTP request, pass it to the HTTP handler, and return the response.
func (handler TestHandler) request(method, path string, headers map[string]string) TestResponse {
	instance, err := aetest.NewInstance(nil)
	if err != nil {
		handler.t.Fatalf("Failed to create instance: %v", err)
	}
	defer instance.Close()

	request, err := instance.NewRequest(method, path, nil)
	if err != nil {
		handler.t.Fatal(err)
	}

	// Set request headers, if any.
	for header, value := range headers {
		request.Header.Set(header, value)
	}

	// Set an arbitrary remote address for logging purposes, etc.
	request.RemoteAddr = "1.2.3.4:80"

	response := httptest.NewRecorder()
	handler.router().ServeHTTP(response, request)
	return TestResponse{handler.t, response}
}

// Make a GET request to the HTTP handler, and return the response.
func (handler TestHandler) Get(path string, headers map[string]string) TestResponse {
	return handler.request(http.MethodGet, path, headers)
}

// Make a POST request to the HTTP handler, and return the response.
func (handler TestHandler) Post(path string, headers map[string]string) TestResponse {
	return handler.request(http.MethodPost, path, headers)
}

type TestResponse struct {
	t *testing.T

	// The HTTP response being asserted against.
	r *httptest.ResponseRecorder
}

// Ensure that the response contains the expected status code.
func (response TestResponse) AssertStatusEquals(expected int) {
	if response.r.Code != expected {
		response.t.Errorf(
			"Handler returned unexpected status code: got `%v` want `%v`",
			response.r.Code, expected)
	}
}

// Ensure that the response body is exactly as expected.
func (response TestResponse) AssertBodyEquals(expected string) {
	if actual := response.r.Body.String(); actual != expected {
		response.t.Errorf(
			"Handler returned unexpected body: got `%v` want `%v`",
			actual, expected)
	}
}

// Ensure that the response body contains a substring.
func (response TestResponse) AssertBodyContains(substr string) {
	if actual := response.r.Body.String(); !strings.Contains(actual, substr) {
		response.t.Errorf(
			"Handler returned unexpected body: did not find `%v` in `%v`",
			substr, actual)
	}
}

// Ensure that the response contains a specific header.
func (response TestResponse) AssertHeaderExists(header string) {
	if _, ok := response.r.Header()[header]; !ok {
		response.t.Errorf(
			"Handler did not set header `%v`",
			header)
	}
}

// Ensure that the response contains a specific header-value pair.
func (response TestResponse) AssertHeaderContains(header, expected string) {
	response.AssertHeaderExists(header)
	actuals, _ := response.r.Header()[header]
	for _, actual := range actuals {
		if actual == expected {
			return
		}
	}

	response.t.Errorf(
		"Handler returned unexpected %v: got `%v` want `%v`",
		header, actuals, expected)
}

func TestGetIndex(t *testing.T) {
	ctx, done, err := aetest.NewContext()
	if err != nil {
		t.Fatal(err)
	}
	defer done()

	response := TestHandler{t, Router, ctx}.Get("/", nil)
	response.AssertStatusEquals(http.StatusOK)
	response.AssertBodyEquals("1.2.3.4\n")
	response.AssertHeaderContains("Content-Type", "text/plain; charset=UTF-8")
}

func TestGetInvalidUrl(t *testing.T) {
	ctx, done, err := aetest.NewContext()
	if err != nil {
		t.Fatal(err)
	}
	defer done()

	response := TestHandler{t, Router, ctx}.Get("/non-existant", nil)
	response.AssertStatusEquals(http.StatusNotFound)
	response.AssertBodyEquals("404 Not Found\n")
	response.AssertHeaderContains("Content-Type", "text/plain; charset=UTF-8")
}
<|endoftext|>


<commit_before>/*
*
*
* Copyright 2012-2016 Viant.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
*  use this file except in compliance with the License. You may obtain a copy of
*  the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
*  License for the specific language governing permissions and limitations under
*  the License.
*
 */
package toolbox

import (
	"reflect"
)

//Iterator represents generic iterator.
type Iterator interface {

	//HasNext returns true if iterator has next element.
	HasNext() bool

	//Next sets item pointer with next element.
	Next(itemPointer interface{})
}

type sliceIterator struct {
	sliceValue reflect.Value
	index      int
}

func (i *sliceIterator) HasNext() bool {
	return i.index < i.sliceValue.Len()
}

func (i *sliceIterator) Next(itemPointer interface{}) {
	value := i.sliceValue.Index(i.index)
	i.index++
	itemPointerValue := reflect.ValueOf(itemPointer)
	itemPointerValue.Elem().Set(value)
}

type stringSliceIterator struct {
	sliceValue []string
	index      int
}

func (i *stringSliceIterator) HasNext() bool {
	return i.index < len(i.sliceValue)
}

func (i *stringSliceIterator) Next(itemPointer interface{}) {
	value := i.sliceValue[i.index]
	i.index++
	if stringPointer, ok  := itemPointer.(*string); ok {
		*stringPointer = value
		return
	}
	interfacePointer:= itemPointer.(*interface{})
	*interfacePointer = value
}

type interfaceSliceIterator struct {
	sliceValue []interface{}
	index      int
}

func (i *interfaceSliceIterator) HasNext() bool {
	return i.index < len(i.sliceValue)
}

func (i *interfaceSliceIterator) Next(itemPointer interface{}) {
	value := i.sliceValue[i.index]
	i.index++
	itemPointerValue := reflect.ValueOf(itemPointer)
	itemPointerValue.Elem().Set(reflect.ValueOf(value))
}

//NewSliceIterator creates a new slice iterator.
func NewSliceIterator(slice interface{}) Iterator {
	if aSlice, ok := slice.([]interface{});ok {
		return &interfaceSliceIterator{aSlice, 0}
	}
	if aSlice, ok := slice.([]string);ok {
		return &stringSliceIterator{aSlice, 0}
	}
	sliceValue := DiscoverValueByKind(reflect.ValueOf(slice), reflect.Slice)
	return &sliceIterator{sliceValue: sliceValue}
}
<commit_msg>patched nil element in integrator<commit_after>/*
*
*
* Copyright 2012-2016 Viant.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
*  use this file except in compliance with the License. You may obtain a copy of
*  the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
*  License for the specific language governing permissions and limitations under
*  the License.
*
 */
package toolbox

import (
	"reflect"
)

//Iterator represents generic iterator.
type Iterator interface {

	//HasNext returns true if iterator has next element.
	HasNext() bool

	//Next sets item pointer with next element.
	Next(itemPointer interface{})
}

type sliceIterator struct {
	sliceValue reflect.Value
	index      int
}

func (i *sliceIterator) HasNext() bool {
	return i.index < i.sliceValue.Len()
}

func (i *sliceIterator) Next(itemPointer interface{}) {
	value := i.sliceValue.Index(i.index)
	i.index++
	itemPointerValue := reflect.ValueOf(itemPointer)
	itemPointerValue.Elem().Set(value)
}

type stringSliceIterator struct {
	sliceValue []string
	index      int
}

func (i *stringSliceIterator) HasNext() bool {
	return i.index < len(i.sliceValue)
}

func (i *stringSliceIterator) Next(itemPointer interface{}) {
	value := i.sliceValue[i.index]
	i.index++
	if stringPointer, ok  := itemPointer.(*string); ok {
		*stringPointer = value
		return
	}
	interfacePointer:= itemPointer.(*interface{})
	*interfacePointer = value
}

type interfaceSliceIterator struct {
	sliceValue []interface{}
	index      int
}

func (i *interfaceSliceIterator) HasNext() bool {
	return i.index < len(i.sliceValue)
}

func (i *interfaceSliceIterator) Next(itemPointer interface{}) {
	value := i.sliceValue[i.index]
	i.index++
	itemPointerValue := reflect.ValueOf(itemPointer)
	if value != nil {
		itemPointerValue.Elem().Set(reflect.ValueOf(value))
	} else {
		itemPointerValue.Elem().Set(reflect.Zero(reflect.TypeOf(itemPointer).Elem()))

	}
}

//NewSliceIterator creates a new slice iterator.
func NewSliceIterator(slice interface{}) Iterator {
	if aSlice, ok := slice.([]interface{});ok {
		return &interfaceSliceIterator{aSlice, 0}
	}
	if aSlice, ok := slice.([]string);ok {
		return &stringSliceIterator{aSlice, 0}
	}
	sliceValue := DiscoverValueByKind(reflect.ValueOf(slice), reflect.Slice)
	return &sliceIterator{sliceValue: sliceValue}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math"
	"net/http"
	"strconv"
	"sync/atomic"
	"time"

	"github.com/grafana/metrictank/clock"
	"github.com/grafana/metrictank/stacktest/graphite"
	"github.com/grafana/metrictank/stats"
	"github.com/grafana/metrictank/util/align"
	log "github.com/sirupsen/logrus"
)

var (
	httpError    = stats.NewCounter32WithTags("parrot.monitoring.error", ";error=http")
	decodeError  = stats.NewCounter32WithTags("parrot.monitoring.error", ";error=decode")
	invalidError = stats.NewCounter32WithTags("parrot.monitoring.error", ";error=invalid")
)

var metricsBySeries []partitionMetrics

type seriesStats struct {
	lastTs           uint32
	nans             int32   // the partition currently being checked - nope?
	deltaSum         float64 // sum of abs(value - ts) across the time series
	numNonMatching   int32   // number of timestamps where value != ts
	lastSeen         uint32  // the last seen non-NaN time stamp (useful for lag)
	correctNumPoints bool    // whether the expected number of points were received
	correctAlignment bool    // whether the last ts matches `now`
	correctSpacing   bool    // whether all points are sorted and 1 period apart
}

type partitionMetrics struct {
	nanCount         *stats.Gauge32 // the number of missing values for each series
	lag              *stats.Gauge32 // time since the last value was recorded
	deltaSum         *stats.Gauge32 // the total amount of drift between expected value and actual values
	nonMatching      *stats.Gauge32 // total number of entries where drift occurred
	correctNumPoints *stats.Bool    // whether the expected number of points were received
	correctAlignment *stats.Bool    // whether the last ts matches `now`
	correctSpacing   *stats.Bool    // whether all points are sorted and 1 period apart
}

func monitor() {
	initMetricsBySeries()
	for tick := range clock.AlignedTickLossless(queryInterval) {

		query := graphite.ExecuteRenderQuery(buildRequest(tick))
		if query.HTTPErr != nil {
			httpError.Inc()
			continue
		}
		if query.DecodeErr != nil {
			decodeError.Inc()
			continue
		}

		for _, s := range query.Decoded {
			processPartitionSeries(s, tick)
		}
		statsGraphite.Report(tick)
	}
}

func processPartitionSeries(s graphite.Series, now time.Time) {
	partition, err := strconv.Atoi(s.Target)
	if err != nil {
		log.Debug("unable to parse partition", err)
		invalidError.Inc()
		return
	}
	if len(s.Datapoints) < 2 {
		log.Debugf("partition has invalid number of datapoints: %d", len(s.Datapoints))
		invalidError.Inc()
		return
	}

	serStats := seriesStats{}
	serStats.lastTs = s.Datapoints[len(s.Datapoints)-1].Ts
	serStats.correctAlignment = int64(serStats.lastTs) == now.Unix()
	serStats.correctNumPoints = len(s.Datapoints) == int(lookbackPeriod/testMetricsInterval)+1
	serStats.correctSpacing = checkSpacing(s.Datapoints)

	for _, dp := range s.Datapoints {
		if math.IsNaN(dp.Val) {
			serStats.nans += 1
			continue
		}
		serStats.lastSeen = dp.Ts
		if diff := dp.Val - float64(dp.Ts); diff != 0 {
			log.Debugf("partition=%d dp.Val=%f dp.Ts=%d diff=%f", partition, dp.Val, dp.Ts, diff)
			serStats.deltaSum += diff
			serStats.numNonMatching += 1
		}
	}

	metrics := metricsBySeries[partition]
	metrics.nanCount.Set(int(serStats.nans))
	lag := atomic.LoadInt64(&lastPublish) - int64(serStats.lastSeen)
	metrics.lag.Set(int(lag))
	metrics.deltaSum.Set(int(serStats.deltaSum))
	metrics.nonMatching.Set(int(serStats.numNonMatching))
	metrics.correctNumPoints.Set(serStats.correctNumPoints)
	metrics.correctAlignment.Set(serStats.correctAlignment)
	metrics.correctSpacing.Set(serStats.correctSpacing)
}

func checkSpacing(points []graphite.Point) bool {
	for i := 1; i < len(points); i++ {
		prev := points[i-1].Ts
		cur := points[i].Ts
		if cur-prev != uint32(testMetricsInterval.Seconds()) {
			return false
		}
	}
	return true
}

func initMetricsBySeries() {
	for p := 0; p < int(partitionCount); p++ {
		metrics := partitionMetrics{
			//TODO enable metrics2docs by adding 'metric' prefix to each metric
			// parrot.monitoring.nancount is the number of missing values for each series
			nanCount: stats.NewGauge32WithTags("parrot.monitoring.nancount", fmt.Sprintf(";partition=%d", p)),
			// parrot.monitoring.lag is the time since the last value was recorded
			lag: stats.NewGauge32WithTags("parrot.monitoring.lag", fmt.Sprintf(";partition=%d", p)),
			// parrot.monitoring.deltaSum is the total amount of drift between expected value and actual values
			deltaSum: stats.NewGauge32WithTags("parrot.monitoring.deltaSum", fmt.Sprintf(";partition=%d", p)),
			// parrot.monitoring.nonMatching is the total number of entries where drift occurred
			nonMatching: stats.NewGauge32WithTags("parrot.monitoring.nonMatching", fmt.Sprintf(";partition=%d", p)),
			// parrot.monitoring.correctNumPoints is whether the expected number of points were received
			correctNumPoints: stats.NewBoolWithTags("parrot.monitoring.correctNumPoints", fmt.Sprintf(";partition=%d", p)),
			// parrot.monitoring.correctAlignment is whether the last ts matches `now`
			correctAlignment: stats.NewBoolWithTags("parrot.monitoring.correctAlignment", fmt.Sprintf(";partition=%d", p)),
			// parrot.monitoring.correctSpacing is whether all points are sorted and 1 period apart
			correctSpacing: stats.NewBoolWithTags("parrot.monitoring.correctSpacing", fmt.Sprintf(";partition=%d", p)),
		}
		metricsBySeries = append(metricsBySeries, metrics)
	}
}

func buildRequest(now time.Time) *http.Request {
	req, _ := http.NewRequest("GET", fmt.Sprintf("%s/render", gatewayAddress), nil)
	q := req.URL.Query()
	q.Set("target", "aliasByNode(parrot.testdata.*.identity.*, 2)")
	q.Set("from", strconv.Itoa(int(now.Add(-1*lookbackPeriod).Unix()-1)))
	q.Set("until", strconv.Itoa(int(now.Unix())))
	q.Set("format", "json")
	q.Set("X-Org-Id", strconv.Itoa(orgId))
	req.URL.RawQuery = q.Encode()
	if len(gatewayKey) != 0 {
		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", gatewayKey))
	}
	return req
}
<commit_msg>consistency between series and partition stats<commit_after>package main

import (
	"fmt"
	"math"
	"net/http"
	"strconv"
	"sync/atomic"
	"time"

	"github.com/grafana/metrictank/clock"
	"github.com/grafana/metrictank/stacktest/graphite"
	"github.com/grafana/metrictank/stats"
	"github.com/grafana/metrictank/util/align"
	log "github.com/sirupsen/logrus"
)

var (
	httpError    = stats.NewCounter32WithTags("parrot.monitoring.error", ";error=http")
	decodeError  = stats.NewCounter32WithTags("parrot.monitoring.error", ";error=decode")
	invalidError = stats.NewCounter32WithTags("parrot.monitoring.error", ";error=invalid")
)

var metricsBySeries []partitionMetrics

type seriesInfo struct {
	lastTs uint32 // last timestamp seen in the response

	// to generate stats from
	lastSeen         uint32  // the last seen non-NaN time stamp (to generate lag from)
	deltaSum         float64 // sum of abs(value - ts) across the time series
	numNans          int32   // number of missing values for each series
	numNonMatching   int32   // number of points where value != ts
	correctNumPoints bool    // whether the expected number of points were received
	correctAlignment bool    // whether the last ts matches `now`
	correctSpacing   bool    // whether all points are sorted and 1 period apart
}

type partitionMetrics struct {
	lag              *stats.Gauge32 // time since the last value was recorded
	deltaSum         *stats.Gauge32 // total amount of drift between expected value and actual values
	numNans          *stats.Gauge32 // number of missing values for each series
	numNonMatching   *stats.Gauge32 // number of points where value != ts
	correctNumPoints *stats.Bool    // whether the expected number of points were received
	correctAlignment *stats.Bool    // whether the last ts matches `now`
	correctSpacing   *stats.Bool    // whether all points are sorted and 1 period apart
}

func monitor() {
	initMetricsBySeries()
	for tick := range clock.AlignedTickLossless(queryInterval) {

		query := graphite.ExecuteRenderQuery(buildRequest(tick))
		if query.HTTPErr != nil {
			httpError.Inc()
			continue
		}
		if query.DecodeErr != nil {
			decodeError.Inc()
			continue
		}

		for _, s := range query.Decoded {
			processPartitionSeries(s, tick)
		}
		statsGraphite.Report(tick)
	}
}

func processPartitionSeries(s graphite.Series, now time.Time) {
	partition, err := strconv.Atoi(s.Target)
	if err != nil {
		log.Debug("unable to parse partition", err)
		invalidError.Inc()
		return
	}
	if len(s.Datapoints) < 2 {
		log.Debugf("partition has invalid number of datapoints: %d", len(s.Datapoints))
		invalidError.Inc()
		return
	}

	serStats := seriesInfo{}
	serStats.lastTs = s.Datapoints[len(s.Datapoints)-1].Ts
	serStats.correctAlignment = int64(serStats.lastTs) == now.Unix()
	serStats.correctNumPoints = len(s.Datapoints) == int(lookbackPeriod/testMetricsInterval)+1
	serStats.correctSpacing = checkSpacing(s.Datapoints)

	for _, dp := range s.Datapoints {
		if math.IsNaN(dp.Val) {
			serStats.numNans += 1
			continue
		}
		serStats.lastSeen = dp.Ts
		if diff := dp.Val - float64(dp.Ts); diff != 0 {
			log.Debugf("partition=%d dp.Val=%f dp.Ts=%d diff=%f", partition, dp.Val, dp.Ts, diff)
			serStats.deltaSum += diff
			serStats.numNonMatching += 1
		}
	}

	metrics := metricsBySeries[partition]
	metrics.numNans.Set(int(serStats.numNans))
	lag := atomic.LoadInt64(&lastPublish) - int64(serStats.lastSeen)
	metrics.lag.Set(int(lag))
	metrics.deltaSum.Set(int(serStats.deltaSum))
	metrics.numNonMatching.Set(int(serStats.numNonMatching))
	metrics.correctNumPoints.Set(serStats.correctNumPoints)
	metrics.correctAlignment.Set(serStats.correctAlignment)
	metrics.correctSpacing.Set(serStats.correctSpacing)
}

func checkSpacing(points []graphite.Point) bool {
	for i := 1; i < len(points); i++ {
		prev := points[i-1].Ts
		cur := points[i].Ts
		if cur-prev != uint32(testMetricsInterval.Seconds()) {
			return false
		}
	}
	return true
}

func initMetricsBySeries() {
	for p := 0; p < int(partitionCount); p++ {
		metrics := partitionMetrics{
			//TODO enable metrics2docs by adding 'metric' prefix to each metric
			// parrot.monitoring.nans is the number of missing values for each series
			numNans: stats.NewGauge32WithTags("parrot.monitoring.nans", fmt.Sprintf(";partition=%d", p)),
			// parrot.monitoring.lag is the time since the last value was recorded
			lag: stats.NewGauge32WithTags("parrot.monitoring.lag", fmt.Sprintf(";partition=%d", p)),
			// parrot.monitoring.deltaSum is the total amount of drift between expected value and actual values
			deltaSum: stats.NewGauge32WithTags("parrot.monitoring.deltaSum", fmt.Sprintf(";partition=%d", p)),
			// parrot.monitoring.nonmatching is the total number of entries where drift occurred
			numNonMatching: stats.NewGauge32WithTags("parrot.monitoring.nonmatching", fmt.Sprintf(";partition=%d", p)),
			// parrot.monitoring.correctNumPoints is whether the expected number of points were received
			correctNumPoints: stats.NewBoolWithTags("parrot.monitoring.correctNumPoints", fmt.Sprintf(";partition=%d", p)),
			// parrot.monitoring.correctAlignment is whether the last ts matches `now`
			correctAlignment: stats.NewBoolWithTags("parrot.monitoring.correctAlignment", fmt.Sprintf(";partition=%d", p)),
			// parrot.monitoring.correctSpacing is whether all points are sorted and 1 period apart
			correctSpacing: stats.NewBoolWithTags("parrot.monitoring.correctSpacing", fmt.Sprintf(";partition=%d", p)),
		}
		metricsBySeries = append(metricsBySeries, metrics)
	}
}

func buildRequest(now time.Time) *http.Request {
	req, _ := http.NewRequest("GET", fmt.Sprintf("%s/render", gatewayAddress), nil)
	q := req.URL.Query()
	q.Set("target", "aliasByNode(parrot.testdata.*.identity.*, 2)")
	q.Set("from", strconv.Itoa(int(now.Add(-1*lookbackPeriod).Unix()-1)))
	q.Set("until", strconv.Itoa(int(now.Unix())))
	q.Set("format", "json")
	q.Set("X-Org-Id", strconv.Itoa(orgId))
	req.URL.RawQuery = q.Encode()
	if len(gatewayKey) != 0 {
		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", gatewayKey))
	}
	return req
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"os"
	"strings"

	. "github.com/tendermint/go-common"
	"github.com/tendermint/tmsp/client"
	"github.com/tendermint/tmsp/types"
	"github.com/urfave/cli"
)

// client is a global variable so it can be reused by the console
var client tmspcli.Client

func main() {
	app := cli.NewApp()
	app.Name = "tmsp-cli"
	app.Usage = "tmsp-cli [command] [args...]"
	app.Version = "0.2.1" // better error handling in console
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "address",
			Value: "tcp://127.0.0.1:46658",
			Usage: "address of application socket",
		},
		cli.StringFlag{
			Name:  "tmsp",
			Value: "socket",
			Usage: "socket or grpc",
		},
		cli.BoolFlag{
			Name:  "verbose",
			Usage: "print the command and results as if it were a console session",
		},
	}
	app.Commands = []cli.Command{
		{
			Name:  "batch",
			Usage: "Run a batch of tmsp commands against an application",
			Action: func(c *cli.Context) error {
				return cmdBatch(app, c)
			},
		},
		{
			Name:  "console",
			Usage: "Start an interactive tmsp console for multiple commands",
			Action: func(c *cli.Context) error {
				return cmdConsole(app, c)
			},
		},
		{
			Name:  "echo",
			Usage: "Have the application echo a message",
			Action: func(c *cli.Context) error {
				return cmdEcho(c)
			},
		},
		{
			Name:  "info",
			Usage: "Get some info about the application",
			Action: func(c *cli.Context) error {
				return cmdInfo(c)
			},
		},
		{
			Name:  "set_option",
			Usage: "Set an option on the application",
			Action: func(c *cli.Context) error {
				return cmdSetOption(c)
			},
		},
		{
			Name:  "append_tx",
			Usage: "Append a new tx to application",
			Action: func(c *cli.Context) error {
				return cmdAppendTx(c)
			},
		},
		{
			Name:  "check_tx",
			Usage: "Validate a tx",
			Action: func(c *cli.Context) error {
				return cmdCheckTx(c)
			},
		},
		{
			Name:  "commit",
			Usage: "Commit the application state and return the Merkle root hash",
			Action: func(c *cli.Context) error {
				return cmdCommit(c)
			},
		},
		{
			Name:  "query",
			Usage: "Query application state",
			Action: func(c *cli.Context) error {
				return cmdQuery(c)
			},
		},
	}
	app.Before = before
	err := app.Run(os.Args)
	if err != nil {
		Exit(err.Error())
	}

}

func before(c *cli.Context) error {
	if client == nil {
		var err error
		client, err = tmspcli.NewClient(c.GlobalString("address"), c.GlobalString("tmsp"), false)
		if err != nil {
			Exit(err.Error())
		}
	}
	return nil
}

// badCmd is called when we invoke with an invalid first argument (just for console for now)
func badCmd(c *cli.Context, cmd string) {
	fmt.Println("Unknown command:", cmd)
	fmt.Println("Please try one of the following:")
	fmt.Println("")
	cli.DefaultAppComplete(c)
}

//--------------------------------------------------------------------------------

func cmdBatch(app *cli.App, c *cli.Context) error {
	bufReader := bufio.NewReader(os.Stdin)
	for {
		line, more, err := bufReader.ReadLine()
		if more {
			return errors.New("Input line is too long")
		} else if err == io.EOF {
			break
		} else if len(line) == 0 {
			continue
		} else if err != nil {
			return err
		}
		args := []string{"tmsp-cli"}
		if c.GlobalBool("verbose") {
			args = append(args, "--verbose")
		}
		args = append(args, strings.Split(string(line), " ")...)
		app.Run(args)
	}
	return nil
}

func cmdConsole(app *cli.App, c *cli.Context) error {
	// don't hard exit on mistyped commands (eg. check vs check_tx)
	app.CommandNotFound = badCmd
	for {
		fmt.Printf("\n> ")
		bufReader := bufio.NewReader(os.Stdin)
		line, more, err := bufReader.ReadLine()
		if more {
			return errors.New("Input is too long")
		} else if err != nil {
			return err
		}

		args := []string{"tmsp-cli"}
		args = append(args, strings.Split(string(line), " ")...)
		if err := app.Run(args); err != nil {
			// if the command doesn't succeed, inform the user without exiting
			fmt.Println("Error:", err.Error())
		}
	}
}

// Have the application echo a message
func cmdEcho(c *cli.Context) error {
	args := c.Args()
	if len(args) != 1 {
		return errors.New("Command echo takes 1 argument")
	}
	res := client.EchoSync(args[0])
	printResponse(c, res, string(res.Data), false)
	return nil
}

// Get some info from the application
func cmdInfo(c *cli.Context) error {
	res, _, _, _ := client.InfoSync()
	printResponse(c, res, string(res.Data), false)
	return nil
}

// Set an option on the application
func cmdSetOption(c *cli.Context) error {
	args := c.Args()
	if len(args) != 2 {
		return errors.New("Command set_option takes 2 arguments (key, value)")
	}
	res := client.SetOptionSync(args[0], args[1])
	printResponse(c, res, Fmt("%s=%s", args[0], args[1]), false)
	return nil
}

// Append a new tx to application
func cmdAppendTx(c *cli.Context) error {
	args := c.Args()
	if len(args) != 1 {
		return errors.New("Command append_tx takes 1 argument")
	}
	txBytes := stringOrHexToBytes(c.Args()[0])
	res := client.AppendTxSync(txBytes)
	printResponse(c, res, string(res.Data), true)
	return nil
}

// Validate a tx
func cmdCheckTx(c *cli.Context) error {
	args := c.Args()
	if len(args) != 1 {
		return errors.New("Command check_tx takes 1 argument")
	}
	txBytes := stringOrHexToBytes(c.Args()[0])
	res := client.CheckTxSync(txBytes)
	printResponse(c, res, string(res.Data), true)
	return nil
}

// Get application Merkle root hash
func cmdCommit(c *cli.Context) error {
	res := client.CommitSync()
	printResponse(c, res, Fmt("0x%X", res.Data), false)
	return nil
}

// Query application state
func cmdQuery(c *cli.Context) error {
	args := c.Args()
	if len(args) != 1 {
		return errors.New("Command query takes 1 argument")
	}
	queryBytes := stringOrHexToBytes(c.Args()[0])
	res := client.QuerySync(queryBytes)
	printResponse(c, res, string(res.Data), true)
	return nil
}

//--------------------------------------------------------------------------------

func printResponse(c *cli.Context, res types.Result, s string, printCode bool) {
	if c.GlobalBool("verbose") {
		fmt.Println(">", c.Command.Name, strings.Join(c.Args(), " "))
	}

	if printCode {
		fmt.Printf("-> code: %s\n", res.Code.String())
	}
	/*if res.Error != "" {
		fmt.Printf("-> error: %s\n", res.Error)
	}*/
	if s != "" {
		fmt.Printf("-> data: %s\n", s)
	}
	if res.Log != "" {
		fmt.Printf("-> log: %s\n", res.Log)
	}

	if c.GlobalBool("verbose") {
		fmt.Println("")
	}

}

// NOTE: s is interpreted as a string unless prefixed with 0x
func stringOrHexToBytes(s string) []byte {
	if len(s) > 2 && s[:2] == "0x" {
		b, err := hex.DecodeString(s[2:])
		if err != nil {
			fmt.Println("Error decoding hex argument:", err.Error())
		}
		return b
	}

	if !strings.HasPrefix(s, "\"") || !strings.HasSuffix(s, "\"") {
		fmt.Printf("Invalid string arg: \"%s\". Must be quoted or a \"0x\"-prefixed hex string\n", s)
		return []byte{}
	}

	// TODO: return errors

	return []byte(s)
}
<commit_msg>Don't include quotes in quoted string args<commit_after>package main

import (
	"bufio"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"os"
	"strings"

	. "github.com/tendermint/go-common"
	"github.com/tendermint/tmsp/client"
	"github.com/tendermint/tmsp/types"
	"github.com/urfave/cli"
)

// client is a global variable so it can be reused by the console
var client tmspcli.Client

func main() {
	app := cli.NewApp()
	app.Name = "tmsp-cli"
	app.Usage = "tmsp-cli [command] [args...]"
	app.Version = "0.2.1" // better error handling in console
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "address",
			Value: "tcp://127.0.0.1:46658",
			Usage: "address of application socket",
		},
		cli.StringFlag{
			Name:  "tmsp",
			Value: "socket",
			Usage: "socket or grpc",
		},
		cli.BoolFlag{
			Name:  "verbose",
			Usage: "print the command and results as if it were a console session",
		},
	}
	app.Commands = []cli.Command{
		{
			Name:  "batch",
			Usage: "Run a batch of tmsp commands against an application",
			Action: func(c *cli.Context) error {
				return cmdBatch(app, c)
			},
		},
		{
			Name:  "console",
			Usage: "Start an interactive tmsp console for multiple commands",
			Action: func(c *cli.Context) error {
				return cmdConsole(app, c)
			},
		},
		{
			Name:  "echo",
			Usage: "Have the application echo a message",
			Action: func(c *cli.Context) error {
				return cmdEcho(c)
			},
		},
		{
			Name:  "info",
			Usage: "Get some info about the application",
			Action: func(c *cli.Context) error {
				return cmdInfo(c)
			},
		},
		{
			Name:  "set_option",
			Usage: "Set an option on the application",
			Action: func(c *cli.Context) error {
				return cmdSetOption(c)
			},
		},
		{
			Name:  "append_tx",
			Usage: "Append a new tx to application",
			Action: func(c *cli.Context) error {
				return cmdAppendTx(c)
			},
		},
		{
			Name:  "check_tx",
			Usage: "Validate a tx",
			Action: func(c *cli.Context) error {
				return cmdCheckTx(c)
			},
		},
		{
			Name:  "commit",
			Usage: "Commit the application state and return the Merkle root hash",
			Action: func(c *cli.Context) error {
				return cmdCommit(c)
			},
		},
		{
			Name:  "query",
			Usage: "Query application state",
			Action: func(c *cli.Context) error {
				return cmdQuery(c)
			},
		},
	}
	app.Before = before
	err := app.Run(os.Args)
	if err != nil {
		Exit(err.Error())
	}

}

func before(c *cli.Context) error {
	if client == nil {
		var err error
		client, err = tmspcli.NewClient(c.GlobalString("address"), c.GlobalString("tmsp"), false)
		if err != nil {
			Exit(err.Error())
		}
	}
	return nil
}

// badCmd is called when we invoke with an invalid first argument (just for console for now)
func badCmd(c *cli.Context, cmd string) {
	fmt.Println("Unknown command:", cmd)
	fmt.Println("Please try one of the following:")
	fmt.Println("")
	cli.DefaultAppComplete(c)
}

//--------------------------------------------------------------------------------

func cmdBatch(app *cli.App, c *cli.Context) error {
	bufReader := bufio.NewReader(os.Stdin)
	for {
		line, more, err := bufReader.ReadLine()
		if more {
			return errors.New("Input line is too long")
		} else if err == io.EOF {
			break
		} else if len(line) == 0 {
			continue
		} else if err != nil {
			return err
		}
		args := []string{"tmsp-cli"}
		if c.GlobalBool("verbose") {
			args = append(args, "--verbose")
		}
		args = append(args, strings.Split(string(line), " ")...)
		app.Run(args)
	}
	return nil
}

func cmdConsole(app *cli.App, c *cli.Context) error {
	// don't hard exit on mistyped commands (eg. check vs check_tx)
	app.CommandNotFound = badCmd
	for {
		fmt.Printf("\n> ")
		bufReader := bufio.NewReader(os.Stdin)
		line, more, err := bufReader.ReadLine()
		if more {
			return errors.New("Input is too long")
		} else if err != nil {
			return err
		}

		args := []string{"tmsp-cli"}
		args = append(args, strings.Split(string(line), " ")...)
		if err := app.Run(args); err != nil {
			// if the command doesn't succeed, inform the user without exiting
			fmt.Println("Error:", err.Error())
		}
	}
}

// Have the application echo a message
func cmdEcho(c *cli.Context) error {
	args := c.Args()
	if len(args) != 1 {
		return errors.New("Command echo takes 1 argument")
	}
	res := client.EchoSync(args[0])
	printResponse(c, res, string(res.Data), false)
	return nil
}

// Get some info from the application
func cmdInfo(c *cli.Context) error {
	res, _, _, _ := client.InfoSync()
	printResponse(c, res, string(res.Data), false)
	return nil
}

// Set an option on the application
func cmdSetOption(c *cli.Context) error {
	args := c.Args()
	if len(args) != 2 {
		return errors.New("Command set_option takes 2 arguments (key, value)")
	}
	res := client.SetOptionSync(args[0], args[1])
	printResponse(c, res, Fmt("%s=%s", args[0], args[1]), false)
	return nil
}

// Append a new tx to application
func cmdAppendTx(c *cli.Context) error {
	args := c.Args()
	if len(args) != 1 {
		return errors.New("Command append_tx takes 1 argument")
	}
	txBytes, err := stringOrHexToBytes(c.Args()[0])
	if err != nil {
		fmt.Println(err.Error())
		return nil
	}
	res := client.AppendTxSync(txBytes)
	printResponse(c, res, string(res.Data), true)
	return nil
}

// Validate a tx
func cmdCheckTx(c *cli.Context) error {
	args := c.Args()
	if len(args) != 1 {
		return errors.New("Command check_tx takes 1 argument")
	}
	txBytes, err := stringOrHexToBytes(c.Args()[0])
	if err != nil {
		fmt.Println(err.Error())
		return nil
	}
	res := client.CheckTxSync(txBytes)
	printResponse(c, res, string(res.Data), true)
	return nil
}

// Get application Merkle root hash
func cmdCommit(c *cli.Context) error {
	res := client.CommitSync()
	printResponse(c, res, Fmt("0x%X", res.Data), false)
	return nil
}

// Query application state
func cmdQuery(c *cli.Context) error {
	args := c.Args()
	if len(args) != 1 {
		return errors.New("Command query takes 1 argument")
	}
	queryBytes, err := stringOrHexToBytes(c.Args()[0])
	if err != nil {
		fmt.Println(err.Error())
		return nil
	}
	res := client.QuerySync(queryBytes)
	printResponse(c, res, string(res.Data), true)
	return nil
}

//--------------------------------------------------------------------------------

func printResponse(c *cli.Context, res types.Result, s string, printCode bool) {
	if c.GlobalBool("verbose") {
		fmt.Println(">", c.Command.Name, strings.Join(c.Args(), " "))
	}

	if printCode {
		fmt.Printf("-> code: %s\n", res.Code.String())
	}
	/*if res.Error != "" {
		fmt.Printf("-> error: %s\n", res.Error)
	}*/
	if s != "" {
		fmt.Printf("-> data: %s\n", s)
	}
	if res.Log != "" {
		fmt.Printf("-> log: %s\n", res.Log)
	}

	if c.GlobalBool("verbose") {
		fmt.Println("")
	}

}

// NOTE: s is interpreted as a string unless prefixed with 0x
func stringOrHexToBytes(s string) ([]byte, error) {
	fmt.Printf("string: %s %x\n", s, []byte(s))

	if len(s) > 2 && strings.ToLower(s[:2]) == "0x" {
		b, err := hex.DecodeString(s[2:])
		if err != nil {
			err = fmt.Errorf("Error decoding hex argument: %s", err.Error())
			return nil, err
		}
		return b, nil
	}

	if !strings.HasPrefix(s, "\"") || !strings.HasSuffix(s, "\"") {
		err := fmt.Errorf("Invalid string arg: \"%s\". Must be quoted or a \"0x\"-prefixed hex string", s)
		return nil, err
	}

	return []byte(s[1 : len(s)-1]), nil
}
<|endoftext|>


<commit_before>package requestid

import (
	"fmt"

	"github.com/rs/xid"
	"golang.org/x/net/context"
	"google.golang.org/grpc/metadata"
)

// RequestIDKey is metadata key name for request ID
var RequestIDKey = "request-id"

func HandleRequestID(ctx context.Context) string {
	md, ok := metadata.FromContext(ctx)
	if !ok {
		return newRequestID()
	}

	header, ok := md[RequestIDKey]
	if !ok || len(header) == 0 {
		return newRequestID()
	}

	requestID := header[0]
	if requestID == "" {
		return newRequestID()
	}

	return requestID
}

func HandleRequestIDChain(ctx context.Context) string {
	md, ok := metadata.FromContext(ctx)
	if !ok {
		return newRequestID()
	}

	header, ok := md[RequestIDKey]
	if !ok || len(header) == 0 {
		return newRequestID()
	}

	requestID := header[0]
	if requestID == "" {
		return newRequestID()
	}

	return fmt.Sprintf("%s,%s", requestID, newRequestID())
}

func newRequestID() string {
	return xid.New().String()
}

func UpdateContextWithRequestID(ctx context.Context, requestID string) context.Context {
	md := metadata.New(map[string]string{RequestIDKey: requestID})
	_md, ok := metadata.FromContext(ctx)
	if ok {
		md = metadata.Join(_md, md)
	}

	ctx = metadata.NewContext(ctx, md)
	ctx = context.WithValue(ctx, RequestIDKey, requestID)
	return ctx
}

func GetRequestID(ctx context.Context) string {
	md, ok := metadata.FromContext(ctx)
	if ok == false {
		return ""
	}

	header, ok := md[RequestIDKey]
	if !ok || len(header) == 0 {
		return ""
	}

	return header[0]
}

func GetRequestIDFromHTTPRequest(ctx context.Context, r *http.Request) (context.Context, string) {
	requestID := r.Header.Get(RequestIDKey)
	if requestID == "" {
		requestID = HandleRequestID(ctx)
	}

	ctx = context.WithValue(ctx, RequestIDKey, requestID)
	return ctx, requestID
}
<commit_msg>update<commit_after>package requestid

import (
	"fmt"
	"net/http"

	"github.com/rs/xid"
	"golang.org/x/net/context"
	"google.golang.org/grpc/metadata"
)

// RequestIDKey is metadata key name for request ID
var RequestIDKey = "request-id"

func HandleRequestID(ctx context.Context) string {
	md, ok := metadata.FromContext(ctx)
	if !ok {
		return newRequestID()
	}

	header, ok := md[RequestIDKey]
	if !ok || len(header) == 0 {
		return newRequestID()
	}

	requestID := header[0]
	if requestID == "" {
		return newRequestID()
	}

	return requestID
}

func HandleRequestIDChain(ctx context.Context) string {
	md, ok := metadata.FromContext(ctx)
	if !ok {
		return newRequestID()
	}

	header, ok := md[RequestIDKey]
	if !ok || len(header) == 0 {
		return newRequestID()
	}

	requestID := header[0]
	if requestID == "" {
		return newRequestID()
	}

	return fmt.Sprintf("%s,%s", requestID, newRequestID())
}

func newRequestID() string {
	return xid.New().String()
}

func UpdateContextWithRequestID(ctx context.Context, requestID string) context.Context {
	md := metadata.New(map[string]string{RequestIDKey: requestID})
	_md, ok := metadata.FromContext(ctx)
	if ok {
		md = metadata.Join(_md, md)
	}

	ctx = metadata.NewContext(ctx, md)
	ctx = context.WithValue(ctx, RequestIDKey, requestID)
	return ctx
}

func GetRequestID(ctx context.Context) string {
	md, ok := metadata.FromContext(ctx)
	if ok == false {
		return ""
	}

	header, ok := md[RequestIDKey]
	if !ok || len(header) == 0 {
		return ""
	}

	return header[0]
}

func GetRequestIDFromHTTPRequest(ctx context.Context, r *http.Request) (context.Context, string) {
	requestID := r.Header.Get(RequestIDKey)
	if requestID == "" {
		requestID = HandleRequestID(ctx)
	}

	ctx = context.WithValue(ctx, RequestIDKey, requestID)
	return ctx, requestID
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

// ClusterAutoscalerVersion contains version of CA.
const ClusterAutoscalerVersion = "0.5.3"
<commit_msg>Bump CA version to 0.5.4<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

// ClusterAutoscalerVersion contains version of CA.
const ClusterAutoscalerVersion = "0.5.4"
<|endoftext|>


<commit_before>package amazonec2

import (
	"errors"
)

type region struct {
	AmiId string
}

// Release 15.10 20151116.1
// See https://cloud-images.ubuntu.com/locator/ec2/
var regionDetails map[string]*region = map[string]*region{
	"ap-northeast-1": {"ami-b36d4edd"},
	"ap-southeast-1": {"ami-1069af73"},
	"ap-southeast-2": {"ami-1d336a7e"},
	"cn-north-1":     {"ami-79eb2214"},
	"eu-west-1":      {"ami-8aa67cf9"},
	"eu-central-1":   {"ami-ab0210c7"},
	"sa-east-1":      {"ami-185de774"},
	"us-east-1":      {"ami-26d5af4c"},
	"us-west-1":      {"ami-9cbcd2fc"},
	"us-west-2":      {"ami-16b1a077"},
	"us-gov-west-1":  {"ami-b0bad893"},
}

func awsRegionsList() []string {
	var list []string

	for k := range regionDetails {
		list = append(list, k)
	}

	return list
}

func validateAwsRegion(region string) (string, error) {
	for _, v := range awsRegionsList() {
		if v == region {
			return region, nil
		}
	}

	return "", errors.New("Invalid region specified")
}
<commit_msg>Fix: missing AMI for ap-northeast-2<commit_after>package amazonec2

import (
	"errors"
)

type region struct {
	AmiId string
}

// Release 15.10 20151116.1
// See https://cloud-images.ubuntu.com/locator/ec2/
var regionDetails map[string]*region = map[string]*region{
	"ap-northeast-1": {"ami-b36d4edd"},
	"ap-northeast-2": {"ami-09dc1267"},
	"ap-southeast-1": {"ami-1069af73"},
	"ap-southeast-2": {"ami-1d336a7e"},
	"cn-north-1":     {"ami-79eb2214"},
	"eu-west-1":      {"ami-8aa67cf9"},
	"eu-central-1":   {"ami-ab0210c7"},
	"sa-east-1":      {"ami-185de774"},
	"us-east-1":      {"ami-26d5af4c"},
	"us-west-1":      {"ami-9cbcd2fc"},
	"us-west-2":      {"ami-16b1a077"},
	"us-gov-west-1":  {"ami-b0bad893"},
}

func awsRegionsList() []string {
	var list []string

	for k := range regionDetails {
		list = append(list, k)
	}

	return list
}

func validateAwsRegion(region string) (string, error) {
	for _, v := range awsRegionsList() {
		if v == region {
			return region, nil
		}
	}

	return "", errors.New("Invalid region specified")
}
<|endoftext|>


<commit_before>package paxos

import (
	"log"
)

type Instance struct {
	cx Cluster

	vin  chan string
	v    string
	done chan int

	// Coordinator
	cPutter PutCloseProcessor

	// Acceptor
	aPutter PutCloser

	// Learner
	lPutter  PutCloser

	logger *log.Logger
}

func NewInstance(cx Cluster, logger *log.Logger) *Instance {
	c := NewC(cx)
	aIns, lIns := make(chan Message), make(chan Message)
	ins := &Instance{
		cx: cx,
		vin: make(chan string),
		done: make(chan int),
		cPutter: c,
		aPutter: ChanPutCloser(aIns),
		lPutter: ChanPutCloser(lIns),
		logger: logger,
	}

	go acceptor(aIns, cx)
	go func() {
		ins.v = learner(uint64(cx.Quorum()), lIns)
		close(ins.done)
	}()

	return ins
}

func (ins *Instance) Put(m Message) {
	ins.cPutter.Put(m)
	ins.aPutter.Put(m)
	ins.lPutter.Put(m)
}

func (ins *Instance) Value() string {
	<-ins.done
	return ins.v
}

func (ins *Instance) Close() {
	ins.cPutter.Close()
	ins.aPutter.Close()
	ins.lPutter.Close()
}

func (ins *Instance) Propose(v string) {
	// TODO make propose into a message type. This becomes:
	//   ins.cPutter.Put(...)
	go ins.cPutter.process(v)
}

<commit_msg>whitespace<commit_after>package paxos

import (
	"log"
)

type Instance struct {
	cx Cluster

	vin  chan string
	v    string
	done chan int

	// Coordinator
	cPutter PutCloseProcessor

	// Acceptor
	aPutter PutCloser

	// Learner
	lPutter  PutCloser

	logger *log.Logger
}

func NewInstance(cx Cluster, logger *log.Logger) *Instance {
	c := NewC(cx)
	aIns, lIns := make(chan Message), make(chan Message)
	ins := &Instance{
		cx: cx,
		vin: make(chan string),
		done: make(chan int),
		cPutter: c,
		aPutter: ChanPutCloser(aIns),
		lPutter: ChanPutCloser(lIns),
		logger: logger,
	}

	go acceptor(aIns, cx)
	go func() {
		ins.v = learner(uint64(cx.Quorum()), lIns)
		close(ins.done)
	}()

	return ins
}

func (ins *Instance) Put(m Message) {
	ins.cPutter.Put(m)
	ins.aPutter.Put(m)
	ins.lPutter.Put(m)
}

func (ins *Instance) Value() string {
	<-ins.done
	return ins.v
}

func (ins *Instance) Close() {
	ins.cPutter.Close()
	ins.aPutter.Close()
	ins.lPutter.Close()
}

func (ins *Instance) Propose(v string) {
	// TODO make propose into a message type. This becomes:
	//   ins.cPutter.Put(...)
	go ins.cPutter.process(v)
}
<|endoftext|>