<commit_before>package conditions

import (
	"fmt"

	"github.com/zimmski/tavor/log"
	"github.com/zimmski/tavor/rand"
	"github.com/zimmski/tavor/token"
	"github.com/zimmski/tavor/token/lists"
	"github.com/zimmski/tavor/token/primitives"
)

// BooleanExpression defines a boolean expression
type BooleanExpression interface {
	token.List

	// Evaluate evaluates the boolean expression and returns its result
	Evaluate() bool
}

// BooleanTrue implements a boolean expression which evaluates to always true
type BooleanTrue struct{}

// NewBooleanTrue returns a new instance of a BooleanTrue token
func NewBooleanTrue() *BooleanTrue {
	return &BooleanTrue{}
}

// Evaluate evaluates the boolean expression and returns its result
func (c *BooleanTrue) Evaluate() bool {
	return true
}

// Token interface methods

// Clone returns a copy of the token and all its children
func (c *BooleanTrue) Clone() token.Token {
	return &BooleanTrue{}
}

// Fuzz fuzzes this token using the random generator by choosing one of the possible permutations for this token
func (c *BooleanTrue) Fuzz(r rand.Rand) {
	// do nothing
}

// FuzzAll calls Fuzz for this token and then FuzzAll for all children of this token
func (c *BooleanTrue) FuzzAll(r rand.Rand) {
	// do nothing
}

// Parse tries to parse the token beginning from the current position in the parser data.
// If the parsing is successful the error argument is nil and the next current position after the token is returned.
func (c *BooleanTrue) Parse(pars *token.InternalParser, cur int) (int, []error) {
	panic("This should never happen")
}

// Permutation sets a specific permutation for this token
func (c *BooleanTrue) Permutation(i uint) error {
	// do nothing

	return nil
}

// Permutations returns the number of permutations for this token
func (c *BooleanTrue) Permutations() uint {
	return 1
}

// PermutationsAll returns the number of all possible permutations for this token including its children
func (c *BooleanTrue) PermutationsAll() uint {
	return 1
}

func (c *BooleanTrue) String() string {
	return "true"
}

// List interface methods

// Get returns the current referenced token at the given index. The error return argument is not nil, if the index is out of bound.
func (c *BooleanTrue) Get(i int) (token.Token, error) {
	return nil, &lists.ListError{
		Type: lists.ListErrorOutOfBound,
	}
}

// Len returns the number of the current referenced tokens
func (c *BooleanTrue) Len() int {
	return 0
}

// InternalGet returns the current referenced internal token at the given index. The error return argument is not nil, if the index is out of bound.
func (c *BooleanTrue) InternalGet(i int) (token.Token, error) {
	return nil, &lists.ListError{
		Type: lists.ListErrorOutOfBound,
	}
}

// InternalLen returns the number of referenced internal tokens
func (c *BooleanTrue) InternalLen() int {
	return 0
}

// InternalLogicalRemove removes the referenced internal token and returns the replacement for the current token or nil if the current token should be removed.
func (c *BooleanTrue) InternalLogicalRemove(tok token.Token) token.Token {
	panic("This should never happen")
}

// InternalReplace replaces an old with a new internal token if it is referenced by this token
func (c *BooleanTrue) InternalReplace(oldToken, newToken token.Token) {
	panic("This should never happen")
}

// BooleanEqual implements a boolean expression which compares the value of two tokens
type BooleanEqual struct {
	a, b token.Token
}

// NewBooleanEqual returns a new instance of a BooleanEqual token referencing two tokens
func NewBooleanEqual(a, b token.Token) *BooleanEqual {
	return &BooleanEqual{
		a: a,
		b: b,
	}
}

// Evaluate evaluates the boolean expression and returns its result
func (c *BooleanEqual) Evaluate() bool {
	return c.a.String() == c.b.String()
}

// Token interface methods

// Clone returns a copy of the token and all its children
func (c *BooleanEqual) Clone() token.Token {
	return &BooleanEqual{
		a: c.a,
		b: c.b,
	}
}

// Fuzz fuzzes this token using the random generator by choosing one of the possible permutations for this token
func (c *BooleanEqual) Fuzz(r rand.Rand) {
	// do nothing
}

// FuzzAll calls Fuzz for this token and then FuzzAll for all children of this token
func (c *BooleanEqual) FuzzAll(r rand.Rand) {
	// do nothing
}

// Parse tries to parse the token beginning from the current position in the parser data.
// If the parsing is successful the error argument is nil and the next current position after the token is returned.
func (c *BooleanEqual) Parse(pars *token.InternalParser, cur int) (int, []error) {
	panic("This should never happen")
}

// Permutation sets a specific permutation for this token
func (c *BooleanEqual) Permutation(i uint) error {
	// do nothing

	return nil
}

// Permutations returns the number of permutations for this token
func (c *BooleanEqual) Permutations() uint {
	return 1
}

// PermutationsAll returns the number of all possible permutations for this token including its children
func (c *BooleanEqual) PermutationsAll() uint {
	return 1
}

func (c *BooleanEqual) String() string {
	return fmt.Sprintf("(%p)%#v == (%p)%#v", c.a, c.a, c.b, c.b)
}

// List interface methods

// Get returns the current referenced token at the given index. The error return argument is not nil, if the index is out of bound.
func (c *BooleanEqual) Get(i int) (token.Token, error) {
	return nil, &lists.ListError{
		Type: lists.ListErrorOutOfBound,
	}
}

// Len returns the number of the current referenced tokens
func (c *BooleanEqual) Len() int {
	return 0
}

// InternalGet returns the current referenced internal token at the given index. The error return argument is not nil, if the index is out of bound.
func (c *BooleanEqual) InternalGet(i int) (token.Token, error) {
	switch i {
	case 0:
		return c.a, nil
	case 1:
		return c.b, nil
	default:
		return nil, &lists.ListError{
			Type: lists.ListErrorOutOfBound,
		}
	}
}

// InternalLen returns the number of referenced internal tokens
func (c *BooleanEqual) InternalLen() int {
	return 2
}

// InternalLogicalRemove removes the referenced internal token and returns the replacement for the current token or nil if the current token should be removed.
func (c *BooleanEqual) InternalLogicalRemove(tok token.Token) token.Token {
	if tok == c.a || tok == c.b {
		return nil
	}

	return c
}

// InternalReplace replaces an old with a new internal token if it is referenced by this token
func (c *BooleanEqual) InternalReplace(oldToken, newToken token.Token) {
	if oldToken == c.a {
		c.a = newToken
	}
	if oldToken == c.b {
		c.b = newToken
	}
}

// VariableDefined implements a boolean expression which evaluates if a variable is defined in a given scope
type VariableDefined struct {
	name          string
	variableScope map[string]token.Token
}

// NewVariableDefined returns a new instance of a VariableDefined token initialzed with the given name and scope
func NewVariableDefined(name string, variableScope map[string]token.Token) *VariableDefined {
	return &VariableDefined{
		name:          name,
		variableScope: variableScope,
	}
}

// Evaluate evaluates the boolean expression and returns its result
func (c *VariableDefined) Evaluate() bool {
	_, ok := c.variableScope[c.name]

	return ok
}

// Token interface methods

// Clone returns a copy of the token and all its children
func (c *VariableDefined) Clone() token.Token {
	return &VariableDefined{
		name:          c.name,
		variableScope: c.variableScope,
	}
}

// Fuzz fuzzes this token using the random generator by choosing one of the possible permutations for this token
func (c *VariableDefined) Fuzz(r rand.Rand) {
	// do nothing
}

// FuzzAll calls Fuzz for this token and then FuzzAll for all children of this token
func (c *VariableDefined) FuzzAll(r rand.Rand) {
	// do nothing
}

// Parse tries to parse the token beginning from the current position in the parser data.
// If the parsing is successful the error argument is nil and the next current position after the token is returned.
func (c *VariableDefined) Parse(pars *token.InternalParser, cur int) (int, []error) {
	panic("This should never happen")
}

// Permutation sets a specific permutation for this token
func (c *VariableDefined) Permutation(i uint) error {
	// do nothing

	return nil
}

// Permutations returns the number of permutations for this token
func (c *VariableDefined) Permutations() uint {
	return 1
}

// PermutationsAll returns the number of all possible permutations for this token including its children
func (c *VariableDefined) PermutationsAll() uint {
	return 1
}

func (c *VariableDefined) String() string {
	return fmt.Sprintf("defined(%q)", c.name)
}

// ScopeToken interface methods

// SetScope sets the scope of the token
func (c *VariableDefined) SetScope(variableScope map[string]token.Token) {
	nScope := make(map[string]token.Token, len(variableScope))
	for k, v := range variableScope {
		nScope[k] = v
	}

	c.variableScope = nScope
}

// ExpressionPointer implements a token pointer to an expression token
type ExpressionPointer struct {
	token token.Token
}

// NewExpressionPointer returns a new instance of a ExpressionPointer token referencing the given token
func NewExpressionPointer(token token.Token) *ExpressionPointer {
	return &ExpressionPointer{
		token: token,
	}
}

// Evaluate evaluates the boolean expression and returns its result
func (c *ExpressionPointer) Evaluate() bool {
	tok := c.token

	if po, ok := tok.(*primitives.Pointer); ok {
		log.Debugf("Found pointer in ExpressionPointer %p(%#v)", c, c)

		for {
			c := po.InternalGet()
			c = c.Clone()
			_ = po.Set(c)

			po, ok = c.(*primitives.Pointer)
			if !ok {
				log.Debugf("Replaced pointer %p(%#v) with %p(%#v)", tok, tok, c, c)

				tok = c

				break
			}
		}
	}

	if t, ok := tok.(BooleanExpression); ok {
		return t.Evaluate()
	}

	panic(fmt.Sprintf("token %p(%#v) does not implement BooleanExpression interface", c.token, c.token))
}

// Token interface methods

// Clone returns a copy of the token and all its children
func (c *ExpressionPointer) Clone() token.Token {
	return &ExpressionPointer{
		token: c.token.Clone(),
	}
}

// Fuzz fuzzes this token using the random generator by choosing one of the possible permutations for this token
func (c *ExpressionPointer) Fuzz(r rand.Rand) {
	// do nothing
}

// FuzzAll calls Fuzz for this token and then FuzzAll for all children of this token
func (c *ExpressionPointer) FuzzAll(r rand.Rand) {
	// do nothing
}

// Parse tries to parse the token beginning from the current position in the parser data.
// If the parsing is successful the error argument is nil and the next current position after the token is returned.
func (c *ExpressionPointer) Parse(pars *token.InternalParser, cur int) (int, []error) {
	panic("This should never happen")
}

// Permutation sets a specific permutation for this token
func (c *ExpressionPointer) Permutation(i uint) error {
	// do nothing

	return nil
}

// Permutations returns the number of permutations for this token
func (c *ExpressionPointer) Permutations() uint {
	return 1
}

// PermutationsAll returns the number of all possible permutations for this token including its children
func (c *ExpressionPointer) PermutationsAll() uint {
	return 1
}

func (c *ExpressionPointer) String() string {
	return c.token.String()
}

// ForwardToken interface methods

// Get returns the current referenced token
func (c *ExpressionPointer) Get() token.Token {
	return nil
}

// InternalGet returns the current referenced internal token
func (c *ExpressionPointer) InternalGet() token.Token {
	return c.token
}

// InternalLogicalRemove removes the referenced internal token and returns the replacement for the current token or nil if the current token should be removed.
func (c *ExpressionPointer) InternalLogicalRemove(tok token.Token) token.Token {
	if c.token == tok {
		return nil
	}

	return c
}

// InternalReplace replaces an old with a new internal token if it is referenced by this token
func (c *ExpressionPointer) InternalReplace(oldToken, newToken token.Token) {
	if c.token == oldToken {
		c.token = newToken
	}
}

// ScopeToken interface methods

// SetScope sets the scope of the token
func (c *ExpressionPointer) SetScope(variableScope map[string]token.Token) {
	tok := c.token

	if po, ok := tok.(*primitives.Pointer); ok {
		log.Debugf("Found pointer in ExpressionPointer %p(%#v)", c, c)

		for {
			c := po.InternalGet()
			c = c.Clone()
			_ = po.Set(c)

			po, ok = c.(*primitives.Pointer)
			if !ok {
				log.Debugf("Replaced pointer %p(%#v) with %p(%#v)", tok, tok, c, c)

				tok = c

				break
			}
		}
	}

	if t, ok := tok.(token.ScopeToken); ok {
		t.SetScope(variableScope)
	}
}
<commit_msg>revert, BooleanExpressions should just implement the Token interface<commit_after>package conditions

import (
	"fmt"

	"github.com/zimmski/tavor/log"
	"github.com/zimmski/tavor/rand"
	"github.com/zimmski/tavor/token"
	"github.com/zimmski/tavor/token/lists"
	"github.com/zimmski/tavor/token/primitives"
)

// BooleanExpression defines a boolean expression
type BooleanExpression interface {
	token.Token

	// Evaluate evaluates the boolean expression and returns its result
	Evaluate() bool
}

// BooleanTrue implements a boolean expression which evaluates to always true
type BooleanTrue struct{}

// NewBooleanTrue returns a new instance of a BooleanTrue token
func NewBooleanTrue() *BooleanTrue {
	return &BooleanTrue{}
}

// Evaluate evaluates the boolean expression and returns its result
func (c *BooleanTrue) Evaluate() bool {
	return true
}

// Token interface methods

// Clone returns a copy of the token and all its children
func (c *BooleanTrue) Clone() token.Token {
	return &BooleanTrue{}
}

// Fuzz fuzzes this token using the random generator by choosing one of the possible permutations for this token
func (c *BooleanTrue) Fuzz(r rand.Rand) {
	// do nothing
}

// FuzzAll calls Fuzz for this token and then FuzzAll for all children of this token
func (c *BooleanTrue) FuzzAll(r rand.Rand) {
	// do nothing
}

// Parse tries to parse the token beginning from the current position in the parser data.
// If the parsing is successful the error argument is nil and the next current position after the token is returned.
func (c *BooleanTrue) Parse(pars *token.InternalParser, cur int) (int, []error) {
	panic("This should never happen")
}

// Permutation sets a specific permutation for this token
func (c *BooleanTrue) Permutation(i uint) error {
	// do nothing

	return nil
}

// Permutations returns the number of permutations for this token
func (c *BooleanTrue) Permutations() uint {
	return 1
}

// PermutationsAll returns the number of all possible permutations for this token including its children
func (c *BooleanTrue) PermutationsAll() uint {
	return 1
}

func (c *BooleanTrue) String() string {
	return "true"
}

// List interface methods

// Get returns the current referenced token at the given index. The error return argument is not nil, if the index is out of bound.
func (c *BooleanTrue) Get(i int) (token.Token, error) {
	return nil, &lists.ListError{
		Type: lists.ListErrorOutOfBound,
	}
}

// Len returns the number of the current referenced tokens
func (c *BooleanTrue) Len() int {
	return 0
}

// InternalGet returns the current referenced internal token at the given index. The error return argument is not nil, if the index is out of bound.
func (c *BooleanTrue) InternalGet(i int) (token.Token, error) {
	return nil, &lists.ListError{
		Type: lists.ListErrorOutOfBound,
	}
}

// InternalLen returns the number of referenced internal tokens
func (c *BooleanTrue) InternalLen() int {
	return 0
}

// InternalLogicalRemove removes the referenced internal token and returns the replacement for the current token or nil if the current token should be removed.
func (c *BooleanTrue) InternalLogicalRemove(tok token.Token) token.Token {
	panic("This should never happen")
}

// InternalReplace replaces an old with a new internal token if it is referenced by this token
func (c *BooleanTrue) InternalReplace(oldToken, newToken token.Token) {
	panic("This should never happen")
}

// BooleanEqual implements a boolean expression which compares the value of two tokens
type BooleanEqual struct {
	a, b token.Token
}

// NewBooleanEqual returns a new instance of a BooleanEqual token referencing two tokens
func NewBooleanEqual(a, b token.Token) *BooleanEqual {
	return &BooleanEqual{
		a: a,
		b: b,
	}
}

// Evaluate evaluates the boolean expression and returns its result
func (c *BooleanEqual) Evaluate() bool {
	return c.a.String() == c.b.String()
}

// Token interface methods

// Clone returns a copy of the token and all its children
func (c *BooleanEqual) Clone() token.Token {
	return &BooleanEqual{
		a: c.a,
		b: c.b,
	}
}

// Fuzz fuzzes this token using the random generator by choosing one of the possible permutations for this token
func (c *BooleanEqual) Fuzz(r rand.Rand) {
	// do nothing
}

// FuzzAll calls Fuzz for this token and then FuzzAll for all children of this token
func (c *BooleanEqual) FuzzAll(r rand.Rand) {
	// do nothing
}

// Parse tries to parse the token beginning from the current position in the parser data.
// If the parsing is successful the error argument is nil and the next current position after the token is returned.
func (c *BooleanEqual) Parse(pars *token.InternalParser, cur int) (int, []error) {
	panic("This should never happen")
}

// Permutation sets a specific permutation for this token
func (c *BooleanEqual) Permutation(i uint) error {
	// do nothing

	return nil
}

// Permutations returns the number of permutations for this token
func (c *BooleanEqual) Permutations() uint {
	return 1
}

// PermutationsAll returns the number of all possible permutations for this token including its children
func (c *BooleanEqual) PermutationsAll() uint {
	return 1
}

func (c *BooleanEqual) String() string {
	return fmt.Sprintf("(%p)%#v == (%p)%#v", c.a, c.a, c.b, c.b)
}

// List interface methods

// Get returns the current referenced token at the given index. The error return argument is not nil, if the index is out of bound.
func (c *BooleanEqual) Get(i int) (token.Token, error) {
	return nil, &lists.ListError{
		Type: lists.ListErrorOutOfBound,
	}
}

// Len returns the number of the current referenced tokens
func (c *BooleanEqual) Len() int {
	return 0
}

// InternalGet returns the current referenced internal token at the given index. The error return argument is not nil, if the index is out of bound.
func (c *BooleanEqual) InternalGet(i int) (token.Token, error) {
	switch i {
	case 0:
		return c.a, nil
	case 1:
		return c.b, nil
	default:
		return nil, &lists.ListError{
			Type: lists.ListErrorOutOfBound,
		}
	}
}

// InternalLen returns the number of referenced internal tokens
func (c *BooleanEqual) InternalLen() int {
	return 2
}

// InternalLogicalRemove removes the referenced internal token and returns the replacement for the current token or nil if the current token should be removed.
func (c *BooleanEqual) InternalLogicalRemove(tok token.Token) token.Token {
	if tok == c.a || tok == c.b {
		return nil
	}

	return c
}

// InternalReplace replaces an old with a new internal token if it is referenced by this token
func (c *BooleanEqual) InternalReplace(oldToken, newToken token.Token) {
	if oldToken == c.a {
		c.a = newToken
	}
	if oldToken == c.b {
		c.b = newToken
	}
}

// VariableDefined implements a boolean expression which evaluates if a variable is defined in a given scope
type VariableDefined struct {
	name          string
	variableScope map[string]token.Token
}

// NewVariableDefined returns a new instance of a VariableDefined token initialzed with the given name and scope
func NewVariableDefined(name string, variableScope map[string]token.Token) *VariableDefined {
	return &VariableDefined{
		name:          name,
		variableScope: variableScope,
	}
}

// Evaluate evaluates the boolean expression and returns its result
func (c *VariableDefined) Evaluate() bool {
	_, ok := c.variableScope[c.name]

	return ok
}

// Token interface methods

// Clone returns a copy of the token and all its children
func (c *VariableDefined) Clone() token.Token {
	return &VariableDefined{
		name:          c.name,
		variableScope: c.variableScope,
	}
}

// Fuzz fuzzes this token using the random generator by choosing one of the possible permutations for this token
func (c *VariableDefined) Fuzz(r rand.Rand) {
	// do nothing
}

// FuzzAll calls Fuzz for this token and then FuzzAll for all children of this token
func (c *VariableDefined) FuzzAll(r rand.Rand) {
	// do nothing
}

// Parse tries to parse the token beginning from the current position in the parser data.
// If the parsing is successful the error argument is nil and the next current position after the token is returned.
func (c *VariableDefined) Parse(pars *token.InternalParser, cur int) (int, []error) {
	panic("This should never happen")
}

// Permutation sets a specific permutation for this token
func (c *VariableDefined) Permutation(i uint) error {
	// do nothing

	return nil
}

// Permutations returns the number of permutations for this token
func (c *VariableDefined) Permutations() uint {
	return 1
}

// PermutationsAll returns the number of all possible permutations for this token including its children
func (c *VariableDefined) PermutationsAll() uint {
	return 1
}

func (c *VariableDefined) String() string {
	return fmt.Sprintf("defined(%q)", c.name)
}

// ScopeToken interface methods

// SetScope sets the scope of the token
func (c *VariableDefined) SetScope(variableScope map[string]token.Token) {
	nScope := make(map[string]token.Token, len(variableScope))
	for k, v := range variableScope {
		nScope[k] = v
	}

	c.variableScope = nScope
}

// ExpressionPointer implements a token pointer to an expression token
type ExpressionPointer struct {
	token token.Token
}

// NewExpressionPointer returns a new instance of a ExpressionPointer token referencing the given token
func NewExpressionPointer(token token.Token) *ExpressionPointer {
	return &ExpressionPointer{
		token: token,
	}
}

// Evaluate evaluates the boolean expression and returns its result
func (c *ExpressionPointer) Evaluate() bool {
	tok := c.token

	if po, ok := tok.(*primitives.Pointer); ok {
		log.Debugf("Found pointer in ExpressionPointer %p(%#v)", c, c)

		for {
			c := po.InternalGet()
			c = c.Clone()
			_ = po.Set(c)

			po, ok = c.(*primitives.Pointer)
			if !ok {
				log.Debugf("Replaced pointer %p(%#v) with %p(%#v)", tok, tok, c, c)

				tok = c

				break
			}
		}
	}

	if t, ok := tok.(BooleanExpression); ok {
		return t.Evaluate()
	}

	panic(fmt.Sprintf("token %p(%#v) does not implement BooleanExpression interface", c.token, c.token))
}

// Token interface methods

// Clone returns a copy of the token and all its children
func (c *ExpressionPointer) Clone() token.Token {
	return &ExpressionPointer{
		token: c.token.Clone(),
	}
}

// Fuzz fuzzes this token using the random generator by choosing one of the possible permutations for this token
func (c *ExpressionPointer) Fuzz(r rand.Rand) {
	// do nothing
}

// FuzzAll calls Fuzz for this token and then FuzzAll for all children of this token
func (c *ExpressionPointer) FuzzAll(r rand.Rand) {
	// do nothing
}

// Parse tries to parse the token beginning from the current position in the parser data.
// If the parsing is successful the error argument is nil and the next current position after the token is returned.
func (c *ExpressionPointer) Parse(pars *token.InternalParser, cur int) (int, []error) {
	panic("This should never happen")
}

// Permutation sets a specific permutation for this token
func (c *ExpressionPointer) Permutation(i uint) error {
	// do nothing

	return nil
}

// Permutations returns the number of permutations for this token
func (c *ExpressionPointer) Permutations() uint {
	return 1
}

// PermutationsAll returns the number of all possible permutations for this token including its children
func (c *ExpressionPointer) PermutationsAll() uint {
	return 1
}

func (c *ExpressionPointer) String() string {
	return c.token.String()
}

// ForwardToken interface methods

// Get returns the current referenced token
func (c *ExpressionPointer) Get() token.Token {
	return nil
}

// InternalGet returns the current referenced internal token
func (c *ExpressionPointer) InternalGet() token.Token {
	return c.token
}

// InternalLogicalRemove removes the referenced internal token and returns the replacement for the current token or nil if the current token should be removed.
func (c *ExpressionPointer) InternalLogicalRemove(tok token.Token) token.Token {
	if c.token == tok {
		return nil
	}

	return c
}

// InternalReplace replaces an old with a new internal token if it is referenced by this token
func (c *ExpressionPointer) InternalReplace(oldToken, newToken token.Token) {
	if c.token == oldToken {
		c.token = newToken
	}
}

// ScopeToken interface methods

// SetScope sets the scope of the token
func (c *ExpressionPointer) SetScope(variableScope map[string]token.Token) {
	tok := c.token

	if po, ok := tok.(*primitives.Pointer); ok {
		log.Debugf("Found pointer in ExpressionPointer %p(%#v)", c, c)

		for {
			c := po.InternalGet()
			c = c.Clone()
			_ = po.Set(c)

			po, ok = c.(*primitives.Pointer)
			if !ok {
				log.Debugf("Replaced pointer %p(%#v) with %p(%#v)", tok, tok, c, c)

				tok = c

				break
			}
		}
	}

	if t, ok := tok.(token.ScopeToken); ok {
		t.SetScope(variableScope)
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package web

import (
	"net/http"
	"net/http/httptest"
	"testing"
	"time"

	"github.com/issue9/assert"
)

var f1 = func(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(1)
}

var h1 = http.HandlerFunc(f1)

func TestMiddleware(t *testing.T) {
	a := assert.New(t)
	m := func(h http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			w.Header().Set("Date", "1111")
			h.ServeHTTP(w, r)
		})
	}
	app, err := newApp("./testdata", m)
	a.NotError(err).NotNil(app)

	app.router.GetFunc("/middleware", f1)
	go func() {
		// 不判断返回值，在被关闭或是重启时，会返回 http.ErrServerClosed 错误
		app.Run()
	}()

	// 等待 Run() 启动完毕，不同机器可能需要的时间会不同
	time.Sleep(500 * time.Millisecond)

	// 正常访问
	resp, err := http.Get("http://localhost:8082/middleware")
	a.NotError(err).NotNil(resp)
	a.Equal(resp.Header.Get("Date"), "1111")
	app.Close()
}

func TestApp_Close(t *testing.T) {
	a := assert.New(t)
	app, err := newApp("./testdata", nil)
	a.NotError(err).NotNil(app)

	app.mux.GetFunc("/test", f1)
	app.mux.GetFunc("/close", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("closed"))
		app.Close()
	})

	go func() {
		err := app.Run()
		a.Error(err).ErrorType(err, http.ErrServerClosed, "错误信息为:%v", err)
	}()

	// 等待 app.run() 启动完毕，不同机器可能需要的时间会不同
	time.Sleep(50 * time.Microsecond)

	resp, err := http.Get("http://localhost:8082/test")
	a.NotError(err).NotNil(resp)
	a.Equal(resp.StatusCode, 1)

	resp, err = http.Get("http://localhost:8082/close")
	a.Error(err).Nil(resp)

	resp, err = http.Get("http://localhost:8082/test")
	a.Error(err).Nil(resp)
}

func TestApp_Shutdown_timeout(t *testing.T) {
	a := assert.New(t)
	app, err := newApp("./testdata", nil)
	a.NotError(err).NotNil(app)

	app.mux.GetFunc("/test", f1)
	app.mux.GetFunc("/close", func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusCreated)
		w.Write([]byte("closed"))
		app.Shutdown()
	})

	go func() {
		err := app.Run()
		a.Error(err).ErrorType(err, http.ErrServerClosed, "错误信息为:%v", err)
	}()

	// 等待 app.run() 启动完毕，不同机器可能需要的时间会不同
	time.Sleep(50 * time.Microsecond)

	resp, err := http.Get("http://localhost:8082/test")
	a.NotError(err).NotNil(resp)
	a.Equal(resp.StatusCode, 1)

	// 关闭指令可以正常执行
	resp, err = http.Get("http://localhost:8082/close")
	a.NotError(err).NotNil(resp)
	a.Equal(resp.StatusCode, http.StatusCreated)

	// 拒绝访问
	resp, err = http.Get("http://localhost:8082/test")
	a.Error(err).Nil(resp)

	// 已被关闭
	time.Sleep(30 * time.Microsecond)
	resp, err = http.Get("http://localhost:8082/test")
	a.Error(err).Nil(resp)
}

func TestApp_Run(t *testing.T) {
	a := assert.New(t)
	app, err := newApp("./testdata", nil)
	a.NotError(err).NotNil(app)

	app.mux.GetFunc("/test", f1)

	go func() {
		err := app.Run()
		a.ErrorType(err, http.ErrServerClosed, "assert.ErrorType 错误，%v", err.Error())
	}()

	time.Sleep(50 * time.Microsecond)
	resp, err := http.Get("http://localhost:8082/test")
	a.NotError(err).NotNil(resp)
	a.Equal(resp.StatusCode, 1)

	resp, err = http.Get("http://localhost:8082/client/file1.txt")
	a.NotError(err).NotNil(resp)
	a.Equal(resp.StatusCode, http.StatusOK)

	resp, err = http.Get("http://localhost:8082/client/dir/file2.txt")
	a.NotError(err).NotNil(resp)
	a.Equal(resp.StatusCode, http.StatusOK)

	app.Close()
}

func TestApp_NewContext(t *testing.T) {
	a := assert.New(t)
	r := httptest.NewRequest(http.MethodGet, "/path", nil)
	w := httptest.NewRecorder()
	app, err := newApp("./testdata", nil)
	a.NotError(err).NotNil(app)

	// 少报头 accept
	ctx := app.NewContext(w, r)
	a.Nil(ctx)

	r = httptest.NewRequest(http.MethodGet, "/path", nil)
	r.Header.Set("Accept", "*/*")
	ctx = app.NewContext(w, r)
	a.NotNil(ctx)
}
<commit_msg>修正测试错误<commit_after>// Copyright 2017 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package web

import (
	"net/http"
	"net/http/httptest"
	"testing"
	"time"

	"github.com/issue9/assert"
)

var f1 = func(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(1)
}

var h1 = http.HandlerFunc(f1)

func TestMiddleware(t *testing.T) {
	a := assert.New(t)
	m := func(h http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			w.Header().Set("Date", "1111")
			h.ServeHTTP(w, r)
		})
	}
	app, err := newApp("./testdata", m)
	a.NotError(err).NotNil(app)

	app.router.GetFunc("/middleware", f1)
	go func() {
		// 不判断返回值，在被关闭或是重启时，会返回 http.ErrServerClosed 错误
		app.Run()
	}()

	// 等待 Run() 启动完毕，不同机器可能需要的时间会不同
	time.Sleep(500 * time.Millisecond)

	// 正常访问
	resp, err := http.Get("http://localhost:8082/middleware")
	a.NotError(err).NotNil(resp)
	a.Equal(resp.Header.Get("Date"), "1111")
	app.Close()
}

func TestApp_Close(t *testing.T) {
	a := assert.New(t)
	app, err := newApp("./testdata", nil)
	a.NotError(err).NotNil(app)

	app.mux.GetFunc("/test", f1)
	app.mux.GetFunc("/close", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("closed"))
		app.Close()
	})

	go func() {
		err := app.Run()
		a.Error(err).ErrorType(err, http.ErrServerClosed, "错误信息为:%v", err)
	}()

	// 等待 app.run() 启动完毕，不同机器可能需要的时间会不同
	time.Sleep(500 * time.Microsecond)

	resp, err := http.Get("http://localhost:8082/test")
	a.NotError(err).NotNil(resp)
	a.Equal(resp.StatusCode, 1)

	resp, err = http.Get("http://localhost:8082/close")
	a.Error(err).Nil(resp)

	resp, err = http.Get("http://localhost:8082/test")
	a.Error(err).Nil(resp)
}

func TestApp_Shutdown_timeout(t *testing.T) {
	a := assert.New(t)
	app, err := newApp("./testdata", nil)
	a.NotError(err).NotNil(app)

	app.mux.GetFunc("/test", f1)
	app.mux.GetFunc("/close", func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusCreated)
		w.Write([]byte("closed"))
		app.Shutdown()
	})

	go func() {
		err := app.Run()
		a.Error(err).ErrorType(err, http.ErrServerClosed, "错误信息为:%v", err)
	}()

	// 等待 app.run() 启动完毕，不同机器可能需要的时间会不同
	time.Sleep(500 * time.Microsecond)

	resp, err := http.Get("http://localhost:8082/test")
	a.NotError(err).NotNil(resp)
	a.Equal(resp.StatusCode, 1)

	// 关闭指令可以正常执行
	resp, err = http.Get("http://localhost:8082/close")
	a.NotError(err).NotNil(resp)
	a.Equal(resp.StatusCode, http.StatusCreated)

	// 拒绝访问
	resp, err = http.Get("http://localhost:8082/test")
	a.Error(err).Nil(resp)

	// 已被关闭
	time.Sleep(30 * time.Microsecond)
	resp, err = http.Get("http://localhost:8082/test")
	a.Error(err).Nil(resp)
}

func TestApp_Run(t *testing.T) {
	a := assert.New(t)
	app, err := newApp("./testdata", nil)
	a.NotError(err).NotNil(app)

	app.mux.GetFunc("/test", f1)

	go func() {
		err := app.Run()
		a.ErrorType(err, http.ErrServerClosed, "assert.ErrorType 错误，%v", err.Error())
	}()

	time.Sleep(500 * time.Microsecond)
	resp, err := http.Get("http://localhost:8082/test")
	a.NotError(err).NotNil(resp)
	a.Equal(resp.StatusCode, 1)

	resp, err = http.Get("http://localhost:8082/client/file1.txt")
	a.NotError(err).NotNil(resp)
	a.Equal(resp.StatusCode, http.StatusOK)

	resp, err = http.Get("http://localhost:8082/client/dir/file2.txt")
	a.NotError(err).NotNil(resp)
	a.Equal(resp.StatusCode, http.StatusOK)

	app.Close()
}

func TestApp_NewContext(t *testing.T) {
	a := assert.New(t)
	r := httptest.NewRequest(http.MethodGet, "/path", nil)
	w := httptest.NewRecorder()
	app, err := newApp("./testdata", nil)
	a.NotError(err).NotNil(app)

	// 少报头 accept
	ctx := app.NewContext(w, r)
	a.Nil(ctx)

	r = httptest.NewRequest(http.MethodGet, "/path", nil)
	r.Header.Set("Accept", "*/*")
	ctx = app.NewContext(w, r)
	a.NotNil(ctx)
}
<|endoftext|>


<commit_before>// Copyright (C) 2020 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// The embed command is used to embed text files into Go executables as strings.
package main

import (
	"context"
	"encoding/csv"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"strconv"

	"github.com/google/gapid/core/app"
	"github.com/google/gapid/core/log"
	"github.com/google/gapid/core/os/file"
	"github.com/google/gapid/core/os/shell"
)

var (
	gapit string
	root  string
)

func main() {
	flag.StringVar(&gapit, "gapit", "gapit", "the path to the gapit command")
	flag.StringVar(&root, "root", "", "the root directory to resolves paths against")
	app.ShortHelp = "benchmark: A tool to run and summarize gapit benchmarks."
	app.Run(run)
}

func run(ctx context.Context) error {
	cfg, err := readConfig()
	if err != nil {
		return err
	}

	tmpOut, err := ioutil.TempDir("", "benchmark")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tmpOut)

	res := []*results{}
	for i := range cfg.Traces {
		r, err := runTrace(ctx, cfg, i, file.Abs(tmpOut))
		if err != nil {
			return err
		}
		res = append(res, r)
	}

	fmt.Println("------------------------")
	printResults(res)
	fmt.Println("------------------------")
	printSummary(res)

	return nil
}

type results struct {
	name   string
	titles []string
	values [][]float64
}

func (r *results) append(path file.Path) error {
	in, err := os.Open(path.System())
	if err != nil {
		return err
	}
	defer in.Close()

	records, err := csv.NewReader(in).ReadAll()
	if err != nil {
		return err
	}

	if len(records) != 2 {
		return fmt.Errorf("Expected two summary rows, got %d", len(records))
	}

	if r.titles != nil && len(r.titles) != len(records[0]) {
		return fmt.Errorf("Unmatched number of titles: got %d, expected %d", len(records[0]), len(r.titles))
	}
	r.titles = records[0]

	if len(records[0]) != len(records[1]) {
		return fmt.Errorf("Unmatched number of values: got %d, expected %d", len(records[1]), len(records[0]))
	}

	values := make([]float64, len(records[1]))
	for i := range values {
		v, err := strconv.ParseFloat(records[1][i], 64)
		if err != nil {
			return fmt.Errorf("Failed to parse summary value \"%s\": %v", records[1][i], err)
		}
		values[i] = v
	}
	r.values = append(r.values, values)

	return nil
}

func runTrace(ctx context.Context, cfg *config, idx int, tmpOut file.Path) (*results, error) {
	trace := cfg.Traces[idx]
	log.I(ctx, "Measuring %v (%v)...", trace.Name, trace.Input)

	res := &results{
		name: trace.Name,
	}
	for i := 0; i < cfg.Iterations; i++ {
		summaryOut := tmpOut.Join(fmt.Sprintf("summry_%d_%d.csv", idx, i))
		args := []string{
			"benchmark2",
			"--numdraws", strconv.Itoa(cfg.Draws),
			"--summaryout", summaryOut.System(),
		}
		if trace.Secondary {
			args = append(args, "--secondary")
		}
		for _, path := range trace.Paths {
			args = append(args, "--paths", path)
		}
		args = append(args, file.Abs(root).Join(trace.Input).System())

		err := shell.Command(gapit, args...).
			Verbose().
			In(root).
			Run(ctx)
		if err != nil {
			return nil, err
		}

		if err := res.append(summaryOut); err != nil {
			return nil, err
		}
	}

	return res, nil
}

type config struct {
	Iterations int
	Draws      int
	Traces     []struct {
		Name      string
		Input     string
		Paths     []string
		Secondary bool
	}
}

func readConfig() (*config, error) {
	args := flag.Args()
	if len(args) != 1 {
		return nil, errors.New("Expected a config file as a paramter")
	}

	in, err := os.Open(args[0])
	if err != nil {
		return nil, err
	}
	defer in.Close()

	dec := json.NewDecoder(in)
	c := config{
		Iterations: 1,
	}
	if err := dec.Decode(&c); err != nil {
		return nil, err
	}

	if len(c.Traces) == 0 {
		return nil, errors.New("No traces in config file, expected at least one")
	}

	return &c, nil
}

func printResults(rs []*results) {
	out := csv.NewWriter(os.Stdout)
	out.Write(append([]string{"Application"}, rs[0].titles...))

	for _, r := range rs {
		for _, vs := range r.values {
			printResultRow(out, r.name, vs)
		}
	}

	out.Flush()
}

func printSummary(rs []*results) {
	out := csv.NewWriter(os.Stdout)
	out.Write(append([]string{"Application"}, rs[0].titles...))

	for _, r := range rs {
		avgs := make([]float64, len(r.values[0]))
		for _, vs := range r.values {
			for i := range avgs {
				avgs[i] += vs[i]
			}
		}
		for i := range avgs {
			avgs[i] /= float64(len(r.values))
		}
		printResultRow(out, r.name, avgs)
	}

	out.Flush()
}

func printResultRow(out *csv.Writer, name string, row []float64) {
	r := make([]string, 1+len(row))
	r[0] = name
	for i, v := range row {
		r[i+1] = fmt.Sprintf("%0.3f", v)
	}
	out.Write(r)
}
<commit_msg>Fix benchmark runner's invokation of gapit benchmark.<commit_after>// Copyright (C) 2020 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// The embed command is used to embed text files into Go executables as strings.
package main

import (
	"context"
	"encoding/csv"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"strconv"

	"github.com/google/gapid/core/app"
	"github.com/google/gapid/core/log"
	"github.com/google/gapid/core/os/file"
	"github.com/google/gapid/core/os/shell"
)

var (
	gapit string
	root  string
)

func main() {
	flag.StringVar(&gapit, "gapit", "gapit", "the path to the gapit command")
	flag.StringVar(&root, "root", "", "the root directory to resolves paths against")
	app.ShortHelp = "benchmark: A tool to run and summarize gapit benchmarks."
	app.Run(run)
}

func run(ctx context.Context) error {
	cfg, err := readConfig()
	if err != nil {
		return err
	}

	tmpOut, err := ioutil.TempDir("", "benchmark")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tmpOut)

	res := []*results{}
	for i := range cfg.Traces {
		r, err := runTrace(ctx, cfg, i, file.Abs(tmpOut))
		if err != nil {
			return err
		}
		res = append(res, r)
	}

	fmt.Println("------------------------")
	printResults(res)
	fmt.Println("------------------------")
	printSummary(res)

	return nil
}

type results struct {
	name   string
	titles []string
	values [][]float64
}

func (r *results) append(path file.Path) error {
	in, err := os.Open(path.System())
	if err != nil {
		return err
	}
	defer in.Close()

	records, err := csv.NewReader(in).ReadAll()
	if err != nil {
		return err
	}

	if len(records) != 2 {
		return fmt.Errorf("Expected two summary rows, got %d", len(records))
	}

	if r.titles != nil && len(r.titles) != len(records[0]) {
		return fmt.Errorf("Unmatched number of titles: got %d, expected %d", len(records[0]), len(r.titles))
	}
	r.titles = records[0]

	if len(records[0]) != len(records[1]) {
		return fmt.Errorf("Unmatched number of values: got %d, expected %d", len(records[1]), len(records[0]))
	}

	values := make([]float64, len(records[1]))
	for i := range values {
		v, err := strconv.ParseFloat(records[1][i], 64)
		if err != nil {
			return fmt.Errorf("Failed to parse summary value \"%s\": %v", records[1][i], err)
		}
		values[i] = v
	}
	r.values = append(r.values, values)

	return nil
}

func runTrace(ctx context.Context, cfg *config, idx int, tmpOut file.Path) (*results, error) {
	trace := cfg.Traces[idx]
	log.I(ctx, "Measuring %v (%v)...", trace.Name, trace.Input)

	res := &results{
		name: trace.Name,
	}
	for i := 0; i < cfg.Iterations; i++ {
		summaryOut := tmpOut.Join(fmt.Sprintf("summry_%d_%d.csv", idx, i))
		args := []string{
			"benchmark",
			"--numdraws", strconv.Itoa(cfg.Draws),
			"--summaryout", summaryOut.System(),
		}
		if trace.Secondary {
			args = append(args, "--secondary")
		}
		for _, path := range trace.Paths {
			args = append(args, "--paths", path)
		}
		args = append(args, file.Abs(root).Join(trace.Input).System())

		err := shell.Command(gapit, args...).
			Verbose().
			In(root).
			Run(ctx)
		if err != nil {
			return nil, err
		}

		if err := res.append(summaryOut); err != nil {
			return nil, err
		}
	}

	return res, nil
}

type config struct {
	Iterations int
	Draws      int
	Traces     []struct {
		Name      string
		Input     string
		Paths     []string
		Secondary bool
	}
}

func readConfig() (*config, error) {
	args := flag.Args()
	if len(args) != 1 {
		return nil, errors.New("Expected a config file as a paramter")
	}

	in, err := os.Open(args[0])
	if err != nil {
		return nil, err
	}
	defer in.Close()

	dec := json.NewDecoder(in)
	c := config{
		Iterations: 1,
	}
	if err := dec.Decode(&c); err != nil {
		return nil, err
	}

	if len(c.Traces) == 0 {
		return nil, errors.New("No traces in config file, expected at least one")
	}

	return &c, nil
}

func printResults(rs []*results) {
	out := csv.NewWriter(os.Stdout)
	out.Write(append([]string{"Application"}, rs[0].titles...))

	for _, r := range rs {
		for _, vs := range r.values {
			printResultRow(out, r.name, vs)
		}
	}

	out.Flush()
}

func printSummary(rs []*results) {
	out := csv.NewWriter(os.Stdout)
	out.Write(append([]string{"Application"}, rs[0].titles...))

	for _, r := range rs {
		avgs := make([]float64, len(r.values[0]))
		for _, vs := range r.values {
			for i := range avgs {
				avgs[i] += vs[i]
			}
		}
		for i := range avgs {
			avgs[i] /= float64(len(r.values))
		}
		printResultRow(out, r.name, avgs)
	}

	out.Flush()
}

func printResultRow(out *csv.Writer, name string, row []float64) {
	r := make([]string, 1+len(row))
	r[0] = name
	for i, v := range row {
		r[i+1] = fmt.Sprintf("%0.3f", v)
	}
	out.Write(r)
}
<|endoftext|>


<commit_before>/*
 * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"context"
	"encoding/hex"
	"encoding/json"
	"net/http"
	"strings"
	"sync"
	"time"

	minio "github.com/minio/minio-go/v7"
	miniogo "github.com/minio/minio-go/v7"
	"github.com/minio/minio-go/v7/pkg/credentials"
	"github.com/minio/minio/cmd/crypto"
	"github.com/minio/minio/pkg/bucket/versioning"
	"github.com/minio/minio/pkg/madmin"
	sha256 "github.com/minio/sha256-simd"
)

// BucketTargetSys represents bucket targets subsystem
type BucketTargetSys struct {
	sync.RWMutex
	arnRemotesMap map[string]*miniogo.Core
	targetsMap    map[string][]madmin.BucketTarget
}

// ListTargets lists bucket targets across tenant or for individual bucket, and returns
// results filtered by arnType
func (sys *BucketTargetSys) ListTargets(ctx context.Context, bucket, arnType string) (targets []madmin.BucketTarget) {
	if bucket != "" {
		if ts, err := sys.ListBucketTargets(ctx, bucket); err == nil {
			for _, t := range ts.Targets {
				if string(t.Type) == arnType || arnType == "" {
					targets = append(targets, t.Clone())
				}
			}
		}
		return targets
	}
	sys.RLock()
	defer sys.RUnlock()
	for _, tgts := range sys.targetsMap {
		for _, t := range tgts {
			if string(t.Type) == arnType || arnType == "" {
				targets = append(targets, t.Clone())
			}
		}
	}
	return
}

// ListBucketTargets - gets list of bucket targets for this bucket.
func (sys *BucketTargetSys) ListBucketTargets(ctx context.Context, bucket string) (*madmin.BucketTargets, error) {
	sys.RLock()
	defer sys.RUnlock()

	tgts, ok := sys.targetsMap[bucket]
	if ok {
		return &madmin.BucketTargets{Targets: tgts}, nil
	}
	return nil, BucketRemoteTargetNotFound{Bucket: bucket}
}

// SetTarget - sets a new minio-go client target for this bucket.
func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *madmin.BucketTarget, update bool) error {
	if globalIsGateway {
		return nil
	}
	if !tgt.Type.IsValid() && !update {
		return BucketRemoteArnTypeInvalid{Bucket: bucket}
	}
	clnt, err := sys.getRemoteTargetClient(tgt)
	if err != nil {
		return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
	}
	// validate if target credentials are ok
	if _, err = clnt.BucketExists(ctx, tgt.TargetBucket); err != nil {
		if minio.ToErrorResponse(err).Code == "NoSuchBucket" {
			return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
		}
		return BucketRemoteConnectionErr{Bucket: tgt.TargetBucket}
	}
	if tgt.Type == madmin.ReplicationService {
		if !globalIsErasure {
			return NotImplemented{}
		}
		if !globalBucketVersioningSys.Enabled(bucket) {
			return BucketReplicationSourceNotVersioned{Bucket: bucket}
		}
		vcfg, err := clnt.GetBucketVersioning(ctx, tgt.TargetBucket)
		if err != nil {
			return BucketRemoteConnectionErr{Bucket: tgt.TargetBucket}
		}
		if vcfg.Status != string(versioning.Enabled) {
			return BucketRemoteTargetNotVersioned{Bucket: tgt.TargetBucket}
		}
	}
	if tgt.Type == madmin.ILMService {
		if globalBucketVersioningSys.Enabled(bucket) {
			vcfg, err := clnt.GetBucketVersioning(ctx, tgt.TargetBucket)
			if err != nil {
				if minio.ToErrorResponse(err).Code == "NoSuchBucket" {
					return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
				}
				return BucketRemoteConnectionErr{Bucket: tgt.TargetBucket}
			}
			if vcfg.Status != string(versioning.Enabled) {
				return BucketRemoteTargetNotVersioned{Bucket: tgt.TargetBucket}
			}
		}
	}
	sys.Lock()
	defer sys.Unlock()

	tgts, ok := sys.targetsMap[bucket]
	if !ok {
		return BucketRemoteTargetNotFound{Bucket: bucket}
	}

	newtgts := make([]madmin.BucketTarget, len(tgts))
	labels := make(map[string]struct{}, len(tgts))
	found := false
	for idx, t := range tgts {
		labels[t.Label] = struct{}{}
		if t.Type == tgt.Type {
			if t.Arn == tgt.Arn && !update {
				return BucketRemoteAlreadyExists{Bucket: t.TargetBucket}
			}
			if t.Label == tgt.Label && !update {
				return BucketRemoteLabelInUse{Bucket: t.TargetBucket}
			}
			newtgts[idx] = *tgt
			found = true
			continue
		}
		newtgts[idx] = t
	}
	if _, ok := labels[tgt.Label]; ok && !update {
		return BucketRemoteLabelInUse{Bucket: tgt.TargetBucket}
	}
	if !found && !update {
		newtgts = append(newtgts, *tgt)
	}

	sys.targetsMap[bucket] = newtgts
	sys.arnRemotesMap[tgt.Arn] = clnt
	return nil
}

// RemoveTarget - removes a remote bucket target for this source bucket.
func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr string) error {
	if globalIsGateway {
		return nil
	}
	if arnStr == "" {
		return BucketRemoteArnInvalid{Bucket: bucket}
	}
	arn, err := madmin.ParseARN(arnStr)
	if err != nil {
		return BucketRemoteArnInvalid{Bucket: bucket}
	}
	if arn.Type == madmin.ReplicationService {
		if !globalIsErasure {
			return NotImplemented{}
		}
		// reject removal of remote target if replication configuration is present
		rcfg, err := getReplicationConfig(ctx, bucket)
		if err == nil && rcfg.RoleArn == arnStr {
			if _, ok := sys.arnRemotesMap[arnStr]; ok {
				return BucketRemoteRemoveDisallowed{Bucket: bucket}
			}
		}
	}
	if arn.Type == madmin.ILMService {
		// reject removal of remote target if lifecycle transition uses this arn
		config, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
		if err == nil && transitionSCInUse(ctx, config, bucket, arnStr) {
			if _, ok := sys.arnRemotesMap[arnStr]; ok {
				return BucketRemoteRemoveDisallowed{Bucket: bucket}
			}
		}
	}

	// delete ARN type from list of matching targets
	sys.Lock()
	defer sys.Unlock()
	found := false
	tgts, ok := sys.targetsMap[bucket]
	if !ok {
		return BucketRemoteTargetNotFound{Bucket: bucket}
	}
	targets := make([]madmin.BucketTarget, 0, len(tgts))
	for _, tgt := range tgts {
		if tgt.Arn != arnStr {
			targets = append(targets, tgt)
			continue
		}
		found = true
	}
	if !found {
		return BucketRemoteTargetNotFound{Bucket: bucket}
	}
	sys.targetsMap[bucket] = targets
	delete(sys.arnRemotesMap, arnStr)
	return nil
}

// GetRemoteTargetClient returns minio-go client for replication target instance
func (sys *BucketTargetSys) GetRemoteTargetClient(ctx context.Context, arn string) *miniogo.Core {
	sys.RLock()
	defer sys.RUnlock()
	return sys.arnRemotesMap[arn]
}

// GetRemoteTargetWithLabel returns bucket target given a target label
func (sys *BucketTargetSys) GetRemoteTargetWithLabel(ctx context.Context, bucket, targetLabel string) *madmin.BucketTarget {
	sys.RLock()
	defer sys.RUnlock()
	for _, t := range sys.targetsMap[bucket] {
		if strings.ToUpper(t.Label) == strings.ToUpper(targetLabel) {
			tgt := t.Clone()
			return &tgt
		}
	}
	return nil
}

// GetRemoteArnWithLabel returns bucket target's ARN given its target label
func (sys *BucketTargetSys) GetRemoteArnWithLabel(ctx context.Context, bucket, tgtLabel string) *madmin.ARN {
	tgt := sys.GetRemoteTargetWithLabel(ctx, bucket, tgtLabel)
	if tgt == nil {
		return nil
	}
	arn, err := madmin.ParseARN(tgt.Arn)
	if err != nil {
		return nil
	}
	return arn
}

// GetRemoteLabelWithArn returns a bucket target's label given its ARN
func (sys *BucketTargetSys) GetRemoteLabelWithArn(ctx context.Context, bucket, arnStr string) string {
	sys.RLock()
	defer sys.RUnlock()
	for _, t := range sys.targetsMap[bucket] {
		if t.Arn == arnStr {
			return t.Label
		}
	}
	return ""
}

// NewBucketTargetSys - creates new replication system.
func NewBucketTargetSys() *BucketTargetSys {
	return &BucketTargetSys{
		arnRemotesMap: make(map[string]*miniogo.Core),
		targetsMap:    make(map[string][]madmin.BucketTarget),
	}
}

// Init initializes the bucket targets subsystem for buckets which have targets configured.
func (sys *BucketTargetSys) Init(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) error {
	if objAPI == nil {
		return errServerNotInitialized
	}

	// In gateway mode, bucket targets is not supported.
	if globalIsGateway {
		return nil
	}

	// Load bucket targets once during boot in background.
	go sys.load(ctx, buckets, objAPI)
	return nil
}

// UpdateAllTargets updates target to reflect metadata updates
func (sys *BucketTargetSys) UpdateAllTargets(bucket string, tgts *madmin.BucketTargets) {
	if sys == nil {
		return
	}
	sys.Lock()
	defer sys.Unlock()
	if tgts == nil || tgts.Empty() {
		// remove target and arn association
		if tgts, ok := sys.targetsMap[bucket]; ok {
			for _, t := range tgts {
				delete(sys.arnRemotesMap, t.Arn)
			}
		}
		delete(sys.targetsMap, bucket)
		return
	}

	if len(tgts.Targets) > 0 {
		sys.targetsMap[bucket] = tgts.Targets
	}
	for _, tgt := range tgts.Targets {
		tgtClient, err := sys.getRemoteTargetClient(&tgt)
		if err != nil {
			continue
		}
		sys.arnRemotesMap[tgt.Arn] = tgtClient
	}
	sys.targetsMap[bucket] = tgts.Targets
}

// create minio-go clients for buckets having remote targets
func (sys *BucketTargetSys) load(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) {
	for _, bucket := range buckets {
		cfg, err := globalBucketMetadataSys.GetBucketTargetsConfig(bucket.Name)
		if err != nil {
			continue
		}
		if cfg == nil || cfg.Empty() {
			continue
		}
		if len(cfg.Targets) > 0 {
			sys.targetsMap[bucket.Name] = cfg.Targets
		}
		for _, tgt := range cfg.Targets {
			tgtClient, err := sys.getRemoteTargetClient(&tgt)
			if err != nil {
				continue
			}
			sys.arnRemotesMap[tgt.Arn] = tgtClient
		}
		sys.targetsMap[bucket.Name] = cfg.Targets
	}
}

// getRemoteTargetInstanceTransport contains a singleton roundtripper.
var getRemoteTargetInstanceTransport http.RoundTripper
var getRemoteTargetInstanceTransportOnce sync.Once

// Returns a minio-go Client configured to access remote host described in replication target config.
func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*miniogo.Core, error) {
	config := tcfg.Credentials
	creds := credentials.NewStaticV4(config.AccessKey, config.SecretKey, "")

	getRemoteTargetInstanceTransportOnce.Do(func() {
		getRemoteTargetInstanceTransport = newGatewayHTTPTransport(10 * time.Minute)
	})

	core, err := miniogo.NewCore(tcfg.URL().Host, &miniogo.Options{
		Creds:     creds,
		Secure:    tcfg.Secure,
		Transport: getRemoteTargetInstanceTransport,
	})
	return core, err
}

// getRemoteARN gets existing ARN for an endpoint or generates a new one.
func (sys *BucketTargetSys) getRemoteARN(bucket string, target *madmin.BucketTarget) string {
	if target == nil {
		return ""
	}
	tgts := sys.targetsMap[bucket]
	for _, tgt := range tgts {
		if tgt.Type == target.Type && tgt.TargetBucket == target.TargetBucket && target.URL().String() == tgt.URL().String() {
			return tgt.Arn
		}
	}
	if !madmin.ServiceType(target.Type).IsValid() {
		return ""
	}
	return generateARN(target)
}

// generate ARN that is unique to this target type
func generateARN(t *madmin.BucketTarget) string {
	hash := sha256.New()
	hash.Write([]byte(t.Type))
	hash.Write([]byte(t.Region))
	hash.Write([]byte(t.TargetBucket))
	hashSum := hex.EncodeToString(hash.Sum(nil))
	arn := madmin.ARN{
		Type:   t.Type,
		ID:     hashSum,
		Region: t.Region,
		Bucket: t.TargetBucket,
	}
	return arn.String()
}

// Returns parsed target config. If KMS is configured, remote target is decrypted
func parseBucketTargetConfig(bucket string, cdata, cmetadata []byte) (*madmin.BucketTargets, error) {
	var (
		data []byte
		err  error
		t    madmin.BucketTargets
		meta map[string]string
	)
	if len(cdata) == 0 {
		return nil, nil
	}
	data = cdata
	if len(cmetadata) != 0 {
		if err := json.Unmarshal(cmetadata, &meta); err != nil {
			return nil, err
		}
		if crypto.S3.IsEncrypted(meta) {
			if data, err = decryptBucketMetadata(cdata, bucket, meta, crypto.Context{bucket: bucket, bucketTargetsFile: bucketTargetsFile}); err != nil {
				return nil, err
			}
		}
	}

	if err = json.Unmarshal(data, &t); err != nil {
		return nil, err
	}
	return &t, nil
}
<commit_msg>fix: regression in adding new replication targets (#11257)<commit_after>/*
 * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"context"
	"encoding/hex"
	"encoding/json"
	"net/http"
	"strings"
	"sync"
	"time"

	minio "github.com/minio/minio-go/v7"
	miniogo "github.com/minio/minio-go/v7"
	"github.com/minio/minio-go/v7/pkg/credentials"
	"github.com/minio/minio/cmd/crypto"
	"github.com/minio/minio/pkg/bucket/versioning"
	"github.com/minio/minio/pkg/madmin"
	sha256 "github.com/minio/sha256-simd"
)

// BucketTargetSys represents bucket targets subsystem
type BucketTargetSys struct {
	sync.RWMutex
	arnRemotesMap map[string]*miniogo.Core
	targetsMap    map[string][]madmin.BucketTarget
}

// ListTargets lists bucket targets across tenant or for individual bucket, and returns
// results filtered by arnType
func (sys *BucketTargetSys) ListTargets(ctx context.Context, bucket, arnType string) (targets []madmin.BucketTarget) {
	if bucket != "" {
		if ts, err := sys.ListBucketTargets(ctx, bucket); err == nil {
			for _, t := range ts.Targets {
				if string(t.Type) == arnType || arnType == "" {
					targets = append(targets, t.Clone())
				}
			}
		}
		return targets
	}
	sys.RLock()
	defer sys.RUnlock()
	for _, tgts := range sys.targetsMap {
		for _, t := range tgts {
			if string(t.Type) == arnType || arnType == "" {
				targets = append(targets, t.Clone())
			}
		}
	}
	return
}

// ListBucketTargets - gets list of bucket targets for this bucket.
func (sys *BucketTargetSys) ListBucketTargets(ctx context.Context, bucket string) (*madmin.BucketTargets, error) {
	sys.RLock()
	defer sys.RUnlock()

	tgts, ok := sys.targetsMap[bucket]
	if ok {
		return &madmin.BucketTargets{Targets: tgts}, nil
	}
	return nil, BucketRemoteTargetNotFound{Bucket: bucket}
}

// SetTarget - sets a new minio-go client target for this bucket.
func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *madmin.BucketTarget, update bool) error {
	if globalIsGateway {
		return nil
	}
	if !tgt.Type.IsValid() && !update {
		return BucketRemoteArnTypeInvalid{Bucket: bucket}
	}
	clnt, err := sys.getRemoteTargetClient(tgt)
	if err != nil {
		return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
	}
	// validate if target credentials are ok
	if _, err = clnt.BucketExists(ctx, tgt.TargetBucket); err != nil {
		if minio.ToErrorResponse(err).Code == "NoSuchBucket" {
			return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
		}
		return BucketRemoteConnectionErr{Bucket: tgt.TargetBucket}
	}
	if tgt.Type == madmin.ReplicationService {
		if !globalIsErasure {
			return NotImplemented{}
		}
		if !globalBucketVersioningSys.Enabled(bucket) {
			return BucketReplicationSourceNotVersioned{Bucket: bucket}
		}
		vcfg, err := clnt.GetBucketVersioning(ctx, tgt.TargetBucket)
		if err != nil {
			return BucketRemoteConnectionErr{Bucket: tgt.TargetBucket}
		}
		if vcfg.Status != string(versioning.Enabled) {
			return BucketRemoteTargetNotVersioned{Bucket: tgt.TargetBucket}
		}
	}
	if tgt.Type == madmin.ILMService {
		if globalBucketVersioningSys.Enabled(bucket) {
			vcfg, err := clnt.GetBucketVersioning(ctx, tgt.TargetBucket)
			if err != nil {
				if minio.ToErrorResponse(err).Code == "NoSuchBucket" {
					return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
				}
				return BucketRemoteConnectionErr{Bucket: tgt.TargetBucket}
			}
			if vcfg.Status != string(versioning.Enabled) {
				return BucketRemoteTargetNotVersioned{Bucket: tgt.TargetBucket}
			}
		}
	}
	sys.Lock()
	defer sys.Unlock()

	tgts := sys.targetsMap[bucket]
	newtgts := make([]madmin.BucketTarget, len(tgts))
	labels := make(map[string]struct{}, len(tgts))
	found := false
	for idx, t := range tgts {
		labels[t.Label] = struct{}{}
		if t.Type == tgt.Type {
			if t.Arn == tgt.Arn && !update {
				return BucketRemoteAlreadyExists{Bucket: t.TargetBucket}
			}
			if t.Label == tgt.Label && !update {
				return BucketRemoteLabelInUse{Bucket: t.TargetBucket}
			}
			newtgts[idx] = *tgt
			found = true
			continue
		}
		newtgts[idx] = t
	}
	if _, ok := labels[tgt.Label]; ok && !update {
		return BucketRemoteLabelInUse{Bucket: tgt.TargetBucket}
	}
	if !found && !update {
		newtgts = append(newtgts, *tgt)
	}

	sys.targetsMap[bucket] = newtgts
	sys.arnRemotesMap[tgt.Arn] = clnt
	return nil
}

// RemoveTarget - removes a remote bucket target for this source bucket.
func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr string) error {
	if globalIsGateway {
		return nil
	}
	if arnStr == "" {
		return BucketRemoteArnInvalid{Bucket: bucket}
	}
	arn, err := madmin.ParseARN(arnStr)
	if err != nil {
		return BucketRemoteArnInvalid{Bucket: bucket}
	}
	if arn.Type == madmin.ReplicationService {
		if !globalIsErasure {
			return NotImplemented{}
		}
		// reject removal of remote target if replication configuration is present
		rcfg, err := getReplicationConfig(ctx, bucket)
		if err == nil && rcfg.RoleArn == arnStr {
			if _, ok := sys.arnRemotesMap[arnStr]; ok {
				return BucketRemoteRemoveDisallowed{Bucket: bucket}
			}
		}
	}
	if arn.Type == madmin.ILMService {
		// reject removal of remote target if lifecycle transition uses this arn
		config, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
		if err == nil && transitionSCInUse(ctx, config, bucket, arnStr) {
			if _, ok := sys.arnRemotesMap[arnStr]; ok {
				return BucketRemoteRemoveDisallowed{Bucket: bucket}
			}
		}
	}

	// delete ARN type from list of matching targets
	sys.Lock()
	defer sys.Unlock()
	found := false
	tgts, ok := sys.targetsMap[bucket]
	if !ok {
		return BucketRemoteTargetNotFound{Bucket: bucket}
	}
	targets := make([]madmin.BucketTarget, 0, len(tgts))
	for _, tgt := range tgts {
		if tgt.Arn != arnStr {
			targets = append(targets, tgt)
			continue
		}
		found = true
	}
	if !found {
		return BucketRemoteTargetNotFound{Bucket: bucket}
	}
	sys.targetsMap[bucket] = targets
	delete(sys.arnRemotesMap, arnStr)
	return nil
}

// GetRemoteTargetClient returns minio-go client for replication target instance
func (sys *BucketTargetSys) GetRemoteTargetClient(ctx context.Context, arn string) *miniogo.Core {
	sys.RLock()
	defer sys.RUnlock()
	return sys.arnRemotesMap[arn]
}

// GetRemoteTargetWithLabel returns bucket target given a target label
func (sys *BucketTargetSys) GetRemoteTargetWithLabel(ctx context.Context, bucket, targetLabel string) *madmin.BucketTarget {
	sys.RLock()
	defer sys.RUnlock()
	for _, t := range sys.targetsMap[bucket] {
		if strings.ToUpper(t.Label) == strings.ToUpper(targetLabel) {
			tgt := t.Clone()
			return &tgt
		}
	}
	return nil
}

// GetRemoteArnWithLabel returns bucket target's ARN given its target label
func (sys *BucketTargetSys) GetRemoteArnWithLabel(ctx context.Context, bucket, tgtLabel string) *madmin.ARN {
	tgt := sys.GetRemoteTargetWithLabel(ctx, bucket, tgtLabel)
	if tgt == nil {
		return nil
	}
	arn, err := madmin.ParseARN(tgt.Arn)
	if err != nil {
		return nil
	}
	return arn
}

// GetRemoteLabelWithArn returns a bucket target's label given its ARN
func (sys *BucketTargetSys) GetRemoteLabelWithArn(ctx context.Context, bucket, arnStr string) string {
	sys.RLock()
	defer sys.RUnlock()
	for _, t := range sys.targetsMap[bucket] {
		if t.Arn == arnStr {
			return t.Label
		}
	}
	return ""
}

// NewBucketTargetSys - creates new replication system.
func NewBucketTargetSys() *BucketTargetSys {
	return &BucketTargetSys{
		arnRemotesMap: make(map[string]*miniogo.Core),
		targetsMap:    make(map[string][]madmin.BucketTarget),
	}
}

// Init initializes the bucket targets subsystem for buckets which have targets configured.
func (sys *BucketTargetSys) Init(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) error {
	if objAPI == nil {
		return errServerNotInitialized
	}

	// In gateway mode, bucket targets is not supported.
	if globalIsGateway {
		return nil
	}

	// Load bucket targets once during boot in background.
	go sys.load(ctx, buckets, objAPI)
	return nil
}

// UpdateAllTargets updates target to reflect metadata updates
func (sys *BucketTargetSys) UpdateAllTargets(bucket string, tgts *madmin.BucketTargets) {
	if sys == nil {
		return
	}
	sys.Lock()
	defer sys.Unlock()
	if tgts == nil || tgts.Empty() {
		// remove target and arn association
		if tgts, ok := sys.targetsMap[bucket]; ok {
			for _, t := range tgts {
				delete(sys.arnRemotesMap, t.Arn)
			}
		}
		delete(sys.targetsMap, bucket)
		return
	}

	if len(tgts.Targets) > 0 {
		sys.targetsMap[bucket] = tgts.Targets
	}
	for _, tgt := range tgts.Targets {
		tgtClient, err := sys.getRemoteTargetClient(&tgt)
		if err != nil {
			continue
		}
		sys.arnRemotesMap[tgt.Arn] = tgtClient
	}
	sys.targetsMap[bucket] = tgts.Targets
}

// create minio-go clients for buckets having remote targets
func (sys *BucketTargetSys) load(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) {
	for _, bucket := range buckets {
		cfg, err := globalBucketMetadataSys.GetBucketTargetsConfig(bucket.Name)
		if err != nil {
			continue
		}
		if cfg == nil || cfg.Empty() {
			continue
		}
		if len(cfg.Targets) > 0 {
			sys.targetsMap[bucket.Name] = cfg.Targets
		}
		for _, tgt := range cfg.Targets {
			tgtClient, err := sys.getRemoteTargetClient(&tgt)
			if err != nil {
				continue
			}
			sys.arnRemotesMap[tgt.Arn] = tgtClient
		}
		sys.targetsMap[bucket.Name] = cfg.Targets
	}
}

// getRemoteTargetInstanceTransport contains a singleton roundtripper.
var getRemoteTargetInstanceTransport http.RoundTripper
var getRemoteTargetInstanceTransportOnce sync.Once

// Returns a minio-go Client configured to access remote host described in replication target config.
func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*miniogo.Core, error) {
	config := tcfg.Credentials
	creds := credentials.NewStaticV4(config.AccessKey, config.SecretKey, "")

	getRemoteTargetInstanceTransportOnce.Do(func() {
		getRemoteTargetInstanceTransport = newGatewayHTTPTransport(10 * time.Minute)
	})

	core, err := miniogo.NewCore(tcfg.URL().Host, &miniogo.Options{
		Creds:     creds,
		Secure:    tcfg.Secure,
		Transport: getRemoteTargetInstanceTransport,
	})
	return core, err
}

// getRemoteARN gets existing ARN for an endpoint or generates a new one.
func (sys *BucketTargetSys) getRemoteARN(bucket string, target *madmin.BucketTarget) string {
	if target == nil {
		return ""
	}
	tgts := sys.targetsMap[bucket]
	for _, tgt := range tgts {
		if tgt.Type == target.Type && tgt.TargetBucket == target.TargetBucket && target.URL().String() == tgt.URL().String() {
			return tgt.Arn
		}
	}
	if !madmin.ServiceType(target.Type).IsValid() {
		return ""
	}
	return generateARN(target)
}

// generate ARN that is unique to this target type
func generateARN(t *madmin.BucketTarget) string {
	hash := sha256.New()
	hash.Write([]byte(t.Type))
	hash.Write([]byte(t.Region))
	hash.Write([]byte(t.TargetBucket))
	hashSum := hex.EncodeToString(hash.Sum(nil))
	arn := madmin.ARN{
		Type:   t.Type,
		ID:     hashSum,
		Region: t.Region,
		Bucket: t.TargetBucket,
	}
	return arn.String()
}

// Returns parsed target config. If KMS is configured, remote target is decrypted
func parseBucketTargetConfig(bucket string, cdata, cmetadata []byte) (*madmin.BucketTargets, error) {
	var (
		data []byte
		err  error
		t    madmin.BucketTargets
		meta map[string]string
	)
	if len(cdata) == 0 {
		return nil, nil
	}
	data = cdata
	if len(cmetadata) != 0 {
		if err := json.Unmarshal(cmetadata, &meta); err != nil {
			return nil, err
		}
		if crypto.S3.IsEncrypted(meta) {
			if data, err = decryptBucketMetadata(cdata, bucket, meta, crypto.Context{bucket: bucket, bucketTargetsFile: bucketTargetsFile}); err != nil {
				return nil, err
			}
		}
	}

	if err = json.Unmarshal(data, &t); err != nil {
		return nil, err
	}
	return &t, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"github.com/containers/image/transports"
	"github.com/containers/storage/pkg/archive"
	"github.com/projectatomic/buildah"
	"github.com/urfave/cli"
)

var (
	commitFlags = []cli.Flag{
		cli.StringFlag{
			Name:  "name",
			Usage: "`name or ID` of the working container",
		},
		cli.StringFlag{
			Name:  "root",
			Usage: "root `directory` of the working container",
		},
		cli.StringFlag{
			Name:  "link",
			Usage: "`pathname` of a symbolic link to the root directory of the working container",
		},
		cli.BoolFlag{
			Name:  "do-not-compress",
			Usage: "don't compress layers",
		},
		cli.StringFlag{
			Name:  "output",
			Usage: "`name` of output image to write",
		},
		cli.StringFlag{
			Name:  "signature-policy",
			Usage: "`pathname` of signature policy file (not usually used)",
		},
	}
	commitDescription = "Writes a new image using the container's read-write layer and, if it is based\n   on an image, the layers of that image"
)

func commitCmd(c *cli.Context) error {
	args := c.Args()
	name := ""
	if c.IsSet("name") {
		name = c.String("name")
	}
	root := ""
	if c.IsSet("root") {
		root = c.String("root")
	}
	link := ""
	if c.IsSet("link") {
		link = c.String("link")
	}
	output := ""
	if c.IsSet("output") {
		output = c.String("output")
	}
	signaturePolicy := ""
	if c.IsSet("signature-policy") {
		signaturePolicy = c.String("signature-policy")
	}
	compress := archive.Uncompressed
	if !c.IsSet("do-not-compress") || !c.Bool("do-not-compress") {
		compress = archive.Gzip
	}
	if name == "" && root == "" && link == "" {
		if len(args) == 0 {
			return fmt.Errorf("either a container name or --root or --link, or some combination, must be specified")
		}
		name = args[0]
		args = args.Tail()
	}
	if output == "" {
		if len(args) == 0 {
			return fmt.Errorf("an image name or the --output flag must be specified")
		}
		output = args[0]
		args = args.Tail()
	}

	store, err := getStore(c)
	if err != nil {
		return err
	}

	builder, err := openBuilder(store, name, root, link)
	if err != nil {
		return fmt.Errorf("error reading build container %q: %v", name, err)
	}

	dest, err := transports.ParseImageName(output)
	if err != nil {
		return fmt.Errorf("error parsing target image name %q: %v", output, err)
	}

	options := buildah.CommitOptions{
		Compression:         compress,
		SignaturePolicyPath: signaturePolicy,
	}
	err = builder.Commit(dest, options)
	if err != nil {
		return fmt.Errorf("error committing container %q to %q: %v", builder.Container, output, err)
	}

	return nil
}
<commit_msg>commit: "do-not-compress" -> "disable-compression"<commit_after>package main

import (
	"fmt"

	"github.com/containers/image/transports"
	"github.com/containers/storage/pkg/archive"
	"github.com/projectatomic/buildah"
	"github.com/urfave/cli"
)

var (
	commitFlags = []cli.Flag{
		cli.StringFlag{
			Name:  "name",
			Usage: "`name or ID` of the working container",
		},
		cli.StringFlag{
			Name:  "root",
			Usage: "root `directory` of the working container",
		},
		cli.StringFlag{
			Name:  "link",
			Usage: "`pathname` of a symbolic link to the root directory of the working container",
		},
		cli.BoolFlag{
			Name:  "disable-compression",
			Usage: "don't compress layers",
		},
		cli.StringFlag{
			Name:  "output",
			Usage: "`name` of output image to write",
		},
		cli.StringFlag{
			Name:  "signature-policy",
			Usage: "`pathname` of signature policy file (not usually used)",
		},
	}
	commitDescription = "Writes a new image using the container's read-write layer and, if it is based\n   on an image, the layers of that image"
)

func commitCmd(c *cli.Context) error {
	args := c.Args()
	name := ""
	if c.IsSet("name") {
		name = c.String("name")
	}
	root := ""
	if c.IsSet("root") {
		root = c.String("root")
	}
	link := ""
	if c.IsSet("link") {
		link = c.String("link")
	}
	output := ""
	if c.IsSet("output") {
		output = c.String("output")
	}
	signaturePolicy := ""
	if c.IsSet("signature-policy") {
		signaturePolicy = c.String("signature-policy")
	}
	compress := archive.Uncompressed
	if !c.IsSet("disable-compression") || !c.Bool("disable-compression") {
		compress = archive.Gzip
	}
	if name == "" && root == "" && link == "" {
		if len(args) == 0 {
			return fmt.Errorf("either a container name or --root or --link, or some combination, must be specified")
		}
		name = args[0]
		args = args.Tail()
	}
	if output == "" {
		if len(args) == 0 {
			return fmt.Errorf("an image name or the --output flag must be specified")
		}
		output = args[0]
		args = args.Tail()
	}

	store, err := getStore(c)
	if err != nil {
		return err
	}

	builder, err := openBuilder(store, name, root, link)
	if err != nil {
		return fmt.Errorf("error reading build container %q: %v", name, err)
	}

	dest, err := transports.ParseImageName(output)
	if err != nil {
		return fmt.Errorf("error parsing target image name %q: %v", output, err)
	}

	options := buildah.CommitOptions{
		Compression:         compress,
		SignaturePolicyPath: signaturePolicy,
	}
	err = builder.Commit(dest, options)
	if err != nil {
		return fmt.Errorf("error committing container %q to %q: %v", builder.Container, output, err)
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"context"
	"sync"
)

func (er erasureObjects) getOnlineDisks() (newDisks []StorageAPI) {
	disks := er.getDisks()
	var wg sync.WaitGroup
	var mu sync.Mutex
	for _, i := range hashOrder(UTCNow().String(), len(disks)) {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			if disks[i-1] == nil {
				return
			}
			di, err := disks[i-1].DiskInfo(context.Background())
			if err != nil || di.Healing {
				// - Do not consume disks which are not reachable
				//   unformatted or simply not accessible for some reason.
				//
				// - Do not consume disks which are being healed
				//
				// - Future: skip busy disks
				return
			}

			mu.Lock()
			newDisks = append(newDisks, disks[i-1])
			mu.Unlock()
		}()
	}
	wg.Wait()
	return newDisks
}

func (er erasureObjects) getLoadBalancedLocalDisks() (newDisks []StorageAPI) {
	disks := er.getDisks()
	// Based on the random shuffling return back randomized disks.
	for _, i := range hashOrder(UTCNow().String(), len(disks)) {
		if disks[i-1] != nil && disks[i-1].IsLocal() {
			if disks[i-1].Healing() == nil && disks[i-1].IsOnline() {
				newDisks = append(newDisks, disks[i-1])
			}
		}
	}
	return newDisks
}

// getLoadBalancedDisks - fetches load balanced (sufficiently randomized) disk slice.
// ensures to skip disks if they are not healing and online.
func (er erasureObjects) getLoadBalancedDisks(optimized bool) []StorageAPI {
	disks := er.getDisks()

	if !optimized {
		var newDisks []StorageAPI
		for _, i := range hashOrder(UTCNow().String(), len(disks)) {
			newDisks = append(newDisks, disks[i-1])
		}
		return newDisks
	}

	var wg sync.WaitGroup
	var mu sync.Mutex
	newDisks := map[uint64][]StorageAPI{}
	// Based on the random shuffling return back randomized disks.
	for _, i := range hashOrder(UTCNow().String(), len(disks)) {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			if disks[i-1] == nil {
				return
			}
			di, err := disks[i-1].DiskInfo(context.Background())
			if err != nil || di.Healing {
				// - Do not consume disks which are not reachable
				//   unformatted or simply not accessible for some reason.
				//
				// - Do not consume disks which are being healed
				//
				// - Future: skip busy disks
				return
			}

			mu.Lock()
			// Capture disks usage wise upto resolution of MiB
			newDisks[di.Used/1024/1024] = append(newDisks[di.Used/1024/1024], disks[i-1])
			mu.Unlock()
		}()
	}
	wg.Wait()

	var max uint64
	for k := range newDisks {
		if k > max {
			max = k
		}
	}

	// Return disks which have maximum disk usage common.
	return newDisks[max]
}
<commit_msg>do not skip healing disks during deletes (#14394)<commit_after>// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"context"
	"sync"
)

func (er erasureObjects) getOnlineDisks() (newDisks []StorageAPI) {
	disks := er.getDisks()
	var wg sync.WaitGroup
	var mu sync.Mutex
	for _, i := range hashOrder(UTCNow().String(), len(disks)) {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			if disks[i-1] == nil {
				return
			}
			di, err := disks[i-1].DiskInfo(context.Background())
			if err != nil || di.Healing {
				// - Do not consume disks which are not reachable
				//   unformatted or simply not accessible for some reason.
				//
				// - Do not consume disks which are being healed
				//
				// - Future: skip busy disks
				return
			}

			mu.Lock()
			newDisks = append(newDisks, disks[i-1])
			mu.Unlock()
		}()
	}
	wg.Wait()
	return newDisks
}

func (er erasureObjects) getLoadBalancedLocalDisks() (newDisks []StorageAPI) {
	disks := er.getDisks()
	// Based on the random shuffling return back randomized disks.
	for _, i := range hashOrder(UTCNow().String(), len(disks)) {
		if disks[i-1] != nil && disks[i-1].IsLocal() {
			newDisks = append(newDisks, disks[i-1])
		}
	}
	return newDisks
}

// getLoadBalancedDisks - fetches load balanced (sufficiently randomized) disk slice.
// ensures to skip disks if they are not healing and online.
func (er erasureObjects) getLoadBalancedDisks(optimized bool) []StorageAPI {
	disks := er.getDisks()

	if !optimized {
		var newDisks []StorageAPI
		for _, i := range hashOrder(UTCNow().String(), len(disks)) {
			newDisks = append(newDisks, disks[i-1])
		}
		return newDisks
	}

	var wg sync.WaitGroup
	var mu sync.Mutex
	newDisks := map[uint64][]StorageAPI{}
	// Based on the random shuffling return back randomized disks.
	for _, i := range hashOrder(UTCNow().String(), len(disks)) {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			if disks[i-1] == nil {
				return
			}
			di, err := disks[i-1].DiskInfo(context.Background())
			if err != nil || di.Healing {
				// - Do not consume disks which are not reachable
				//   unformatted or simply not accessible for some reason.
				//
				// - Do not consume disks which are being healed
				//
				// - Future: skip busy disks
				return
			}

			mu.Lock()
			// Capture disks usage wise upto resolution of MiB
			newDisks[di.Used/1024/1024] = append(newDisks[di.Used/1024/1024], disks[i-1])
			mu.Unlock()
		}()
	}
	wg.Wait()

	var max uint64
	for k := range newDisks {
		if k > max {
			max = k
		}
	}

	// Return disks which have maximum disk usage common.
	return newDisks[max]
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/briandowns/spinner"
	"github.com/urfave/cli"
)

type resultOutput struct {
	HostIdentifier string              `json:"host"`
	Rows           []map[string]string `json:"rows"`
}

func queryCommand() cli.Command {
	var (
		flHosts, flLabels, flQuery string
		flDebug                    bool
	)
	return cli.Command{
		Name:      "query",
		Usage:     "Run a live query",
		UsageText: `fleetctl query [options]`,
		Flags: []cli.Flag{
			configFlag(),
			contextFlag(),
			cli.StringFlag{
				Name:        "hosts",
				EnvVar:      "HOSTS",
				Value:       "",
				Destination: &flHosts,
				Usage:       "Comma separated hostnames to target",
			},
			cli.StringFlag{
				Name:        "labels",
				EnvVar:      "LABELS",
				Value:       "",
				Destination: &flLabels,
				Usage:       "Comma separated label names to target",
			},
			cli.StringFlag{
				Name:        "query",
				EnvVar:      "QUERY",
				Value:       "",
				Destination: &flQuery,
				Usage:       "Query to run",
			},
			cli.BoolFlag{
				Name:        "debug",
				EnvVar:      "DEBUG",
				Destination: &flDebug,
				Usage:       "Whether or not to enable debug logging",
			},
		},
		Action: func(c *cli.Context) error {
			fleet, err := clientFromCLI(c)
			if err != nil {
				return err
			}

			if flHosts == "" && flLabels == "" {
				return errors.New("No hosts or labels targeted")
			}

			if flQuery == "" {
				return errors.New("No query specified")
			}

			hosts := strings.Split(flHosts, ",")
			labels := strings.Split(flLabels, ",")

			res, err := fleet.LiveQuery(flQuery, labels, hosts)
			if err != nil {
				return err
			}

			tick := time.NewTicker(100 * time.Millisecond)
			defer tick.Stop()

			// See charsets at
			// https://godoc.org/github.com/briandowns/spinner#pkg-variables
			s := spinner.New(spinner.CharSets[24], 200*time.Millisecond)
			s.Writer = os.Stderr
			s.Start()

			for {
				select {
				case hostResult := <-res.Results():
					out := resultOutput{hostResult.Host.HostName, hostResult.Rows}
					if err := json.NewEncoder(os.Stdout).Encode(out); err != nil {
						fmt.Fprintf(os.Stderr, "Error writing output: %s\n", err)
					}

				case err := <-res.Errors():
					fmt.Fprintf(os.Stderr, "Error talking to server: %s\n", err.Error())

				case <-tick.C:
					// Print status message to stderr
					status := res.Status()
					totals := res.Totals()
					var percentTotal, percentOnline float64
					var responded, total, online uint
					if status != nil && totals != nil {
						total = totals.Total
						online = totals.Online
						responded = status.ActualResults
						if total > 0 {
							percentTotal = 100 * float64(responded) / float64(total)
						}
						if online > 0 {
							percentOnline = 100 * float64(responded) / float64(online)
						}
					}
					s.Suffix = fmt.Sprintf("  %.f%% responded (%.f%% online) | %d/%d targeted hosts (%d/%d online)", percentTotal, percentOnline, responded, total, responded, online)
				}
			}
		},
	}
}
<commit_msg>Add --quiet and --exit arguments for fleetctl query (#1887)<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/briandowns/spinner"
	"github.com/urfave/cli"
)

type resultOutput struct {
	HostIdentifier string              `json:"host"`
	Rows           []map[string]string `json:"rows"`
}

func queryCommand() cli.Command {
	var (
		flHosts, flLabels, flQuery string
		flDebug, flQuiet, flExit   bool
	)
	return cli.Command{
		Name:      "query",
		Usage:     "Run a live query",
		UsageText: `fleetctl query [options]`,
		Flags: []cli.Flag{
			configFlag(),
			contextFlag(),
			cli.StringFlag{
				Name:        "hosts",
				EnvVar:      "HOSTS",
				Value:       "",
				Destination: &flHosts,
				Usage:       "Comma separated hostnames to target",
			},
			cli.StringFlag{
				Name:        "labels",
				EnvVar:      "LABELS",
				Value:       "",
				Destination: &flLabels,
				Usage:       "Comma separated label names to target",
			},
			cli.BoolFlag{
				Name:        "quiet",
				EnvVar:      "QUIET",
				Destination: &flQuiet,
				Usage:       "Only print results (no status information)",
			},
			cli.BoolFlag{
				Name:        "exit",
				EnvVar:      "EXIT",
				Destination: &flExit,
				Usage:       "Exit when 100% of online hosts have results returned",
			},
			cli.StringFlag{
				Name:        "query",
				EnvVar:      "QUERY",
				Value:       "",
				Destination: &flQuery,
				Usage:       "Query to run",
			},
			cli.BoolFlag{
				Name:        "debug",
				EnvVar:      "DEBUG",
				Destination: &flDebug,
				Usage:       "Whether or not to enable debug logging",
			},
		},
		Action: func(c *cli.Context) error {
			fleet, err := clientFromCLI(c)
			if err != nil {
				return err
			}

			if flHosts == "" && flLabels == "" {
				return errors.New("No hosts or labels targeted")
			}

			if flQuery == "" {
				return errors.New("No query specified")
			}

			hosts := strings.Split(flHosts, ",")
			labels := strings.Split(flLabels, ",")

			res, err := fleet.LiveQuery(flQuery, labels, hosts)
			if err != nil {
				return err
			}

			tick := time.NewTicker(100 * time.Millisecond)
			defer tick.Stop()

			// See charsets at
			// https://godoc.org/github.com/briandowns/spinner#pkg-variables
			s := spinner.New(spinner.CharSets[24], 200*time.Millisecond)
			s.Writer = os.Stderr
			if !flQuiet {
				s.Start()
			}

			for {
				select {
				case hostResult := <-res.Results():
					out := resultOutput{hostResult.Host.HostName, hostResult.Rows}
					if err := json.NewEncoder(os.Stdout).Encode(out); err != nil {
						fmt.Fprintf(os.Stderr, "Error writing output: %s\n", err)
					}

				case err := <-res.Errors():
					fmt.Fprintf(os.Stderr, "Error talking to server: %s\n", err.Error())

				case <-tick.C:
					// Print status message to stderr
					status := res.Status()
					totals := res.Totals()
					var percentTotal, percentOnline float64
					var responded, total, online uint
					if status != nil && totals != nil {
						total = totals.Total
						online = totals.Online
						responded = status.ActualResults
						if total > 0 {
							percentTotal = 100 * float64(responded) / float64(total)
						}
						if online > 0 {
							percentOnline = 100 * float64(responded) / float64(online)
						}
					}

					if responded >= online && flExit {
						return nil
					}

					if !flQuiet {
						s.Suffix = fmt.Sprintf(
							"  %.f%% responded (%.f%% online) | %d/%d targeted hosts (%d/%d online)",
							percentTotal, percentOnline,
							responded, total,
							responded, online,
						)
					}
				}
			}
		},
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"archive/tar"
	"archive/zip"
	"compress/gzip"
	"crypto/sha256"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"
)

const (
	currentVersionURL = "https://golang.org/VERSION?m=text"
	downloadURLPrefix = "https://storage.googleapis.com/golang"
)

// downloadGoVersion downloads and upacks the specific go version to dest/go.
func downloadGoVersion(version, ops, arch, dest string) error {
	suffix := "tar.gz"
	if ops == "windows" {
		suffix = "zip"
	}
	uri := fmt.Sprintf("%s/%s.%s-%s.%s", downloadURLPrefix, version, ops, arch, suffix)

	verbosef("Downloading %s", uri)

	resp, err := http.Get(uri)
	if err != nil {
		return fmt.Errorf("Downloading Go from %s failed: %v", uri, err)
	}
	if resp.StatusCode > 299 {
		return fmt.Errorf("Downloading Go from %s failed with HTTP status %s", resp.Status)
	}
	defer resp.Body.Close()

	tmpf, err := ioutil.TempFile("", "go")
	if err != nil {
		return err
	}
	defer os.Remove(tmpf.Name())

	h := sha256.New()

	w := io.MultiWriter(tmpf, h)
	if _, err := io.Copy(w, resp.Body); err != nil {
		return err
	}

	verbosef("Downloading SHA %s.sha256", uri)

	sresp, err := http.Get(uri + ".sha256")
	if err != nil {
		return fmt.Errorf("Downloading Go sha256 from %s.sha256 failed: %v", uri, err)
	}
	defer sresp.Body.Close()
	if sresp.StatusCode > 299 {
		return fmt.Errorf("Downloading Go sha256 from %s.sha256 failed with HTTP status %s", sresp.Status)
	}

	shasum, err := ioutil.ReadAll(sresp.Body)
	if err != nil {
		return err
	}

	// Check the shasum.
	sum := fmt.Sprintf("%x", h.Sum(nil))
	if sum != string(shasum) {
		return fmt.Errorf("Shasum mismatch %s vs. %s", sum, string(shasum))
	}

	unpackFunc := unpackTar
	if ops == "windows" {
		unpackFunc = unpackZip
	}
	if err := unpackFunc(tmpf.Name(), dest); err != nil {
		return fmt.Errorf("Unpacking Go to %s failed: %v", dest, err)
	}
	return nil
}

func unpack(dest, name string, fi os.FileInfo, r io.Reader) error {
	if strings.HasPrefix(name, "go/") {
		name = name[len("go/"):]
	}

	path := filepath.Join(dest, name)
	if fi.IsDir() {
		return os.MkdirAll(path, fi.Mode())
	}

	f, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, fi.Mode())
	if err != nil {
		return err
	}
	defer f.Close()

	_, err = io.Copy(f, r)
	return err
}

func unpackTar(src, dest string) error {
	r, err := os.Open(src)
	if err != nil {
		return err
	}
	defer r.Close()

	archive, err := gzip.NewReader(r)
	if err != nil {
		return err
	}
	defer archive.Close()

	tarReader := tar.NewReader(archive)

	for {
		header, err := tarReader.Next()
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}

		if err := unpack(dest, header.Name, header.FileInfo(), tarReader); err != nil {
			return err
		}
	}

	return nil
}

func unpackZip(src, dest string) error {
	zr, err := zip.OpenReader(src)
	if err != nil {
		return err
	}

	for _, f := range zr.File {
		fr, err := f.Open()
		if err != nil {
			return err
		}
		if err := unpack(dest, f.Name, f.FileInfo(), fr); err != nil {
			return err
		}
		fr.Close()
	}

	return nil
}

func getLatestGoVersion() (string, error) {
	resp, err := http.Get(currentVersionURL)
	if err != nil {
		return "", fmt.Errorf("Getting current Go version failed: %v", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode > 299 {
		b, _ := ioutil.ReadAll(io.LimitReader(resp.Body, 1024))
		return "", fmt.Errorf("Could not get current Go version: HTTP %d: %q", resp.StatusCode, b)
	}
	version, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	return strings.TrimSpace(string(version)), nil
}
<commit_msg>cmd/getgo: add a user-agent to download requests<commit_after>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"archive/tar"
	"archive/zip"
	"compress/gzip"
	"crypto/sha256"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"
)

const (
	currentVersionURL = "https://golang.org/VERSION?m=text"
	downloadURLPrefix = "https://storage.googleapis.com/golang"
)

// downloadGoVersion downloads and upacks the specific go version to dest/go.
func downloadGoVersion(version, ops, arch, dest string) error {
	suffix := "tar.gz"
	if ops == "windows" {
		suffix = "zip"
	}
	uri := fmt.Sprintf("%s/%s.%s-%s.%s", downloadURLPrefix, version, ops, arch, suffix)

	verbosef("Downloading %s", uri)

	req, err := http.NewRequest("GET", uri, nil)
	if err != nil {
		return err
	}
	req.Header.Add("User-Agent", fmt.Sprintf("golang.org-getgo/%s", version))

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return fmt.Errorf("Downloading Go from %s failed: %v", uri, err)
	}
	if resp.StatusCode > 299 {
		return fmt.Errorf("Downloading Go from %s failed with HTTP status %s", resp.Status)
	}
	defer resp.Body.Close()

	tmpf, err := ioutil.TempFile("", "go")
	if err != nil {
		return err
	}
	defer os.Remove(tmpf.Name())

	h := sha256.New()

	w := io.MultiWriter(tmpf, h)
	if _, err := io.Copy(w, resp.Body); err != nil {
		return err
	}

	verbosef("Downloading SHA %s.sha256", uri)

	sresp, err := http.Get(uri + ".sha256")
	if err != nil {
		return fmt.Errorf("Downloading Go sha256 from %s.sha256 failed: %v", uri, err)
	}
	defer sresp.Body.Close()
	if sresp.StatusCode > 299 {
		return fmt.Errorf("Downloading Go sha256 from %s.sha256 failed with HTTP status %s", sresp.Status)
	}

	shasum, err := ioutil.ReadAll(sresp.Body)
	if err != nil {
		return err
	}

	// Check the shasum.
	sum := fmt.Sprintf("%x", h.Sum(nil))
	if sum != string(shasum) {
		return fmt.Errorf("Shasum mismatch %s vs. %s", sum, string(shasum))
	}

	unpackFunc := unpackTar
	if ops == "windows" {
		unpackFunc = unpackZip
	}
	if err := unpackFunc(tmpf.Name(), dest); err != nil {
		return fmt.Errorf("Unpacking Go to %s failed: %v", dest, err)
	}
	return nil
}

func unpack(dest, name string, fi os.FileInfo, r io.Reader) error {
	if strings.HasPrefix(name, "go/") {
		name = name[len("go/"):]
	}

	path := filepath.Join(dest, name)
	if fi.IsDir() {
		return os.MkdirAll(path, fi.Mode())
	}

	f, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, fi.Mode())
	if err != nil {
		return err
	}
	defer f.Close()

	_, err = io.Copy(f, r)
	return err
}

func unpackTar(src, dest string) error {
	r, err := os.Open(src)
	if err != nil {
		return err
	}
	defer r.Close()

	archive, err := gzip.NewReader(r)
	if err != nil {
		return err
	}
	defer archive.Close()

	tarReader := tar.NewReader(archive)

	for {
		header, err := tarReader.Next()
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}

		if err := unpack(dest, header.Name, header.FileInfo(), tarReader); err != nil {
			return err
		}
	}

	return nil
}

func unpackZip(src, dest string) error {
	zr, err := zip.OpenReader(src)
	if err != nil {
		return err
	}

	for _, f := range zr.File {
		fr, err := f.Open()
		if err != nil {
			return err
		}
		if err := unpack(dest, f.Name, f.FileInfo(), fr); err != nil {
			return err
		}
		fr.Close()
	}

	return nil
}

func getLatestGoVersion() (string, error) {
	resp, err := http.Get(currentVersionURL)
	if err != nil {
		return "", fmt.Errorf("Getting current Go version failed: %v", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode > 299 {
		b, _ := ioutil.ReadAll(io.LimitReader(resp.Body, 1024))
		return "", fmt.Errorf("Could not get current Go version: HTTP %d: %q", resp.StatusCode, b)
	}
	version, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	return strings.TrimSpace(string(version)), nil
}
<|endoftext|>